2023-09-23T20:09:18.388 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T20:09:18.388 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832 --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --owner scheduled_teuthology@teuthology --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi --verbose -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832/orig.config.yaml 2023-09-23T20:09:18.453 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T20:09:18.536 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7401832' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi121.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi123.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T20:09:18.537 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T20:09:18.538 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T20:09:18.538 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T20:09:18.539 INFO:teuthology.task.internal:Checking packages... 2023-09-23T20:09:18.559 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T20:09:18.559 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:09:18.559 INFO:teuthology.packaging:ref: None 2023-09-23T20:09:18.559 INFO:teuthology.packaging:tag: None 2023-09-23T20:09:18.559 INFO:teuthology.packaging:branch: reef 2023-09-23T20:09:18.560 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:09:18.560 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T20:09:18.903 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460.gbb0daeff 2023-09-23T20:09:18.903 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T20:09:18.909 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T20:09:18.909 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T20:09:18.913 INFO:teuthology.task.internal:Saving configuration 2023-09-23T20:09:18.933 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T20:09:18.938 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T20:09:18.964 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi121.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:04:22.227064', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T20:09:18.992 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi123.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:04:22.228906', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T20:09:18.992 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T20:09:18.997 INFO:teuthology.task.internal:roles: ubuntu@smithi121.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T20:09:18.997 INFO:teuthology.task.internal:roles: ubuntu@smithi123.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T20:09:18.997 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T20:09:19.069 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f993042baf0>, signals=[15]) 2023-09-23T20:09:19.069 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T20:09:19.076 INFO:teuthology.task.internal:Opening connections... 2023-09-23T20:09:19.076 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi121.front.sepia.ceph.com 2023-09-23T20:09:19.077 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:09:19.150 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi123.front.sepia.ceph.com 2023-09-23T20:09:19.150 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:09:19.216 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T20:09:19.222 DEBUG:teuthology.orchestra.run.smithi121:> uname -m 2023-09-23T20:09:19.243 INFO:teuthology.orchestra.run.smithi121.stdout:x86_64 2023-09-23T20:09:19.243 DEBUG:teuthology.orchestra.run.smithi121:> cat /etc/os-release 2023-09-23T20:09:19.305 INFO:teuthology.orchestra.run.smithi121.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T20:09:19.305 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T20:09:19.305 INFO:teuthology.orchestra.run.smithi121.stdout:ID="rhel" 2023-09-23T20:09:19.305 INFO:teuthology.orchestra.run.smithi121.stdout:ID_LIKE="fedora" 2023-09-23T20:09:19.305 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION_ID="8.6" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:PLATFORM_ID="platform:el8" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:ANSI_COLOR="0;31" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T20:09:19.306 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T20:09:19.307 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T20:09:19.307 INFO:teuthology.lock.ops:Updating smithi121.front.sepia.ceph.com on lock server 2023-09-23T20:09:19.332 DEBUG:teuthology.orchestra.run.smithi123:> uname -m 2023-09-23T20:09:19.352 INFO:teuthology.orchestra.run.smithi123.stdout:x86_64 2023-09-23T20:09:19.352 DEBUG:teuthology.orchestra.run.smithi123:> cat /etc/os-release 2023-09-23T20:09:19.411 INFO:teuthology.orchestra.run.smithi123.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:ID="rhel" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:ID_LIKE="fedora" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION_ID="8.6" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:PLATFORM_ID="platform:el8" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:ANSI_COLOR="0;31" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T20:09:19.412 INFO:teuthology.orchestra.run.smithi123.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T20:09:19.413 INFO:teuthology.orchestra.run.smithi123.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T20:09:19.413 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:09:19.413 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T20:09:19.413 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T20:09:19.413 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T20:09:19.413 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T20:09:19.413 INFO:teuthology.lock.ops:Updating smithi123.front.sepia.ceph.com on lock server 2023-09-23T20:09:19.431 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T20:09:19.441 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T20:09:19.494 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T20:09:19.495 DEBUG:teuthology.orchestra.run.smithi121:> test '!' -e /home/ubuntu/cephtest 2023-09-23T20:09:19.497 DEBUG:teuthology.orchestra.run.smithi123:> test '!' -e /home/ubuntu/cephtest 2023-09-23T20:09:19.516 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T20:09:19.524 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T20:09:19.524 DEBUG:teuthology.orchestra.run.smithi121:> test -z $(ls -A /var/lib/ceph) 2023-09-23T20:09:19.559 DEBUG:teuthology.orchestra.run.smithi123:> test -z $(ls -A /var/lib/ceph) 2023-09-23T20:09:19.589 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T20:09:19.645 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T20:09:19.659 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T20:09:19.659 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T20:09:19.660 DEBUG:teuthology.orchestra.run.smithi121:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T20:09:19.660 DEBUG:teuthology.orchestra.run.smithi123:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T20:09:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:09:19.678 DEBUG:teuthology.orchestra.run.smithi121:> uname -r 2023-09-23T20:09:19.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:09:19.679 DEBUG:teuthology.orchestra.run.smithi123:> uname -r 2023-09-23T20:09:19.740 INFO:teuthology.orchestra.run.smithi123.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:19.740 INFO:teuthology.task.kernel:Running kernel on smithi123: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:19.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum install -y kernel 2023-09-23T20:09:19.741 INFO:teuthology.orchestra.run.smithi121.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:19.741 INFO:teuthology.task.kernel:Running kernel on smithi121: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:19.741 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum install -y kernel 2023-09-23T20:09:20.934 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:09:20.934 INFO:teuthology.orchestra.run.smithi123.stdout:Unable to read consumer identity 2023-09-23T20:09:20.971 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:09:20.972 INFO:teuthology.orchestra.run.smithi121.stdout:Unable to read consumer identity 2023-09-23T20:09:22.433 INFO:teuthology.orchestra.run.smithi123.stdout:Copr repo for python3-asyncssh owned by ceph 13 kB/s | 3.5 kB 00:00 2023-09-23T20:09:22.605 INFO:teuthology.orchestra.run.smithi121.stdout:Copr repo for python3-asyncssh owned by ceph 11 kB/s | 3.5 kB 00:00 2023-09-23T20:09:23.675 INFO:teuthology.orchestra.run.smithi123.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-09-23T20:09:24.564 INFO:teuthology.orchestra.run.smithi121.stdout:Extra Packages for Enterprise Linux 8.7 MB/s | 16 MB 00:01 2023-09-23T20:09:27.409 INFO:teuthology.orchestra.run.smithi123.stdout:lab-extras 89 kB/s | 24 kB 00:00 2023-09-23T20:09:28.021 INFO:teuthology.orchestra.run.smithi121.stdout:lab-extras 460 kB/s | 24 kB 00:00 2023-09-23T20:09:28.103 INFO:teuthology.orchestra.run.smithi123.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T20:09:28.164 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:09:28.165 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:09:28.165 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:09:28.294 DEBUG:teuthology.orchestra.run.smithi123:> echo no | sudo yum reinstall kernel || true 2023-09-23T20:09:28.624 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:09:28.624 INFO:teuthology.orchestra.run.smithi123.stdout:Unable to read consumer identity 2023-09-23T20:09:28.674 INFO:teuthology.orchestra.run.smithi121.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T20:09:28.725 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:09:28.726 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:09:28.726 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:09:28.868 DEBUG:teuthology.orchestra.run.smithi121:> echo no | sudo yum reinstall kernel || true 2023-09-23T20:09:28.882 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 08:09:27 PM UTC. 2023-09-23T20:09:28.922 INFO:teuthology.orchestra.run.smithi123.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:09:28.961 INFO:teuthology.orchestra.run.smithi123.stderr:Error: No packages marked for reinstall. 2023-09-23T20:09:29.008 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum reinstall -y kernel || true 2023-09-23T20:09:29.239 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:09:29.239 INFO:teuthology.orchestra.run.smithi121.stdout:Unable to read consumer identity 2023-09-23T20:09:29.339 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:09:29.339 INFO:teuthology.orchestra.run.smithi123.stdout:Unable to read consumer identity 2023-09-23T20:09:29.506 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 08:09:27 PM UTC. 2023-09-23T20:09:29.549 INFO:teuthology.orchestra.run.smithi121.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:09:29.584 INFO:teuthology.orchestra.run.smithi121.stderr:Error: No packages marked for reinstall. 2023-09-23T20:09:29.602 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 08:09:27 PM UTC. 2023-09-23T20:09:29.631 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum reinstall -y kernel || true 2023-09-23T20:09:29.645 INFO:teuthology.orchestra.run.smithi123.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:09:29.665 INFO:teuthology.orchestra.run.smithi123.stderr:Error: No packages marked for reinstall. 2023-09-23T20:09:29.711 DEBUG:teuthology.orchestra.run.smithi123:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T20:09:29.791 INFO:teuthology.orchestra.run.smithi123.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:29.792 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:29.792 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T20:09:29.792 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T20:09:29.792 DEBUG:teuthology.orchestra.run.smithi123:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T20:09:29.826 INFO:teuthology.orchestra.run.smithi123.stdout:ttyS1 2023-09-23T20:09:29.838 DEBUG:teuthology.parallel:result is None 2023-09-23T20:09:29.964 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:09:29.965 INFO:teuthology.orchestra.run.smithi121.stdout:Unable to read consumer identity 2023-09-23T20:09:30.222 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:00:03 ago on Sat 23 Sep 2023 08:09:27 PM UTC. 2023-09-23T20:09:30.261 INFO:teuthology.orchestra.run.smithi121.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T20:09:30.282 INFO:teuthology.orchestra.run.smithi121.stderr:Error: No packages marked for reinstall. 2023-09-23T20:09:30.333 DEBUG:teuthology.orchestra.run.smithi121:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T20:09:30.414 INFO:teuthology.orchestra.run.smithi121.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:30.414 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T20:09:30.414 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T20:09:30.414 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T20:09:30.414 DEBUG:teuthology.orchestra.run.smithi121:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T20:09:30.440 INFO:teuthology.orchestra.run.smithi121.stdout:ttyS1 2023-09-23T20:09:30.457 DEBUG:teuthology.parallel:result is None 2023-09-23T20:09:30.457 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T20:09:30.463 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T20:09:30.464 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T20:09:30.498 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T20:09:30.516 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T20:09:30.523 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T20:09:30.533 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T20:09:30.533 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T20:09:30.555 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T20:09:30.580 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T20:09:30.587 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T20:09:30.587 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T20:09:30.618 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T20:09:30.664 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:09:30.666 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:09:30.674 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:09:30.677 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T20:09:30.680 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T20:09:30.686 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T20:09:30.687 DEBUG:teuthology.orchestra.run.smithi121:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T20:09:30.689 DEBUG:teuthology.orchestra.run.smithi123:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T20:09:30.716 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T20:09:30.724 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T20:09:30.724 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T20:09:30.756 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T20:09:30.773 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:09:30.854 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:09:30.909 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:09:30.992 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:09:31.035 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:09:31.035 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T20:09:31.113 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:09:31.159 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T20:09:31.211 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:09:31.296 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T20:09:31.338 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:09:31.338 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T20:09:31.406 DEBUG:teuthology.orchestra.run.smithi121:> sudo service rsyslog restart 2023-09-23T20:09:31.408 DEBUG:teuthology.orchestra.run.smithi123:> sudo service rsyslog restart 2023-09-23T20:09:31.458 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T20:09:31.501 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T20:09:31.923 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T20:09:31.930 INFO:teuthology.task.internal:Starting timer... 2023-09-23T20:09:31.931 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T20:09:31.940 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T20:09:31.947 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T20:09:31.947 DEBUG:teuthology.orchestra.run.smithi121:> sudo service auditd rotate 2023-09-23T20:09:32.004 INFO:teuthology.orchestra.run.smithi121.stdout:Rotating logs: 2023-09-23T20:09:32.005 DEBUG:teuthology.orchestra.run.smithi123:> sudo service auditd rotate 2023-09-23T20:09:32.070 INFO:teuthology.orchestra.run.smithi123.stdout:Rotating logs: 2023-09-23T20:09:32.072 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T20:09:32.072 DEBUG:teuthology.orchestra.run.smithi121:> /usr/sbin/getenforce 2023-09-23T20:09:32.100 INFO:teuthology.orchestra.run.smithi121.stdout:Permissive 2023-09-23T20:09:32.100 DEBUG:teuthology.orchestra.run.smithi123:> /usr/sbin/getenforce 2023-09-23T20:09:32.123 INFO:teuthology.orchestra.run.smithi123.stdout:Permissive 2023-09-23T20:09:32.123 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi121.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi123.front.sepia.ceph.com': 'permissive'} 2023-09-23T20:09:32.123 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T20:09:32.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:09:32.168 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T20:09:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:09:32.195 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T20:09:32.195 DEBUG:teuthology.orchestra.run.smithi121:> sudo /usr/sbin/setenforce permissive 2023-09-23T20:09:32.234 DEBUG:teuthology.orchestra.run.smithi123:> sudo /usr/sbin/setenforce permissive 2023-09-23T20:09:32.262 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T20:09:32.271 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T20:09:32.276 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T20:09:32.384 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T20:09:32.399 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T20:09:32.400 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi121.front.sepia.ceph.com,smithi123.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T20:18:06.550 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi121.front.sepia.ceph.com'), Remote(name='ubuntu@smithi123.front.sepia.ceph.com')] 2023-09-23T20:18:06.551 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2023-09-23T20:18:06.551 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:18:06.621 DEBUG:teuthology.orchestra.run.smithi121:> true 2023-09-23T20:18:06.688 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi121.front.sepia.ceph.com' 2023-09-23T20:18:06.688 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2023-09-23T20:18:06.689 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:18:06.758 DEBUG:teuthology.orchestra.run.smithi123:> true 2023-09-23T20:18:06.824 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi123.front.sepia.ceph.com' 2023-09-23T20:18:06.825 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T20:18:06.835 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T20:18:06.835 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T20:18:06.836 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:18:06.838 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T20:18:06.838 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:18:06.870 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T20:18:06.886 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T20:18:06.896 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T20:18:06.911 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T20:18:06.953 INFO:teuthology.orchestra.run.smithi121.stderr:sudo: ntpd: command not found 2023-09-23T20:18:06.965 INFO:teuthology.orchestra.run.smithi121.stdout:506 Cannot talk to daemon 2023-09-23T20:18:06.980 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T20:18:06.988 INFO:teuthology.orchestra.run.smithi123.stderr:sudo: ntpd: command not found 2023-09-23T20:18:06.994 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T20:18:07.000 INFO:teuthology.orchestra.run.smithi123.stdout:506 Cannot talk to daemon 2023-09-23T20:18:07.016 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T20:18:07.030 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T20:18:07.055 INFO:teuthology.orchestra.run.smithi121.stderr:bash: ntpq: command not found 2023-09-23T20:18:07.059 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:18:07.059 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2023-09-23T20:18:07.059 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.059 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.059 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.059 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.111 INFO:teuthology.orchestra.run.smithi123.stderr:bash: ntpq: command not found 2023-09-23T20:18:07.114 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:18:07.115 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2023-09-23T20:18:07.115 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.115 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.115 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.115 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:18:07.116 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T20:18:07.125 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T20:18:07.126 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi121.front.sepia.ceph.com 2023-09-23T20:18:07.126 DEBUG:teuthology.orchestra.run.smithi121:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T20:18:07.126 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi123.front.sepia.ceph.com 2023-09-23T20:18:07.126 DEBUG:teuthology.orchestra.run.smithi123:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T20:18:07.494 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:07.530 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:08.281 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:24 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:08.336 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:24 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:10.028 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:18:10.029 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:18:10.029 INFO:teuthology.orchestra.run.smithi121.stdout: Package Architecture Version Repository Size 2023-09-23T20:18:10.029 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:18:10.030 INFO:teuthology.orchestra.run.smithi121.stdout:Resetting modules: 2023-09-23T20:18:10.030 INFO:teuthology.orchestra.run.smithi121.stdout: container-tools 2023-09-23T20:18:10.030 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:18:10.030 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:18:10.030 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:18:10.030 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:18:10.077 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:18:10.078 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:18:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: Package Architecture Version Repository Size 2023-09-23T20:18:10.078 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:18:10.078 INFO:teuthology.orchestra.run.smithi123.stdout:Resetting modules: 2023-09-23T20:18:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: container-tools 2023-09-23T20:18:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:18:10.079 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:18:10.079 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:18:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:18:10.991 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:27 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:11.038 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:27 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:14.073 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:18:14.131 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:18:14.512 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:14.823 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:15.313 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:31 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:15.628 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:32 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:17.144 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:18:17.237 INFO:teuthology.orchestra.run.smithi121.stdout:========================================================================================================================== 2023-09-23T20:18:17.237 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:18:17.237 INFO:teuthology.orchestra.run.smithi121.stdout:========================================================================================================================== 2023-09-23T20:18:17.237 INFO:teuthology.orchestra.run.smithi121.stdout:Installing group/module packages: 2023-09-23T20:18:17.237 INFO:teuthology.orchestra.run.smithi121.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout:Downgrading: 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T20:18:17.238 INFO:teuthology.orchestra.run.smithi121.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout:Installing module profiles: 2023-09-23T20:18:17.239 INFO:teuthology.orchestra.run.smithi121.stdout: container-tools/common 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout:Enabling module streams: 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout: container-tools 3.0 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout:========================================================================================================================== 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout:Install 7 Packages 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout:Downgrade 12 Packages 2023-09-23T20:18:17.240 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:18:17.241 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 51 M 2023-09-23T20:18:17.241 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:18:17.318 INFO:teuthology.orchestra.run.smithi121.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 690 kB/s | 53 kB 00:00 2023-09-23T20:18:17.321 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:18:17.335 INFO:teuthology.orchestra.run.smithi121.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 548 kB/s | 51 kB 00:00 2023-09-23T20:18:17.360 INFO:teuthology.orchestra.run.smithi121.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 764 kB/s | 91 kB 00:00 2023-09-23T20:18:17.394 INFO:teuthology.orchestra.run.smithi121.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 6.6 MB/s | 511 kB 00:00 2023-09-23T20:18:17.414 INFO:teuthology.orchestra.run.smithi123.stdout:========================================================================================================================== 2023-09-23T20:18:17.414 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:18:17.414 INFO:teuthology.orchestra.run.smithi123.stdout:========================================================================================================================== 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout:Installing group/module packages: 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:18:17.415 INFO:teuthology.orchestra.run.smithi123.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout:Downgrading: 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T20:18:17.416 INFO:teuthology.orchestra.run.smithi123.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout:Installing module profiles: 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout: container-tools/common 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout:Enabling module streams: 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout: container-tools 3.0 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout:========================================================================================================================== 2023-09-23T20:18:17.417 INFO:teuthology.orchestra.run.smithi123.stdout:Install 7 Packages 2023-09-23T20:18:17.418 INFO:teuthology.orchestra.run.smithi123.stdout:Downgrade 12 Packages 2023-09-23T20:18:17.418 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:18:17.418 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 51 M 2023-09-23T20:18:17.418 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:18:17.453 INFO:teuthology.orchestra.run.smithi121.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 584 kB/s | 69 kB 00:00 2023-09-23T20:18:17.486 INFO:teuthology.orchestra.run.smithi121.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 784 kB/s | 72 kB 00:00 2023-09-23T20:18:17.491 INFO:teuthology.orchestra.run.smithi123.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 706 kB/s | 51 kB 00:00 2023-09-23T20:18:17.516 INFO:teuthology.orchestra.run.smithi123.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 539 kB/s | 53 kB 00:00 2023-09-23T20:18:17.541 INFO:teuthology.orchestra.run.smithi123.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 738 kB/s | 91 kB 00:00 2023-09-23T20:18:17.567 INFO:teuthology.orchestra.run.smithi123.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.3 MB/s | 69 kB 00:00 2023-09-23T20:18:17.569 INFO:teuthology.orchestra.run.smithi121.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 14 MB/s | 2.9 MB 00:00 2023-09-23T20:18:17.594 INFO:teuthology.orchestra.run.smithi121.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 515 kB/s | 56 kB 00:00 2023-09-23T20:18:17.601 INFO:teuthology.orchestra.run.smithi123.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 4.6 MB/s | 511 kB 00:00 2023-09-23T20:18:17.626 INFO:teuthology.orchestra.run.smithi123.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-09-23T20:18:17.628 INFO:teuthology.orchestra.run.smithi121.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 868 kB/s | 51 kB 00:00 2023-09-23T20:18:17.668 INFO:teuthology.orchestra.run.smithi123.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.3 MB/s | 56 kB 00:00 2023-09-23T20:18:17.687 INFO:teuthology.orchestra.run.smithi121.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 5.4 MB/s | 322 kB 00:00 2023-09-23T20:18:17.718 INFO:teuthology.orchestra.run.smithi123.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 16 MB/s | 2.9 MB 00:00 2023-09-23T20:18:17.737 INFO:teuthology.orchestra.run.smithi121.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 817 kB/s | 41 kB 00:00 2023-09-23T20:18:17.743 INFO:teuthology.orchestra.run.smithi123.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 677 kB/s | 51 kB 00:00 2023-09-23T20:18:17.794 INFO:teuthology.orchestra.run.smithi123.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-09-23T20:18:17.836 INFO:teuthology.orchestra.run.smithi123.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 982 kB/s | 41 kB 00:00 2023-09-23T20:18:17.854 INFO:teuthology.orchestra.run.smithi121.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 27 MB/s | 11 MB 00:00 2023-09-23T20:18:17.920 INFO:teuthology.orchestra.run.smithi123.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 26 MB/s | 2.2 MB 00:00 2023-09-23T20:18:17.921 INFO:teuthology.orchestra.run.smithi121.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 12 MB/s | 2.2 MB 00:00 2023-09-23T20:18:18.086 INFO:teuthology.orchestra.run.smithi123.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 23 MB/s | 11 MB 00:00 2023-09-23T20:18:18.113 INFO:teuthology.orchestra.run.smithi121.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 25 MB/s | 6.4 MB 00:00 2023-09-23T20:18:18.239 INFO:teuthology.orchestra.run.smithi121.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 21 MB/s | 6.8 MB 00:00 2023-09-23T20:18:18.253 INFO:teuthology.orchestra.run.smithi123.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 19 MB/s | 6.4 MB 00:00 2023-09-23T20:18:18.362 INFO:teuthology.orchestra.run.smithi123.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 25 MB/s | 6.8 MB 00:00 2023-09-23T20:18:18.395 INFO:teuthology.orchestra.run.smithi123.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 7.5 MB/s | 1.1 MB 00:00 2023-09-23T20:18:18.421 INFO:teuthology.orchestra.run.smithi123.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 3.1 MB/s | 185 kB 00:00 2023-09-23T20:18:18.447 INFO:teuthology.orchestra.run.smithi121.stdout:(16/19): containernetworking-plugins-0.9.1-1.mo 22 MB/s | 19 MB 00:00 2023-09-23T20:18:18.454 INFO:teuthology.orchestra.run.smithi123.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 864 kB/s | 51 kB 00:00 2023-09-23T20:18:18.472 INFO:teuthology.orchestra.run.smithi121.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 792 kB/s | 185 kB 00:00 2023-09-23T20:18:18.498 INFO:teuthology.orchestra.run.smithi121.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.0 MB/s | 51 kB 00:00 2023-09-23T20:18:18.531 INFO:teuthology.orchestra.run.smithi121.stdout:(19/19): cockpit-podman-29-2.module+el8.6.0+148 2.6 MB/s | 1.1 MB 00:00 2023-09-23T20:18:18.532 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:18:18.532 INFO:teuthology.orchestra.run.smithi121.stdout:Total 39 MB/s | 51 MB 00:01 2023-09-23T20:18:18.638 INFO:teuthology.orchestra.run.smithi123.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 21 MB/s | 19 MB 00:00 2023-09-23T20:18:18.638 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:18:18.638 INFO:teuthology.orchestra.run.smithi123.stdout:Total 41 MB/s | 51 MB 00:01 2023-09-23T20:18:19.009 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:18:19.051 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:18:19.051 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:18:19.117 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:18:19.160 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:18:19.160 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:18:19.473 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:18:19.477 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:18:19.587 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:18:19.591 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:18:20.654 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:18:20.702 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T20:18:20.743 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:18:20.800 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T20:18:20.867 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T20:18:20.877 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T20:18:20.899 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T20:18:20.983 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T20:18:20.992 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T20:18:21.012 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T20:18:21.108 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T20:18:21.182 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T20:18:38.100 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T20:18:38.236 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T20:18:39.850 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T20:18:40.044 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T20:18:40.070 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T20:18:40.282 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T20:18:40.289 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T20:18:40.461 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T20:18:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T20:18:40.663 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T20:18:40.663 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T20:18:40.663 INFO:teuthology.orchestra.run.smithi123.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T20:18:40.663 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:18:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T20:18:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T20:18:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T20:18:40.921 INFO:teuthology.orchestra.run.smithi121.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T20:18:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:18:41.446 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T20:18:41.662 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T20:18:41.748 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T20:18:41.987 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T20:18:43.117 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T20:18:43.405 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T20:18:43.465 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T20:18:43.778 INFO:teuthology.orchestra.run.smithi123.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T20:18:43.866 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T20:18:44.082 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T20:18:44.279 INFO:teuthology.orchestra.run.smithi121.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T20:18:44.544 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T20:18:44.794 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T20:18:45.304 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T20:18:45.507 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T20:18:45.736 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T20:18:45.919 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T20:18:46.027 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T20:18:46.120 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T20:18:46.243 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T20:18:46.244 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T20:18:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T20:18:46.473 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T20:18:46.529 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T20:18:46.679 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T20:18:46.720 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T20:18:46.720 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T20:18:46.852 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T20:18:46.852 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T20:18:46.896 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T20:18:47.054 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T20:18:47.197 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T20:18:47.213 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T20:18:47.379 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T20:18:47.404 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T20:18:47.405 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T20:18:47.586 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T20:18:47.646 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T20:18:47.746 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T20:18:47.820 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T20:18:47.862 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T20:18:47.971 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T20:18:48.028 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T20:18:48.029 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T20:18:48.179 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T20:18:48.195 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T20:18:48.387 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T20:18:48.579 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T20:18:48.712 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T20:18:48.754 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T20:18:48.892 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T20:18:48.893 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T20:18:49.036 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T20:18:49.135 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T20:18:49.136 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T20:18:49.137 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T20:18:49.138 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T20:18:49.138 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T20:18:49.138 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T20:18:49.138 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T20:18:49.138 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T20:18:49.564 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T20:18:49.960 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T20:18:49.960 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T20:18:49.960 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T20:18:49.961 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T20:18:49.962 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T20:18:49.962 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T20:18:49.962 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T20:18:49.962 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T20:18:49.962 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T20:18:49.962 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T20:18:49.962 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T20:18:49.963 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T20:18:49.963 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T20:18:49.963 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T20:18:49.963 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T20:18:49.963 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T20:18:49.963 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T20:18:49.964 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T20:18:49.964 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T20:18:49.964 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T20:18:49.964 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T20:18:49.965 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T20:18:49.965 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T20:18:49.965 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T20:18:50.544 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T20:18:50.544 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:07 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:51.451 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T20:18:51.451 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:07 ago on Sat 23 Sep 2023 08:16:43 PM UTC. 2023-09-23T20:18:52.866 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:18:53.060 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:18:53.060 INFO:teuthology.orchestra.run.smithi123.stdout:Downgraded: 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.061 INFO:teuthology.orchestra.run.smithi123.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:18:53.062 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:18:53.246 DEBUG:teuthology.parallel:result is None 2023-09-23T20:18:53.853 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:18:54.072 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:18:54.072 INFO:teuthology.orchestra.run.smithi121.stdout:Downgraded: 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.073 INFO:teuthology.orchestra.run.smithi121.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:18:54.074 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:18:54.264 DEBUG:teuthology.parallel:result is None 2023-09-23T20:18:54.265 INFO:teuthology.run_tasks:Running task install... 2023-09-23T20:18:54.275 DEBUG:teuthology.task.install:project ceph 2023-09-23T20:18:54.275 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T20:18:54.275 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T20:18:54.275 INFO:teuthology.task.install:Using flavor: default 2023-09-23T20:18:54.281 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T20:18:54.281 INFO:teuthology.task.install:extra packages: [] 2023-09-23T20:18:54.281 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T20:18:54.281 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:18:54.281 INFO:teuthology.packaging:ref: None 2023-09-23T20:18:54.281 INFO:teuthology.packaging:tag: None 2023-09-23T20:18:54.281 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:18:54.282 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:18:54.282 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:18:54.283 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T20:18:54.284 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:18:54.284 INFO:teuthology.packaging:ref: None 2023-09-23T20:18:54.284 INFO:teuthology.packaging:tag: None 2023-09-23T20:18:54.284 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:18:54.284 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:18:54.284 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:18:54.539 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T20:18:54.540 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T20:18:54.693 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T20:18:54.694 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T20:18:54.746 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T20:18:54.747 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:18:54.747 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T20:18:54.776 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T20:18:54.776 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:18:54.776 INFO:teuthology.packaging:ref: None 2023-09-23T20:18:54.776 INFO:teuthology.packaging:tag: None 2023-09-23T20:18:54.776 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:18:54.776 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:18:54.776 DEBUG:teuthology.orchestra.run.smithi121:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T20:18:54.803 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T20:18:54.803 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:18:54.803 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T20:18:54.837 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T20:18:54.837 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:18:54.837 INFO:teuthology.packaging:ref: None 2023-09-23T20:18:54.837 INFO:teuthology.packaging:tag: None 2023-09-23T20:18:54.837 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:18:54.838 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:18:54.838 DEBUG:teuthology.orchestra.run.smithi123:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T20:18:54.843 DEBUG:teuthology.orchestra.run.smithi121:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T20:18:54.911 DEBUG:teuthology.orchestra.run.smithi123:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T20:18:54.921 DEBUG:teuthology.orchestra.run.smithi121:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T20:18:54.986 INFO:teuthology.orchestra.run.smithi121.stdout:check_obsoletes = 1 2023-09-23T20:18:54.988 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum clean all 2023-09-23T20:18:54.997 DEBUG:teuthology.orchestra.run.smithi123:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T20:18:55.028 INFO:teuthology.orchestra.run.smithi123.stdout:check_obsoletes = 1 2023-09-23T20:18:55.030 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum clean all 2023-09-23T20:18:55.357 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:55.425 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:55.632 INFO:teuthology.orchestra.run.smithi121.stdout:44 files removed 2023-09-23T20:18:55.679 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-radosgw 2023-09-23T20:18:55.689 INFO:teuthology.orchestra.run.smithi123.stdout:44 files removed 2023-09-23T20:18:55.736 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-radosgw 2023-09-23T20:18:56.011 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:56.067 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:18:56.713 INFO:teuthology.orchestra.run.smithi121.stdout:ceph packages for x86_64 248 kB/s | 78 kB 00:00 2023-09-23T20:18:56.754 INFO:teuthology.orchestra.run.smithi123.stdout:ceph packages for x86_64 256 kB/s | 78 kB 00:00 2023-09-23T20:18:57.022 INFO:teuthology.orchestra.run.smithi121.stdout:ceph noarch packages 33 kB/s | 9.1 kB 00:00 2023-09-23T20:18:57.163 INFO:teuthology.orchestra.run.smithi123.stdout:ceph noarch packages 24 kB/s | 9.1 kB 00:00 2023-09-23T20:18:57.413 INFO:teuthology.orchestra.run.smithi123.stdout:ceph source packages 6.3 kB/s | 1.5 kB 00:00 2023-09-23T20:18:57.448 INFO:teuthology.orchestra.run.smithi121.stdout:ceph source packages 3.6 kB/s | 1.5 kB 00:00 2023-09-23T20:18:57.589 INFO:teuthology.orchestra.run.smithi123.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-23T20:18:57.632 INFO:teuthology.orchestra.run.smithi121.stdout:Copr repo for python3-asyncssh owned by ceph 21 kB/s | 3.5 kB 00:00 2023-09-23T20:18:59.282 INFO:teuthology.orchestra.run.smithi121.stdout:Extra Packages for Enterprise Linux 9.7 MB/s | 16 MB 00:01 2023-09-23T20:19:00.090 INFO:teuthology.orchestra.run.smithi123.stdout:Extra Packages for Enterprise Linux 6.4 MB/s | 16 MB 00:02 2023-09-23T20:19:02.732 INFO:teuthology.orchestra.run.smithi121.stdout:lab-extras 365 kB/s | 24 kB 00:00 2023-09-23T20:19:03.598 INFO:teuthology.orchestra.run.smithi123.stdout:lab-extras 406 kB/s | 24 kB 00:00 2023-09-23T20:19:03.717 INFO:teuthology.orchestra.run.smithi121.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 58 MB/s | 53 MB 00:00 2023-09-23T20:19:04.450 INFO:teuthology.orchestra.run.smithi123.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 61 MB/s | 47 MB 00:00 2023-09-23T20:19:11.826 INFO:teuthology.orchestra.run.smithi121.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 57 MB/s | 47 MB 00:00 2023-09-23T20:19:12.268 INFO:teuthology.orchestra.run.smithi123.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 66 MB/s | 53 MB 00:00 2023-09-23T20:19:22.723 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:19:22.725 INFO:teuthology.orchestra.run.smithi121.stdout:==================================================================================================== 2023-09-23T20:19:22.725 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout:==================================================================================================== 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout:Upgrading: 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T20:19:22.726 INFO:teuthology.orchestra.run.smithi121.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T20:19:22.727 INFO:teuthology.orchestra.run.smithi121.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout:==================================================================================================== 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout:Install 21 Packages 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout:Upgrade 2 Packages 2023-09-23T20:19:22.728 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:19:22.729 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 63 M 2023-09-23T20:19:22.729 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:19:23.587 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:19:23.590 INFO:teuthology.orchestra.run.smithi123.stdout:==================================================================================================== 2023-09-23T20:19:23.590 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:19:23.590 INFO:teuthology.orchestra.run.smithi123.stdout:==================================================================================================== 2023-09-23T20:19:23.591 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:19:23.591 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:19:23.591 INFO:teuthology.orchestra.run.smithi123.stdout:Upgrading: 2023-09-23T20:19:23.591 INFO:teuthology.orchestra.run.smithi123.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:19:23.591 INFO:teuthology.orchestra.run.smithi123.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T20:19:23.591 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:19:23.591 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T20:19:23.592 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T20:19:23.592 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T20:19:23.592 INFO:teuthology.orchestra.run.smithi123.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T20:19:23.592 INFO:teuthology.orchestra.run.smithi123.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T20:19:23.592 INFO:teuthology.orchestra.run.smithi123.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T20:19:23.592 INFO:teuthology.orchestra.run.smithi123.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T20:19:23.593 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T20:19:23.594 INFO:teuthology.orchestra.run.smithi123.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T20:19:23.594 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T20:19:23.594 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T20:19:23.594 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T20:19:23.594 INFO:teuthology.orchestra.run.smithi123.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T20:19:23.594 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:19:23.594 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:19:23.595 INFO:teuthology.orchestra.run.smithi123.stdout:==================================================================================================== 2023-09-23T20:19:23.595 INFO:teuthology.orchestra.run.smithi123.stdout:Install 21 Packages 2023-09-23T20:19:23.595 INFO:teuthology.orchestra.run.smithi123.stdout:Upgrade 2 Packages 2023-09-23T20:19:23.595 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:19:23.596 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 63 M 2023-09-23T20:19:23.596 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:19:24.709 INFO:teuthology.orchestra.run.smithi121.stdout:(1/23): ceph-common-17.2.6-1123.g61431f6b.el8.x 15 MB/s | 23 MB 00:01 2023-09-23T20:19:24.776 INFO:teuthology.orchestra.run.smithi121.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 364 kB/s | 24 kB 00:00 2023-09-23T20:19:24.901 INFO:teuthology.orchestra.run.smithi121.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 6.2 MB/s | 796 kB 00:00 2023-09-23T20:19:25.010 INFO:teuthology.orchestra.run.smithi121.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 4.6 MB/s | 506 kB 00:00 2023-09-23T20:19:25.460 INFO:teuthology.orchestra.run.smithi121.stdout:(5/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8. 7.1 MB/s | 17 MB 00:02 2023-09-23T20:19:25.569 INFO:teuthology.orchestra.run.smithi121.stdout:(6/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 9.9 MB/s | 5.5 MB 00:00 2023-09-23T20:19:25.628 INFO:teuthology.orchestra.run.smithi121.stdout:(7/23): python3-ceph-argparse-17.2.6-1123.g6143 267 kB/s | 45 kB 00:00 2023-09-23T20:19:25.653 INFO:teuthology.orchestra.run.smithi121.stdout:(8/23): python3-ceph-common-17.2.6-1123.g61431f 1.4 MB/s | 115 kB 00:00 2023-09-23T20:19:25.728 INFO:teuthology.orchestra.run.smithi121.stdout:(9/23): python3-rados-17.2.6-1123.g61431f6b.el8 4.8 MB/s | 371 kB 00:00 2023-09-23T20:19:25.778 INFO:teuthology.orchestra.run.smithi121.stdout:(10/23): python3-cephfs-17.2.6-1123.g61431f6b.e 1.2 MB/s | 190 kB 00:00 2023-09-23T20:19:25.803 INFO:teuthology.orchestra.run.smithi121.stdout:(11/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 4.6 MB/s | 358 kB 00:00 2023-09-23T20:19:25.871 INFO:teuthology.orchestra.run.smithi121.stdout:(12/23): ceph-base-17.2.6-1123.g61431f6b.el8.x8 1.9 MB/s | 5.2 MB 00:02 2023-09-23T20:19:25.895 INFO:teuthology.orchestra.run.smithi121.stdout:(13/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 911 kB/s | 106 kB 00:00 2023-09-23T20:19:25.979 INFO:teuthology.orchestra.run.smithi121.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 542 kB/s | 59 kB 00:00 2023-09-23T20:19:26.005 INFO:teuthology.orchestra.run.smithi121.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 691 kB/s | 75 kB 00:00 2023-09-23T20:19:26.030 INFO:teuthology.orchestra.run.smithi121.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.3 MB/s | 306 kB 00:00 2023-09-23T20:19:26.097 INFO:teuthology.orchestra.run.smithi121.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 510 kB/s | 47 kB 00:00 2023-09-23T20:19:26.122 INFO:teuthology.orchestra.run.smithi121.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.8 MB/s | 259 kB 00:00 2023-09-23T20:19:26.156 INFO:teuthology.orchestra.run.smithi121.stdout:(19/23): thrift-0.13.0-2.el8.x86_64.rpm 9.9 MB/s | 1.7 MB 00:00 2023-09-23T20:19:26.181 INFO:teuthology.orchestra.run.smithi121.stdout:(20/23): python3-prettytable-0.7.2-14.el8.noarc 524 kB/s | 44 kB 00:00 2023-09-23T20:19:26.206 INFO:teuthology.orchestra.run.smithi121.stdout:(21/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.1 MB/s | 354 kB 00:00 2023-09-23T20:19:26.276 INFO:teuthology.orchestra.run.smithi123.stdout:(1/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8. 6.7 MB/s | 17 MB 00:02 2023-09-23T20:19:26.335 INFO:teuthology.orchestra.run.smithi123.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 419 kB/s | 24 kB 00:00 2023-09-23T20:19:26.410 INFO:teuthology.orchestra.run.smithi123.stdout:(3/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 2.0 MB/s | 5.2 MB 00:02 2023-09-23T20:19:26.443 INFO:teuthology.orchestra.run.smithi123.stdout:(4/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 7.2 MB/s | 796 kB 00:00 2023-09-23T20:19:26.465 INFO:teuthology.orchestra.run.smithi121.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 13 MB/s | 3.6 MB 00:00 2023-09-23T20:19:26.569 INFO:teuthology.orchestra.run.smithi123.stdout:(5/23): libradosstriper1-17.2.6-1123.g61431f6b. 3.1 MB/s | 506 kB 00:00 2023-09-23T20:19:26.669 INFO:teuthology.orchestra.run.smithi123.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 446 kB/s | 45 kB 00:00 2023-09-23T20:19:26.828 INFO:teuthology.orchestra.run.smithi123.stdout:(7/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 14 MB/s | 5.5 MB 00:00 2023-09-23T20:19:26.853 INFO:teuthology.orchestra.run.smithi123.stdout:(8/23): python3-ceph-common-17.2.6-1123.g61431f 628 kB/s | 115 kB 00:00 2023-09-23T20:19:26.878 INFO:teuthology.orchestra.run.smithi123.stdout:(9/23): python3-cephfs-17.2.6-1123.g61431f6b.el 3.7 MB/s | 190 kB 00:00 2023-09-23T20:19:26.945 INFO:teuthology.orchestra.run.smithi123.stdout:(10/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 5.2 MB/s | 358 kB 00:00 2023-09-23T20:19:26.970 INFO:teuthology.orchestra.run.smithi123.stdout:(11/23): python3-rados-17.2.6-1123.g61431f6b.el 3.1 MB/s | 371 kB 00:00 2023-09-23T20:19:26.973 INFO:teuthology.orchestra.run.smithi121.stdout:(23/23): librados2-17.2.6-1123.g61431f6b.el8.x8 4.2 MB/s | 3.5 MB 00:00 2023-09-23T20:19:26.976 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:19:26.976 INFO:teuthology.orchestra.run.smithi121.stdout:Total 15 MB/s | 63 MB 00:04 2023-09-23T20:19:27.004 INFO:teuthology.orchestra.run.smithi123.stdout:(12/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 1.8 MB/s | 106 kB 00:00 2023-09-23T20:19:27.022 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:19:27.067 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:19:27.067 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:19:27.255 INFO:teuthology.orchestra.run.smithi123.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 235 kB/s | 59 kB 00:00 2023-09-23T20:19:27.338 INFO:teuthology.orchestra.run.smithi123.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 834 kB/s | 306 kB 00:00 2023-09-23T20:19:27.355 INFO:teuthology.orchestra.run.smithi123.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 750 kB/s | 75 kB 00:00 2023-09-23T20:19:27.408 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:19:27.411 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:19:27.414 INFO:teuthology.orchestra.run.smithi123.stdout:(16/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 4.3 MB/s | 259 kB 00:00 2023-09-23T20:19:27.447 INFO:teuthology.orchestra.run.smithi123.stdout:(17/23): python3-prettytable-0.7.2-14.el8.noarc 1.3 MB/s | 44 kB 00:00 2023-09-23T20:19:27.498 INFO:teuthology.orchestra.run.smithi123.stdout:(18/23): librdkafka-0.11.4-3.el8.x86_64.rpm 6.9 MB/s | 354 kB 00:00 2023-09-23T20:19:27.531 INFO:teuthology.orchestra.run.smithi123.stdout:(19/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 1.4 MB/s | 47 kB 00:00 2023-09-23T20:19:27.590 INFO:teuthology.orchestra.run.smithi123.stdout:(20/23): thrift-0.13.0-2.el8.x86_64.rpm 6.9 MB/s | 1.7 MB 00:00 2023-09-23T20:19:28.040 INFO:teuthology.orchestra.run.smithi123.stdout:(21/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 8.0 MB/s | 3.6 MB 00:00 2023-09-23T20:19:28.307 INFO:teuthology.orchestra.run.smithi123.stdout:(22/23): ceph-common-17.2.6-1123.g61431f6b.el8. 5.2 MB/s | 23 MB 00:04 2023-09-23T20:19:28.491 INFO:teuthology.orchestra.run.smithi123.stdout:(23/23): librados2-17.2.6-1123.g61431f6b.el8.x8 3.6 MB/s | 3.5 MB 00:00 2023-09-23T20:19:28.494 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:19:28.495 INFO:teuthology.orchestra.run.smithi123.stdout:Total 13 MB/s | 63 MB 00:04 2023-09-23T20:19:28.506 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:19:28.542 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:19:28.589 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:19:28.589 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:19:28.702 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:19:28.941 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:19:28.944 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:19:29.079 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:19:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:19:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:19:29.600 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:19:29.883 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T20:19:29.986 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:19:30.110 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:19:30.180 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:19:30.540 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:19:30.547 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:19:30.729 INFO:teuthology.orchestra.run.smithi123.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:19:30.760 INFO:teuthology.orchestra.run.smithi121.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:19:30.815 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:19:30.895 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:19:30.985 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:19:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:19:31.162 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:19:31.187 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T20:19:31.362 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:19:31.489 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T20:19:31.748 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:19:31.963 INFO:teuthology.orchestra.run.smithi123.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:19:32.013 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:19:32.191 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T20:19:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-09-23T20:19:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:19:32.379 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-23T20:19:32.460 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:19:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T20:19:32.606 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-23T20:19:32.812 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:19:33.020 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:19:33.252 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T20:19:33.256 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T20:19:33.412 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:19:33.426 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:19:33.468 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:19:33.481 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:19:33.595 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T20:19:33.600 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T20:19:33.765 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:19:33.821 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T20:19:33.972 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:19:34.055 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T20:19:34.170 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T20:19:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T20:19:34.312 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:19:34.380 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:19:34.505 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T20:19:34.698 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T20:19:34.919 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T20:19:35.279 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T20:19:38.165 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:19:38.433 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:19:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:19:39.016 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:19:39.203 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:19:39.301 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:19:39.386 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:19:39.558 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:19:39.765 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:19:40.036 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:19:40.103 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:19:40.252 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:19:58.613 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:19:58.613 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /sys 2023-09-23T20:19:58.614 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /proc 2023-09-23T20:19:58.614 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /dev 2023-09-23T20:19:58.614 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /run 2023-09-23T20:19:58.614 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /mnt 2023-09-23T20:19:58.614 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /var/tmp 2023-09-23T20:19:58.614 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /home 2023-09-23T20:19:58.615 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /tmp 2023-09-23T20:19:58.615 INFO:teuthology.orchestra.run.smithi121.stdout:skipping the directory /dev 2023-09-23T20:19:58.615 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:19:59.227 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:19:59.227 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /sys 2023-09-23T20:19:59.227 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /proc 2023-09-23T20:19:59.227 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /dev 2023-09-23T20:19:59.227 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /run 2023-09-23T20:19:59.228 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /mnt 2023-09-23T20:19:59.228 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /var/tmp 2023-09-23T20:19:59.228 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /home 2023-09-23T20:19:59.228 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /tmp 2023-09-23T20:19:59.228 INFO:teuthology.orchestra.run.smithi123.stdout:skipping the directory /dev 2023-09-23T20:19:59.228 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:00.372 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:20:00.454 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:20:00.454 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:20:00.697 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:20:00.698 INFO:teuthology.orchestra.run.smithi121.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:20:00.891 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:20:00.972 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:20:00.972 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:20:01.185 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:20:01.185 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:20:01.332 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:20:01.332 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T20:20:01.332 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:20:01.332 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:20:01.333 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T20:20:01.334 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T20:20:01.758 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:20:01.758 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T20:20:01.759 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-09-23T20:20:01.760 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-09-23T20:20:01.761 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-09-23T20:20:01.761 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-09-23T20:20:01.761 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:20:01.761 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T20:20:01.761 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T20:20:02.760 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:20:02.761 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:00:51 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:03.070 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:20:03.070 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:00:51 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:05.180 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:20:05.367 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout:Upgraded: 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.368 INFO:teuthology.orchestra.run.smithi121.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.369 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.370 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.370 INFO:teuthology.orchestra.run.smithi121.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T20:20:05.370 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:05.370 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:20:05.477 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:20:05.526 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-test 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout:Upgraded: 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.663 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.664 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.665 INFO:teuthology.orchestra.run.smithi123.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T20:20:05.665 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.665 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.665 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:05.665 INFO:teuthology.orchestra.run.smithi123.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T20:20:05.665 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:05.665 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:20:05.813 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-test 2023-09-23T20:20:05.873 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:06.161 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:06.646 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:00:55 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:06.967 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:00:54 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:08.008 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout:============================================================================================ 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout:============================================================================================ 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T20:20:08.009 INFO:teuthology.orchestra.run.smithi121.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T20:20:08.010 INFO:teuthology.orchestra.run.smithi121.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T20:20:08.010 INFO:teuthology.orchestra.run.smithi121.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T20:20:08.010 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:08.010 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:20:08.010 INFO:teuthology.orchestra.run.smithi121.stdout:============================================================================================ 2023-09-23T20:20:08.010 INFO:teuthology.orchestra.run.smithi121.stdout:Install 6 Packages 2023-09-23T20:20:08.010 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:08.011 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 53 M 2023-09-23T20:20:08.011 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 233 M 2023-09-23T20:20:08.011 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:20:08.050 INFO:teuthology.orchestra.run.smithi121.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.7 MB/s | 70 kB 00:00 2023-09-23T20:20:08.142 INFO:teuthology.orchestra.run.smithi121.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 3.8 MB/s | 323 kB 00:00 2023-09-23T20:20:08.193 INFO:teuthology.orchestra.run.smithi121.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2023-09-23T20:20:08.226 INFO:teuthology.orchestra.run.smithi121.stdout:(4/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 810 kB/s | 174 kB 00:00 2023-09-23T20:20:08.251 INFO:teuthology.orchestra.run.smithi121.stdout:(5/6): jq-1.6-3.el8.x86_64.rpm 3.4 MB/s | 202 kB 00:00 2023-09-23T20:20:08.314 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:20:08.315 INFO:teuthology.orchestra.run.smithi123.stdout:============================================================================================ 2023-09-23T20:20:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:20:08.315 INFO:teuthology.orchestra.run.smithi123.stdout:============================================================================================ 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout:============================================================================================ 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout:Install 6 Packages 2023-09-23T20:20:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:08.317 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 53 M 2023-09-23T20:20:08.317 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 233 M 2023-09-23T20:20:08.317 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:20:08.353 INFO:teuthology.orchestra.run.smithi123.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.0 MB/s | 70 kB 00:00 2023-09-23T20:20:08.428 INFO:teuthology.orchestra.run.smithi123.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.5 MB/s | 323 kB 00:00 2023-09-23T20:20:08.470 INFO:teuthology.orchestra.run.smithi123.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-09-23T20:20:08.512 INFO:teuthology.orchestra.run.smithi123.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-23T20:20:08.554 INFO:teuthology.orchestra.run.smithi123.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 739 kB/s | 174 kB 00:00 2023-09-23T20:20:12.118 INFO:teuthology.orchestra.run.smithi121.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 13 MB/s | 52 MB 00:04 2023-09-23T20:20:12.120 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:20:12.120 INFO:teuthology.orchestra.run.smithi121.stdout:Total 13 MB/s | 53 MB 00:04 2023-09-23T20:20:12.157 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:20:12.178 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:20:12.178 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:20:12.450 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:20:12.454 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:20:12.645 INFO:teuthology.orchestra.run.smithi123.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 12 MB/s | 52 MB 00:04 2023-09-23T20:20:12.647 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:20:12.647 INFO:teuthology.orchestra.run.smithi123.stdout:Total 12 MB/s | 53 MB 00:04 2023-09-23T20:20:12.683 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:20:12.704 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:20:12.704 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:20:12.903 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:20:12.977 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:20:12.980 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:20:13.067 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:20:13.137 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:20:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T20:20:13.450 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:20:13.518 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T20:20:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:20:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:20:13.725 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T20:20:13.850 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T20:20:13.859 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:20:14.014 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T20:20:14.155 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T20:20:14.305 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:20:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:20:19.063 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:20:19.230 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:20:19.281 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:20:19.281 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T20:20:19.281 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T20:20:19.281 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T20:20:19.281 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T20:20:19.281 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T20:20:19.519 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:20:19.739 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:20:19.740 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T20:20:19.740 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T20:20:19.740 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T20:20:19.740 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T20:20:19.740 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T20:20:20.379 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T20:20:20.379 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:09 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:20.843 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T20:20:20.844 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:08 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:22.706 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:20:22.887 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:22.887 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:20:22.887 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:22.887 INFO:teuthology.orchestra.run.smithi121.stdout: jq-1.6-3.el8.x86_64 2023-09-23T20:20:22.888 INFO:teuthology.orchestra.run.smithi121.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:22.888 INFO:teuthology.orchestra.run.smithi121.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T20:20:22.888 INFO:teuthology.orchestra.run.smithi121.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T20:20:22.888 INFO:teuthology.orchestra.run.smithi121.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T20:20:22.888 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:22.888 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:20:23.015 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph 2023-09-23T20:20:23.148 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: jq-1.6-3.el8.x86_64 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T20:20:23.331 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:23.332 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:20:23.349 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:23.468 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph 2023-09-23T20:20:23.801 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:24.138 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:13 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:24.683 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:12 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:25.497 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:20:25.500 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================= 2023-09-23T20:20:25.500 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:20:25.500 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================= 2023-09-23T20:20:25.500 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:20:25.500 INFO:teuthology.orchestra.run.smithi121.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T20:20:25.500 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:20:25.500 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T20:20:25.501 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T20:20:25.502 INFO:teuthology.orchestra.run.smithi121.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================= 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout:Install 32 Packages 2023-09-23T20:20:25.503 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:25.504 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 28 M 2023-09-23T20:20:25.504 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 110 M 2023-09-23T20:20:25.504 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:20:26.043 INFO:teuthology.orchestra.run.smithi121.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 33 kB/s | 6.4 kB 00:00 2023-09-23T20:20:26.108 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================= 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================= 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T20:20:26.112 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T20:20:26.113 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T20:20:26.114 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T20:20:26.115 INFO:teuthology.orchestra.run.smithi123.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T20:20:26.116 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:26.116 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:20:26.116 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================= 2023-09-23T20:20:26.116 INFO:teuthology.orchestra.run.smithi123.stdout:Install 32 Packages 2023-09-23T20:20:26.116 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:26.116 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 28 M 2023-09-23T20:20:26.117 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 110 M 2023-09-23T20:20:26.117 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:20:26.359 INFO:teuthology.orchestra.run.smithi121.stdout:(2/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 4.5 MB/s | 2.3 MB 00:00 2023-09-23T20:20:26.401 INFO:teuthology.orchestra.run.smithi121.stdout:(3/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 2.8 MB/s | 1.6 MB 00:00 2023-09-23T20:20:26.412 INFO:teuthology.orchestra.run.smithi123.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 47 kB/s | 6.4 kB 00:00 2023-09-23T20:20:26.502 INFO:teuthology.orchestra.run.smithi121.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 2.3 MB/s | 238 kB 00:00 2023-09-23T20:20:26.593 INFO:teuthology.orchestra.run.smithi121.stdout:(5/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 7.1 MB/s | 3.9 MB 00:00 2023-09-23T20:20:26.702 INFO:teuthology.orchestra.run.smithi121.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 218 kB/s | 44 kB 00:00 2023-09-23T20:20:26.736 INFO:teuthology.orchestra.run.smithi121.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.3 MB/s | 185 kB 00:00 2023-09-23T20:20:26.795 INFO:teuthology.orchestra.run.smithi121.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 1.8 MB/s | 173 kB 00:00 2023-09-23T20:20:26.820 INFO:teuthology.orchestra.run.smithi121.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 4.5 MB/s | 384 kB 00:00 2023-09-23T20:20:26.845 INFO:teuthology.orchestra.run.smithi121.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 221 kB/s | 11 kB 00:00 2023-09-23T20:20:26.870 INFO:teuthology.orchestra.run.smithi121.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 350 kB/s | 18 kB 00:00 2023-09-23T20:20:26.895 INFO:teuthology.orchestra.run.smithi121.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 973 kB/s | 49 kB 00:00 2023-09-23T20:20:26.947 INFO:teuthology.orchestra.run.smithi121.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 767 kB/s | 59 kB 00:00 2023-09-23T20:20:26.979 INFO:teuthology.orchestra.run.smithi121.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.3 MB/s | 283 kB 00:00 2023-09-23T20:20:27.005 INFO:teuthology.orchestra.run.smithi121.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 272 kB/s | 16 kB 00:00 2023-09-23T20:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 373 kB/s | 19 kB 00:00 2023-09-23T20:20:27.055 INFO:teuthology.orchestra.run.smithi121.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 479 kB/s | 24 kB 00:00 2023-09-23T20:20:27.081 INFO:teuthology.orchestra.run.smithi121.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 580 kB/s | 29 kB 00:00 2023-09-23T20:20:27.087 INFO:teuthology.orchestra.run.smithi123.stdout:(2/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 1.9 MB/s | 1.6 MB 00:00 2023-09-23T20:20:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 545 kB/s | 27 kB 00:00 2023-09-23T20:20:27.148 INFO:teuthology.orchestra.run.smithi121.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 3.7 MB/s | 255 kB 00:00 2023-09-23T20:20:27.174 INFO:teuthology.orchestra.run.smithi121.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 3.7 MB/s | 251 kB 00:00 2023-09-23T20:20:27.199 INFO:teuthology.orchestra.run.smithi121.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.7 MB/s | 85 kB 00:00 2023-09-23T20:20:27.224 INFO:teuthology.orchestra.run.smithi121.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 459 kB/s | 23 kB 00:00 2023-09-23T20:20:27.308 INFO:teuthology.orchestra.run.smithi121.stdout:(24/32): python3-pycparser-2.14-14.el8.noarch.r 1.0 MB/s | 109 kB 00:00 2023-09-23T20:20:27.341 INFO:teuthology.orchestra.run.smithi121.stdout:(25/32): python3-cryptography-3.2.1-5.el8.x86_6 4.7 MB/s | 559 kB 00:00 2023-09-23T20:20:27.375 INFO:teuthology.orchestra.run.smithi121.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 3.5 MB/s | 238 kB 00:00 2023-09-23T20:20:27.400 INFO:teuthology.orchestra.run.smithi121.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 924 kB/s | 54 kB 00:00 2023-09-23T20:20:27.504 INFO:teuthology.orchestra.run.smithi123.stdout:(3/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 3.6 MB/s | 3.9 MB 00:01 2023-09-23T20:20:27.554 INFO:teuthology.orchestra.run.smithi123.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 4.7 MB/s | 238 kB 00:00 2023-09-23T20:20:27.595 INFO:teuthology.orchestra.run.smithi121.stdout:(28/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86 14 MB/s | 17 MB 00:01 2023-09-23T20:20:27.626 INFO:teuthology.orchestra.run.smithi121.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 155 kB/s | 39 kB 00:00 2023-09-23T20:20:27.638 INFO:teuthology.orchestra.run.smithi123.stdout:(5/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 1.7 MB/s | 2.3 MB 00:01 2023-09-23T20:20:27.651 INFO:teuthology.orchestra.run.smithi121.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 627 kB/s | 157 kB 00:00 2023-09-23T20:20:27.693 INFO:teuthology.orchestra.run.smithi121.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 6.7 MB/s | 457 kB 00:00 2023-09-23T20:20:27.709 INFO:teuthology.orchestra.run.smithi121.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.2 MB/s | 103 kB 00:00 2023-09-23T20:20:27.712 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:20:27.712 INFO:teuthology.orchestra.run.smithi121.stdout:Total 13 MB/s | 28 MB 00:02 2023-09-23T20:20:27.804 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:20:27.840 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:20:27.840 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:20:28.043 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:20:28.047 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:20:28.389 INFO:teuthology.orchestra.run.smithi123.stdout:(6/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86_ 13 MB/s | 17 MB 00:01 2023-09-23T20:20:28.514 INFO:teuthology.orchestra.run.smithi123.stdout:(7/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 45 kB/s | 44 kB 00:00 2023-09-23T20:20:28.997 INFO:teuthology.orchestra.run.smithi123.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 136 kB/s | 185 kB 00:01 2023-09-23T20:20:29.222 INFO:teuthology.orchestra.run.smithi123.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 49 kB/s | 11 kB 00:00 2023-09-23T20:20:29.414 INFO:teuthology.orchestra.run.smithi123.stdout:(10/32): python3-cherrypy-18.4.0-1.el8.noarch.r 427 kB/s | 384 kB 00:00 2023-09-23T20:20:29.448 INFO:teuthology.orchestra.run.smithi123.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 78 kB/s | 18 kB 00:00 2023-09-23T20:20:29.552 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:20:29.648 INFO:teuthology.orchestra.run.smithi123.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 209 kB/s | 49 kB 00:00 2023-09-23T20:20:29.707 INFO:teuthology.orchestra.run.smithi123.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 227 kB/s | 59 kB 00:00 2023-09-23T20:20:29.749 INFO:teuthology.orchestra.run.smithi123.stdout:(14/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 127 kB/s | 173 kB 00:01 2023-09-23T20:20:29.793 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T20:20:29.940 INFO:teuthology.orchestra.run.smithi123.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 67 kB/s | 16 kB 00:00 2023-09-23T20:20:29.991 INFO:teuthology.orchestra.run.smithi123.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 77 kB/s | 19 kB 00:00 2023-09-23T20:20:30.104 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T20:20:30.158 INFO:teuthology.orchestra.run.smithi123.stdout:(17/32): python3-pecan-1.3.2-9.el8.noarch.rpm 555 kB/s | 283 kB 00:00 2023-09-23T20:20:30.174 INFO:teuthology.orchestra.run.smithi123.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 102 kB/s | 24 kB 00:00 2023-09-23T20:20:30.233 INFO:teuthology.orchestra.run.smithi123.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 121 kB/s | 29 kB 00:00 2023-09-23T20:20:30.351 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T20:20:30.392 INFO:teuthology.orchestra.run.smithi123.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 118 kB/s | 27 kB 00:00 2023-09-23T20:20:30.574 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T20:20:30.584 INFO:teuthology.orchestra.run.smithi123.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 624 kB/s | 255 kB 00:00 2023-09-23T20:20:30.642 INFO:teuthology.orchestra.run.smithi123.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 341 kB/s | 85 kB 00:00 2023-09-23T20:20:30.668 INFO:teuthology.orchestra.run.smithi123.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 578 kB/s | 251 kB 00:00 2023-09-23T20:20:30.743 INFO:teuthology.orchestra.run.smithi123.stdout:(24/32): python3-pytz-2017.2-9.el8.noarch.rpm 539 kB/s | 54 kB 00:00 2023-09-23T20:20:30.768 INFO:teuthology.orchestra.run.smithi123.stdout:(25/32): python3-markupsafe-0.23-19.el8.x86_64. 386 kB/s | 39 kB 00:00 2023-09-23T20:20:30.785 INFO:teuthology.orchestra.run.smithi123.stdout:(26/32): python3-mako-1.0.6-13.el8.noarch.rpm 3.7 MB/s | 157 kB 00:00 2023-09-23T20:20:30.811 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T20:20:30.819 INFO:teuthology.orchestra.run.smithi123.stdout:(27/32): python3-werkzeug-0.12.2-4.el8.noarch.r 8.9 MB/s | 457 kB 00:00 2023-09-23T20:20:30.844 INFO:teuthology.orchestra.run.smithi123.stdout:(28/32): python3-zc-lockfile-2.0-2.el8.noarch.r 88 kB/s | 23 kB 00:00 2023-09-23T20:20:30.870 INFO:teuthology.orchestra.run.smithi123.stdout:(29/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.2 MB/s | 103 kB 00:00 2023-09-23T20:20:30.903 INFO:teuthology.orchestra.run.smithi123.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 1.3 MB/s | 109 kB 00:00 2023-09-23T20:20:30.920 INFO:teuthology.orchestra.run.smithi123.stdout:(31/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 5.2 MB/s | 238 kB 00:00 2023-09-23T20:20:30.945 INFO:teuthology.orchestra.run.smithi123.stdout:(32/32): python3-cryptography-3.2.1-5.el8.x86_6 5.4 MB/s | 559 kB 00:00 2023-09-23T20:20:30.950 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:20:30.950 INFO:teuthology.orchestra.run.smithi123.stdout:Total 5.9 MB/s | 28 MB 00:04 2023-09-23T20:20:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T20:20:31.040 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:20:31.077 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:20:31.077 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:20:31.223 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T20:20:31.281 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:20:31.285 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:20:31.584 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T20:20:31.858 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T20:20:32.062 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T20:20:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T20:20:32.356 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:20:32.539 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T20:20:32.565 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T20:20:32.760 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T20:20:32.797 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T20:20:32.971 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T20:20:33.004 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-09-23T20:20:33.185 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T20:20:33.331 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-09-23T20:20:33.341 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T20:20:33.533 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T20:20:33.588 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-09-23T20:20:33.694 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T20:20:33.760 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-09-23T20:20:33.925 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T20:20:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T20:20:34.491 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-09-23T20:20:34.895 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T20:20:34.897 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-09-23T20:20:35.106 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-09-23T20:20:35.357 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-09-23T20:20:35.399 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T20:20:35.549 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-09-23T20:20:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T20:20:35.718 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-09-23T20:20:35.890 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T20:20:35.938 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T20:20:36.109 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T20:20:36.237 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T20:20:36.306 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T20:20:36.513 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T20:20:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T20:20:36.672 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T20:20:36.799 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T20:20:36.893 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T20:20:37.030 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T20:20:37.063 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:20:37.255 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T20:20:37.432 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T20:20:37.746 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T20:20:38.024 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T20:20:38.219 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T20:20:38.523 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T20:20:38.534 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:20:38.746 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:20:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T20:20:39.138 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:20:39.165 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T20:20:39.304 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:20:39.326 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:20:39.519 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:20:39.670 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:20:39.688 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:20:39.876 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T20:20:40.241 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T20:20:40.242 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T20:20:40.243 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T20:20:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:20:40.976 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:20:41.368 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:20:41.510 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:20:41.725 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:20:41.761 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T20:20:41.761 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:30 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:41.876 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:20:41.894 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:20:42.041 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:20:42.380 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:20:42.380 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T20:20:42.380 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T20:20:42.380 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T20:20:42.380 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T20:20:42.380 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T20:20:42.381 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-09-23T20:20:42.382 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-09-23T20:20:42.383 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-09-23T20:20:43.915 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-09-23T20:20:43.915 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:31 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:44.054 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:20:44.238 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:44.238 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T20:20:44.239 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T20:20:44.240 INFO:teuthology.orchestra.run.smithi121.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:44.241 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:20:44.386 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-base 2023-09-23T20:20:44.734 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:45.520 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:46.272 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:20:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:46.460 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:20:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T20:20:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T20:20:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T20:20:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T20:20:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T20:20:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T20:20:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T20:20:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T20:20:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:46.463 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:20:46.617 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-base 2023-09-23T20:20:46.849 INFO:teuthology.orchestra.run.smithi121.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:20:46.958 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:20:46.959 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:20:46.959 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:20:46.962 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:47.074 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install cephadm 2023-09-23T20:20:47.418 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:47.757 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:35 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:48.242 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:37 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:49.082 INFO:teuthology.orchestra.run.smithi123.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:20:49.190 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:20:49.191 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:20:49.191 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:20:49.305 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install cephadm 2023-09-23T20:20:49.604 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:20:49.605 INFO:teuthology.orchestra.run.smithi121.stdout:Install 1 Package 2023-09-23T20:20:49.606 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:49.606 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 80 k 2023-09-23T20:20:49.606 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 363 k 2023-09-23T20:20:49.606 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:20:49.648 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:49.766 INFO:teuthology.orchestra.run.smithi121.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 498 kB/s | 80 kB 00:00 2023-09-23T20:20:49.767 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:20:49.767 INFO:teuthology.orchestra.run.smithi121.stdout:Total 494 kB/s | 80 kB 00:00 2023-09-23T20:20:49.767 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:20:49.775 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:20:49.775 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:20:49.777 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:20:49.781 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:20:49.846 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:20:50.466 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:38 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:51.663 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:51.883 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:20:51.883 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout:Install 1 Package 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 80 k 2023-09-23T20:20:51.884 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 363 k 2023-09-23T20:20:51.885 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:20:52.278 INFO:teuthology.orchestra.run.smithi123.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 204 kB/s | 80 kB 00:00 2023-09-23T20:20:52.278 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:20:52.279 INFO:teuthology.orchestra.run.smithi123.stdout:Total 203 kB/s | 80 kB 00:00 2023-09-23T20:20:52.279 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:20:52.287 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:20:52.287 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:20:52.289 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:20:52.293 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:20:52.341 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:20:52.974 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:53.142 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:53.965 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:54.118 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:54.205 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:54.205 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:54.286 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:55.330 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:20:55.331 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:56.784 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:20:56.961 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:56.961 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:20:56.961 INFO:teuthology.orchestra.run.smithi121.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:20:56.961 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:56.961 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:20:57.086 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T20:20:57.443 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:57.765 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:20:57.943 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:57.943 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:20:57.944 INFO:teuthology.orchestra.run.smithi123.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:20:57.944 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:20:57.944 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:20:58.083 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T20:20:58.232 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:47 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:20:58.439 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:20:59.249 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:47 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:20:59.607 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:20:59.607 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:20:59.607 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repo Size 2023-09-23T20:20:59.607 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:20:59.608 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:20:59.608 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T20:20:59.608 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:59.608 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:20:59.608 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:20:59.608 INFO:teuthology.orchestra.run.smithi121.stdout:Install 1 Package 2023-09-23T20:20:59.608 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:20:59.609 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 161 k 2023-09-23T20:20:59.609 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 490 k 2023-09-23T20:20:59.609 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:20:59.804 INFO:teuthology.orchestra.run.smithi121.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 825 kB/s | 161 kB 00:00 2023-09-23T20:20:59.805 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:20:59.805 INFO:teuthology.orchestra.run.smithi121.stdout:Total 820 kB/s | 161 kB 00:00 2023-09-23T20:20:59.805 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:20:59.815 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:20:59.816 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:20:59.867 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:20:59.871 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:20:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:21:00.183 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:21:00.572 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:21:00.618 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repo Size 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:21:00.619 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:21:00.620 INFO:teuthology.orchestra.run.smithi123.stdout:Install 1 Package 2023-09-23T20:21:00.620 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:00.620 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 161 k 2023-09-23T20:21:00.620 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 490 k 2023-09-23T20:21:00.620 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:21:00.799 INFO:teuthology.orchestra.run.smithi123.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 902 kB/s | 161 kB 00:00 2023-09-23T20:21:00.800 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:00.801 INFO:teuthology.orchestra.run.smithi123.stdout:Total 895 kB/s | 161 kB 00:00 2023-09-23T20:21:00.801 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:21:00.810 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:21:00.810 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:21:00.862 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:21:00.866 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:21:00.983 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:21:01.174 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:21:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:21:01.616 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:21:01.617 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:50 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:02.537 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:21:02.537 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:50 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:03.956 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:21:04.135 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:04.136 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:21:04.136 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:21:04.136 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:04.136 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:21:04.286 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-mgr 2023-09-23T20:21:04.631 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:04.933 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:21:05.110 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:05.111 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:21:05.111 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:21:05.111 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:05.111 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:21:05.247 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-mgr 2023-09-23T20:21:05.485 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:05.592 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:06.814 INFO:teuthology.orchestra.run.smithi121.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:21:06.924 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:21:06.925 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:21:06.926 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:21:07.044 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T20:21:07.213 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:55 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:07.386 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:08.430 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:08.541 INFO:teuthology.orchestra.run.smithi123.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:21:08.652 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:21:08.653 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:21:08.653 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:21:08.775 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T20:21:09.116 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:09.789 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:21:09.790 INFO:teuthology.orchestra.run.smithi121.stdout:=================================================================================================== 2023-09-23T20:21:09.790 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:21:09.790 INFO:teuthology.orchestra.run.smithi121.stdout:=================================================================================================== 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:21:09.791 INFO:teuthology.orchestra.run.smithi121.stdout:=================================================================================================== 2023-09-23T20:21:09.792 INFO:teuthology.orchestra.run.smithi121.stdout:Install 6 Packages 2023-09-23T20:21:09.792 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:09.792 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 2.0 M 2023-09-23T20:21:09.792 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 8.0 M 2023-09-23T20:21:09.792 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:21:09.963 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:10.560 INFO:teuthology.orchestra.run.smithi121.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 67 kB/s | 23 kB 00:00 2023-09-23T20:21:10.577 INFO:teuthology.orchestra.run.smithi121.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 40 kB/s | 14 kB 00:00 2023-09-23T20:21:10.669 INFO:teuthology.orchestra.run.smithi121.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 3.7 MB/s | 1.7 MB 00:00 2023-09-23T20:21:10.744 INFO:teuthology.orchestra.run.smithi121.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 570 kB/s | 43 kB 00:00 2023-09-23T20:21:10.886 INFO:teuthology.orchestra.run.smithi121.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 103 kB/s | 34 kB 00:00 2023-09-23T20:21:11.070 INFO:teuthology.orchestra.run.smithi121.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 399 kB/s | 196 kB 00:00 2023-09-23T20:21:11.073 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:11.073 INFO:teuthology.orchestra.run.smithi121.stdout:Total 1.6 MB/s | 2.0 MB 00:01 2023-09-23T20:21:11.086 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:21:11.096 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:21:11.096 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:21:11.158 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:21:11.162 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:21:11.328 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:21:11.329 INFO:teuthology.orchestra.run.smithi123.stdout:=================================================================================================== 2023-09-23T20:21:11.329 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:21:11.329 INFO:teuthology.orchestra.run.smithi123.stdout:=================================================================================================== 2023-09-23T20:21:11.329 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout:=================================================================================================== 2023-09-23T20:21:11.330 INFO:teuthology.orchestra.run.smithi123.stdout:Install 6 Packages 2023-09-23T20:21:11.331 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:11.331 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 2.0 M 2023-09-23T20:21:11.331 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 8.0 M 2023-09-23T20:21:11.331 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:21:11.485 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:21:11.521 INFO:teuthology.orchestra.run.smithi123.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1123.g6143 100 kB/s | 14 kB 00:00 2023-09-23T20:21:11.546 INFO:teuthology.orchestra.run.smithi123.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1123.g614 136 kB/s | 23 kB 00:00 2023-09-23T20:21:11.689 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T20:21:11.747 INFO:teuthology.orchestra.run.smithi123.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 4.6 MB/s | 1.7 MB 00:00 2023-09-23T20:21:11.796 INFO:teuthology.orchestra.run.smithi123.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 858 kB/s | 43 kB 00:00 2023-09-23T20:21:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T20:21:11.884 INFO:teuthology.orchestra.run.smithi123.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 93 kB/s | 34 kB 00:00 2023-09-23T20:21:12.082 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T20:21:12.203 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T20:21:12.541 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T20:21:12.760 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:21:12.905 INFO:teuthology.orchestra.run.smithi123.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 145 kB/s | 196 kB 00:01 2023-09-23T20:21:12.907 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:12.908 INFO:teuthology.orchestra.run.smithi123.stdout:Total 1.3 MB/s | 2.0 MB 00:01 2023-09-23T20:21:12.920 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:21:12.930 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:21:12.930 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:21:12.983 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:21:12.983 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T20:21:12.984 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T20:21:12.984 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T20:21:12.984 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T20:21:12.984 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T20:21:12.991 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:21:12.995 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:21:13.316 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:21:13.480 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T20:21:13.694 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T20:21:14.035 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T20:21:14.035 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:03 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:14.335 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T20:21:14.844 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T20:21:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T20:21:15.389 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:21:15.602 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:21:15.602 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T20:21:15.603 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T20:21:15.603 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T20:21:15.603 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T20:21:15.603 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T20:21:16.350 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:21:16.530 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:16.530 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:21:16.530 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:16.530 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:16.530 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:16.530 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T20:21:16.530 INFO:teuthology.orchestra.run.smithi121.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T20:21:16.531 INFO:teuthology.orchestra.run.smithi121.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T20:21:16.531 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:16.531 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:21:16.660 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T20:21:16.669 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T20:21:16.669 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:04 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:17.007 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:17.828 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:06 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:19.123 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:21:19.205 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:21:19.206 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================================= 2023-09-23T20:21:19.206 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================================= 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T20:21:19.207 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================================= 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout:Install 8 Packages 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 35 M 2023-09-23T20:21:19.208 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 210 M 2023-09-23T20:21:19.209 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:21:19.303 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:19.303 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:19.304 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:21:19.328 INFO:teuthology.orchestra.run.smithi121.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 5.3 MB/s | 644 kB 00:00 2023-09-23T20:21:19.375 INFO:teuthology.orchestra.run.smithi121.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.0 MB/s | 171 kB 00:00 2023-09-23T20:21:19.433 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T20:21:19.500 INFO:teuthology.orchestra.run.smithi121.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 30 MB/s | 3.7 MB 00:00 2023-09-23T20:21:19.559 INFO:teuthology.orchestra.run.smithi121.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 3.8 MB/s | 225 kB 00:00 2023-09-23T20:21:19.734 INFO:teuthology.orchestra.run.smithi121.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 34 MB/s | 14 MB 00:00 2023-09-23T20:21:19.778 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:19.851 INFO:teuthology.orchestra.run.smithi121.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 17 MB/s | 4.9 MB 00:00 2023-09-23T20:21:19.935 INFO:teuthology.orchestra.run.smithi121.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 24 MB/s | 4.7 MB 00:00 2023-09-23T20:21:20.384 INFO:teuthology.orchestra.run.smithi121.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 6.3 MB/s | 7.4 MB 00:01 2023-09-23T20:21:20.386 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:20.386 INFO:teuthology.orchestra.run.smithi121.stdout:Total 30 MB/s | 35 MB 00:01 2023-09-23T20:21:20.585 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:08 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:20.622 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:21:20.640 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:21:20.640 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:21:20.934 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:21:20.937 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:21:21.510 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:21:21.677 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:21:21.818 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:21:21.952 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================================= 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================================= 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T20:21:21.954 INFO:teuthology.orchestra.run.smithi123.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================================= 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout:Install 8 Packages 2023-09-23T20:21:21.955 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:21.956 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 35 M 2023-09-23T20:21:21.956 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 210 M 2023-09-23T20:21:21.956 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:21:22.002 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:21:22.095 INFO:teuthology.orchestra.run.smithi123.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 27 MB/s | 3.7 MB 00:00 2023-09-23T20:21:22.137 INFO:teuthology.orchestra.run.smithi123.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-09-23T20:21:22.270 INFO:teuthology.orchestra.run.smithi123.stdout:(3/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 36 MB/s | 4.9 MB 00:00 2023-09-23T20:21:22.404 INFO:teuthology.orchestra.run.smithi123.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 31 MB/s | 14 MB 00:00 2023-09-23T20:21:22.454 INFO:teuthology.orchestra.run.smithi123.stdout:(5/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 13 MB/s | 644 kB 00:00 2023-09-23T20:21:22.496 INFO:teuthology.orchestra.run.smithi123.stdout:(6/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 4.0 MB/s | 171 kB 00:00 2023-09-23T20:21:22.580 INFO:teuthology.orchestra.run.smithi123.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.7 MB 00:00 2023-09-23T20:21:22.600 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:21:22.769 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:21:22.789 INFO:teuthology.orchestra.run.smithi123.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 8.9 MB/s | 7.4 MB 00:00 2023-09-23T20:21:22.789 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:22.789 INFO:teuthology.orchestra.run.smithi123.stdout:Total 42 MB/s | 35 MB 00:00 2023-09-23T20:21:23.028 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:21:23.046 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:21:23.046 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:21:23.339 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:21:23.343 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:21:23.365 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:21:23.593 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:21:23.891 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:21:24.027 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:21:24.165 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:21:24.262 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:21:24.302 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:21:24.919 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:21:25.035 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:21:25.086 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:21:25.277 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:21:26.072 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:21:26.269 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:21:26.940 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:21:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:21:27.237 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:21:27.435 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:21:28.291 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T20:21:28.484 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:21:28.750 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:21:28.751 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T20:21:28.751 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:21:28.751 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T20:21:28.751 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T20:21:28.751 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:21:28.751 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:21:28.751 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T20:21:29.193 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:21:29.828 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T20:21:29.828 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:18 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:30.433 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T20:21:30.601 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:21:30.843 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:21:30.844 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T20:21:30.844 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-09-23T20:21:30.844 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-09-23T20:21:30.844 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-09-23T20:21:30.844 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-09-23T20:21:30.844 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-09-23T20:21:30.844 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-09-23T20:21:31.898 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-09-23T20:21:31.917 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:19 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:32.190 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T20:21:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T20:21:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:32.373 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:21:32.526 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-mgr-rook 2023-09-23T20:21:32.871 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:33.668 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:22 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:34.190 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T20:21:34.378 INFO:teuthology.orchestra.run.smithi123.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T20:21:34.379 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:34.379 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:21:34.537 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-mgr-rook 2023-09-23T20:21:34.882 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:35.094 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:21:35.096 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================== 2023-09-23T20:21:35.096 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:21:35.096 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================== 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T20:21:35.097 INFO:teuthology.orchestra.run.smithi121.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout:======================================================================================================== 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout:Install 13 Packages 2023-09-23T20:21:35.098 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:35.099 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 1.9 M 2023-09-23T20:21:35.099 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 25 M 2023-09-23T20:21:35.099 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:21:35.423 INFO:teuthology.orchestra.run.smithi121.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 180 kB/s | 16 kB 00:00 2023-09-23T20:21:35.439 INFO:teuthology.orchestra.run.smithi121.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 307 kB/s | 33 kB 00:00 2023-09-23T20:21:35.490 INFO:teuthology.orchestra.run.smithi121.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.3 MB/s | 81 kB 00:00 2023-09-23T20:21:35.565 INFO:teuthology.orchestra.run.smithi121.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 857 kB/s | 64 kB 00:00 2023-09-23T20:21:35.599 INFO:teuthology.orchestra.run.smithi121.stdout:(5/13): python3-kubernetes-11.0.0-6.el8.noarch. 7.3 MB/s | 1.2 MB 00:00 2023-09-23T20:21:35.615 INFO:teuthology.orchestra.run.smithi121.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 1.2 MB/s | 61 kB 00:00 2023-09-23T20:21:35.682 INFO:teuthology.orchestra.run.smithi121.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 2.3 MB/s | 155 kB 00:00 2023-09-23T20:21:35.687 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:23 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:35.699 INFO:teuthology.orchestra.run.smithi121.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 425 kB/s | 43 kB 00:00 2023-09-23T20:21:35.733 INFO:teuthology.orchestra.run.smithi121.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 2.1 MB/s | 110 kB 00:00 2023-09-23T20:21:35.758 INFO:teuthology.orchestra.run.smithi121.stdout:(10/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el 111 kB/s | 47 kB 00:00 2023-09-23T20:21:35.792 INFO:teuthology.orchestra.run.smithi121.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 288 kB/s | 27 kB 00:00 2023-09-23T20:21:35.825 INFO:teuthology.orchestra.run.smithi121.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.3 MB/s | 126 kB 00:00 2023-09-23T20:21:35.884 INFO:teuthology.orchestra.run.smithi121.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 159 kB/s | 20 kB 00:00 2023-09-23T20:21:35.885 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:35.885 INFO:teuthology.orchestra.run.smithi121.stdout:Total 2.5 MB/s | 1.9 MB 00:00 2023-09-23T20:21:35.946 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:21:35.962 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:21:35.962 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:21:36.032 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:21:36.036 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:21:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:21:36.732 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T20:21:36.993 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T20:21:37.059 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:21:37.061 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================== 2023-09-23T20:21:37.061 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================== 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T20:21:37.062 INFO:teuthology.orchestra.run.smithi123.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout:======================================================================================================== 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout:Install 13 Packages 2023-09-23T20:21:37.063 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:37.064 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 1.9 M 2023-09-23T20:21:37.064 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 25 M 2023-09-23T20:21:37.064 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:21:37.139 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T20:21:37.331 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T20:21:37.460 INFO:teuthology.orchestra.run.smithi123.stdout:(1/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 129 kB/s | 47 kB 00:00 2023-09-23T20:21:37.510 INFO:teuthology.orchestra.run.smithi123.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 39 kB/s | 16 kB 00:00 2023-09-23T20:21:37.536 INFO:teuthology.orchestra.run.smithi123.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 75 kB/s | 33 kB 00:00 2023-09-23T20:21:37.552 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T20:21:37.761 INFO:teuthology.orchestra.run.smithi123.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 285 kB/s | 64 kB 00:00 2023-09-23T20:21:37.794 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T20:21:37.903 INFO:teuthology.orchestra.run.smithi123.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 432 kB/s | 61 kB 00:00 2023-09-23T20:21:37.970 INFO:teuthology.orchestra.run.smithi123.stdout:(6/13): python3-pyasn1-modules-0.3.7-6.el8.noar 1.6 MB/s | 110 kB 00:00 2023-09-23T20:21:37.995 INFO:teuthology.orchestra.run.smithi123.stdout:(7/13): python3-google-auth-1.1.1-10.el8.noarch 152 kB/s | 81 kB 00:00 2023-09-23T20:21:38.018 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T20:21:38.020 INFO:teuthology.orchestra.run.smithi123.stdout:(8/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 529 kB/s | 27 kB 00:00 2023-09-23T20:21:38.071 INFO:teuthology.orchestra.run.smithi123.stdout:(9/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.6 MB/s | 126 kB 00:00 2023-09-23T20:21:38.096 INFO:teuthology.orchestra.run.smithi123.stdout:(10/13): python3-jsonpointer-1.10-11.el8.noarch 287 kB/s | 20 kB 00:00 2023-09-23T20:21:38.121 INFO:teuthology.orchestra.run.smithi123.stdout:(11/13): python3-requests-oauthlib-1.0.0-1.el8. 854 kB/s | 43 kB 00:00 2023-09-23T20:21:38.147 INFO:teuthology.orchestra.run.smithi123.stdout:(12/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 3.0 MB/s | 155 kB 00:00 2023-09-23T20:21:38.180 INFO:teuthology.orchestra.run.smithi123.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 1.7 MB/s | 1.2 MB 00:00 2023-09-23T20:21:38.183 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:38.183 INFO:teuthology.orchestra.run.smithi123.stdout:Total 1.7 MB/s | 1.9 MB 00:01 2023-09-23T20:21:38.247 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T20:21:38.247 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:21:38.263 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:21:38.263 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:21:38.334 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:21:38.338 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:21:38.412 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T20:21:38.635 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T20:21:38.804 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:21:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T20:21:39.244 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T20:21:39.452 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T20:21:39.654 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-09-23T20:21:39.734 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T20:21:39.831 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-09-23T20:21:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-09-23T20:21:40.015 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T20:21:40.144 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-09-23T20:21:40.198 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:21:40.315 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T20:21:40.439 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T20:21:40.440 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T20:21:40.441 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T20:21:40.464 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T20:21:40.655 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T20:21:41.524 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T20:21:41.524 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:30 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:41.720 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T20:21:41.950 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T20:21:42.091 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:21:42.332 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-09-23T20:21:42.333 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-23T20:21:42.334 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-09-23T20:21:43.447 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-09-23T20:21:43.447 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:31 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:43.907 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T20:21:44.088 INFO:teuthology.orchestra.run.smithi121.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:44.089 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:21:44.220 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T20:21:44.571 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:45.458 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:34 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:46.033 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T20:21:46.217 INFO:teuthology.orchestra.run.smithi123.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T20:21:46.218 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T20:21:46.218 INFO:teuthology.orchestra.run.smithi123.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T20:21:46.218 INFO:teuthology.orchestra.run.smithi123.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T20:21:46.218 INFO:teuthology.orchestra.run.smithi123.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T20:21:46.218 INFO:teuthology.orchestra.run.smithi123.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T20:21:46.218 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:46.218 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:21:46.372 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T20:21:46.722 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:46.881 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:21:46.882 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================================== 2023-09-23T20:21:46.882 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:21:46.882 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================================== 2023-09-23T20:21:46.882 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:21:46.882 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T20:21:46.882 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:21:46.882 INFO:teuthology.orchestra.run.smithi121.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================================== 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout:Install 5 Packages 2023-09-23T20:21:46.883 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:46.884 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 6.0 M 2023-09-23T20:21:46.884 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 26 M 2023-09-23T20:21:46.884 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:21:47.174 INFO:teuthology.orchestra.run.smithi121.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 494 kB/s | 60 kB 00:00 2023-09-23T20:21:47.258 INFO:teuthology.orchestra.run.smithi121.stdout:(2/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 589 kB/s | 121 kB 00:00 2023-09-23T20:21:47.283 INFO:teuthology.orchestra.run.smithi121.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 2.0 MB/s | 476 kB 00:00 2023-09-23T20:21:47.317 INFO:teuthology.orchestra.run.smithi121.stdout:(4/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.7 MB/s | 538 kB 00:00 2023-09-23T20:21:47.408 INFO:teuthology.orchestra.run.smithi121.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 32 MB/s | 4.8 MB 00:00 2023-09-23T20:21:47.409 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:47.410 INFO:teuthology.orchestra.run.smithi121.stdout:Total 11 MB/s | 6.0 MB 00:00 2023-09-23T20:21:47.462 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:21:47.474 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:21:47.474 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:21:47.552 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:21:47.555 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:21:47.593 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:35 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:48.520 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:21:48.850 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T20:21:48.956 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:21:48.957 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================================== 2023-09-23T20:21:48.957 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:21:48.957 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================================== 2023-09-23T20:21:48.957 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:21:48.957 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T20:21:48.957 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================================== 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout:Install 5 Packages 2023-09-23T20:21:48.958 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:48.959 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 6.0 M 2023-09-23T20:21:48.959 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 26 M 2023-09-23T20:21:48.959 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:21:49.089 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T20:21:49.179 INFO:teuthology.orchestra.run.smithi123.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 639 kB/s | 121 kB 00:00 2023-09-23T20:21:49.288 INFO:teuthology.orchestra.run.smithi123.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 4.9 MB/s | 538 kB 00:00 2023-09-23T20:21:49.426 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:21:49.454 INFO:teuthology.orchestra.run.smithi123.stdout:(3/5): python3-babel-2.5.1-7.el8.noarch.rpm 29 MB/s | 4.8 MB 00:00 2023-09-23T20:21:49.479 INFO:teuthology.orchestra.run.smithi123.stdout:(4/5): python3-natsort-7.1.1-2.el8.noarch.rpm 123 kB/s | 60 kB 00:00 2023-09-23T20:21:49.723 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T20:21:49.796 INFO:teuthology.orchestra.run.smithi123.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 589 kB/s | 476 kB 00:00 2023-09-23T20:21:49.799 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:49.799 INFO:teuthology.orchestra.run.smithi123.stdout:Total 7.1 MB/s | 6.0 MB 00:00 2023-09-23T20:21:49.852 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:21:49.863 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:21:49.863 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:21:49.929 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:21:49.942 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:21:49.946 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:21:50.161 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:21:50.161 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T20:21:50.161 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T20:21:50.161 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:21:50.161 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T20:21:50.883 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:21:51.201 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T20:21:51.201 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:40 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:51.235 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T20:21:51.482 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T20:21:51.835 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:21:52.083 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T20:21:52.289 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:21:52.522 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:21:52.522 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T20:21:52.522 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T20:21:52.522 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:21:52.522 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T20:21:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T20:21:53.527 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:41 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:53.622 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:21:53.805 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:53.805 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:21:53.805 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:53.805 INFO:teuthology.orchestra.run.smithi121.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T20:21:53.805 INFO:teuthology.orchestra.run.smithi121.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T20:21:53.805 INFO:teuthology.orchestra.run.smithi121.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T20:21:53.805 INFO:teuthology.orchestra.run.smithi121.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T20:21:53.806 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:53.806 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:21:53.949 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install ceph-fuse 2023-09-23T20:21:54.301 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:55.088 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:44 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:55.845 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:56.028 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:21:56.186 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install ceph-fuse 2023-09-23T20:21:56.449 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:21:56.449 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:21:56.449 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2023-09-23T20:21:56.449 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-fuse 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout:Install 2 Packages 2023-09-23T20:21:56.450 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:21:56.451 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 957 k 2023-09-23T20:21:56.451 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 3.0 M 2023-09-23T20:21:56.451 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:21:56.540 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:21:56.986 INFO:teuthology.orchestra.run.smithi121.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 155 kB/s | 83 kB 00:00 2023-09-23T20:21:57.194 INFO:teuthology.orchestra.run.smithi121.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 1.1 MB/s | 874 kB 00:00 2023-09-23T20:21:57.194 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:57.195 INFO:teuthology.orchestra.run.smithi121.stdout:Total 1.3 MB/s | 957 kB 00:00 2023-09-23T20:21:57.208 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:21:57.230 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:21:57.230 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:21:57.305 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:21:57.309 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:21:57.422 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:45 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:21:57.468 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:21:57.762 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T20:21:57.934 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:21:58.286 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:21:58.286 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T20:21:58.775 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repository Size 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-fuse 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2023-09-23T20:21:58.776 INFO:teuthology.orchestra.run.smithi123.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T20:21:58.777 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:58.777 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:21:58.777 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:21:58.777 INFO:teuthology.orchestra.run.smithi123.stdout:Install 2 Packages 2023-09-23T20:21:58.777 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:21:58.777 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 957 k 2023-09-23T20:21:58.778 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 3.0 M 2023-09-23T20:21:58.778 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:21:58.847 INFO:teuthology.orchestra.run.smithi123.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.2 MB/s | 83 kB 00:00 2023-09-23T20:21:59.048 INFO:teuthology.orchestra.run.smithi123.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 3.2 MB/s | 874 kB 00:00 2023-09-23T20:21:59.048 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:21:59.048 INFO:teuthology.orchestra.run.smithi123.stdout:Total 3.4 MB/s | 957 kB 00:00 2023-09-23T20:21:59.059 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:21:59.079 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:21:59.080 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:21:59.152 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:21:59.155 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:21:59.308 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T20:21:59.308 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:48 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:21:59.316 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:21:59.573 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T20:21:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:22:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:22:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T20:22:01.159 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T20:22:01.159 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:49 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:01.797 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:22:01.977 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:01.978 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:22:01.978 INFO:teuthology.orchestra.run.smithi121.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T20:22:01.978 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:01.978 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:02.622 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install librados-devel 2023-09-23T20:22:02.983 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:03.553 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:22:03.736 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:03.736 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:22:03.736 INFO:teuthology.orchestra.run.smithi123.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T20:22:03.736 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:03.736 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:03.813 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:52 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:03.879 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install librados-devel 2023-09-23T20:22:04.236 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:05.071 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:53 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:05.188 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repo Size 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout:Install 1 Package 2023-09-23T20:22:05.189 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:05.190 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 130 k 2023-09-23T20:22:05.190 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 464 k 2023-09-23T20:22:05.190 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:22:05.389 INFO:teuthology.orchestra.run.smithi121.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 651 kB/s | 130 kB 00:00 2023-09-23T20:22:05.390 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:05.390 INFO:teuthology.orchestra.run.smithi121.stdout:Total 647 kB/s | 130 kB 00:00 2023-09-23T20:22:05.390 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:22:05.396 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:22:05.396 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:22:05.449 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:22:05.453 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:22:05.587 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:22:05.790 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:06.026 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:06.501 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:06.502 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:06.502 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repo Size 2023-09-23T20:22:06.502 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:06.502 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:22:06.502 INFO:teuthology.orchestra.run.smithi123.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T20:22:06.502 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:06.502 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:22:06.503 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:06.503 INFO:teuthology.orchestra.run.smithi123.stdout:Install 1 Package 2023-09-23T20:22:06.503 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:06.503 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 130 k 2023-09-23T20:22:06.503 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 464 k 2023-09-23T20:22:06.503 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:22:06.687 INFO:teuthology.orchestra.run.smithi123.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 706 kB/s | 130 kB 00:00 2023-09-23T20:22:06.687 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:06.687 INFO:teuthology.orchestra.run.smithi123.stdout:Total 702 kB/s | 130 kB 00:00 2023-09-23T20:22:06.688 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:22:06.693 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:22:06.693 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:22:06.748 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:22:06.751 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:22:06.878 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:22:07.018 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:07.088 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:07.088 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:02:56 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:07.232 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:08.284 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:02:56 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:09.370 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:22:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:09.550 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:22:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:09.550 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:09.721 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install libcephfs2 2023-09-23T20:22:10.073 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:10.955 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:22:11.022 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:00 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:11.135 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:11.136 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:22:11.136 INFO:teuthology.orchestra.run.smithi123.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:11.136 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:11.136 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:11.273 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install libcephfs2 2023-09-23T20:22:11.626 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:12.345 INFO:teuthology.orchestra.run.smithi121.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:12.453 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:12.454 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:22:12.454 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:12.547 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:00 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:12.563 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install libcephfs-devel 2023-09-23T20:22:12.916 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:13.765 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:02 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:13.803 INFO:teuthology.orchestra.run.smithi123.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:13.913 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:13.914 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:22:13.914 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:14.040 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install libcephfs-devel 2023-09-23T20:22:14.394 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:15.139 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:15.139 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:15.139 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repo Size 2023-09-23T20:22:15.139 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:15.140 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:22:15.140 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T20:22:15.140 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:15.140 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:22:15.140 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:15.140 INFO:teuthology.orchestra.run.smithi121.stdout:Install 1 Package 2023-09-23T20:22:15.140 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:15.141 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 25 k 2023-09-23T20:22:15.141 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 109 k 2023-09-23T20:22:15.141 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:22:15.436 INFO:teuthology.orchestra.run.smithi121.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 84 kB/s | 25 kB 00:00 2023-09-23T20:22:15.437 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:15.437 INFO:teuthology.orchestra.run.smithi121.stdout:Total 84 kB/s | 25 kB 00:00 2023-09-23T20:22:15.437 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:22:15.441 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:22:15.441 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:22:15.443 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:22:15.447 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:22:15.497 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:22:15.515 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:03 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:15.679 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:15.926 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:16.907 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:16.907 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repo Size 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout:Install 1 Package 2023-09-23T20:22:16.908 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:16.909 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 25 k 2023-09-23T20:22:16.909 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 109 k 2023-09-23T20:22:16.909 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:22:16.917 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:16.917 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:05 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:17.159 INFO:teuthology.orchestra.run.smithi123.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 99 kB/s | 25 kB 00:00 2023-09-23T20:22:17.159 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:17.159 INFO:teuthology.orchestra.run.smithi123.stdout:Total 99 kB/s | 25 kB 00:00 2023-09-23T20:22:17.160 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:22:17.163 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:22:17.163 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:22:17.165 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:22:17.169 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:22:17.221 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:22:17.375 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:17.576 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:18.605 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:18.605 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:06 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:19.508 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:22:19.688 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:19.689 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:22:19.689 INFO:teuthology.orchestra.run.smithi121.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:19.689 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:19.689 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:19.821 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install librados2 2023-09-23T20:22:20.176 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:21.027 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:10 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:21.076 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:22:21.256 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:21.256 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:22:21.257 INFO:teuthology.orchestra.run.smithi123.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:21.257 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:21.257 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:21.405 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install librados2 2023-09-23T20:22:21.759 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:22.339 INFO:teuthology.orchestra.run.smithi121.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:22.451 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:22.452 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:22:22.452 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:22.574 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install librbd1 2023-09-23T20:22:22.788 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:10 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:22.925 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:23.917 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:12 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:24.045 INFO:teuthology.orchestra.run.smithi123.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:24.154 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:24.155 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:22:24.155 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:24.372 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install librbd1 2023-09-23T20:22:24.726 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:25.172 INFO:teuthology.orchestra.run.smithi121.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:25.281 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:25.282 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:22:25.282 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:25.399 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install python3-rados 2023-09-23T20:22:25.750 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:25.777 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:13 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:27.026 INFO:teuthology.orchestra.run.smithi123.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:27.137 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:27.139 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:22:27.139 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:27.140 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:16 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:27.273 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install python3-rados 2023-09-23T20:22:27.627 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:28.406 INFO:teuthology.orchestra.run.smithi121.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:28.516 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:28.517 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:22:28.517 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:28.640 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install python3-rgw 2023-09-23T20:22:28.995 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:29.182 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:17 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:30.503 INFO:teuthology.orchestra.run.smithi123.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:30.522 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:19 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:30.610 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:30.612 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:22:30.612 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:30.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install python3-rgw 2023-09-23T20:22:31.090 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:31.788 INFO:teuthology.orchestra.run.smithi121.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:31.897 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:31.898 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:22:31.898 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:32.005 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install python3-cephfs 2023-09-23T20:22:32.209 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:20 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:32.361 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:33.239 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:33.456 INFO:teuthology.orchestra.run.smithi123.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:33.564 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:33.565 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:22:33.566 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:33.686 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install python3-cephfs 2023-09-23T20:22:34.039 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:34.507 INFO:teuthology.orchestra.run.smithi121.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:34.615 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:34.616 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:22:34.616 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:34.740 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install python3-rbd 2023-09-23T20:22:34.900 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:35.093 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:24 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:36.156 INFO:teuthology.orchestra.run.smithi123.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:36.269 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:36.270 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:22:36.270 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:36.399 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install python3-rbd 2023-09-23T20:22:36.752 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:37.276 INFO:teuthology.orchestra.run.smithi121.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:37.385 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:37.386 INFO:teuthology.orchestra.run.smithi121.stdout:Nothing to do. 2023-09-23T20:22:37.386 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:37.504 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install rbd-fuse 2023-09-23T20:22:37.662 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:25 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:37.859 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:38.653 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:27 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:38.914 INFO:teuthology.orchestra.run.smithi123.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:22:39.021 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:39.022 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2023-09-23T20:22:39.022 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:39.143 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install rbd-fuse 2023-09-23T20:22:39.498 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:40.080 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:40.081 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:40.081 INFO:teuthology.orchestra.run.smithi121.stdout: Package Architecture Version Repository Size 2023-09-23T20:22:40.081 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:40.081 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:22:40.081 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T20:22:40.081 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:40.082 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:22:40.082 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:40.082 INFO:teuthology.orchestra.run.smithi121.stdout:Install 1 Package 2023-09-23T20:22:40.082 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:40.082 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 90 k 2023-09-23T20:22:40.082 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 264 k 2023-09-23T20:22:40.083 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:22:40.247 INFO:teuthology.orchestra.run.smithi121.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 549 kB/s | 90 kB 00:00 2023-09-23T20:22:40.247 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:40.248 INFO:teuthology.orchestra.run.smithi121.stdout:Total 546 kB/s | 90 kB 00:00 2023-09-23T20:22:40.248 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:22:40.257 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:22:40.257 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:22:40.299 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:40.311 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:22:40.315 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:22:40.431 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:22:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:41.723 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout: Package Architecture Version Repository Size 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:22:41.724 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:41.725 INFO:teuthology.orchestra.run.smithi123.stdout:Install 1 Package 2023-09-23T20:22:41.725 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:41.725 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 90 k 2023-09-23T20:22:41.725 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 264 k 2023-09-23T20:22:41.725 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:22:41.928 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:41.929 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:30 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:42.020 INFO:teuthology.orchestra.run.smithi123.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 306 kB/s | 90 kB 00:00 2023-09-23T20:22:42.021 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:42.021 INFO:teuthology.orchestra.run.smithi123.stdout:Total 305 kB/s | 90 kB 00:00 2023-09-23T20:22:42.021 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:22:42.030 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:22:42.030 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:22:42.083 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:22:42.087 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:22:42.191 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:22:42.337 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:42.574 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:43.603 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:43.603 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:31 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:44.420 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:22:44.601 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:44.601 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:22:44.601 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:44.601 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:44.601 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:44.746 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install rbd-mirror 2023-09-23T20:22:45.102 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:45.982 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:34 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:46.144 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:22:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:46.329 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:22:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:46.329 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:46.464 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install rbd-mirror 2023-09-23T20:22:46.817 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:47.417 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repo Size 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:47.418 INFO:teuthology.orchestra.run.smithi121.stdout:Install 1 Package 2023-09-23T20:22:47.419 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:47.419 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 3.5 M 2023-09-23T20:22:47.419 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 16 M 2023-09-23T20:22:47.419 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:22:47.711 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:35 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:48.100 INFO:teuthology.orchestra.run.smithi121.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 5.1 MB/s | 3.5 MB 00:00 2023-09-23T20:22:48.100 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:48.100 INFO:teuthology.orchestra.run.smithi121.stdout:Total 5.1 MB/s | 3.5 MB 00:00 2023-09-23T20:22:48.101 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:22:48.112 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:22:48.113 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:22:48.179 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:22:48.183 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:22:48.628 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:22:48.842 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:49.082 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:49.083 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:49.083 INFO:teuthology.orchestra.run.smithi123.stdout: Package Arch Version Repo Size 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout:Install 1 Package 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 3.5 M 2023-09-23T20:22:49.084 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 16 M 2023-09-23T20:22:49.085 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:22:49.199 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:49.748 INFO:teuthology.orchestra.run.smithi123.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 5.2 MB/s | 3.5 MB 00:00 2023-09-23T20:22:49.748 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:49.749 INFO:teuthology.orchestra.run.smithi123.stdout:Total 5.2 MB/s | 3.5 MB 00:00 2023-09-23T20:22:49.749 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:22:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:22:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:22:49.827 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:22:49.831 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:22:50.232 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:50.233 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:39 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:22:50.455 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:50.815 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:51.865 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:51.865 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:39 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:53.367 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:22:53.547 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:53.548 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:22:53.548 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:53.548 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:53.548 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:22:53.679 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum -y install rbd-nbd 2023-09-23T20:22:54.034 INFO:teuthology.orchestra.run.smithi121.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:54.770 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:22:54.915 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:43 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:54.952 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:54.953 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:22:54.953 INFO:teuthology.orchestra.run.smithi123.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:22:54.953 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:54.953 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:22:55.084 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum -y install rbd-nbd 2023-09-23T20:22:55.439 INFO:teuthology.orchestra.run.smithi123.stdout:Updating Subscription Management repositories. 2023-09-23T20:22:56.270 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2023-09-23T20:22:56.271 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:56.271 INFO:teuthology.orchestra.run.smithi121.stdout: Package Architecture Version Repository Size 2023-09-23T20:22:56.271 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:56.271 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2023-09-23T20:22:56.271 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T20:22:56.271 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:56.272 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2023-09-23T20:22:56.272 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2023-09-23T20:22:56.272 INFO:teuthology.orchestra.run.smithi121.stdout:Install 1 Package 2023-09-23T20:22:56.272 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:22:56.272 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 173 k 2023-09-23T20:22:56.272 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 552 k 2023-09-23T20:22:56.273 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2023-09-23T20:22:56.326 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:44 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:22:56.460 INFO:teuthology.orchestra.run.smithi121.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 921 kB/s | 173 kB 00:00 2023-09-23T20:22:56.461 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:56.461 INFO:teuthology.orchestra.run.smithi121.stdout:Total 915 kB/s | 173 kB 00:00 2023-09-23T20:22:56.461 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2023-09-23T20:22:56.472 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2023-09-23T20:22:56.472 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2023-09-23T20:22:56.527 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2023-09-23T20:22:56.530 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2023-09-23T20:22:56.654 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2023-09-23T20:22:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:57.013 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:57.691 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2023-09-23T20:22:57.691 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:57.691 INFO:teuthology.orchestra.run.smithi123.stdout: Package Architecture Version Repository Size 2023-09-23T20:22:57.691 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:57.692 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2023-09-23T20:22:57.692 INFO:teuthology.orchestra.run.smithi123.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T20:22:57.692 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:57.692 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2023-09-23T20:22:57.692 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2023-09-23T20:22:57.692 INFO:teuthology.orchestra.run.smithi123.stdout:Install 1 Package 2023-09-23T20:22:57.692 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:22:57.693 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 173 k 2023-09-23T20:22:57.693 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 552 k 2023-09-23T20:22:57.693 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2023-09-23T20:22:58.058 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:58.059 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:03:47 ago on Sat 23 Sep 2023 08:19:11 PM UTC. 2023-09-23T20:22:58.067 INFO:teuthology.orchestra.run.smithi123.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 462 kB/s | 173 kB 00:00 2023-09-23T20:22:58.068 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:22:58.068 INFO:teuthology.orchestra.run.smithi123.stdout:Total 460 kB/s | 173 kB 00:00 2023-09-23T20:22:58.069 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2023-09-23T20:22:58.079 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2023-09-23T20:22:58.079 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2023-09-23T20:22:58.134 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2023-09-23T20:22:58.138 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2023-09-23T20:22:58.256 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2023-09-23T20:22:58.401 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:58.638 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:59.688 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:22:59.688 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:03:47 ago on Sat 23 Sep 2023 08:19:12 PM UTC. 2023-09-23T20:23:00.634 INFO:teuthology.orchestra.run.smithi121.stdout:Installed products updated. 2023-09-23T20:23:00.817 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:23:00.817 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2023-09-23T20:23:00.817 INFO:teuthology.orchestra.run.smithi121.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:23:00.817 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:23:00.817 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2023-09-23T20:23:00.954 DEBUG:teuthology.parallel:result is None 2023-09-23T20:23:02.280 INFO:teuthology.orchestra.run.smithi123.stdout:Installed products updated. 2023-09-23T20:23:02.463 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:23:02.463 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2023-09-23T20:23:02.463 INFO:teuthology.orchestra.run.smithi123.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:23:02.464 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:23:02.464 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2023-09-23T20:23:02.609 DEBUG:teuthology.parallel:result is None 2023-09-23T20:23:02.609 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:23:02.609 INFO:teuthology.packaging:ref: None 2023-09-23T20:23:02.609 INFO:teuthology.packaging:tag: None 2023-09-23T20:23:02.609 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:23:02.609 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:23:02.610 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:23:02.855 DEBUG:teuthology.orchestra.run.smithi121:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T20:23:02.882 INFO:teuthology.orchestra.run.smithi121.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T20:23:02.883 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T20:23:02.883 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T20:23:02.883 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:23:02.883 INFO:teuthology.packaging:ref: None 2023-09-23T20:23:02.883 INFO:teuthology.packaging:tag: None 2023-09-23T20:23:02.883 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:23:02.883 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:23:02.883 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:23:03.134 DEBUG:teuthology.orchestra.run.smithi123:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T20:23:03.157 INFO:teuthology.orchestra.run.smithi123.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T20:23:03.157 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T20:23:03.157 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T20:23:03.157 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T20:23:03.158 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:23:03.158 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T20:23:03.190 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:23:03.190 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T20:23:03.226 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T20:23:03.226 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:23:03.226 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T20:23:03.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T20:23:03.332 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:23:03.332 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T20:23:03.359 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T20:23:03.426 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T20:23:03.426 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:23:03.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T20:23:03.455 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T20:23:03.524 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:23:03.524 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T20:23:03.551 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T20:23:03.617 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T20:23:03.618 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:23:03.618 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T20:23:03.646 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T20:23:03.714 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:23:03.714 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T20:23:03.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T20:23:03.806 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T20:23:03.880 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T20:23:03.881 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T20:23:03.881 INFO:tasks.cephadm:Cluster fsid is ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:23:03.881 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T20:23:03.882 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.121', 'mon.c': '[v2:172.21.15.121:3301,v1:172.21.15.121:6790]', 'mon.b': '172.21.15.123'} 2023-09-23T20:23:03.882 INFO:tasks.cephadm:First mon is mon.a on smithi121 2023-09-23T20:23:03.882 INFO:tasks.cephadm:First mgr is y 2023-09-23T20:23:03.882 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T20:23:03.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo hostname $(hostname -s) 2023-09-23T20:23:03.911 DEBUG:teuthology.orchestra.run.smithi123:> sudo hostname $(hostname -s) 2023-09-23T20:23:03.938 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T20:23:03.938 DEBUG:teuthology.orchestra.run.smithi121:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T20:23:04.102 INFO:teuthology.orchestra.run.smithi121.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 20:23 /home/ubuntu/cephtest/cephadm 2023-09-23T20:23:04.102 DEBUG:teuthology.orchestra.run.smithi123:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T20:23:04.239 INFO:teuthology.orchestra.run.smithi123.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 20:23 /home/ubuntu/cephtest/cephadm 2023-09-23T20:23:04.240 DEBUG:teuthology.orchestra.run.smithi121:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T20:23:04.258 DEBUG:teuthology.orchestra.run.smithi123:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T20:23:04.285 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T20:23:04.286 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T20:23:04.300 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T20:23:05.415 INFO:teuthology.orchestra.run.smithi123.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:23:05.563 INFO:teuthology.orchestra.run.smithi121.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:23:43.719 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:23:43.719 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T20:23:43.719 INFO:teuthology.orchestra.run.smithi123.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T20:23:43.719 INFO:teuthology.orchestra.run.smithi123.stdout: "repo_digests": [ 2023-09-23T20:23:43.719 INFO:teuthology.orchestra.run.smithi123.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T20:23:43.719 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:23:43.719 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:23:47.199 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:23:47.199 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T20:23:47.199 INFO:teuthology.orchestra.run.smithi121.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T20:23:47.199 INFO:teuthology.orchestra.run.smithi121.stdout: "repo_digests": [ 2023-09-23T20:23:47.199 INFO:teuthology.orchestra.run.smithi121.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T20:23:47.199 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:23:47.200 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:23:47.224 DEBUG:teuthology.orchestra.run.smithi121:> sudo mkdir -p /etc/ceph 2023-09-23T20:23:47.256 DEBUG:teuthology.orchestra.run.smithi123:> sudo mkdir -p /etc/ceph 2023-09-23T20:23:47.293 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 777 /etc/ceph 2023-09-23T20:23:47.323 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 777 /etc/ceph 2023-09-23T20:23:47.365 INFO:tasks.cephadm:Writing seed config... 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T20:23:47.366 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T20:23:47.367 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T20:23:47.367 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T20:23:47.367 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:23:47.367 DEBUG:teuthology.orchestra.run.smithi121:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T20:23:47.384 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ffb133f0-5a4e-11ee-8db4-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T20:23:47.385 DEBUG:teuthology.orchestra.run.smithi121:mon.a> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T20:23:47.427 DEBUG:teuthology.orchestra.run.smithi121:mgr.y> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T20:23:47.452 INFO:journalctl@ceph.mon.a.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:23:47.454 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T20:23:47.454 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.121 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:23:47.479 INFO:journalctl@ceph.mgr.y.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:23:47.651 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:23:47.652 INFO:teuthology.orchestra.run.smithi121.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'ffb133f0-5a4e-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.121', '--skip-admin-label'] 2023-09-23T20:23:47.713 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stdout 3.0.1 2023-09-23T20:23:47.714 INFO:teuthology.orchestra.run.smithi121.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T20:23:47.714 INFO:teuthology.orchestra.run.smithi121.stdout:Verifying podman|docker is present... 2023-09-23T20:23:47.775 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stdout 3.0.1 2023-09-23T20:23:47.776 INFO:teuthology.orchestra.run.smithi121.stdout:Verifying lvm2 is present... 2023-09-23T20:23:47.776 INFO:teuthology.orchestra.run.smithi121.stdout:Verifying time synchronization is in place... 2023-09-23T20:23:47.783 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T20:23:47.783 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T20:23:47.789 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T20:23:47.789 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout inactive 2023-09-23T20:23:47.795 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout enabled 2023-09-23T20:23:47.800 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout active 2023-09-23T20:23:47.800 INFO:teuthology.orchestra.run.smithi121.stdout:Unit chronyd.service is enabled and running 2023-09-23T20:23:47.801 INFO:teuthology.orchestra.run.smithi121.stdout:Repeating the final host check... 2023-09-23T20:23:47.855 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stdout 3.0.1 2023-09-23T20:23:47.855 INFO:teuthology.orchestra.run.smithi121.stdout:podman (/bin/podman) version 3.0.1 is present 2023-09-23T20:23:47.856 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl is present 2023-09-23T20:23:47.856 INFO:teuthology.orchestra.run.smithi121.stdout:lvcreate is present 2023-09-23T20:23:47.862 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T20:23:47.862 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T20:23:47.867 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T20:23:47.867 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout inactive 2023-09-23T20:23:47.873 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout enabled 2023-09-23T20:23:47.879 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout active 2023-09-23T20:23:47.879 INFO:teuthology.orchestra.run.smithi121.stdout:Unit chronyd.service is enabled and running 2023-09-23T20:23:47.879 INFO:teuthology.orchestra.run.smithi121.stdout:Host looks OK 2023-09-23T20:23:47.879 INFO:teuthology.orchestra.run.smithi121.stdout:Cluster fsid: ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:23:47.880 INFO:teuthology.orchestra.run.smithi121.stdout:Acquiring lock 139642168822248 on /run/cephadm/ffb133f0-5a4e-11ee-8db4-212e2dc638e7.lock 2023-09-23T20:23:47.880 INFO:teuthology.orchestra.run.smithi121.stdout:Lock 139642168822248 acquired on /run/cephadm/ffb133f0-5a4e-11ee-8db4-212e2dc638e7.lock 2023-09-23T20:23:47.880 INFO:teuthology.orchestra.run.smithi121.stdout:Verifying IP 172.21.15.121 port 3300 ... 2023-09-23T20:23:47.880 INFO:teuthology.orchestra.run.smithi121.stdout:Verifying IP 172.21.15.121 port 6789 ... 2023-09-23T20:23:47.880 INFO:teuthology.orchestra.run.smithi121.stdout:Base mon IP(s) is [172.21.15.121:3300, 172.21.15.121:6789], mon addrv is [v2:172.21.15.121:3300,v1:172.21.15.121:6789] 2023-09-23T20:23:47.884 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-09-23T20:23:47.884 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.121 metric 100 2023-09-23T20:23:47.886 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T20:23:47.886 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-09-23T20:23:47.886 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-09-23T20:23:47.889 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T20:23:47.889 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T20:23:47.889 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T20:23:47.889 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-09-23T20:23:47.889 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout inet6 fe80::4981:6028:a169:615d/64 scope link noprefixroute 2023-09-23T20:23:47.889 INFO:teuthology.orchestra.run.smithi121.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T20:23:47.890 INFO:teuthology.orchestra.run.smithi121.stdout:Mon IP `172.21.15.121` is in CIDR network `172.21.0.0/20` 2023-09-23T20:23:47.890 INFO:teuthology.orchestra.run.smithi121.stdout:Mon IP `172.21.15.121` is in CIDR network `172.21.0.0/20` 2023-09-23T20:23:47.890 INFO:teuthology.orchestra.run.smithi121.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T20:23:47.890 INFO:teuthology.orchestra.run.smithi121.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T20:23:47.891 INFO:teuthology.orchestra.run.smithi121.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:23:48.286 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stdout 0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T20:23:48.286 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:23:48.286 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T20:23:48.286 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stderr Copying blob sha256:af7d028f70e220a466755c6119effe51eebe359d9c00eb6b4883db3fabc6becd 2023-09-23T20:23:48.286 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T20:23:48.286 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stderr Copying config sha256:0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T20:23:48.287 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T20:23:48.287 INFO:teuthology.orchestra.run.smithi121.stdout:/bin/podman: stderr Storing signatures 2023-09-23T20:23:49.351 INFO:teuthology.orchestra.run.smithi121.stdout:ceph: stdout ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T20:23:49.351 INFO:teuthology.orchestra.run.smithi121.stdout:Ceph version: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T20:23:49.352 INFO:teuthology.orchestra.run.smithi121.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T20:23:50.290 INFO:teuthology.orchestra.run.smithi121.stdout:stat: stdout 167 167 2023-09-23T20:23:50.290 INFO:teuthology.orchestra.run.smithi121.stdout:Creating initial keys... 2023-09-23T20:23:51.281 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph-authtool: stdout AQBWSQ9lH7sdKBAAeXQj4/FlLANcTC79/c4xVQ== 2023-09-23T20:23:52.221 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph-authtool: stdout AQBXSQ9lnFeiJxAAw5aFPYhr2VOMyDnSgslIUw== 2023-09-23T20:23:53.265 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph-authtool: stdout AQBYSQ9lgJ3ZJBAA5H2MXJ5Z1tdoBEtNZkdiYA== 2023-09-23T20:23:53.265 INFO:teuthology.orchestra.run.smithi121.stdout:Creating initial monmap... 2023-09-23T20:23:54.299 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T20:23:54.299 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout:monmaptool for a [v2:172.21.15.121:3300,v1:172.21.15.121:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout:setting min_mon_release = octopus 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/monmaptool: set fsid to ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:23:54.300 INFO:teuthology.orchestra.run.smithi121.stdout:Creating mon... 2023-09-23T20:23:55.388 INFO:teuthology.orchestra.run.smithi121.stdout:create mon.a on 2023-09-23T20:23:55.484 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T20:23:55.568 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T20:23:55.656 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7.target. 2023-09-23T20:23:55.656 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7.target. 2023-09-23T20:23:55.772 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T20:23:55.773 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T20:23:55.857 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7.target.wants/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@.service. 2023-09-23T20:23:55.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:55 smithi121 systemd[1]: Starting Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:23:56.477 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:23:56.477 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout disabled 2023-09-23T20:23:56.484 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:23:56.484 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout inactive 2023-09-23T20:23:56.484 INFO:teuthology.orchestra.run.smithi121.stdout:firewalld.service is not enabled 2023-09-23T20:23:56.484 INFO:teuthology.orchestra.run.smithi121.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T20:23:56.485 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for mon to start... 2023-09-23T20:23:56.485 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for mon... 2023-09-23T20:23:56.707 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:56 smithi121 ceph-mon[99306]: mkfs ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:23:56.708 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:56 smithi121 ceph-mon[99306]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:23:56.708 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:56 smithi121 bash[99240]: 2429cebcc726bcf2a15cd00526bb0b797dcc759a8a25b75c6a9a6647f13f4c56 2023-09-23T20:23:56.708 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:56 smithi121 systemd[1]: Started Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:23:57.857 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T20:23:57.857 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout id: ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:23:57.857 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout services: 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.577538s) 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout data: 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:23:57.858 INFO:teuthology.orchestra.run.smithi121.stdout:mon is available 2023-09-23T20:23:57.859 INFO:teuthology.orchestra.run.smithi121.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T20:23:57.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:57 smithi121 ceph-mon[99306]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:23:57.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:57 smithi121 ceph-mon[99306]: monmap e1: 1 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0]} removed_ranks: {} 2023-09-23T20:23:57.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:57 smithi121 ceph-mon[99306]: fsmap 2023-09-23T20:23:57.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:57 smithi121 ceph-mon[99306]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T20:23:57.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:57 smithi121 ceph-mon[99306]: mgrmap e1: no daemons active 2023-09-23T20:23:57.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:57 smithi121 ceph-mon[99306]: from='client.? 172.21.15.121:0/2302579097' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T20:23:58.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:58 smithi121 ceph-mon[99306]: from='client.? 172.21.15.121:0/2732088849' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T20:23:58.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:58 smithi121 ceph-mon[99306]: from='client.? 172.21.15.121:0/2732088849' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T20:23:59.023 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout fsid = ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.121:3300,v1:172.21.15.121:6789] 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:23:59.024 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T20:23:59.025 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T20:23:59.025 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T20:23:59.025 INFO:teuthology.orchestra.run.smithi121.stdout:Generating new minimal ceph.conf... 2023-09-23T20:23:59.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:23:59 smithi121 ceph-mon[99306]: from='client.? 172.21.15.121:0/159339203' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:24:00.372 INFO:teuthology.orchestra.run.smithi121.stdout:Restarting the monitor... 2023-09-23T20:24:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:00 smithi121 systemd[1]: Stopping Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:00 smithi121 conmon[99282]: 2023-09-23T20:24:00.625+0000 7f63c4dda700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:24:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:00 smithi121 conmon[99282]: 2023-09-23T20:24:00.625+0000 7f63c4dda700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T20:24:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:00 smithi121 bash[99862]: 2429cebcc726bcf2a15cd00526bb0b797dcc759a8a25b75c6a9a6647f13f4c56 2023-09-23T20:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:01 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T20:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:01 smithi121 systemd[1]: Stopped Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:01 smithi121 systemd[1]: Starting Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:02.100 INFO:teuthology.orchestra.run.smithi121.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T20:24:02.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:24:02.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:24:02.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: pidfile_write: ignore empty --pid-file 2023-09-23T20:24:02.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: load: jerasure load: lrc 2023-09-23T20:24:02.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:24:02.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: DB SUMMARY 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: DB Session ID: EUQQGTVATON373QBY9F2 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83613 ; 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:24:02.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.env: 0x55730e21ba80 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.fs: Posix File System 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.info_log: 0x55730f4285a0 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.statistics: (nil) 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.use_fsync: 0 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:24:02.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.db_log_dir: 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:24:02.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:24:02.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:24:02.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.write_buffer_manager: 0x55731019a900 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:24:02.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.unordered_write: 0 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.row_cache: None 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.wal_filter: None 2023-09-23T20:24:02.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:24:02.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:24:02.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_open_files: -1 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Compression algorithms supported: 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kZSTD supported: 0 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:24:02.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.merge_operator: 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_filter: None 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:24:02.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55730f47afb8) 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: index_type: 0 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: data_block_index_type: 0 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: index_shortening: 1 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: hash_index_allow_collision: 1 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: checksum: 1 2023-09-23T20:24:02.324 INFO:journalctl@ceph.mon.a.smithi121.stdout: no_block_cache: 0 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache: 0x55730f4ad090 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache_options: 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: capacity : 536870912 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: num_shard_bits : 4 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: strict_capacity_limit : 0 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache_compressed: (nil) 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: persistent_cache: (nil) 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_size: 4096 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_size_deviation: 10 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_restart_interval: 16 2023-09-23T20:24:02.325 INFO:journalctl@ceph.mon.a.smithi121.stdout: index_block_restart_interval: 1 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: metadata_block_size: 4096 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: partition_filters: 0 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: use_delta_encoding: 1 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: whole_key_filtering: 1 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: verify_compression: 0 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: format_version: 4 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: enable_index_compression: 1 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_align: 0 2023-09-23T20:24:02.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression: NoCompression 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.num_levels: 7 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:24:02.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:24:02.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:24:02.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:24:02.330 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:24:02.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.ttl: 2592000 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:24:02.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500642047824, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500642048476, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80160, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78428, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72636, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695500642, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "d3cdedbb-126c-42ce-8a13-1eb8400b9e28", "db_session_id": "EUQQGTVATON373QBY9F2"}} 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500642048800, "job": 1, "event": "recovery_finished"} 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55730f441180 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: DB pointer 0x55730f516000 2023-09-23T20:24:02.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** DB Stats ** 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:24:02.334 INFO:journalctl@ceph.mon.a.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.1 0.00 0.00 1 0.000 0 0 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.1 0.00 0.00 1 0.000 0 0 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.1 0.00 0.00 1 0.000 0 0 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 174.1 0.00 0.00 1 0.000 0 0 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:24:02.335 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative compaction: 0.00 GB write, 25.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval compaction: 0.00 GB write, 25.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:24:02.336 INFO:journalctl@ceph.mon.a.smithi121.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.1 0.00 0.00 1 0.000 0 0 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.1 0.00 0.00 1 0.000 0 0 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 174.1 0.00 0.00 1 0.000 0 0 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:24:02.337 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative compaction: 0.00 GB write, 25.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: starting mon.a rank 0 at public addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] at bind addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???) e1 preinit fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???).mds e1 new map 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???).mds e1 print_map 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: e1 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:24:02.338 INFO:journalctl@ceph.mon.a.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout: legacy client fscid: -1 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout: No filesystems configured 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 bash[100193]: 081ecc327acfa7958e19348e2e028fe1a2d042dc6edd34bfcea72a95e3fdf195 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 systemd[1]: Started Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: monmap e1: 1 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0]} removed_ranks: {} 2023-09-23T20:24:02.339 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: fsmap 2023-09-23T20:24:02.340 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T20:24:02.340 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:02 smithi121 ceph-mon[100258]: mgrmap e1: no daemons active 2023-09-23T20:24:03.347 INFO:teuthology.orchestra.run.smithi121.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T20:24:03.347 INFO:teuthology.orchestra.run.smithi121.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:24:03.347 INFO:teuthology.orchestra.run.smithi121.stdout:Creating mgr... 2023-09-23T20:24:03.348 INFO:teuthology.orchestra.run.smithi121.stdout:Verifying port 9283 ... 2023-09-23T20:24:03.475 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T20:24:03.475 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T20:24:03.563 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7.target.wants/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@.service. 2023-09-23T20:24:03.653 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:03 smithi121 systemd[1]: Starting Ceph mgr.y for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:24:03.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1420390990' entity='client.admin' 2023-09-23T20:24:04.202 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:24:04.202 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout disabled 2023-09-23T20:24:04.209 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:24:04.210 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout inactive 2023-09-23T20:24:04.210 INFO:teuthology.orchestra.run.smithi121.stdout:firewalld.service is not enabled 2023-09-23T20:24:04.210 INFO:teuthology.orchestra.run.smithi121.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T20:24:04.219 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:24:04.219 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout disabled 2023-09-23T20:24:04.227 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:24:04.227 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout inactive 2023-09-23T20:24:04.228 INFO:teuthology.orchestra.run.smithi121.stdout:firewalld.service is not enabled 2023-09-23T20:24:04.228 INFO:teuthology.orchestra.run.smithi121.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T20:24:04.228 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for mgr to start... 2023-09-23T20:24:04.228 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for mgr... 2023-09-23T20:24:04.305 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:04 smithi121 bash[100640]: 666708c47463a034a01e0f6dbf6473f8a19bb7a5e56032e8c503c568ba5e4977 2023-09-23T20:24:04.305 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:04 smithi121 systemd[1]: Started Ceph mgr.y for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:24:04.575 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:04 smithi121 conmon[100681]: 2023-09-23T20:24:04.305+0000 7f8cca49e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:24:04.575 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:04 smithi121 conmon[100681]: 2023-09-23T20:24:04.382+0000 7f8cca49e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:24:04.575 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:04 smithi121 conmon[100681]: 2023-09-23T20:24:04.474+0000 7f8cca49e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:24:04.575 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:04 smithi121 conmon[100681]: 2023-09-23T20:24:04.575+0000 7f8cca49e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:24:04.862 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:04 smithi121 conmon[100681]: 2023-09-23T20:24:04.746+0000 7f8cca49e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:24:05.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:05 smithi121 conmon[100681]: 2023-09-23T20:24:05.075+0000 7f8cca49e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:24:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2313182180' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:24:05.570 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:24:05.573 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:05.573 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsid": "ffb133f0-5a4e-11ee-8db4-212e2dc638e7", 2023-09-23T20:24:05.573 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:24:05.573 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T20:24:05.574 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:24:05.575 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:24:05.578 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:24:05.579 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:05.580 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:23:56.454616+0000", 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:05.581 INFO:teuthology.orchestra.run.smithi121.stdout:mgr not available, waiting (1/15)... 2023-09-23T20:24:05.581 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:05 smithi121 conmon[100681]: 2023-09-23T20:24:05.162+0000 7f8cca49e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:24:05.581 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:05 smithi121 conmon[100681]: 2023-09-23T20:24:05.288+0000 7f8cca49e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:24:05.581 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:05 smithi121 conmon[100681]: 2023-09-23T20:24:05.379+0000 7f8cca49e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:24:05.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:05 smithi121 conmon[100681]: 2023-09-23T20:24:05.796+0000 7f8cca49e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:24:06.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:06 smithi121 conmon[100681]: 2023-09-23T20:24:06.356+0000 7f8cca49e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:24:06.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:06 smithi121 conmon[100681]: 2023-09-23T20:24:06.521+0000 7f8cca49e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:24:07.532 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:07 smithi121 conmon[100681]: 2023-09-23T20:24:07.281+0000 7f8cca49e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:24:07.532 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:07 smithi121 conmon[100681]: 2023-09-23T20:24:07.369+0000 7f8cca49e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:24:07.786 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:07 smithi121 conmon[100681]: 2023-09-23T20:24:07.532+0000 7f8cca49e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:24:08.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:07 smithi121 conmon[100681]: 2023-09-23T20:24:07.816+0000 7f8cca49e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:24:08.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:07 smithi121 conmon[100681]: 2023-09-23T20:24:07.905+0000 7f8cca49e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:24:08.467 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:08 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1722279508' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:24:08.467 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:08 smithi121 conmon[100681]: 2023-09-23T20:24:08.201+0000 7f8cca49e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:24:08.808 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsid": "ffb133f0-5a4e-11ee-8db4-212e2dc638e7", 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:24:08.809 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:08.810 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:24:08.811 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:24:08.812 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:23:56.454616+0000", 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:mgr not available, waiting (2/15)... 2023-09-23T20:24:08.814 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:08 smithi121 conmon[100681]: 2023-09-23T20:24:08.467+0000 7f8cca49e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:24:09.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:09 smithi121 conmon[100681]: 2023-09-23T20:24:09.174+0000 7f8cca49e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:24:09.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:09 smithi121 conmon[100681]: 2023-09-23T20:24:09.302+0000 7f8cca49e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:24:09.810 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:09 smithi121 conmon[100681]: 2023-09-23T20:24:09.488+0000 7f8cca49e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:24:09.811 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:09 smithi121 conmon[100681]: 2023-09-23T20:24:09.600+0000 7f8cca49e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:24:10.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:09 smithi121 conmon[100681]: 2023-09-23T20:24:09.810+0000 7f8cca49e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:24:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: Activating manager daemon y 2023-09-23T20:24:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: mgrmap e2: y(active, starting, since 0.00271516s) 2023-09-23T20:24:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:24:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:24:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:24:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: Manager daemon y is now available 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:10 smithi121 ceph-mon[100258]: from='mgr.14100 172.21.15.121:0/914604055' entity='mgr.y' 2023-09-23T20:24:10.914 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:10 smithi121 conmon[100681]: 2023-09-23T20:24:10.646+0000 7f8cca49e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsid": "ffb133f0-5a4e-11ee-8db4-212e2dc638e7", 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:24:11.962 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:23:56.454616+0000", 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:11.963 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:24:11.964 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:11.964 INFO:teuthology.orchestra.run.smithi121.stdout:mgr not available, waiting (3/15)... 2023-09-23T20:24:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:11 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2080126197' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:24:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:11 smithi121 ceph-mon[100258]: mgrmap e3: y(active, since 1.0056s) 2023-09-23T20:24:14.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:14 smithi121 ceph-mon[100258]: mgrmap e4: y(active, since 2s) 2023-09-23T20:24:15.348 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:24:15.348 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:15.348 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsid": "ffb133f0-5a4e-11ee-8db4-212e2dc638e7", 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:24:15.349 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:24:15.350 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:24:15.351 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:15.352 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:24:15.353 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:24:15.353 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:23:56.454616+0000", 2023-09-23T20:24:15.353 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:24:15.353 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:24:15.353 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:24:15.353 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:15.353 INFO:teuthology.orchestra.run.smithi121.stdout:mgr is available 2023-09-23T20:24:15.353 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:15 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2698391880' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:24:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:16 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3889478911' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T20:24:16.686 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:24:16.686 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T20:24:16.686 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout fsid = ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T20:24:16.687 INFO:teuthology.orchestra.run.smithi121.stdout:Enabling cephadm module... 2023-09-23T20:24:18.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:18 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/425721271' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T20:24:18.390 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: ignoring --setuser ceph since I am not root 2023-09-23T20:24:18.390 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: ignoring --setgroup ceph since I am not root 2023-09-23T20:24:18.390 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: 2023-09-23T20:24:18.219+0000 7f6e110ba1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:24:18.390 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: 2023-09-23T20:24:18.298+0000 7f6e110ba1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:24:18.390 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: 2023-09-23T20:24:18.389+0000 7f6e110ba1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:24:18.647 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: 2023-09-23T20:24:18.481+0000 7f6e110ba1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:24:18.908 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: 2023-09-23T20:24:18.647+0000 7f6e110ba1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:24:19.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:19 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/425721271' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T20:24:19.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:19 smithi121 ceph-mon[100258]: mgrmap e5: y(active, since 7s) 2023-09-23T20:24:19.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:18 smithi121 conmon[100681]: 2023-09-23T20:24:18.973+0000 7f6e110ba1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:24:19.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:19 smithi121 conmon[100681]: 2023-09-23T20:24:19.060+0000 7f6e110ba1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:24:19.500 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:19 smithi121 conmon[100681]: 2023-09-23T20:24:19.192+0000 7f6e110ba1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:24:19.501 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:19 smithi121 conmon[100681]: 2023-09-23T20:24:19.292+0000 7f6e110ba1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:24:19.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:19 smithi121 conmon[100681]: 2023-09-23T20:24:19.729+0000 7f6e110ba1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:24:20.051 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:20.052 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T20:24:20.052 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:24:20.052 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T20:24:20.052 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T20:24:20.052 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:20.052 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for the mgr to restart... 2023-09-23T20:24:20.052 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for mgr epoch 5... 2023-09-23T20:24:20.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3239776798' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T20:24:20.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:20 smithi121 conmon[100681]: 2023-09-23T20:24:20.311+0000 7f6e110ba1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:24:20.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:20 smithi121 conmon[100681]: 2023-09-23T20:24:20.484+0000 7f6e110ba1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:24:21.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:21 smithi121 conmon[100681]: 2023-09-23T20:24:21.237+0000 7f6e110ba1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:24:21.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:21 smithi121 conmon[100681]: 2023-09-23T20:24:21.324+0000 7f6e110ba1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:24:21.757 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:21 smithi121 conmon[100681]: 2023-09-23T20:24:21.489+0000 7f6e110ba1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:24:22.123 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:21 smithi121 conmon[100681]: 2023-09-23T20:24:21.756+0000 7f6e110ba1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:24:22.123 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:21 smithi121 conmon[100681]: 2023-09-23T20:24:21.841+0000 7f6e110ba1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:24:22.379 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:22 smithi121 conmon[100681]: 2023-09-23T20:24:22.122+0000 7f6e110ba1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:24:22.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:22 smithi121 conmon[100681]: 2023-09-23T20:24:22.377+0000 7f6e110ba1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:24:23.385 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:23 smithi121 conmon[100681]: 2023-09-23T20:24:23.070+0000 7f6e110ba1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:24:23.385 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:23 smithi121 conmon[100681]: 2023-09-23T20:24:23.198+0000 7f6e110ba1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:24:23.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:23 smithi121 conmon[100681]: 2023-09-23T20:24:23.384+0000 7f6e110ba1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:24:23.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:23 smithi121 conmon[100681]: 2023-09-23T20:24:23.496+0000 7f6e110ba1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:24:24.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:23 smithi121 conmon[100681]: 2023-09-23T20:24:23.709+0000 7f6e110ba1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:24 smithi121 conmon[100681]: 2023-09-23T20:24:24.546+0000 7f6e110ba1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: Active manager daemon y restarted 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: Activating manager daemon y 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: mgrmap e6: y(active, starting, since 0.00525957s) 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:24:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:24:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:24:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:24:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: Manager daemon y is now available 2023-09-23T20:24:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:24 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:25 smithi121 ceph-mon[100258]: Found migration_current of "None". Setting to last migration. 2023-09-23T20:24:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:25 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:25 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:24:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:25 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:24:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:25 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:25.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:25 smithi121 ceph-mon[100258]: mgrmap e7: y(active, since 1.00567s) 2023-09-23T20:24:25.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:25 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:26.167 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:26.167 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T20:24:26.167 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T20:24:26.167 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:26.167 INFO:teuthology.orchestra.run.smithi121.stdout:mgr epoch 5 is available 2023-09-23T20:24:26.167 INFO:teuthology.orchestra.run.smithi121.stdout:Setting orchestrator backend to cephadm... 2023-09-23T20:24:26.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:26 smithi121 ceph-mon[100258]: [23/Sep/2023:20:24:25] ENGINE Bus STARTING 2023-09-23T20:24:26.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:26 smithi121 ceph-mon[100258]: [23/Sep/2023:20:24:25] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:24:26.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:26 smithi121 ceph-mon[100258]: [23/Sep/2023:20:24:25] ENGINE Bus STARTED 2023-09-23T20:24:26.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:26 smithi121 ceph-mon[100258]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T20:24:26.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:26 smithi121 ceph-mon[100258]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T20:24:27.728 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:27 smithi121 ceph-mon[100258]: mgrmap e8: y(active, since 2s) 2023-09-23T20:24:27.728 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:27 smithi121 ceph-mon[100258]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:27.728 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:27 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:27.728 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:27 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:28.757 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T20:24:28.757 INFO:teuthology.orchestra.run.smithi121.stdout:Generating ssh key... 2023-09-23T20:24:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:29 smithi121 ceph-mon[100258]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:30.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: Generating public/private rsa key pair. 2023-09-23T20:24:30.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: Your identification has been saved in /tmp/tmp2m4bbk6l/key. 2023-09-23T20:24:30.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: Your public key has been saved in /tmp/tmp2m4bbk6l/key.pub. 2023-09-23T20:24:30.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: The key fingerprint is: 2023-09-23T20:24:30.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: SHA256:QIFl2j+f0m56j29MagTHqtds7R9I/5tTK4mDQXsXHfw ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:24:30.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: The key's randomart image is: 2023-09-23T20:24:30.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: +---[RSA 3072]----+ 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | o=. . | 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | .= o | 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | . o . . o| 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | o..o . .E| 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | S+. . . | 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | .*.+.+ .| 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | ..+B=+ + o| 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | . .+O++o =.| 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: | ..*o==.oo+| 2023-09-23T20:24:30.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:29 smithi121 conmon[100681]: +----[SHA256]-----+ 2023-09-23T20:24:31.025 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:30 smithi121 ceph-mon[100258]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:31.025 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:30 smithi121 ceph-mon[100258]: Generating ssh key... 2023-09-23T20:24:31.025 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:30 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:31.025 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:30 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:31.631 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIiTk795WqAC44WhhmFQm1qgPcHmo6GAQ3lriaUFNZqwb66mGbWJJ6sWztP+G5kOf0W3ugXoDRO1denEhMQhUKkwzt/VaoaEi5DrnNKoSRVi3YWK4xxPnTQ/GgIKmR/F/znqgAHsZIgxkMcYUSP5C4CE0HlR1lqC74djFS+vAvL2Arkk4HhxJyaqsX0rL1mD7sLI/fD18w6dTBbMPJ2k2a+aXu5uNaeBYIsxAY/WrAYkJbjrbD9ptldNLjoTn/ABnZ2wWLAGB4dUVO1vGHDZsZPMgzh/aAyOXxR55OsEj9q6yliUtVLJG/NCG6XM+CYvJBc6UCqsI4z/hX2rMGsI6QRJOI1epYXMSIJUAr/GEuvMDOsvlIpuHWLluIVH5miwHsGRUgefQF7N7IDS9y+jgzgfq8QZcz0QrZh4DY+cha93oOJYH0LVt3QtGTY5GRG5V/xJFRC4L+Rc8w0xiQQh7p3Bn+FSUsW+YN/4aYVCjxiSssI2NlLGLHDX1txefeNfU= ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:24:31.632 INFO:teuthology.orchestra.run.smithi121.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:24:31.632 INFO:teuthology.orchestra.run.smithi121.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T20:24:31.633 INFO:teuthology.orchestra.run.smithi121.stdout:Adding host smithi121... 2023-09-23T20:24:31.888 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:31 smithi121 ceph-mon[100258]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:33.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:33 smithi121 ceph-mon[100258]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi121", "addr": "172.21.15.121", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:33 smithi121 ceph-mon[100258]: Deploying cephadm binary to smithi121 2023-09-23T20:24:34.756 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout Added host 'smithi121' with addr '172.21.15.121' 2023-09-23T20:24:34.756 INFO:teuthology.orchestra.run.smithi121.stdout:Deploying unmanaged mon service... 2023-09-23T20:24:35.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:35 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:35 smithi121 ceph-mon[100258]: Added host smithi121 2023-09-23T20:24:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:35 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:36.493 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T20:24:36.494 INFO:teuthology.orchestra.run.smithi121.stdout:Deploying unmanaged mgr service... 2023-09-23T20:24:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:36 smithi121 ceph-mon[100258]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:36 smithi121 ceph-mon[100258]: Saving service mon spec with placement count:5 2023-09-23T20:24:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:36 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:38.122 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T20:24:38.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:37 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:38.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:37 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:38.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:37 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:39.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:38 smithi121 ceph-mon[100258]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:39.096 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:38 smithi121 ceph-mon[100258]: Saving service mgr spec with placement count:2 2023-09-23T20:24:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:39 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3183977720' entity='client.admin' 2023-09-23T20:24:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:39 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:39 smithi121 ceph-mon[100258]: from='mgr.14120 172.21.15.121:0/4233846220' entity='mgr.y' 2023-09-23T20:24:41.072 INFO:teuthology.orchestra.run.smithi121.stdout:Enabling the dashboard module... 2023-09-23T20:24:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:41 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1809970312' entity='client.admin' 2023-09-23T20:24:43.312 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:43 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/158185598' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T20:24:43.312 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: ignoring --setuser ceph since I am not root 2023-09-23T20:24:43.312 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: ignoring --setgroup ceph since I am not root 2023-09-23T20:24:43.312 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: 2023-09-23T20:24:43.242+0000 7f81dc7951c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:24:43.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: 2023-09-23T20:24:43.322+0000 7f81dc7951c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:24:43.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: 2023-09-23T20:24:43.409+0000 7f81dc7951c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:24:43.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: 2023-09-23T20:24:43.500+0000 7f81dc7951c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:24:43.953 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: 2023-09-23T20:24:43.666+0000 7f81dc7951c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:24:44.221 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:43 smithi121 conmon[100681]: 2023-09-23T20:24:43.997+0000 7f81dc7951c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:24:44.221 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:44 smithi121 conmon[100681]: 2023-09-23T20:24:44.088+0000 7f81dc7951c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:24:44.221 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:44 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/158185598' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T20:24:44.221 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:44 smithi121 ceph-mon[100258]: mgrmap e9: y(active, since 18s) 2023-09-23T20:24:44.512 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:44 smithi121 conmon[100681]: 2023-09-23T20:24:44.219+0000 7f81dc7951c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:24:44.513 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:44 smithi121 conmon[100681]: 2023-09-23T20:24:44.317+0000 7f81dc7951c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:24:44.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:44 smithi121 conmon[100681]: 2023-09-23T20:24:44.742+0000 7f81dc7951c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:24:45.190 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:45.190 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T20:24:45.191 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:24:45.191 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T20:24:45.191 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T20:24:45.191 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:45.191 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for the mgr to restart... 2023-09-23T20:24:45.191 INFO:teuthology.orchestra.run.smithi121.stdout:Waiting for mgr epoch 9... 2023-09-23T20:24:45.319 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:45 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3737287704' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T20:24:45.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:45 smithi121 conmon[100681]: 2023-09-23T20:24:45.318+0000 7f81dc7951c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:24:45.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:45 smithi121 conmon[100681]: 2023-09-23T20:24:45.499+0000 7f81dc7951c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:24:46.512 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:46 smithi121 conmon[100681]: 2023-09-23T20:24:46.256+0000 7f81dc7951c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:24:46.512 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:46 smithi121 conmon[100681]: 2023-09-23T20:24:46.345+0000 7f81dc7951c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:24:46.512 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:46 smithi121 conmon[100681]: 2023-09-23T20:24:46.511+0000 7f81dc7951c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:24:47.147 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:46 smithi121 conmon[100681]: 2023-09-23T20:24:46.780+0000 7f81dc7951c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:24:47.147 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:46 smithi121 conmon[100681]: 2023-09-23T20:24:46.866+0000 7f81dc7951c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:24:47.147 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:47 smithi121 conmon[100681]: 2023-09-23T20:24:47.146+0000 7f81dc7951c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:24:47.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:47 smithi121 conmon[100681]: 2023-09-23T20:24:47.399+0000 7f81dc7951c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:24:48.399 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:48 smithi121 conmon[100681]: 2023-09-23T20:24:48.085+0000 7f81dc7951c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:24:48.399 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:48 smithi121 conmon[100681]: 2023-09-23T20:24:48.213+0000 7f81dc7951c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:24:48.399 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:48 smithi121 conmon[100681]: 2023-09-23T20:24:48.398+0000 7f81dc7951c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:24:48.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:48 smithi121 conmon[100681]: 2023-09-23T20:24:48.510+0000 7f81dc7951c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:24:49.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:48 smithi121 conmon[100681]: 2023-09-23T20:24:48.721+0000 7f81dc7951c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:24:49 smithi121 conmon[100681]: 2023-09-23T20:24:49.558+0000 7f81dc7951c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: Active manager daemon y restarted 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: Activating manager daemon y 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: mgrmap e10: y(active, starting, since 0.00310363s) 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:24:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:24:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:24:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:24:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: Manager daemon y is now available 2023-09-23T20:24:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:50.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:50 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:24:50.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:50 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:24:50.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:50 smithi121 ceph-mon[100258]: mgrmap e11: y(active, since 1.00549s) 2023-09-23T20:24:51.227 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:24:51.228 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T20:24:51.228 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T20:24:51.228 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:24:51.228 INFO:teuthology.orchestra.run.smithi121.stdout:mgr epoch 9 is available 2023-09-23T20:24:51.228 INFO:teuthology.orchestra.run.smithi121.stdout:Generating a dashboard self-signed certificate... 2023-09-23T20:24:52.046 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:51 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:24:52.046 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:51 smithi121 ceph-mon[100258]: [23/Sep/2023:20:24:50] ENGINE Bus STARTING 2023-09-23T20:24:52.046 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:51 smithi121 ceph-mon[100258]: [23/Sep/2023:20:24:50] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:24:52.046 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:51 smithi121 ceph-mon[100258]: [23/Sep/2023:20:24:50] ENGINE Bus STARTED 2023-09-23T20:24:52.046 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:51 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:24:52.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:51 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:24:53.144 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T20:24:53.144 INFO:teuthology.orchestra.run.smithi121.stdout:Creating initial admin user... 2023-09-23T20:24:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:53 smithi121 ceph-mon[100258]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:53 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:24:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:53 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:24:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:53 smithi121 ceph-mon[100258]: mgrmap e12: y(active, since 2s) 2023-09-23T20:24:54.752 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$A9dYPq0YNGkSTX4xgjbJRuf6aeZIaWZb0YAMlHf1qoMQ8uJG6N4zu", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695500694, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T20:24:54.752 INFO:teuthology.orchestra.run.smithi121.stdout:Fetching dashboard port number... 2023-09-23T20:24:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:55 smithi121 ceph-mon[100258]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:55 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:24:56.026 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T20:24:56.036 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:24:56.036 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout disabled 2023-09-23T20:24:56.044 INFO:teuthology.orchestra.run.smithi121.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:24:56.044 INFO:teuthology.orchestra.run.smithi121.stdout:systemctl: stdout inactive 2023-09-23T20:24:56.044 INFO:teuthology.orchestra.run.smithi121.stdout:firewalld.service is not enabled 2023-09-23T20:24:56.044 INFO:teuthology.orchestra.run.smithi121.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T20:24:56.045 INFO:teuthology.orchestra.run.smithi121.stdout:Ceph Dashboard is now available at: 2023-09-23T20:24:56.046 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:56.046 INFO:teuthology.orchestra.run.smithi121.stdout: URL: https://smithi121:8443/ 2023-09-23T20:24:56.046 INFO:teuthology.orchestra.run.smithi121.stdout: User: admin 2023-09-23T20:24:56.046 INFO:teuthology.orchestra.run.smithi121.stdout: Password: 1yrjngsymx 2023-09-23T20:24:56.046 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:56.047 INFO:teuthology.orchestra.run.smithi121.stdout:Saving cluster configuration to /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config directory 2023-09-23T20:24:56.047 INFO:teuthology.orchestra.run.smithi121.stdout:Enabling autotune for osd_memory_target 2023-09-23T20:24:56.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:56 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1588275315' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T20:24:57.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:57 smithi121 ceph-mon[100258]: mgrmap e13: y(active, since 6s) 2023-09-23T20:24:58.717 INFO:teuthology.orchestra.run.smithi121.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T20:24:58.718 INFO:teuthology.orchestra.run.smithi121.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T20:24:58.718 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.718 INFO:teuthology.orchestra.run.smithi121.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:24:58.718 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.718 INFO:teuthology.orchestra.run.smithi121.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: ceph telemetry on 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout:For more information see: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:24:58.719 INFO:teuthology.orchestra.run.smithi121.stdout:Bootstrap complete. 2023-09-23T20:24:58.756 INFO:tasks.cephadm:Fetching config... 2023-09-23T20:24:58.756 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:24:58.756 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T20:24:58.772 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T20:24:58.773 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:24:58.773 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T20:24:58.830 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T20:24:58.831 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:24:58.831 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T20:24:58.898 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T20:24:58.899 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:24:58.899 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T20:24:58.955 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T20:24:58.955 DEBUG:teuthology.orchestra.run.smithi121:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIiTk795WqAC44WhhmFQm1qgPcHmo6GAQ3lriaUFNZqwb66mGbWJJ6sWztP+G5kOf0W3ugXoDRO1denEhMQhUKkwzt/VaoaEi5DrnNKoSRVi3YWK4xxPnTQ/GgIKmR/F/znqgAHsZIgxkMcYUSP5C4CE0HlR1lqC74djFS+vAvL2Arkk4HhxJyaqsX0rL1mD7sLI/fD18w6dTBbMPJ2k2a+aXu5uNaeBYIsxAY/WrAYkJbjrbD9ptldNLjoTn/ABnZ2wWLAGB4dUVO1vGHDZsZPMgzh/aAyOXxR55OsEj9q6yliUtVLJG/NCG6XM+CYvJBc6UCqsI4z/hX2rMGsI6QRJOI1epYXMSIJUAr/GEuvMDOsvlIpuHWLluIVH5miwHsGRUgefQF7N7IDS9y+jgzgfq8QZcz0QrZh4DY+cha93oOJYH0LVt3QtGTY5GRG5V/xJFRC4L+Rc8w0xiQQh7p3Bn+FSUsW+YN/4aYVCjxiSssI2NlLGLHDX1txefeNfU= ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T20:24:59.035 INFO:teuthology.orchestra.run.smithi121.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIiTk795WqAC44WhhmFQm1qgPcHmo6GAQ3lriaUFNZqwb66mGbWJJ6sWztP+G5kOf0W3ugXoDRO1denEhMQhUKkwzt/VaoaEi5DrnNKoSRVi3YWK4xxPnTQ/GgIKmR/F/znqgAHsZIgxkMcYUSP5C4CE0HlR1lqC74djFS+vAvL2Arkk4HhxJyaqsX0rL1mD7sLI/fD18w6dTBbMPJ2k2a+aXu5uNaeBYIsxAY/WrAYkJbjrbD9ptldNLjoTn/ABnZ2wWLAGB4dUVO1vGHDZsZPMgzh/aAyOXxR55OsEj9q6yliUtVLJG/NCG6XM+CYvJBc6UCqsI4z/hX2rMGsI6QRJOI1epYXMSIJUAr/GEuvMDOsvlIpuHWLluIVH5miwHsGRUgefQF7N7IDS9y+jgzgfq8QZcz0QrZh4DY+cha93oOJYH0LVt3QtGTY5GRG5V/xJFRC4L+Rc8w0xiQQh7p3Bn+FSUsW+YN/4aYVCjxiSssI2NlLGLHDX1txefeNfU= ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:24:59.047 DEBUG:teuthology.orchestra.run.smithi123:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIiTk795WqAC44WhhmFQm1qgPcHmo6GAQ3lriaUFNZqwb66mGbWJJ6sWztP+G5kOf0W3ugXoDRO1denEhMQhUKkwzt/VaoaEi5DrnNKoSRVi3YWK4xxPnTQ/GgIKmR/F/znqgAHsZIgxkMcYUSP5C4CE0HlR1lqC74djFS+vAvL2Arkk4HhxJyaqsX0rL1mD7sLI/fD18w6dTBbMPJ2k2a+aXu5uNaeBYIsxAY/WrAYkJbjrbD9ptldNLjoTn/ABnZ2wWLAGB4dUVO1vGHDZsZPMgzh/aAyOXxR55OsEj9q6yliUtVLJG/NCG6XM+CYvJBc6UCqsI4z/hX2rMGsI6QRJOI1epYXMSIJUAr/GEuvMDOsvlIpuHWLluIVH5miwHsGRUgefQF7N7IDS9y+jgzgfq8QZcz0QrZh4DY+cha93oOJYH0LVt3QtGTY5GRG5V/xJFRC4L+Rc8w0xiQQh7p3Bn+FSUsW+YN/4aYVCjxiSssI2NlLGLHDX1txefeNfU= ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T20:24:59.093 INFO:teuthology.orchestra.run.smithi123.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIiTk795WqAC44WhhmFQm1qgPcHmo6GAQ3lriaUFNZqwb66mGbWJJ6sWztP+G5kOf0W3ugXoDRO1denEhMQhUKkwzt/VaoaEi5DrnNKoSRVi3YWK4xxPnTQ/GgIKmR/F/znqgAHsZIgxkMcYUSP5C4CE0HlR1lqC74djFS+vAvL2Arkk4HhxJyaqsX0rL1mD7sLI/fD18w6dTBbMPJ2k2a+aXu5uNaeBYIsxAY/WrAYkJbjrbD9ptldNLjoTn/ABnZ2wWLAGB4dUVO1vGHDZsZPMgzh/aAyOXxR55OsEj9q6yliUtVLJG/NCG6XM+CYvJBc6UCqsI4z/hX2rMGsI6QRJOI1epYXMSIJUAr/GEuvMDOsvlIpuHWLluIVH5miwHsGRUgefQF7N7IDS9y+jgzgfq8QZcz0QrZh4DY+cha93oOJYH0LVt3QtGTY5GRG5V/xJFRC4L+Rc8w0xiQQh7p3Bn+FSUsW+YN/4aYVCjxiSssI2NlLGLHDX1txefeNfU= ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:24:59.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T20:24:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:24:59 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/795839557' entity='client.admin' 2023-09-23T20:24:59.441 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi121", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:25:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1604829800' entity='client.admin' 2023-09-23T20:25:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:02.334 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T20:25:02.334 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T20:25:03.206 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:05.651 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi123 2023-09-23T20:25:05.651 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:25:05.651 DEBUG:teuthology.orchestra.run.smithi123:> dd of=/etc/ceph/ceph.conf 2023-09-23T20:25:05.672 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:25:05.672 DEBUG:teuthology.orchestra.run.smithi123:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:25:05.732 INFO:tasks.cephadm:Adding host smithi123 to orchestrator... 2023-09-23T20:25:05.732 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi123 2023-09-23T20:25:05.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:06.076 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:06 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:06 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:25:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:06 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:25:08.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:08 smithi121 ceph-mon[100258]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi123", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:09.621 INFO:teuthology.orchestra.run.smithi121.stdout:Added host 'smithi123' with addr '172.21.15.123' 2023-09-23T20:25:09.652 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:09 smithi121 ceph-mon[100258]: Deploying cephadm binary to smithi123 2023-09-23T20:25:10.304 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T20:25:10.640 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:10.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:10 smithi121 ceph-mon[100258]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:10.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:10.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:10 smithi121 ceph-mon[100258]: Added host smithi123 2023-09-23T20:25:10.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:10.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:10.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:12.367 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:25:12.367 INFO:teuthology.orchestra.run.smithi121.stdout:[{"addr": "172.21.15.121", "hostname": "smithi121", "labels": [], "status": ""}, {"addr": "172.21.15.123", "hostname": "smithi123", "labels": [], "status": ""}] 2023-09-23T20:25:12.387 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:12 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:12.387 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:12 smithi121 ceph-mon[100258]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:13.036 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T20:25:13.036 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T20:25:13.367 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:13.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:13 smithi121 ceph-mon[100258]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:25:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:14 smithi121 ceph-mon[100258]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:15.341 INFO:teuthology.orchestra.run.smithi121.stderr:adjusted tunables profile to default 2023-09-23T20:25:15.587 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:25:15.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:25:15.589 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/778578278' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T20:25:15.589 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:15.589 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:15.589 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:15 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:16.027 INFO:tasks.cephadm:Adding mon.a on smithi121 2023-09-23T20:25:16.028 INFO:tasks.cephadm:Adding mon.c on smithi121 2023-09-23T20:25:16.028 INFO:tasks.cephadm:Adding mon.b on smithi123 2023-09-23T20:25:16.028 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi121:172.21.15.121=a;smithi121:[v2:172.21.15.121:3301,v1:172.21.15.121:6790]=c;smithi123:172.21.15.123=b' 2023-09-23T20:25:16.401 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:16.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:16 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/778578278' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:25:16.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:16 smithi121 ceph-mon[100258]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:16.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:16 smithi121 ceph-mon[100258]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:17.995 INFO:teuthology.orchestra.run.smithi123.stdout:Scheduled mon update... 2023-09-23T20:25:18.672 DEBUG:teuthology.orchestra.run.smithi121:mon.c> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T20:25:18.675 DEBUG:teuthology.orchestra.run.smithi123:mon.b> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T20:25:18.678 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:25:18.679 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:25:18.707 INFO:journalctl@ceph.mon.b.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:25:18.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi121:172.21.15.121=a;smithi121:[v2:172.21.15.121:3301,v1:172.21.15.121:6790]=c;smithi123:172.21.15.123=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: Saving service mon spec with placement smithi121:172.21.15.121=a;smithi121:[v2:172.21.15.121:3301,v1:172.21.15.121:6790]=c;smithi123:172.21.15.123=b;count:3 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:18 smithi121 ceph-mon[100258]: Deploying daemon mon.b on smithi123 2023-09-23T20:25:19.645 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:20 smithi121 ceph-mon[100258]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:21.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:21 smithi123 systemd[1]: Starting Ceph mon.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:22.003 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:25:22.003 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:23:53.656082Z","created":"2023-09-23T20:23:53.656082Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T20:25:22.005 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2023-09-23T20:25:22.305 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: pidfile_write: ignore empty --pid-file 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: load: jerasure load: lrc 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: DB SUMMARY 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: DB Session ID: FT8641ARA2A202NKV2OF 2023-09-23T20:25:22.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.env: 0x55c64c3a6a80 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.fs: Posix File System 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.info_log: 0x55c64e7565a0 2023-09-23T20:25:22.307 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.statistics: (nil) 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.use_fsync: 0 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:25:22.308 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.db_log_dir: 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:25:22.309 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.write_buffer_manager: 0x55c64f4c8900 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.unordered_write: 0 2023-09-23T20:25:22.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.row_cache: None 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.wal_filter: None 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:25:22.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:25:22.312 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_open_files: -1 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:25:22.313 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Compression algorithms supported: 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kZSTD supported: 0 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:25:22.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.merge_operator: 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_filter: None 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c64e7a8fb8) 2023-09-23T20:25:22.315 INFO:journalctl@ceph.mon.b.smithi123.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: index_type: 0 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: data_block_index_type: 0 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: index_shortening: 1 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: hash_index_allow_collision: 1 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: checksum: 1 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: no_block_cache: 0 2023-09-23T20:25:22.316 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache: 0x55c64e7db090 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache_options: 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: capacity : 536870912 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: num_shard_bits : 4 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: strict_capacity_limit : 0 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache_compressed: (nil) 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: persistent_cache: (nil) 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_size: 4096 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_size_deviation: 10 2023-09-23T20:25:22.317 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_restart_interval: 16 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: index_block_restart_interval: 1 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: metadata_block_size: 4096 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: partition_filters: 0 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: use_delta_encoding: 1 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: whole_key_filtering: 1 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: verify_compression: 0 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: format_version: 4 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: enable_index_compression: 1 2023-09-23T20:25:22.318 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_align: 0 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression: NoCompression 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.num_levels: 7 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:25:22.319 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:25:22.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:25:22.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:25:22.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:25:22.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:25:22.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:25:22.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:25:22.321 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:25:22.321 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:25:22.321 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:25:22.321 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:25:22.321 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:25:22.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:25:22.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:25:22.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.ttl: 2592000 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:25:22.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500722127311, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T20:25:22.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500722127583, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695500722, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "bfcf2e88-0a7d-40ac-a1ff-f75ece987af9", "db_session_id": "FT8641ARA2A202NKV2OF"}} 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500722127916, "job": 1, "event": "recovery_finished"} 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c64e76f180 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: DB pointer 0x55c64e844000 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** DB Stats ** 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:25:22.327 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:22.328 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:25:22.329 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:22.330 INFO:journalctl@ceph.mon.b.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:25:22.331 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: using public_addr v2:172.21.15.123:0/0 -> [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: starting mon.b rank -1 at public addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] at bind addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(???) e0 preinit fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 bash[104884]: 6a36337c715a282e8d4ae3540f8ed0f0d11566b814dfb20eea600ec8478bb04f 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 systemd[1]: Started Ceph mon.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).mds e1 new map 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).mds e1 print_map 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout: e1 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout: legacy client fscid: -1 2023-09-23T20:25:22.332 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout: No filesystems configured 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.333 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/778578278' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/778578278' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi121:172.21.15.121=a;smithi121:[v2:172.21.15.121:3301,v1:172.21.15.121:6790]=c;smithi123:172.21.15.123=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: Saving service mon spec with placement smithi121:172.21.15.121=a;smithi121:[v2:172.21.15.121:3301,v1:172.21.15.121:6790]=c;smithi123:172.21.15.123=b;count:3 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: Deploying daemon mon.b on smithi123 2023-09-23T20:25:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T20:25:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:22 smithi123 ceph-mon[104968]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-23T20:25:23.853 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:25:23.854 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:25:24.205 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: mon.a calling monitor election 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: mon.b calling monitor election 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: monmap e2: 2 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} removed_ranks: {} 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: fsmap 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: mgrmap e13: y(active, since 37s) 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: overall HEALTH_OK 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:27.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: mon.a calling monitor election 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: mon.b calling monitor election 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:25:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: monmap e2: 2 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} removed_ranks: {} 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: fsmap 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: mgrmap e13: y(active, since 37s) 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: overall HEALTH_OK 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:28 smithi121 ceph-mon[100258]: Deploying daemon mon.c on smithi121 2023-09-23T20:25:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:28 smithi121 ceph-mon[100258]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:28 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:28 smithi123 ceph-mon[104968]: Deploying daemon mon.c on smithi121 2023-09-23T20:25:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:28 smithi123 ceph-mon[104968]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:28 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:28.817 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:25:28.817 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":2,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:25:22.152089Z","created":"2023-09-23T20:23:53.656082Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T20:25:28.819 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 2 2023-09-23T20:25:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:29 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/3576247363' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:25:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:29 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/3576247363' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:25:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:29 smithi121 systemd[1]: Starting Ceph mon.c for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:30.360 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:25:30.360 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:25:30.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[100258]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pidfile_write: ignore empty --pid-file 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: load: jerasure load: lrc 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: DB SUMMARY 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: DB Session ID: XCCEO1Q9J3T4NLRQPEAM 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:25:30.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.env: 0x56504841ba80 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.fs: Posix File System 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.info_log: 0x5650498725a0 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.statistics: (nil) 2023-09-23T20:25:30.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.use_fsync: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:25:30.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.db_log_dir: 2023-09-23T20:25:30.529 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T20:25:30.529 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:25:30.529 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:25:30.530 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:25:30.530 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:25:30.530 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:25:30.530 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:25:30.530 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:25:30.530 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:25:30.530 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.write_buffer_manager: 0x56504a5e4930 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.unordered_write: 0 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:25:30.531 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.row_cache: None 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.wal_filter: None 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:25:30.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:25:30.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_open_files: -1 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:25:30.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Compression algorithms supported: 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kZSTD supported: 0 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T20:25:30.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.merge_operator: 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_filter: None 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5650498c4fb8) 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:25:30.536 INFO:journalctl@ceph.mon.c.smithi121.stdout: index_type: 0 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: data_block_index_type: 0 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: index_shortening: 1 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: hash_index_allow_collision: 1 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: checksum: 1 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: no_block_cache: 0 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache: 0x5650498f7090 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache_options: 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: capacity : 536870912 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: num_shard_bits : 4 2023-09-23T20:25:30.537 INFO:journalctl@ceph.mon.c.smithi121.stdout: strict_capacity_limit : 0 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache_compressed: (nil) 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: persistent_cache: (nil) 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_size: 4096 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_size_deviation: 10 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_restart_interval: 16 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: index_block_restart_interval: 1 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: metadata_block_size: 4096 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: partition_filters: 0 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: use_delta_encoding: 1 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:25:30.538 INFO:journalctl@ceph.mon.c.smithi121.stdout: whole_key_filtering: 1 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout: verify_compression: 0 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout: format_version: 4 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout: enable_index_compression: 1 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_align: 0 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression: NoCompression 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:25:30.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.num_levels: 7 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:25:30.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:25:30.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:25:30.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:25:30.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:25:30.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.ttl: 2592000 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:25:30.545 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500730509367, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500730509720, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695500730, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "71cd5bed-8157-48de-9e44-0f2b4de6a315", "db_session_id": "XCCEO1Q9J3T4NLRQPEAM"}} 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500730510777, "job": 1, "event": "recovery_finished"} 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x56504988b180 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: DB pointer 0x565049960000 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:25:30.546 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** DB Stats ** 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:30.547 INFO:journalctl@ceph.mon.c.smithi121.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:25:30.548 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:30.549 INFO:journalctl@ceph.mon.c.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:25:30.550 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: using public_addrv [v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0] 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: starting mon.c rank -1 at public addrs [v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0] at bind addrs [v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(???) e0 preinit fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:25:30.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 bash[118163]: a0406cc2877167ac48c6de134c35cbaedd111239f5d8ac3109e37b9f5ea0858f 2023-09-23T20:25:30.697 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:30 smithi123 ceph-mon[104968]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T20:25:30.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T20:25:30.913 INFO:journalctl@ceph.mon.c.smithi121.stdout: e1 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout: legacy client fscid: -1 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout: No filesystems configured 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T20:25:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/778578278' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/778578278' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi121:172.21.15.121=a;smithi121:[v2:172.21.15.121:3301,v1:172.21.15.121:6790]=c;smithi123:172.21.15.123=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: Saving service mon spec with placement smithi121:172.21.15.121=a;smithi121:[v2:172.21.15.121:3301,v1:172.21.15.121:6790]=c;smithi123:172.21.15.123=b;count:3 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: Deploying daemon mon.b on smithi123 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.a calling monitor election 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:30.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.b calling monitor election 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: monmap e2: 2 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} removed_ranks: {} 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: fsmap 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:30.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mgrmap e13: y(active, since 37s) 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: overall HEALTH_OK 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: Deploying daemon mon.c on smithi121 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/3576247363' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T20:25:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 ceph-mon[118228]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T20:25:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:30 smithi121 systemd[1]: Started Ceph mon.c for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:25:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:25:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: mon.a calling monitor election 2023-09-23T20:25:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: mon.b calling monitor election 2023-09-23T20:25:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: mon.c calling monitor election 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: fsmap 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: mgrmap e13: y(active, since 46s) 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: overall HEALTH_OK 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: mon.a calling monitor election 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: mon.b calling monitor election 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: mon.c calling monitor election 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: fsmap 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: mgrmap e13: y(active, since 46s) 2023-09-23T20:25:35.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: overall HEALTH_OK 2023-09-23T20:25:35.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.924 INFO:teuthology.orchestra.run.smithi123.stdout: 2023-09-23T20:25:35.925 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":3,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:25:30.535176Z","created":"2023-09-23T20:23:53.656082Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3301","nonce":0},{"type":"v1","addr":"172.21.15.121:6790","nonce":0}]},"addr":"172.21.15.121:6790/0","public_addr":"172.21.15.121:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-23T20:25:35.927 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 3 2023-09-23T20:25:35.957 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: mon.a calling monitor election 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: mon.b calling monitor election 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.958 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: mon.c calling monitor election 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: fsmap 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: mgrmap e13: y(active, since 46s) 2023-09-23T20:25:35.959 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: overall HEALTH_OK 2023-09-23T20:25:35.960 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.960 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:35.960 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:36.565 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T20:25:36.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[100258]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/109657893' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[118228]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:36.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/109657893' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:25:36.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:36 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:36.923 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:36.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:36 smithi123 ceph-mon[104968]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:36.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:36 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:36 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:36 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/109657893' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:25:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:36 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:25:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:38.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:38.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:37 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:38.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:38.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:37 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:38.884 INFO:teuthology.orchestra.run.smithi121.stdout:# minimal ceph.conf for ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:25:38.885 INFO:teuthology.orchestra.run.smithi121.stdout:[global] 2023-09-23T20:25:38.885 INFO:teuthology.orchestra.run.smithi121.stdout: fsid = ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:25:38.885 INFO:teuthology.orchestra.run.smithi121.stdout: mon_host = [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] [v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0] 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[100258]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[100258]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[100258]: Reconfiguring daemon mon.a on smithi121 2023-09-23T20:25:39.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:25:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:25:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[118228]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[118228]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:25:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:38 smithi121 ceph-mon[118228]: Reconfiguring daemon mon.a on smithi121 2023-09-23T20:25:39.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:38 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:25:39.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:38 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:25:39.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:38 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:38 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:25:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:38 smithi123 ceph-mon[104968]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:38 smithi123 ceph-mon[104968]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:25:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:38 smithi123 ceph-mon[104968]: Reconfiguring daemon mon.a on smithi121 2023-09-23T20:25:39.856 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T20:25:39.856 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:25:39.856 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T20:25:39.885 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:25:39.886 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:25:39.953 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:25:39.953 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T20:25:39.989 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:25:39.989 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:25:40.060 INFO:tasks.cephadm:Adding mgr.y on smithi121 2023-09-23T20:25:40.060 INFO:tasks.cephadm:Adding mgr.x on smithi123 2023-09-23T20:25:40.060 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi121=y;smithi123=x' 2023-09-23T20:25:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3333300706' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:40.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3333300706' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:40.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:40.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:40.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:40.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:39 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3333300706' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:40.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:40.427 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:41.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:40 smithi121 ceph-mon[100258]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:41.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:40 smithi121 ceph-mon[100258]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:25:41.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:40 smithi121 ceph-mon[100258]: Reconfiguring daemon mon.c on smithi121 2023-09-23T20:25:41.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:40 smithi121 ceph-mon[118228]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:41.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:40 smithi121 ceph-mon[118228]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:25:41.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:40 smithi121 ceph-mon[118228]: Reconfiguring daemon mon.c on smithi121 2023-09-23T20:25:41.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:40 smithi123 ceph-mon[104968]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:41.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:40 smithi123 ceph-mon[104968]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:25:41.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:40 smithi123 ceph-mon[104968]: Reconfiguring daemon mon.c on smithi121 2023-09-23T20:25:42.195 INFO:teuthology.orchestra.run.smithi123.stdout:Scheduled mgr update... 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[100258]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[100258]: Reconfiguring daemon mon.b on smithi123 2023-09-23T20:25:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[118228]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:25:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:42 smithi121 ceph-mon[118228]: Reconfiguring daemon mon.b on smithi123 2023-09-23T20:25:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:42 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:42 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:42 smithi123 ceph-mon[104968]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:25:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:42 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:25:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:42 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:25:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:42 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:42 smithi123 ceph-mon[104968]: Reconfiguring daemon mon.b on smithi123 2023-09-23T20:25:43.008 DEBUG:teuthology.orchestra.run.smithi123:mgr.x> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T20:25:43.010 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T20:25:43.010 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:25:43.010 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T20:25:43.031 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T20:25:43.031 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_1 2023-09-23T20:25:43.038 INFO:journalctl@ceph.mgr.x.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:25:43.092 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T20:25:43.092 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:43.093 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 384634 Links: 1 2023-09-23T20:25:43.093 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:43.093 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:43.093 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2023-09-23 20:25:29.926070092 +0000 2023-09-23T20:25:43.093 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2023-09-23 20:17:37.636018325 +0000 2023-09-23T20:25:43.093 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2023-09-23 20:17:37.636018325 +0000 2023-09-23T20:25:43.093 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2023-09-23T20:25:43.093 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T20:25:43.162 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2023-09-23T20:25:43.162 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2023-09-23T20:25:43.162 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000253397 s, 2.0 MB/s 2023-09-23T20:25:43.163 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T20:25:43.223 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_2 2023-09-23T20:25:43.281 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T20:25:43.281 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:43.281 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 389694 Links: 1 2023-09-23T20:25:43.281 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:43.281 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:43.281 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2023-09-23 20:25:29.926070092 +0000 2023-09-23T20:25:43.281 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2023-09-23 20:17:38.159011008 +0000 2023-09-23T20:25:43.282 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2023-09-23 20:17:38.159011008 +0000 2023-09-23T20:25:43.282 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2023-09-23T20:25:43.282 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T20:25:43.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:43.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:43.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:43.348 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:43.348 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:43.348 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.348 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:25:43.348 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:25:43.348 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:25:43.348 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:43.348 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2023-09-23T20:25:43.349 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2023-09-23T20:25:43.349 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000210663 s, 2.4 MB/s 2023-09-23T20:25:43.350 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T20:25:43.409 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_3 2023-09-23T20:25:43.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:43.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:25:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:25:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:25:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:43.467 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T20:25:43.467 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:43.467 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 387057 Links: 1 2023-09-23T20:25:43.467 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:43.468 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:43.468 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2023-09-23 20:25:29.926070092 +0000 2023-09-23T20:25:43.468 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2023-09-23 20:17:38.653004097 +0000 2023-09-23T20:25:43.468 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2023-09-23 20:17:38.653004097 +0000 2023-09-23T20:25:43.468 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2023-09-23T20:25:43.468 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T20:25:43.534 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2023-09-23T20:25:43.534 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2023-09-23T20:25:43.535 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000299076 s, 1.7 MB/s 2023-09-23T20:25:43.536 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T20:25:43.594 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_4 2023-09-23T20:25:43.653 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T20:25:43.653 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:43.654 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 385800 Links: 1 2023-09-23T20:25:43.654 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:43.654 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:43.654 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2023-09-23 20:25:29.926070092 +0000 2023-09-23T20:25:43.654 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2023-09-23 20:17:39.150997130 +0000 2023-09-23T20:25:43.654 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2023-09-23 20:17:39.150997130 +0000 2023-09-23T20:25:43.655 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2023-09-23T20:25:43.655 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T20:25:43.719 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2023-09-23T20:25:43.719 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2023-09-23T20:25:43.719 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000176836 s, 2.9 MB/s 2023-09-23T20:25:43.720 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T20:25:43.777 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:25:43.777 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T20:25:43.794 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T20:25:43.794 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_1 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 6h/6d Inode: 390463 Links: 1 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2023-09-23 20:25:21.499794657 +0000 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2023-09-23 20:17:37.639048767 +0000 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2023-09-23 20:17:37.639048767 +0000 2023-09-23T20:25:43.852 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: - 2023-09-23T20:25:43.853 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T20:25:43.920 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2023-09-23T20:25:43.920 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2023-09-23T20:25:43.920 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000247156 s, 2.1 MB/s 2023-09-23T20:25:43.921 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T20:25:43.980 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_2 2023-09-23T20:25:44.036 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T20:25:44.036 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:44.036 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 6h/6d Inode: 385670 Links: 1 2023-09-23T20:25:44.036 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:44.037 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:44.037 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2023-09-23 20:25:21.499794657 +0000 2023-09-23T20:25:44.037 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2023-09-23 20:17:38.128036898 +0000 2023-09-23T20:25:44.037 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2023-09-23 20:17:38.128036898 +0000 2023-09-23T20:25:44.037 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: - 2023-09-23T20:25:44.037 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T20:25:44.102 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2023-09-23T20:25:44.102 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2023-09-23T20:25:44.102 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000218202 s, 2.3 MB/s 2023-09-23T20:25:44.103 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T20:25:44.161 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_3 2023-09-23T20:25:44.225 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T20:25:44.225 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:44.225 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 6h/6d Inode: 382495 Links: 1 2023-09-23T20:25:44.225 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:44.225 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:44.225 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2023-09-23 20:25:21.499794657 +0000 2023-09-23T20:25:44.225 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2023-09-23 20:17:38.623024884 +0000 2023-09-23T20:25:44.226 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2023-09-23 20:17:38.623024884 +0000 2023-09-23T20:25:44.226 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: - 2023-09-23T20:25:44.226 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T20:25:44.297 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2023-09-23T20:25:44.297 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2023-09-23T20:25:44.297 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000207937 s, 2.5 MB/s 2023-09-23T20:25:44.298 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T20:25:44.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:44 smithi123 ceph-mon[104968]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi121=y;smithi123=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:44.325 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:44 smithi123 ceph-mon[104968]: Saving service mgr spec with placement smithi121=y;smithi123=x;count:2 2023-09-23T20:25:44.326 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:44 smithi123 ceph-mon[104968]: Deploying daemon mgr.x on smithi123 2023-09-23T20:25:44.348 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_4 2023-09-23T20:25:44.405 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T20:25:44.405 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:25:44.405 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 6h/6d Inode: 385749 Links: 1 2023-09-23T20:25:44.405 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:25:44.405 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:25:44.405 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2023-09-23 20:25:44.379253964 +0000 2023-09-23T20:25:44.406 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2023-09-23 20:17:39.101013282 +0000 2023-09-23T20:25:44.406 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2023-09-23 20:17:39.101013282 +0000 2023-09-23T20:25:44.406 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: - 2023-09-23T20:25:44.406 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T20:25:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:44 smithi121 ceph-mon[100258]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi121=y;smithi123=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:44 smithi121 ceph-mon[100258]: Saving service mgr spec with placement smithi121=y;smithi123=x;count:2 2023-09-23T20:25:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:44 smithi121 ceph-mon[100258]: Deploying daemon mgr.x on smithi123 2023-09-23T20:25:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:44 smithi121 ceph-mon[118228]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi121=y;smithi123=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:44 smithi121 ceph-mon[118228]: Saving service mgr spec with placement smithi121=y;smithi123=x;count:2 2023-09-23T20:25:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:44 smithi121 ceph-mon[118228]: Deploying daemon mgr.x on smithi123 2023-09-23T20:25:44.476 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2023-09-23T20:25:44.476 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2023-09-23T20:25:44.476 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000178488 s, 2.9 MB/s 2023-09-23T20:25:44.477 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T20:25:44.535 INFO:tasks.cephadm:Deploying osd.0 on smithi121 with /dev/vg_nvme/lv_4... 2023-09-23T20:25:44.536 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T20:25:44.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:44 smithi123 systemd[1]: Starting Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:25:44.880 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:45.306 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:45 smithi123 ceph-mon[104968]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:45.306 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 ceph-mgr[108208]: -- 172.21.15.123:0/2621437412 <== mon.1 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e51a8dc9c0 con 0x55e51a8b5000 2023-09-23T20:25:45.306 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 bash[108142]: 24010296b9ffab77d8e6deec78d3cb5c33aa3d5f96c4c9fb66d48069cecaee08 2023-09-23T20:25:45.307 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 systemd[1]: Started Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:25:45.307 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 conmon[108184]: 2023-09-23T20:25:45.135+0000 7f855dc871c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:25:45.307 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 conmon[108184]: 2023-09-23T20:25:45.214+0000 7f855dc871c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:25:45.307 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 conmon[108184]: 2023-09-23T20:25:45.306+0000 7f855dc871c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:25:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:45 smithi121 ceph-mon[100258]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:45.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:45 smithi121 ceph-mon[118228]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:45.594 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 conmon[108184]: 2023-09-23T20:25:45.401+0000 7f855dc871c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:25:45.933 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 conmon[108184]: 2023-09-23T20:25:45.594+0000 7f855dc871c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:25:46.190 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.191 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:46 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:46.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:46 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:46 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:46 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:46 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:46.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:46 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:46.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:45 smithi123 conmon[108184]: 2023-09-23T20:25:45.933+0000 7f855dc871c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:25:46.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:46 smithi123 conmon[108184]: 2023-09-23T20:25:46.023+0000 7f855dc871c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:25:46.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:46 smithi123 conmon[108184]: 2023-09-23T20:25:46.159+0000 7f855dc871c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:25:46.672 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:46 smithi123 conmon[108184]: 2023-09-23T20:25:46.256+0000 7f855dc871c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:25:46.689 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T20:25:46.689 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T20:25:46.689 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2023-09-23T20:25:46.689 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2023-09-23T20:25:46.690 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135697 s, 773 MB/s 2023-09-23T20:25:46.690 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2023-09-23T20:25:46.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:46 smithi123 conmon[108184]: 2023-09-23T20:25:46.672+0000 7f855dc871c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:25:47.397 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:47 smithi123 conmon[108184]: 2023-09-23T20:25:47.232+0000 7f855dc871c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:25:47.397 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:47.397 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.397 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.397 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:47.398 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:47.398 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.398 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:25:47.398 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:25:47.398 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:25:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:25:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:25:47.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:47.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi121:vg_nvme/lv_4 2023-09-23T20:25:47.728 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:47 smithi123 conmon[108184]: 2023-09-23T20:25:47.397+0000 7f855dc871c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:25:48.185 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:48.392 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:48 smithi123 conmon[108184]: 2023-09-23T20:25:48.138+0000 7f855dc871c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:25:48.393 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:48 smithi123 conmon[108184]: 2023-09-23T20:25:48.226+0000 7f855dc871c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:25:48.393 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:48 smithi123 ceph-mon[104968]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:25:48.393 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:48 smithi123 ceph-mon[104968]: Reconfiguring daemon mgr.y on smithi121 2023-09-23T20:25:48.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:48 smithi121 ceph-mon[118228]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:25:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:48 smithi121 ceph-mon[118228]: Reconfiguring daemon mgr.y on smithi121 2023-09-23T20:25:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:48 smithi121 ceph-mon[100258]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:25:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:48 smithi121 ceph-mon[100258]: Reconfiguring daemon mgr.y on smithi121 2023-09-23T20:25:48.659 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:48 smithi123 conmon[108184]: 2023-09-23T20:25:48.392+0000 7f855dc871c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:25:48.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:48 smithi123 conmon[108184]: 2023-09-23T20:25:48.659+0000 7f855dc871c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:25:48.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:48 smithi123 conmon[108184]: 2023-09-23T20:25:48.743+0000 7f855dc871c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:25:49.275 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:49 smithi123 conmon[108184]: 2023-09-23T20:25:49.022+0000 7f855dc871c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:25:49.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:49 smithi123 ceph-mon[104968]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:49.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:49 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:49 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:49 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:49.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:49 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:49.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:49 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:49.276 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:49 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[118228]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[100258]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:49.728 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:49 smithi123 conmon[108184]: 2023-09-23T20:25:49.274+0000 7f855dc871c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:25:50.228 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:49 smithi123 conmon[108184]: 2023-09-23T20:25:49.960+0000 7f855dc871c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:25:50.228 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:50 smithi123 conmon[108184]: 2023-09-23T20:25:50.087+0000 7f855dc871c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:25:50.589 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:50 smithi123 conmon[108184]: 2023-09-23T20:25:50.269+0000 7f855dc871c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:25:50.590 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:50 smithi123 conmon[108184]: 2023-09-23T20:25:50.379+0000 7f855dc871c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:25:50.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[118228]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[118228]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[100258]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[100258]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:50.797 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:25:50.798 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:25:50.798 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:50 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:50.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:50 smithi123 conmon[108184]: 2023-09-23T20:25:50.589+0000 7f855dc871c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:25:50.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:50 smithi123 ceph-mon[104968]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:50.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:50 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:25:50.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:50 smithi123 ceph-mon[104968]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:25:50.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:50 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:25:50.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:50 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:25:50.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:50 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:51.728 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:25:51 smithi123 conmon[108184]: 2023-09-23T20:25:51.422+0000 7f855dc871c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:25:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:51 smithi123 ceph-mon[104968]: Standby manager daemon x started 2023-09-23T20:25:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:51 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:25:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:51 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:25:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:51 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:25:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:51 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:25:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[100258]: Standby manager daemon x started 2023-09-23T20:25:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:25:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:25:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:25:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:25:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[118228]: Standby manager daemon x started 2023-09-23T20:25:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:25:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:25:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:25:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:51 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2236728981' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:25:52.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: mgrmap e14: y(active, since 62s), standbys: x 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/68433284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]: dispatch 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]: dispatch 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]': finished 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:52.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: mgrmap e14: y(active, since 62s), standbys: x 2023-09-23T20:25:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:25:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/68433284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]: dispatch 2023-09-23T20:25:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]: dispatch 2023-09-23T20:25:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]': finished 2023-09-23T20:25:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:25:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:25:52.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:52.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: mgrmap e14: y(active, since 62s), standbys: x 2023-09-23T20:25:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:25:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/68433284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]: dispatch 2023-09-23T20:25:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]: dispatch 2023-09-23T20:25:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9367e9c4-5ec4-4c89-a35d-16375c1fee90"}]': finished 2023-09-23T20:25:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:25:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:25:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:53 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1497351755' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:25:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:53 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1497351755' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:25:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:53 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1497351755' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:25:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:54 smithi121 ceph-mon[100258]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:54.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:54 smithi121 ceph-mon[118228]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:54 smithi123 ceph-mon[104968]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:56 smithi121 ceph-mon[100258]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:56 smithi121 ceph-mon[118228]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:56 smithi123 ceph-mon[104968]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:59.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:59 smithi121 ceph-mon[100258]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:59.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:59 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:25:59.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:25:59 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:59.320 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:59 smithi121 ceph-mon[118228]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:59.320 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:59 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:25:59.320 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:25:59 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:59 smithi123 ceph-mon[104968]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:25:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:59 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:25:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:25:59 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:00.354 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:00 smithi121 ceph-mon[100258]: Deploying daemon osd.0 on smithi121 2023-09-23T20:26:00.354 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:00 smithi121 ceph-mon[118228]: Deploying daemon osd.0 on smithi121 2023-09-23T20:26:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:00 smithi123 ceph-mon[104968]: Deploying daemon osd.0 on smithi121 2023-09-23T20:26:01.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:01 smithi121 ceph-mon[100258]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:01.253 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:01 smithi121 ceph-mon[118228]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:01.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:01 smithi123 ceph-mon[104968]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:03 smithi121 ceph-mon[100258]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:03 smithi121 ceph-mon[118228]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:03.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:03 smithi123 ceph-mon[104968]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[100258]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:04 smithi121 ceph-mon[118228]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:04 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:04 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:04 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:04 smithi123 ceph-mon[104968]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:06.386 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 0 on host 'smithi121' 2023-09-23T20:26:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:06 smithi123 ceph-mon[104968]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.562 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[118228]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:06.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:06.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:06.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.563 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.563 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:06.564 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[100258]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:06.564 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:06.564 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:06.564 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:07.064 DEBUG:teuthology.orchestra.run.smithi121:osd.0> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T20:26:07.067 INFO:tasks.cephadm:Deploying osd.1 on smithi121 with /dev/vg_nvme/lv_3... 2023-09-23T20:26:07.067 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T20:26:07.113 INFO:journalctl@ceph.osd.0.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:26:07.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:07.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:07.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:07.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[100258]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:26:07.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:07.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:07.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:07.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:07 smithi121 ceph-mon[118228]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:26:07.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:07 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:07.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:07 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:07.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:07 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:07.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:07 smithi123 ceph-mon[104968]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:26:08.257 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:08.478 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:26:08 smithi121 conmon[124538]: 2023-09-23T20:26:08.369+0000 7f6428030700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T20:26:08.478 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:26:08 smithi121 conmon[124538]: 2023-09-23T20:26:08.371+0000 7f6420e27700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[100258]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[100258]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[100258]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[100258]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[118228]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[118228]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:08.479 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[118228]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:08.480 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:08 smithi121 ceph-mon[118228]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:08 smithi123 ceph-mon[104968]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:26:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:08 smithi123 ceph-mon[104968]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:26:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:08 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:08.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:08 smithi123 ceph-mon[104968]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:08.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:08 smithi123 ceph-mon[104968]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[118228]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[118228]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:26:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:09.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[100258]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:09.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[100258]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:26:09.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:09.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:09 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:09 smithi123 ceph-mon[104968]: from='osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:09 smithi123 ceph-mon[104968]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:26:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:09 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:09 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:10.386 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255] boot 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi121 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi121 to 16144M 2023-09-23T20:26:10.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255] boot 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi121 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:10.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi121 to 16144M 2023-09-23T20:26:10.389 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.389 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:10.389 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:10.389 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:10.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:10.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: osd.0 [v2:172.21.15.121:6802/801467255,v1:172.21.15.121:6803/801467255] boot 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi121 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi121 to 16144M 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:10.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:10.737 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T20:26:10.737 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T20:26:10.737 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2023-09-23T20:26:10.737 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2023-09-23T20:26:10.737 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124932 s, 839 MB/s 2023-09-23T20:26:10.737 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2023-09-23T20:26:11.408 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi121:vg_nvme/lv_3 2023-09-23T20:26:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:11 smithi121 ceph-mon[118228]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:26:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:11 smithi121 ceph-mon[100258]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:26:11.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:11 smithi123 ceph-mon[104968]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:26:11.783 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:13.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:13 smithi121 ceph-mon[100258]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:13.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:13 smithi121 ceph-mon[118228]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:13 smithi123 ceph-mon[104968]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:14.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:14 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:26:14.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:14 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:26:14.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:14 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:14.375 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:14 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:26:14.375 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:14 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:26:14.375 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:14 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:14 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:26:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:14 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:26:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:14 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:15.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:15 smithi121 ceph-mon[118228]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:26:15.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:15 smithi121 ceph-mon[118228]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:15 smithi121 ceph-mon[100258]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:26:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:15 smithi121 ceph-mon[100258]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:15 smithi123 ceph-mon[104968]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:26:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:15 smithi123 ceph-mon[104968]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3564586069' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "798606f1-4cc7-4397-9f99-1f3d8259eba1"}]: dispatch 2023-09-23T20:26:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3564586069' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "798606f1-4cc7-4397-9f99-1f3d8259eba1"}]': finished 2023-09-23T20:26:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[100258]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:26:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3564586069' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "798606f1-4cc7-4397-9f99-1f3d8259eba1"}]: dispatch 2023-09-23T20:26:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3564586069' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "798606f1-4cc7-4397-9f99-1f3d8259eba1"}]': finished 2023-09-23T20:26:16.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[118228]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:26:16.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:16 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:16 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3564586069' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "798606f1-4cc7-4397-9f99-1f3d8259eba1"}]: dispatch 2023-09-23T20:26:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:16 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3564586069' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "798606f1-4cc7-4397-9f99-1f3d8259eba1"}]': finished 2023-09-23T20:26:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:16 smithi123 ceph-mon[104968]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:26:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:16 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:17 smithi121 ceph-mon[100258]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:17 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1591025405' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:26:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:17 smithi121 ceph-mon[118228]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:17 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1591025405' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:26:17.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:17 smithi123 ceph-mon[104968]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:17.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:17 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1591025405' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:26:19.353 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:19 smithi121 ceph-mon[118228]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:19.354 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:19 smithi121 ceph-mon[100258]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:19.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:19 smithi123 ceph-mon[104968]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:21 smithi121 ceph-mon[100258]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:21.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:21 smithi121 ceph-mon[118228]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:21 smithi123 ceph-mon[104968]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:26:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:26:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:26:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:23.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:23 smithi121 ceph-mon[100258]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:23.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:23 smithi121 ceph-mon[100258]: Deploying daemon osd.1 on smithi121 2023-09-23T20:26:23.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:23 smithi121 ceph-mon[118228]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:23.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:23 smithi121 ceph-mon[118228]: Deploying daemon osd.1 on smithi121 2023-09-23T20:26:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:23 smithi123 ceph-mon[104968]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:23 smithi123 ceph-mon[104968]: Deploying daemon osd.1 on smithi121 2023-09-23T20:26:25.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:25 smithi121 ceph-mon[100258]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:25.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:25 smithi121 ceph-mon[118228]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:25 smithi123 ceph-mon[104968]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:27 smithi121 ceph-mon[100258]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:27 smithi121 ceph-mon[118228]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:27 smithi123 ceph-mon[104968]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:28.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:28.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:28.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:28.581 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[100258]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:28.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:28.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:28.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:28.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:28 smithi121 ceph-mon[118228]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:28 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:28 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:28 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:28 smithi123 ceph-mon[104968]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:30.401 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 1 on host 'smithi121' 2023-09-23T20:26:30.441 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.441 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.441 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:30.441 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:30.441 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.441 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[118228]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:30.442 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.442 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.442 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:30.442 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:30.442 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.442 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:30 smithi121 ceph-mon[100258]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:30 smithi123 ceph-mon[104968]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:31.059 DEBUG:teuthology.orchestra.run.smithi121:osd.1> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T20:26:31.061 INFO:tasks.cephadm:Deploying osd.2 on smithi121 with /dev/vg_nvme/lv_2... 2023-09-23T20:26:31.061 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T20:26:31.090 INFO:journalctl@ceph.osd.1.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:26:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[100258]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:26:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:31 smithi121 ceph-mon[118228]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:26:31.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:31 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:31.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:31 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:31 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:31 smithi123 ceph-mon[104968]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:26:32.208 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:32.475 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[100258]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:26:32.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[100258]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:26:32.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:32.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[100258]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:32.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[100258]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:32.476 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[118228]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:26:32.476 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[118228]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:26:32.477 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:32.477 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[118228]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:32.477 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:32 smithi121 ceph-mon[118228]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:32.477 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:26:32 smithi121 conmon[133456]: 2023-09-23T20:26:32.392+0000 7f190127c700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T20:26:32.477 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:26:32 smithi121 conmon[133456]: 2023-09-23T20:26:32.396+0000 7f18fb876700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:26:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:32 smithi123 ceph-mon[104968]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:26:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:32 smithi123 ceph-mon[104968]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:26:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:32 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:32 smithi123 ceph-mon[104968]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:32 smithi123 ceph-mon[104968]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:33.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[118228]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:33.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[118228]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:26:33.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:33.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:33.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[100258]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:33.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[100258]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:26:33.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:33.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:33 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:33 smithi123 ceph-mon[104968]: from='osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:33 smithi123 ceph-mon[104968]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:26:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:33 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:33 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:26:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:26:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[118228]: osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368] boot 2023-09-23T20:26:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[118228]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:26:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[118228]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:34.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:26:34.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:26:34.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:34.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[100258]: osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368] boot 2023-09-23T20:26:34.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[100258]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:26:34.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:34.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:34 smithi121 ceph-mon[100258]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:34.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:34 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:26:34.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:34 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:26:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:34 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:34 smithi123 ceph-mon[104968]: osd.1 [v2:172.21.15.121:6810/3606674368,v1:172.21.15.121:6811/3606674368] boot 2023-09-23T20:26:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:34 smithi123 ceph-mon[104968]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:26:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:34 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:26:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:34 smithi123 ceph-mon[104968]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:26:34.794 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T20:26:34.794 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T20:26:34.794 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2023-09-23T20:26:34.795 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2023-09-23T20:26:34.795 INFO:teuthology.orchestra.run.smithi121.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110437 s, 949 MB/s 2023-09-23T20:26:34.795 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 2023-09-23T20:26:34.795 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2023-09-23T20:26:35.487 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi121:vg_nvme/lv_2 2023-09-23T20:26:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi121 2023-09-23T20:26:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi121 to 8072M 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi121 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi121 to 8072M 2023-09-23T20:26:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:26:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:35 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi121 2023-09-23T20:26:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi121 to 8072M 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:35 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:35.829 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:36.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:36 smithi121 ceph-mon[100258]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:36.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:36 smithi121 ceph-mon[118228]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:36 smithi123 ceph-mon[104968]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:38.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:38 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:26:38.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:38 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:26:38.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:38 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:38.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:38 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:26:38.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:38 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:26:38.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:38 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:38 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:26:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:38 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:26:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:38 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:39 smithi121 ceph-mon[100258]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:26:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:39 smithi121 ceph-mon[100258]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:39 smithi121 ceph-mon[118228]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:26:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:39 smithi121 ceph-mon[118228]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:39 smithi123 ceph-mon[104968]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:26:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:39 smithi123 ceph-mon[104968]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1344048558' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "03ecb78d-0bc7-43d1-8c25-1c6051697c2f"}]: dispatch 2023-09-23T20:26:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1344048558' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "03ecb78d-0bc7-43d1-8c25-1c6051697c2f"}]': finished 2023-09-23T20:26:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[100258]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:26:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1344048558' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "03ecb78d-0bc7-43d1-8c25-1c6051697c2f"}]: dispatch 2023-09-23T20:26:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1344048558' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "03ecb78d-0bc7-43d1-8c25-1c6051697c2f"}]': finished 2023-09-23T20:26:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[118228]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:26:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:40 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:40 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1344048558' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "03ecb78d-0bc7-43d1-8c25-1c6051697c2f"}]: dispatch 2023-09-23T20:26:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:40 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1344048558' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "03ecb78d-0bc7-43d1-8c25-1c6051697c2f"}]': finished 2023-09-23T20:26:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:40 smithi123 ceph-mon[104968]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:26:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:40 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:41 smithi121 ceph-mon[100258]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:41 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2247114835' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:26:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:41 smithi121 ceph-mon[118228]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:41 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2247114835' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:26:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:41 smithi123 ceph-mon[104968]: pgmap v59: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:41 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2247114835' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:26:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:43 smithi121 ceph-mon[100258]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:43 smithi121 ceph-mon[118228]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:43 smithi123 ceph-mon[104968]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:45.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:45 smithi121 ceph-mon[118228]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:45 smithi121 ceph-mon[100258]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:45.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:45 smithi123 ceph-mon[104968]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:46.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:46 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:26:46.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:46 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:46.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:46 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:26:46.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:46 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:46 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:26:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:46 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:47 smithi121 ceph-mon[100258]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:47 smithi121 ceph-mon[100258]: Deploying daemon osd.2 on smithi121 2023-09-23T20:26:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:47 smithi121 ceph-mon[118228]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:47 smithi121 ceph-mon[118228]: Deploying daemon osd.2 on smithi121 2023-09-23T20:26:47.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:47 smithi123 ceph-mon[104968]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:47.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:47 smithi123 ceph-mon[104968]: Deploying daemon osd.2 on smithi121 2023-09-23T20:26:49.405 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:49 smithi121 ceph-mon[100258]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:49.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:49 smithi121 ceph-mon[118228]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:49 smithi123 ceph-mon[104968]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:51 smithi121 ceph-mon[100258]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:51 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:51 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:51 smithi121 ceph-mon[118228]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:51 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:51 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:51 smithi123 ceph-mon[104968]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:51 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:51 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:52.112 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:52.113 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:53.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:53 smithi121 ceph-mon[100258]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:53 smithi121 ceph-mon[118228]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:53 smithi123 ceph-mon[104968]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:54.315 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 2 on host 'smithi121' 2023-09-23T20:26:54.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:54.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:54.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[118228]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[118228]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[100258]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:54.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:54 smithi121 ceph-mon[100258]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:26:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:54 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:54 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:54 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:26:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:54 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:26:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:54 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:54 smithi123 ceph-mon[104968]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:54 smithi123 ceph-mon[104968]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:26:54.979 DEBUG:teuthology.orchestra.run.smithi121:osd.2> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T20:26:54.981 INFO:tasks.cephadm:Deploying osd.3 on smithi121 with /dev/vg_nvme/lv_1... 2023-09-23T20:26:54.981 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T20:26:55.010 INFO:journalctl@ceph.osd.2.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:26:55.264 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[100258]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:26:55.264 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[100258]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:26:55.264 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:55.264 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[100258]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:55.264 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:55.264 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:55.264 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:55.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[118228]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:26:55.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[118228]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:26:55.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:55.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[118228]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:55.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:55.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:55.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:55 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:55.663 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:26:55 smithi121 conmon[142308]: 2023-09-23T20:26:55.260+0000 7fe7055f1700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T20:26:55.663 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:26:55 smithi121 conmon[142308]: 2023-09-23T20:26:55.270+0000 7fe700bed700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:26:55.697 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:55 smithi123 ceph-mon[104968]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:26:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:55 smithi123 ceph-mon[104968]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:26:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:55 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:55 smithi123 ceph-mon[104968]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:26:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:55 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:55 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:26:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:55 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:56.576 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[100258]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:56.576 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[100258]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:26:56.576 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:56.576 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:56.577 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[100258]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:56.577 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[118228]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:56.577 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[118228]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:26:56.577 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:56.577 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:56.577 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:56 smithi121 ceph-mon[118228]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:56.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:56 smithi123 ceph-mon[104968]: from='osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:26:56.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:56 smithi123 ceph-mon[104968]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:26:56.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:56 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:56.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:56 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:56.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:56 smithi123 ceph-mon[104968]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:26:57.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:26:57.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[100258]: osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487] boot 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[100258]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[118228]: osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487] boot 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[118228]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:26:57.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:57 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:57.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:57 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:26:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:57 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:26:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:57 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:57 smithi123 ceph-mon[104968]: osd.2 [v2:172.21.15.121:6818/4028229487,v1:172.21.15.121:6819/4028229487] boot 2023-09-23T20:26:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:57 smithi123 ceph-mon[104968]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:26:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:57 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:26:58.184 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T20:26:58.185 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T20:26:58.185 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2023-09-23T20:26:58.185 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2023-09-23T20:26:58.185 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125079 s, 838 MB/s 2023-09-23T20:26:58.185 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2023-09-23T20:26:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:58 smithi121 ceph-mon[118228]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:26:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:58 smithi121 ceph-mon[118228]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:26:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:58 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:26:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:58 smithi121 ceph-mon[100258]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:26:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:58 smithi121 ceph-mon[100258]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:26:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:58 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:26:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:58 smithi123 ceph-mon[104968]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:26:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:58 smithi123 ceph-mon[104968]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:26:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:58 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:26:59.283 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:59 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:26:59.283 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:59 smithi121 ceph-mon[118228]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:26:59.283 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:26:59 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:26:59.283 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:59 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:26:59.283 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:59 smithi121 ceph-mon[100258]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:26:59.283 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:26:59 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:26:59.285 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi121:vg_nvme/lv_1 2023-09-23T20:26:59.632 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:59 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:26:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:59 smithi123 ceph-mon[104968]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:26:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:26:59 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi121 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi121 to 5381M 2023-09-23T20:27:00.558 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[100258]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi121 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.559 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi121 to 5381M 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:00 smithi121 ceph-mon[118228]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi121 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi121 to 5381M 2023-09-23T20:27:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:00.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:00.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:00.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:00 smithi123 ceph-mon[104968]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:00.826 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148314]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:27:00.826 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148314]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:00.826 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148314]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:00.827 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148314]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:01.104 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148374]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:27:01.104 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148374]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:01.104 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148374]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:01.104 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:27:00 smithi121 sudo[148374]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:01.377 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148404]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:27:01.377 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148404]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:01.377 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148404]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:01.377 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148404]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:01.634 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148427]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:27:01.635 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148427]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:01.635 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148427]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:01.635 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148427]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:01.635 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[118228]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T20:27:01.635 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:01.635 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:01.635 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:01.636 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:01.636 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[100258]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T20:27:01.636 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:01.636 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:01.636 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:01.636 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:01 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:01.908 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148504]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:27:01.908 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148504]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:01.908 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148504]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:01.908 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148504]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:01 smithi123 ceph-mon[104968]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T20:27:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:01 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:01 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:01 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:01 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:02.162 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148652]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:27:02.162 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148652]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:02.163 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148652]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:02.163 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:27:01 smithi121 sudo[148652]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:02.431 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148681]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:27:02.431 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148681]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:02.431 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148681]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:02.431 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148681]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:02.885 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148730]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:27:02.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148730]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:02.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148730]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:02.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 sudo[148730]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:02.928 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109084]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:27:02.928 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109084]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:02.928 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109084]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:02.928 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109084]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:03.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:03.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:03.151 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109087]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109087]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109087]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 sudo[109087]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148808]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:27:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148808]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148808]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148808]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148821]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:27:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148821]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:27:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148821]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:27:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 sudo[148821]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:04.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:04.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4175576330' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6711d54-f7b7-4876-924e-e9b4c58f9e2f"}]: dispatch 2023-09-23T20:27:04.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4175576330' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6711d54-f7b7-4876-924e-e9b4c58f9e2f"}]': finished 2023-09-23T20:27:04.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:27:04.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:27:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4175576330' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6711d54-f7b7-4876-924e-e9b4c58f9e2f"}]: dispatch 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4175576330' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6711d54-f7b7-4876-924e-e9b4c58f9e2f"}]': finished 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4175576330' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6711d54-f7b7-4876-924e-e9b4c58f9e2f"}]: dispatch 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4175576330' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6711d54-f7b7-4876-924e-e9b4c58f9e2f"}]': finished 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:27:04.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:04 smithi123 ceph-mon[104968]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:04 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3398468107' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:04 smithi121 ceph-mon[100258]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3398468107' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:04 smithi121 ceph-mon[118228]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:04 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3398468107' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:06 smithi121 ceph-mon[100258]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:06 smithi121 ceph-mon[118228]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:06 smithi123 ceph-mon[104968]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:08 smithi123 ceph-mon[104968]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:08.876 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:08 smithi121 ceph-mon[118228]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:08.876 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:08 smithi121 ceph-mon[100258]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:10.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:10 smithi121 ceph-mon[100258]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:27:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:10 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:10.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:10 smithi121 ceph-mon[118228]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:10.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:27:10.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:10 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:10 smithi123 ceph-mon[104968]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:27:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:10 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:11.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:11 smithi121 ceph-mon[100258]: Deploying daemon osd.3 on smithi121 2023-09-23T20:27:11.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:11 smithi121 ceph-mon[118228]: Deploying daemon osd.3 on smithi121 2023-09-23T20:27:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:11 smithi123 ceph-mon[104968]: Deploying daemon osd.3 on smithi121 2023-09-23T20:27:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:12 smithi121 ceph-mon[100258]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:12 smithi121 ceph-mon[118228]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:12.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:12 smithi123 ceph-mon[104968]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:14.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:14 smithi121 ceph-mon[100258]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:14.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:14 smithi121 ceph-mon[118228]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:14 smithi123 ceph-mon[104968]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:16.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:16.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:16.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:16.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[100258]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:16.556 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:16.557 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:16.557 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:16.557 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:16 smithi121 ceph-mon[118228]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:16 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:16.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:16 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:16.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:16 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:16.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:16 smithi123 ceph-mon[104968]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:18.488 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 3 on host 'smithi121' 2023-09-23T20:27:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[100258]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:18.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:18.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:18.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:18 smithi121 ceph-mon[118228]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:27:18.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:18 smithi123 ceph-mon[104968]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:27:19.140 DEBUG:teuthology.orchestra.run.smithi121:osd.3> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T20:27:19.142 INFO:tasks.cephadm:Deploying osd.4 on smithi123 with /dev/vg_nvme/lv_4... 2023-09-23T20:27:19.142 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T20:27:19.173 INFO:journalctl@ceph.osd.3.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:27:19.497 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:20.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:20 smithi123 ceph-mon[104968]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:27:20.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:20 smithi123 ceph-mon[104968]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:27:20.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:20 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:20.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:20 smithi123 ceph-mon[104968]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:27:20.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:20 smithi123 ceph-mon[104968]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[100258]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:27:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[100258]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[100258]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[100258]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[118228]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[118228]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[118228]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:27:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:20 smithi121 ceph-mon[118228]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:20.915 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:27:20 smithi121 conmon[151425]: 2023-09-23T20:27:20.480+0000 7fef34031700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T20:27:20.915 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:27:20 smithi121 conmon[151425]: 2023-09-23T20:27:20.488+0000 7fef2f62d700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:27:21.165 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T20:27:21.165 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T20:27:21.165 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2023-09-23T20:27:21.165 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2023-09-23T20:27:21.165 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0146917 s, 714 MB/s 2023-09-23T20:27:21.165 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2023-09-23T20:27:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[100258]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:27:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[100258]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T20:27:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:21.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[118228]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:27:21.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[118228]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T20:27:21.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:21.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:21 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:21.793 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:21 smithi123 ceph-mon[104968]: from='osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2023-09-23T20:27:21.794 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:21 smithi123 ceph-mon[104968]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T20:27:21.794 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:21 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:21.794 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:21 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:21.806 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi123:vg_nvme/lv_4 2023-09-23T20:27:22.139 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:22.684 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:27:22.684 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:27:22.684 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780] boot 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi121 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.685 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi121 to 4036M 2023-09-23T20:27:22.686 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.686 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:22.686 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:22.686 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:22 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:27:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:27:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780] boot 2023-09-23T20:27:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:27:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi121 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi121 to 4036M 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: osd.3 [v2:172.21.15.121:6826/2305478780,v1:172.21.15.121:6827/2305478780] boot 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi121 2023-09-23T20:27:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi121 to 4036M 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:22 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:23.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:23 smithi121 ceph-mon[100258]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T20:27:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:23 smithi121 ceph-mon[118228]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T20:27:23.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:23 smithi123 ceph-mon[104968]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T20:27:24.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:24 smithi123 ceph-mon[104968]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:24.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:24 smithi123 ceph-mon[104968]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:24.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:24.593 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:24.593 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[100258]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[100258]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:24.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[118228]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:24.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[118228]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:24.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:24.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:24.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[118228]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/2070042642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]: dispatch 2023-09-23T20:27:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]: dispatch 2023-09-23T20:27:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]': finished 2023-09-23T20:27:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[118228]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:27:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/3508146690' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[100258]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/2070042642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]: dispatch 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]: dispatch 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]': finished 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[100258]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:26 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/3508146690' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:26 smithi123 ceph-mon[104968]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:26 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/2070042642' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]: dispatch 2023-09-23T20:27:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:26 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]: dispatch 2023-09-23T20:27:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:26 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84bd1437-f644-42b0-8bc8-6a3e1814ad9d"}]': finished 2023-09-23T20:27:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:26 smithi123 ceph-mon[104968]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:27:26.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:26 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:26.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:26 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/3508146690' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:28 smithi123 ceph-mon[104968]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:28 smithi121 ceph-mon[100258]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:28 smithi121 ceph-mon[118228]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:30.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:30 smithi121 ceph-mon[118228]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:30 smithi121 ceph-mon[100258]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:30 smithi123 ceph-mon[104968]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:32 smithi121 ceph-mon[100258]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:32 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:27:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:32 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:32.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:32 smithi121 ceph-mon[118228]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:32.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:32 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:27:32.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:32 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:32.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:32 smithi123 ceph-mon[104968]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:32 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:27:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:32 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:33 smithi121 ceph-mon[100258]: Deploying daemon osd.4 on smithi123 2023-09-23T20:27:33.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:33 smithi121 ceph-mon[118228]: Deploying daemon osd.4 on smithi123 2023-09-23T20:27:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:33 smithi123 ceph-mon[104968]: Deploying daemon osd.4 on smithi123 2023-09-23T20:27:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:34 smithi121 ceph-mon[100258]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:34.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:34 smithi121 ceph-mon[118228]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:34.968 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:34 smithi123 ceph-mon[104968]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:36 smithi123 ceph-mon[104968]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:36 smithi121 ceph-mon[100258]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:36 smithi121 ceph-mon[118228]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:38 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:38 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:38 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:38 smithi123 ceph-mon[104968]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[100258]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:38.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:38.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:38.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:38.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:38 smithi121 ceph-mon[118228]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:40.226 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 4 on host 'smithi123' 2023-09-23T20:27:40.251 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.251 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.251 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:40.251 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:40.251 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:39 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:39 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:40.802 DEBUG:teuthology.orchestra.run.smithi123:osd.4> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T20:27:40.805 INFO:tasks.cephadm:Deploying osd.5 on smithi123 with /dev/vg_nvme/lv_3... 2023-09-23T20:27:40.805 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T20:27:40.836 INFO:journalctl@ceph.osd.4.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:27:41.089 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:40 smithi123 ceph-mon[104968]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:41.089 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:40 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:41.090 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:40 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:41.090 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:40 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:41.090 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:40 smithi123 ceph-mon[104968]: from='osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:27:41.090 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:40 smithi123 ceph-mon[104968]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[118228]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[118228]: from='osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[118228]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[100258]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:27:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[100258]: from='osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:27:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:40 smithi121 ceph-mon[100258]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:27:41.841 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:42.490 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:42 smithi123 ceph-mon[104968]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:27:42.490 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:42 smithi123 ceph-mon[104968]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:27:42.490 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:42 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:42.490 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:42 smithi123 ceph-mon[104968]: from='osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:27:42.490 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:42 smithi123 ceph-mon[104968]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:27:42.490 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:42 smithi123 ceph-mon[104968]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:42.491 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 20:27:42 smithi123 conmon[112995]: 2023-09-23T20:27:42.424+0000 7fe443233700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T20:27:42.491 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 20:27:42 smithi123 conmon[112995]: 2023-09-23T20:27:42.429+0000 7fe43e02e700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:27:42.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[100258]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:27:42.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[100258]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:27:42.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:42.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[100258]: from='osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:27:42.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[100258]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:27:42.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[100258]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[118228]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:27:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[118228]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:27:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[118228]: from='osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:27:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[118228]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:27:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:42 smithi121 ceph-mon[118228]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:43.641 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:43 smithi123 ceph-mon[104968]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:27:43.641 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:43 smithi123 ceph-mon[104968]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T20:27:43.641 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:43.641 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:43 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:43.787 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T20:27:43.787 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T20:27:43.787 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2023-09-23T20:27:43.787 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2023-09-23T20:27:43.787 INFO:teuthology.orchestra.run.smithi123.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114725 s, 914 MB/s 2023-09-23T20:27:43.787 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 2023-09-23T20:27:43.787 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2023-09-23T20:27:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[100258]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:27:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[100258]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T20:27:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[118228]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:27:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[118228]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T20:27:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:43 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:44.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:27:44.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:27:44.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:44.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785] boot 2023-09-23T20:27:44.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:27:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:44 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.730 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi123:vg_nvme/lv_3 2023-09-23T20:27:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:27:44.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:27:44.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:44.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785] boot 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:44.950 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:44.951 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.951 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:27:44.951 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:27:44.951 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:44.951 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: osd.4 [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785] boot 2023-09-23T20:27:44.951 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:27:44.951 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:44.952 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:44 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:27:45.065 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:45.672 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:45 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi123 2023-09-23T20:27:45.672 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:45 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi123 to 17168M 2023-09-23T20:27:45.672 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:45 smithi123 ceph-mon[104968]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:27:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:45 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi123 2023-09-23T20:27:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:45 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi123 to 17168M 2023-09-23T20:27:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:45 smithi121 ceph-mon[100258]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:27:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:45 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi123 2023-09-23T20:27:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:45 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi123 to 17168M 2023-09-23T20:27:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:45 smithi121 ceph-mon[118228]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:27:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:46 smithi123 ceph-mon[104968]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:27:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:46 smithi123 ceph-mon[104968]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:27:46.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:46 smithi121 ceph-mon[100258]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:27:46.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:46 smithi121 ceph-mon[100258]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:27:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:46 smithi121 ceph-mon[118228]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:27:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:46 smithi121 ceph-mon[118228]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:27:47.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:47 smithi123 ceph-mon[104968]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:27:47.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:47 smithi123 ceph-mon[104968]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:47.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:47.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:47.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:47 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[100258]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:27:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[100258]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[118228]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:27:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[118228]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:27:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:27:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:27:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:47 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:48.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:48 smithi121 ceph-mon[100258]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T20:27:48.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:48 smithi121 ceph-mon[100258]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:27:48.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:48 smithi121 ceph-mon[118228]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T20:27:48.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:48 smithi121 ceph-mon[118228]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:27:48.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:48 smithi123 ceph-mon[104968]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T20:27:48.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:48 smithi123 ceph-mon[104968]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:27:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/3580596323' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]: dispatch 2023-09-23T20:27:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]: dispatch 2023-09-23T20:27:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]': finished 2023-09-23T20:27:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[100258]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:27:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:27:49.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/3580596323' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]: dispatch 2023-09-23T20:27:49.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]: dispatch 2023-09-23T20:27:49.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]': finished 2023-09-23T20:27:49.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[118228]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:27:49.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:49 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:27:49.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:49 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/3580596323' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]: dispatch 2023-09-23T20:27:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:49 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]: dispatch 2023-09-23T20:27:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:49 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eaf000e3-5147-4aa8-a0d3-65c6a38bd803"}]': finished 2023-09-23T20:27:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:49 smithi123 ceph-mon[104968]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:27:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:49 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:27:50.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:50 smithi121 ceph-mon[100258]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 139 KiB/s, 0 objects/s recovering 2023-09-23T20:27:50.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:50 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/57422045' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:50.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:50 smithi121 ceph-mon[118228]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 139 KiB/s, 0 objects/s recovering 2023-09-23T20:27:50.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:50 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/57422045' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:50.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:50 smithi123 ceph-mon[104968]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 139 KiB/s, 0 objects/s recovering 2023-09-23T20:27:50.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:50 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/57422045' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:27:52.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:52 smithi121 ceph-mon[100258]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:27:52.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:52 smithi121 ceph-mon[118228]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:27:52.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:52 smithi123 ceph-mon[104968]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:27:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:54 smithi121 ceph-mon[100258]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T20:27:54.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:54 smithi121 ceph-mon[118228]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T20:27:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:54 smithi123 ceph-mon[104968]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2023-09-23T20:27:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:55 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:27:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:55 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:55 smithi123 ceph-mon[104968]: Deploying daemon osd.5 on smithi123 2023-09-23T20:27:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:55 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:27:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:55 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:55 smithi121 ceph-mon[100258]: Deploying daemon osd.5 on smithi123 2023-09-23T20:27:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:55 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:27:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:55 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:55 smithi121 ceph-mon[118228]: Deploying daemon osd.5 on smithi123 2023-09-23T20:27:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:56 smithi123 ceph-mon[104968]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2023-09-23T20:27:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:56 smithi121 ceph-mon[100258]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2023-09-23T20:27:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:56 smithi121 ceph-mon[118228]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2023-09-23T20:27:58.721 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:27:58 smithi123 ceph-mon[104968]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:27:58.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:27:58 smithi121 ceph-mon[100258]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:27:58.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:27:58 smithi121 ceph-mon[118228]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:28:00.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:00 smithi123 ceph-mon[104968]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-09-23T20:28:00.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:00.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:00.698 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:00 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[100258]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-09-23T20:28:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[118228]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-09-23T20:28:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:00 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:02.886 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 5 on host 'smithi123' 2023-09-23T20:28:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[100258]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[118228]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.173 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:02 smithi123 ceph-mon[104968]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:03.173 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.173 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:03.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:03.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:03.479 DEBUG:teuthology.orchestra.run.smithi123:osd.5> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T20:28:03.481 INFO:tasks.cephadm:Deploying osd.6 on smithi123 with /dev/vg_nvme/lv_2... 2023-09-23T20:28:03.481 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T20:28:03.513 INFO:journalctl@ceph.osd.5.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:28:04.086 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:04.086 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:04.086 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:03 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:04.086 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:03 smithi123 ceph-mon[104968]: from='osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:28:04.086 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:03 smithi123 ceph-mon[104968]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:28:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[100258]: from='osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:28:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[100258]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:28:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[118228]: from='osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:28:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:03 smithi121 ceph-mon[118228]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:28:04.507 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:28:05.154 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:04 smithi123 ceph-mon[104968]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:05.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:04 smithi123 ceph-mon[104968]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:28:05.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:04 smithi123 ceph-mon[104968]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:28:05.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:04 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:05.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:04 smithi123 ceph-mon[104968]: from='osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:05.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:04 smithi123 ceph-mon[104968]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:05.155 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 20:28:04 smithi123 conmon[122599]: 2023-09-23T20:28:04.896+0000 7fe6592c3700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T20:28:05.155 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 20:28:04 smithi123 conmon[122599]: 2023-09-23T20:28:04.901+0000 7fe6558c1700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[100258]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[100258]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[100258]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[100258]: from='osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[100258]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[118228]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[118228]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:28:05.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[118228]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:28:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[118228]: from='osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:04 smithi121 ceph-mon[118228]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[100258]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[100258]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:28:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[118228]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[118228]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:28:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:05 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:05 smithi123 ceph-mon[104968]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:05 smithi123 ceph-mon[104968]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:28:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:05 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:05 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:06.930 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T20:28:06.931 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T20:28:06.931 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2023-09-23T20:28:06.931 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2023-09-23T20:28:06.931 INFO:teuthology.orchestra.run.smithi123.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0115545 s, 908 MB/s 2023-09-23T20:28:06.931 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2023-09-23T20:28:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:28:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:28:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:28:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602] boot 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602] boot 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:07.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:07.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: osd.5 [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602] boot 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:07.660 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi123:vg_nvme/lv_2 2023-09-23T20:28:07.998 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:28:08.201 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:07 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi123 2023-09-23T20:28:08.202 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:07 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi123 to 8584M 2023-09-23T20:28:08.202 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:07 smithi123 ceph-mon[104968]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:28:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:07 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi123 2023-09-23T20:28:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:07 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi123 to 8584M 2023-09-23T20:28:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:07 smithi121 ceph-mon[100258]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:28:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:07 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi123 2023-09-23T20:28:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:07 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi123 to 8584M 2023-09-23T20:28:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:07 smithi121 ceph-mon[118228]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:28:09.212 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:08 smithi123 ceph-mon[104968]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:09.212 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:08 smithi123 ceph-mon[104968]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:28:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:08 smithi121 ceph-mon[100258]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:08 smithi121 ceph-mon[100258]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:28:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:08 smithi121 ceph-mon[118228]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:08 smithi121 ceph-mon[118228]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:28:10.193 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:09 smithi123 ceph-mon[104968]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:28:10.193 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:09 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:28:10.193 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:09 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:28:10.193 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:09 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[100258]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:28:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:28:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:28:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[118228]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:28:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:28:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:28:10.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:09 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:10 smithi123 ceph-mon[104968]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:10 smithi123 ceph-mon[104968]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:28:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:10 smithi123 ceph-mon[104968]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:28:11.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:10 smithi121 ceph-mon[100258]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:11.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:10 smithi121 ceph-mon[100258]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:28:11.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:10 smithi121 ceph-mon[100258]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:28:11.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:10 smithi121 ceph-mon[118228]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:11.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:10 smithi121 ceph-mon[118228]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:28:11.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:10 smithi121 ceph-mon[118228]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:28:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:12 smithi123 ceph-mon[104968]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:12 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/3591127516' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]: dispatch 2023-09-23T20:28:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:12 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]: dispatch 2023-09-23T20:28:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:12 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]': finished 2023-09-23T20:28:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:12 smithi123 ceph-mon[104968]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:28:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:12 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:12 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/659288481' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:28:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[100258]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/3591127516' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]: dispatch 2023-09-23T20:28:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]: dispatch 2023-09-23T20:28:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]': finished 2023-09-23T20:28:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[100258]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/659288481' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[118228]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/3591127516' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]: dispatch 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]: dispatch 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "29ce2c8b-96a4-4b71-97b2-c307f944ea0c"}]': finished 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[118228]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:13.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:12 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/659288481' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:28:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:13 smithi121 ceph-mon[100258]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:28:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:13 smithi121 ceph-mon[100258]: Cluster is now healthy 2023-09-23T20:28:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:13 smithi121 ceph-mon[118228]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:28:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:13 smithi121 ceph-mon[118228]: Cluster is now healthy 2023-09-23T20:28:14.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:13 smithi123 ceph-mon[104968]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:28:14.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:13 smithi123 ceph-mon[104968]: Cluster is now healthy 2023-09-23T20:28:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:14 smithi123 ceph-mon[104968]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T20:28:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:14 smithi121 ceph-mon[100258]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T20:28:15.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:14 smithi121 ceph-mon[118228]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T20:28:17.030 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:16 smithi123 ceph-mon[104968]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-23T20:28:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:16 smithi121 ceph-mon[100258]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-23T20:28:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:16 smithi121 ceph-mon[118228]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-23T20:28:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:18 smithi123 ceph-mon[104968]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:28:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:28:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:18 smithi123 ceph-mon[104968]: Deploying daemon osd.6 on smithi123 2023-09-23T20:28:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[100258]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:28:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:28:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[100258]: Deploying daemon osd.6 on smithi123 2023-09-23T20:28:18.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[118228]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:28:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:28:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:18 smithi121 ceph-mon[118228]: Deploying daemon osd.6 on smithi123 2023-09-23T20:28:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:20 smithi123 ceph-mon[104968]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T20:28:21.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:20 smithi121 ceph-mon[100258]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T20:28:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:20 smithi121 ceph-mon[118228]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T20:28:22.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:22 smithi123 ceph-mon[104968]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:28:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:22 smithi121 ceph-mon[100258]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:28:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:22 smithi121 ceph-mon[118228]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:28:23.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:23 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:23.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:23 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:23.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:23 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:23 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:23 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:23 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:23 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:23 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:23 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:24.932 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:24 smithi123 ceph-mon[104968]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:24.932 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:24.932 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:24.932 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:24.932 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:24.932 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:24 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[100258]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[118228]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:24 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:25.469 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 6 on host 'smithi123' 2023-09-23T20:28:26.026 DEBUG:teuthology.orchestra.run.smithi123:osd.6> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T20:28:26.028 INFO:tasks.cephadm:Deploying osd.7 on smithi123 with /dev/vg_nvme/lv_1... 2023-09-23T20:28:26.028 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T20:28:26.060 INFO:journalctl@ceph.osd.6.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:28:26.574 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:26 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:26.574 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:26 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:26.574 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:26 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:26.574 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:26 smithi123 ceph-mon[104968]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:26.574 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:26 smithi123 ceph-mon[104968]: from='osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:28:26.574 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:26 smithi123 ceph-mon[104968]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:28:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[100258]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[100258]: from='osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:28:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[100258]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:28:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:26.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:26.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:26.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[118228]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:26.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[118228]: from='osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:28:26.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:26 smithi121 ceph-mon[118228]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:28:26.984 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:28:27.729 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 20:28:27 smithi123 conmon[131622]: 2023-09-23T20:28:27.467+0000 7fe793b5c700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T20:28:27.729 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 20:28:27 smithi123 conmon[131622]: 2023-09-23T20:28:27.474+0000 7fe78f158700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:28:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:27 smithi123 ceph-mon[104968]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:28:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:27 smithi123 ceph-mon[104968]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:28:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:27 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:27 smithi123 ceph-mon[104968]: from='osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:27 smithi123 ceph-mon[104968]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[100258]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:28:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[100258]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:28:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[100258]: from='osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[100258]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:27.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[118228]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:28:27.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[118228]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:28:27.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:27.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[118228]: from='osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:27.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:27 smithi121 ceph-mon[118228]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:28 smithi123 ceph-mon[104968]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:28 smithi123 ceph-mon[104968]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:28:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:28 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:28 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:28 smithi123 ceph-mon[104968]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:28.877 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T20:28:28.877 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T20:28:28.877 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2023-09-23T20:28:28.877 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2023-09-23T20:28:28.877 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114535 s, 916 MB/s 2023-09-23T20:28:28.878 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2023-09-23T20:28:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[100258]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[100258]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:28:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[100258]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[118228]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[118228]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:28:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:28 smithi121 ceph-mon[118228]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:28:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:29 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:28:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:29 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:28:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:29 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:29 smithi123 ceph-mon[104968]: osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390] boot 2023-09-23T20:28:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:29 smithi123 ceph-mon[104968]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:28:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:29 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:29.796 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi123:vg_nvme/lv_1 2023-09-23T20:28:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:28:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:28:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[100258]: osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390] boot 2023-09-23T20:28:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[100258]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:28:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:28:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:28:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[118228]: osd.6 [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390] boot 2023-09-23T20:28:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[118228]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:28:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:29 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:28:30.133 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:28:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi123 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi123 to 5722M 2023-09-23T20:28:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:30 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:28:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi123 2023-09-23T20:28:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi123 to 5722M 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:28:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi123 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi123 to 5722M 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:30 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:31 smithi123 ceph-mon[104968]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:28:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:31 smithi121 ceph-mon[100258]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:28:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:31 smithi121 ceph-mon[118228]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:28:32.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:32 smithi123 ceph-mon[104968]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:28:32.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:32 smithi123 ceph-mon[104968]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:32.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:32 smithi123 ceph-mon[104968]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:28:32.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:32 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:28:32.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:32 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:28:32.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:32 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[100258]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:28:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[100258]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[100258]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:28:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:28:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:28:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[118228]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:28:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[118228]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[118228]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:28:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:28:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:28:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:32 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:33 smithi123 ceph-mon[104968]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:28:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:33 smithi121 ceph-mon[100258]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:28:33.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:33 smithi121 ceph-mon[118228]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:28:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[100258]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/1059890366' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]: dispatch 2023-09-23T20:28:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]: dispatch 2023-09-23T20:28:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[100258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]': finished 2023-09-23T20:28:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[100258]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:28:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:34.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[118228]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:34.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/1059890366' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]: dispatch 2023-09-23T20:28:34.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]: dispatch 2023-09-23T20:28:34.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[118228]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]': finished 2023-09-23T20:28:34.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[118228]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:28:34.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:34 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:34 smithi123 ceph-mon[104968]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:34 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/1059890366' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]: dispatch 2023-09-23T20:28:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:34 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]: dispatch 2023-09-23T20:28:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:34 smithi123 ceph-mon[104968]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "69e6dd32-d849-40d2-bef3-a0cf635c0346"}]': finished 2023-09-23T20:28:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:34 smithi123 ceph-mon[104968]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:28:34.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:34 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:35 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/320912287' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:28:35.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:35 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/320912287' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:28:35.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:35 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/320912287' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:28:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:36 smithi121 ceph-mon[100258]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2023-09-23T20:28:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:36 smithi121 ceph-mon[118228]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2023-09-23T20:28:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:36 smithi123 ceph-mon[104968]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2023-09-23T20:28:38.580 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:38 smithi123 ceph-mon[104968]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2023-09-23T20:28:38.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:38 smithi121 ceph-mon[100258]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2023-09-23T20:28:38.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:38 smithi121 ceph-mon[118228]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2023-09-23T20:28:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:40 smithi123 ceph-mon[104968]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:28:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:40 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:28:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:40 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:40 smithi123 ceph-mon[104968]: Deploying daemon osd.7 on smithi123 2023-09-23T20:28:41.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[100258]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:28:41.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:28:41.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:41.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[100258]: Deploying daemon osd.7 on smithi123 2023-09-23T20:28:41.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[118228]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:28:41.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:28:41.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:41.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:40 smithi121 ceph-mon[118228]: Deploying daemon osd.7 on smithi123 2023-09-23T20:28:42.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:42 smithi123 ceph-mon[104968]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-09-23T20:28:43.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:42 smithi121 ceph-mon[100258]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-09-23T20:28:43.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:42 smithi121 ceph-mon[118228]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-09-23T20:28:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:44 smithi123 ceph-mon[104968]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:28:45.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:44 smithi121 ceph-mon[100258]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:28:45.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:44 smithi121 ceph-mon[118228]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:28:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:45 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:45 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:45 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:46.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:45 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:46.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:45 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:46.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:45 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:45 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:45 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:45 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:46.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:46 smithi123 ceph-mon[104968]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-23T20:28:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:46 smithi121 ceph-mon[118228]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-23T20:28:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:46 smithi121 ceph-mon[100258]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-23T20:28:47.753 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 7 on host 'smithi123' 2023-09-23T20:28:48.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:48 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:48.366 DEBUG:teuthology.orchestra.run.smithi123:osd.7> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T20:28:48.368 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T20:28:48.368 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T20:28:48.400 INFO:journalctl@ceph.osd.7.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:28:48.400 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.401 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:48.402 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:48.402 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.402 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.402 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:48.402 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:48 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:48.720 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:49.274 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:49 smithi121 ceph-mon[100258]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:49.275 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:49 smithi121 ceph-mon[100258]: from='osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:28:49.275 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:49 smithi121 ceph-mon[100258]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:28:49.275 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:49 smithi121 ceph-mon[118228]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:49.275 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:49 smithi121 ceph-mon[118228]: from='osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:28:49.275 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:49 smithi121 ceph-mon[118228]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:28:49.410 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:49 smithi123 ceph-mon[104968]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:49.410 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:49 smithi123 ceph-mon[104968]: from='osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:28:49.410 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:49 smithi123 ceph-mon[104968]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:28:50.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[100258]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:28:50.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[100258]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:28:50.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:50.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[100258]: from='osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:50.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[100258]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:50.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[118228]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:28:50.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[118228]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:28:50.023 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:50.023 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[118228]: from='osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:50.023 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:50 smithi121 ceph-mon[118228]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:50.408 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:28:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:50 smithi123 ceph-mon[104968]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:28:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:50 smithi123 ceph-mon[104968]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:28:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:50 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:50 smithi123 ceph-mon[104968]: from='osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:50.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:50 smithi123 ceph-mon[104968]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T20:28:50.480 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 20:28:50 smithi123 conmon[141153]: 2023-09-23T20:28:50.025+0000 7f0715371700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T20:28:50.480 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 20:28:50 smithi123 conmon[141153]: 2023-09-23T20:28:50.033+0000 7f071016c700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:28:51.106 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1695500908,"num_in_osds":8,"osd_in_since":1695500914,"num_remapped_pgs":0} 2023-09-23T20:28:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[100258]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[100258]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[100258]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:28:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1900546199' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:28:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[118228]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[118228]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[118228]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:28:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:51 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1900546199' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:28:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:51 smithi123 ceph-mon[104968]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:28:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:51 smithi123 ceph-mon[104968]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2023-09-23T20:28:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:51 smithi123 ceph-mon[104968]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:28:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:51 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:51 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:51 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1900546199' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:28:52.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: purged_snaps scrub starts 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: purged_snaps scrub ok 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382] boot 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[100258]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: purged_snaps scrub starts 2023-09-23T20:28:52.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: purged_snaps scrub ok 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382] boot 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:52.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:52.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:52 smithi121 ceph-mon[118228]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:28:52.454 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: purged_snaps scrub starts 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: purged_snaps scrub ok 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: osd.7 [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382] boot 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:28:52.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:52.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:52.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:28:52.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:52 smithi123 ceph-mon[104968]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:28:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[100258]: Detected new or changed devices on smithi123 2023-09-23T20:28:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi123 to 4292M 2023-09-23T20:28:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[100258]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[100258]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:28:53.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[118228]: Detected new or changed devices on smithi123 2023-09-23T20:28:53.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi123 to 4292M 2023-09-23T20:28:53.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[118228]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:53.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:53 smithi121 ceph-mon[118228]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:28:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:53 smithi123 ceph-mon[104968]: Detected new or changed devices on smithi123 2023-09-23T20:28:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:53 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi123 to 4292M 2023-09-23T20:28:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:53 smithi123 ceph-mon[104968]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:53 smithi123 ceph-mon[104968]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:28:54.119 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:28:54.847 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500931,"num_in_osds":8,"osd_in_since":1695500914,"num_remapped_pgs":0} 2023-09-23T20:28:54.848 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:28:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:55 smithi121 ceph-mon[118228]: pgmap v169: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:55 smithi121 ceph-mon[118228]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:28:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:55 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3375724891' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:28:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:55 smithi121 ceph-mon[100258]: pgmap v169: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:55 smithi121 ceph-mon[100258]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:28:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:55 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3375724891' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:28:55.190 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:55 smithi123 ceph-mon[104968]: pgmap v169: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:55 smithi123 ceph-mon[104968]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:28:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:55 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3375724891' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:28:56.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:56 smithi121 ceph-mon[100258]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:28:56.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:56 smithi121 ceph-mon[118228]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:28:56.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:56 smithi123 ceph-mon[104968]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:28:56.841 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:28:56.841 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":59,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:28:55.038766+0000","last_up_change":"2023-09-23T20:28:51.017490+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6803","nonce":801467255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6805","nonce":801467255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6809","nonce":801467255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6807","nonce":801467255}]},"public_addr":"172.21.15.121:6803/801467255","cluster_addr":"172.21.15.121:6805/801467255","heartbeat_back_addr":"172.21.15.121:6809/801467255","heartbeat_front_addr":"172.21.15.121:6807/801467255","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6811","nonce":3606674368}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6813","nonce":3606674368}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6817","nonce":3606674368}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6815","nonce":3606674368}]},"public_addr":"172.21.15.121:6811/3606674368","cluster_addr":"172.21.15.121:6813/3606674368","heartbeat_back_addr":"172.21.15.121:6817/3606674368","heartbeat_front_addr":"172.21.15.121:6815/3606674368","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6819","nonce":4028229487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6821","nonce":4028229487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6825","nonce":4028229487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6823","nonce":4028229487}]},"public_addr":"172.21.15.121:6819/4028229487","cluster_addr":"172.21.15.121:6821/4028229487","heartbeat_back_addr":"172.21.15.121:6825/4028229487","heartbeat_front_addr":"172.21.15.121:6823/4028229487","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6827","nonce":2305478780}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6829","nonce":2305478780}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6833","nonce":2305478780}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6831","nonce":2305478780}]},"public_addr":"172.21.15.121:6827/2305478780","cluster_addr":"172.21.15.121:6829/2305478780","heartbeat_back_addr":"172.21.15.121:6833/2305478780","heartbeat_front_addr":"172.21.15.121:6831/2305478780","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:28:57.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:57 smithi121 ceph-mon[100258]: pgmap v172: 1 pgs: 1 peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:57 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4124914540' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:28:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:57 smithi121 ceph-mon[118228]: pgmap v172: 1 pgs: 1 peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:57 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4124914540' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:28:57.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:57 smithi123 ceph-mon[104968]: pgmap v172: 1 pgs: 1 peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:28:57.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:57 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4124914540' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:28:57.558 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T20:26:57.902240+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T20:28:57.558 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T20:28:57.902 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:58 smithi123 ceph-mon[104968]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-09-23T20:28:58.855 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:58 smithi121 ceph-mon[118228]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-09-23T20:28:58.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:58 smithi121 ceph-mon[100258]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2023-09-23T20:28:59.513 INFO:teuthology.orchestra.run.smithi121.stdout:pg_num: 1 2023-09-23T20:28:59.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:28:59 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3383005664' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:28:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:28:59 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3383005664' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:28:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:28:59 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3383005664' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:29:00.255 INFO:tasks.cephadm:Adding prometheus.a on smithi123 2023-09-23T20:29:00.255 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi123=a' 2023-09-23T20:29:00.619 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:00.832 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:00 smithi123 ceph-mon[104968]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-09-23T20:29:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:00 smithi121 ceph-mon[100258]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-09-23T20:29:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:00 smithi121 ceph-mon[118228]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2023-09-23T20:29:02.239 INFO:teuthology.orchestra.run.smithi123.stdout:Scheduled prometheus update... 2023-09-23T20:29:02.887 DEBUG:teuthology.orchestra.run.smithi123:prometheus.a> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T20:29:02.889 INFO:tasks.cephadm:Adding node-exporter.a on smithi121 2023-09-23T20:29:02.889 INFO:tasks.cephadm:Adding node-exporter.b on smithi123 2023-09-23T20:29:02.889 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi121=a;smithi123=b' 2023-09-23T20:29:02.920 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:29:02.942 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:02 smithi123 ceph-mon[104968]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:29:02.942 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:02.943 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:02.943 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:02.943 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:02.943 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:02 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[100258]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[118228]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:29:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:02 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:03.252 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:03 smithi123 ceph-mon[104968]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi123=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:03 smithi123 ceph-mon[104968]: Saving service prometheus spec with placement smithi123=a;count:1 2023-09-23T20:29:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:03 smithi123 ceph-mon[104968]: Deploying daemon prometheus.a on smithi123 2023-09-23T20:29:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:03 smithi121 ceph-mon[100258]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi123=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:03 smithi121 ceph-mon[100258]: Saving service prometheus spec with placement smithi123=a;count:1 2023-09-23T20:29:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:03 smithi121 ceph-mon[100258]: Deploying daemon prometheus.a on smithi123 2023-09-23T20:29:04.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:03 smithi121 ceph-mon[118228]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi123=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:04.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:03 smithi121 ceph-mon[118228]: Saving service prometheus spec with placement smithi123=a;count:1 2023-09-23T20:29:04.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:03 smithi121 ceph-mon[118228]: Deploying daemon prometheus.a on smithi123 2023-09-23T20:29:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:04 smithi123 ceph-mon[104968]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T20:29:05.031 INFO:teuthology.orchestra.run.smithi123.stdout:Scheduled node-exporter update... 2023-09-23T20:29:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:04 smithi121 ceph-mon[100258]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T20:29:05.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:04 smithi121 ceph-mon[118228]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T20:29:06.297 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:06 smithi123 ceph-mon[104968]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi121=a;smithi123=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:06.297 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:06 smithi123 ceph-mon[104968]: Saving service node-exporter spec with placement smithi121=a;smithi123=b;count:2 2023-09-23T20:29:06.297 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:06 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:06.298 DEBUG:teuthology.orchestra.run.smithi121:node-exporter.a> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T20:29:06.301 DEBUG:teuthology.orchestra.run.smithi123:node-exporter.b> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T20:29:06.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:06 smithi121 ceph-mon[100258]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi121=a;smithi123=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:06.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:06 smithi121 ceph-mon[100258]: Saving service node-exporter spec with placement smithi121=a;smithi123=b;count:2 2023-09-23T20:29:06.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:06 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:06.331 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:06 smithi121 ceph-mon[118228]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi121=a;smithi123=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:06.331 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:06 smithi121 ceph-mon[118228]: Saving service node-exporter spec with placement smithi121=a;smithi123=b;count:2 2023-09-23T20:29:06.331 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:06 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:06.337 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:29:06.342 INFO:tasks.cephadm:Adding alertmanager.a on smithi121 2023-09-23T20:29:06.342 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi121=a' 2023-09-23T20:29:06.373 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:29:06.725 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:07 smithi123 ceph-mon[104968]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T20:29:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:07 smithi121 ceph-mon[100258]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T20:29:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:07 smithi121 ceph-mon[118228]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2023-09-23T20:29:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:08 smithi123 ceph-mon[104968]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:29:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:08 smithi121 ceph-mon[100258]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:29:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:08 smithi121 ceph-mon[118228]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:29:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:10 smithi123 ceph-mon[104968]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:10 smithi121 ceph-mon[100258]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:11.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:10 smithi121 ceph-mon[118228]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:12.162 INFO:teuthology.orchestra.run.smithi123.stdout:Scheduled alertmanager update... 2023-09-23T20:29:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[118228]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[118228]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[118228]: Saving service alertmanager spec with placement smithi121=a;count:1 2023-09-23T20:29:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[100258]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[100258]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[100258]: Saving service alertmanager spec with placement smithi121=a;count:1 2023-09-23T20:29:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:13 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:13 smithi123 ceph-mon[104968]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:13 smithi123 ceph-mon[104968]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi121=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:29:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:13 smithi123 ceph-mon[104968]: Saving service alertmanager spec with placement smithi121=a;count:1 2023-09-23T20:29:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:13 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:14.569 DEBUG:teuthology.orchestra.run.smithi121:alertmanager.a> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T20:29:14.572 INFO:tasks.cephadm:Adding grafana.a on smithi123 2023-09-23T20:29:14.572 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi123=a' 2023-09-23T20:29:14.604 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:29:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:15 smithi121 ceph-mon[100258]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:15.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:15 smithi121 ceph-mon[118228]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:15 smithi123 ceph-mon[104968]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:15.644 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:15.906 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:15 smithi123 systemd[1]: Starting Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.006Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.006Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.007Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi123 (none))" 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.007Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.007Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.009Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.009Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.011Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:29:17.166 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.011Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.012Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.012Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.383µs 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.012Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.012Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.012Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.529µs wal_replay_duration=331.271µs wbl_replay_duration=90ns total_replay_duration=386.246µs 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.013Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.013Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.013Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 bash[149389]: 700b419679b409a9ed46f8531a5545a0469e29bb269c9688604da6d8f2ae6cdc 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 systemd[1]: Started Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.028Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.019687ms db_storage=681ns remote_storage=1.009µs web_handler=300ns query_engine=518ns scrape=790.648µs scrape_sd=30.16µs notify=1.062µs notify_sd=2.312µs rules=14.038652ms tracing=8.504µs 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.028Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:29:17.167 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:29:17 smithi123 conmon[149461]: ts=2023-09-23T20:29:17.028Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:29:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:17 smithi123 ceph-mon[104968]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:17.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:17 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:17 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:17 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:17 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:29:17.611 INFO:teuthology.orchestra.run.smithi123.stdout:Scheduled grafana update... 2023-09-23T20:29:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[100258]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:29:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[118228]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:17 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:29:18.241 DEBUG:teuthology.orchestra.run.smithi123:grafana.a> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T20:29:18.243 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T20:29:18.243 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T20:29:18.273 INFO:journalctl@ceph.grafana.a.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T20:29:18.475 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 conmon[108184]: ignoring --setuser ceph since I am not root 2023-09-23T20:29:18.475 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 conmon[108184]: ignoring --setgroup ceph since I am not root 2023-09-23T20:29:18.475 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 ceph-mgr[108208]: -- 172.21.15.123:0/1129141704 <== mon.1 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5651019ce9c0 con 0x5651019a6c00 2023-09-23T20:29:18.475 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 conmon[108184]: 2023-09-23T20:29:18.313+0000 7f2ecc3c71c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:29:18.475 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 conmon[108184]: 2023-09-23T20:29:18.389+0000 7f2ecc3c71c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:29:18.475 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 conmon[108184]: 2023-09-23T20:29:18.474+0000 7f2ecc3c71c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:29:18.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 conmon[100681]: ignoring --setuser ceph since I am not root 2023-09-23T20:29:18.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 conmon[100681]: ignoring --setgroup ceph since I am not root 2023-09-23T20:29:18.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 ceph-mgr[100705]: -- 172.21.15.121:0/1399111949 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5584e37489c0 con 0x5584e3721000 2023-09-23T20:29:18.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 conmon[100681]: 2023-09-23T20:29:18.308+0000 7fb5522561c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:29:18.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 conmon[100681]: 2023-09-23T20:29:18.388+0000 7fb5522561c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:29:18.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 conmon[100681]: 2023-09-23T20:29:18.485+0000 7fb5522561c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:29:18.604 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:18.728 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 conmon[108184]: 2023-09-23T20:29:18.564+0000 7f2ecc3c71c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:29:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:18 smithi123 ceph-mon[104968]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:29:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:18 smithi123 ceph-mon[104968]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T20:29:18.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 conmon[100681]: 2023-09-23T20:29:18.600+0000 7fb5522561c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:29:18.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:18.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:18 smithi121 ceph-mon[118228]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:29:18.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:18 smithi121 ceph-mon[118228]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T20:29:18.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' 2023-09-23T20:29:18.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:18 smithi121 ceph-mon[100258]: from='mgr.14152 172.21.15.121:0/794506255' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:29:18.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:18 smithi121 ceph-mon[100258]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T20:29:19.032 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:18 smithi123 conmon[108184]: 2023-09-23T20:29:18.729+0000 7f2ecc3c71c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:29:19.114 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:18 smithi121 conmon[100681]: 2023-09-23T20:29:18.790+0000 7fb5522561c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:29:19.335 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:19 smithi123 conmon[108184]: 2023-09-23T20:29:19.032+0000 7f2ecc3c71c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:29:19.336 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:19 smithi123 conmon[108184]: 2023-09-23T20:29:19.119+0000 7f2ecc3c71c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:29:19.336 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:19 smithi123 conmon[108184]: 2023-09-23T20:29:19.244+0000 7f2ecc3c71c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:29:19.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:19 smithi121 conmon[100681]: 2023-09-23T20:29:19.112+0000 7fb5522561c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:29:19.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:19 smithi121 conmon[100681]: 2023-09-23T20:29:19.208+0000 7fb5522561c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:29:19.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:19 smithi121 conmon[100681]: 2023-09-23T20:29:19.338+0000 7fb5522561c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:29:19.728 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:19 smithi123 conmon[108184]: 2023-09-23T20:29:19.335+0000 7f2ecc3c71c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:29:19.867 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:19 smithi121 conmon[100681]: 2023-09-23T20:29:19.430+0000 7fb5522561c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:29:20.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:19 smithi121 conmon[100681]: 2023-09-23T20:29:19.865+0000 7fb5522561c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:29:20.228 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:19 smithi123 conmon[108184]: 2023-09-23T20:29:19.749+0000 7f2ecc3c71c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:29:20.452 INFO:teuthology.orchestra.run.smithi121.stdout:[client.0] 2023-09-23T20:29:20.452 INFO:teuthology.orchestra.run.smithi121.stdout: key = AQCgSg9let2/GhAAci3qr9I/wmIpjd22by0kAQ== 2023-09-23T20:29:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:20 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2993992893' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:20.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:20 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2993992893' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:29:20.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:20 smithi123 conmon[108184]: 2023-09-23T20:29:20.303+0000 7f2ecc3c71c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:29:20.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:20 smithi123 conmon[108184]: 2023-09-23T20:29:20.470+0000 7f2ecc3c71c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:29:20.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:20 smithi121 conmon[100681]: 2023-09-23T20:29:20.481+0000 7fb5522561c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:29:20.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:20 smithi121 conmon[100681]: 2023-09-23T20:29:20.659+0000 7fb5522561c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:29:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2993992893' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2993992893' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:29:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:20 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2993992893' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:20 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2993992893' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:29:21.117 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T20:29:21.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T20:29:21.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T20:29:21.158 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T20:29:21.478 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:21 smithi123 conmon[108184]: 2023-09-23T20:29:21.234+0000 7f2ecc3c71c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:29:21.478 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:21 smithi123 conmon[108184]: 2023-09-23T20:29:21.326+0000 7f2ecc3c71c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:29:21.514 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:21.683 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:21 smithi121 conmon[100681]: 2023-09-23T20:29:21.426+0000 7fb5522561c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:29:21.683 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:21 smithi121 conmon[100681]: 2023-09-23T20:29:21.515+0000 7fb5522561c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:29:21.814 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:21 smithi123 conmon[108184]: 2023-09-23T20:29:21.515+0000 7f2ecc3c71c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:29:21.955 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:21 smithi121 conmon[100681]: 2023-09-23T20:29:21.682+0000 7fb5522561c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:29:22.206 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:21 smithi123 conmon[108184]: 2023-09-23T20:29:21.814+0000 7f2ecc3c71c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:29:22.206 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:21 smithi123 conmon[108184]: 2023-09-23T20:29:21.905+0000 7f2ecc3c71c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:29:22.324 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:21 smithi121 conmon[100681]: 2023-09-23T20:29:21.954+0000 7fb5522561c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:29:22.324 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:22 smithi121 conmon[100681]: 2023-09-23T20:29:22.040+0000 7fb5522561c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:29:22.471 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:22 smithi123 conmon[108184]: 2023-09-23T20:29:22.206+0000 7f2ecc3c71c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:29:22.580 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:22 smithi121 conmon[100681]: 2023-09-23T20:29:22.322+0000 7fb5522561c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:29:22.728 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:22 smithi123 conmon[108184]: 2023-09-23T20:29:22.471+0000 7f2ecc3c71c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:29:22.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:22 smithi121 conmon[100681]: 2023-09-23T20:29:22.579+0000 7fb5522561c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:29:23.171 INFO:teuthology.orchestra.run.smithi123.stdout:[client.1] 2023-09-23T20:29:23.171 INFO:teuthology.orchestra.run.smithi123.stdout: key = AQCjSg9lYcgGChAAD8I+C5l9hJihvz4Rq2PIBg== 2023-09-23T20:29:23.467 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:23 smithi123 conmon[108184]: 2023-09-23T20:29:23.212+0000 7f2ecc3c71c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:29:23.468 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:23 smithi123 conmon[108184]: 2023-09-23T20:29:23.350+0000 7f2ecc3c71c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:29:23.597 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:23 smithi121 conmon[100681]: 2023-09-23T20:29:23.277+0000 7fb5522561c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:29:23.597 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:23 smithi121 conmon[100681]: 2023-09-23T20:29:23.407+0000 7fb5522561c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:29:23.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:23 smithi121 ceph-mon[100258]: from='client.? 172.21.15.123:0/357395181' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:23.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:23 smithi121 ceph-mon[100258]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:23.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:23 smithi121 ceph-mon[100258]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:29:23.598 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:23 smithi121 ceph-mon[118228]: from='client.? 172.21.15.123:0/357395181' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:23.598 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:23 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:23.598 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:23 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:29:23.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:23 smithi123 ceph-mon[104968]: from='client.? 172.21.15.123:0/357395181' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:23 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:29:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:23 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:29:23.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:23 smithi123 conmon[108184]: 2023-09-23T20:29:23.545+0000 7f2ecc3c71c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:29:23.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:23 smithi123 conmon[108184]: 2023-09-23T20:29:23.660+0000 7f2ecc3c71c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:29:23.754 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2023-09-23T20:29:23.755 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T20:29:23.755 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T20:29:23.796 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T20:29:23.796 DEBUG:teuthology.orchestra.run.smithi121:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T20:29:23.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:23 smithi121 conmon[100681]: 2023-09-23T20:29:23.595+0000 7fb5522561c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:29:23.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:23 smithi121 conmon[100681]: 2023-09-23T20:29:23.708+0000 7fb5522561c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:29:24.121 INFO:teuthology.misc.health.smithi121.stdout: 2023-09-23T20:29:24.121 INFO:teuthology.misc.health.smithi121.stdout:{"epoch":59,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:28:55.038766+0000","last_up_change":"2023-09-23T20:28:51.017490+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6803","nonce":801467255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6805","nonce":801467255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6809","nonce":801467255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6807","nonce":801467255}]},"public_addr":"172.21.15.121:6803/801467255","cluster_addr":"172.21.15.121:6805/801467255","heartbeat_back_addr":"172.21.15.121:6809/801467255","heartbeat_front_addr":"172.21.15.121:6807/801467255","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6811","nonce":3606674368}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6813","nonce":3606674368}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6817","nonce":3606674368}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6815","nonce":3606674368}]},"public_addr":"172.21.15.121:6811/3606674368","cluster_addr":"172.21.15.121:6813/3606674368","heartbeat_back_addr":"172.21.15.121:6817/3606674368","heartbeat_front_addr":"172.21.15.121:6815/3606674368","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6819","nonce":4028229487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6821","nonce":4028229487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6825","nonce":4028229487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6823","nonce":4028229487}]},"public_addr":"172.21.15.121:6819/4028229487","cluster_addr":"172.21.15.121:6821/4028229487","heartbeat_back_addr":"172.21.15.121:6825/4028229487","heartbeat_front_addr":"172.21.15.121:6823/4028229487","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6827","nonce":2305478780}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6829","nonce":2305478780}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6833","nonce":2305478780}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6831","nonce":2305478780}]},"public_addr":"172.21.15.121:6827/2305478780","cluster_addr":"172.21.15.121:6829/2305478780","heartbeat_back_addr":"172.21.15.121:6833/2305478780","heartbeat_front_addr":"172.21.15.121:6831/2305478780","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:29:24.135 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T20:29:24.135 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T20:29:24.135 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T20:29:24.228 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:23 smithi123 conmon[108184]: 2023-09-23T20:29:23.882+0000 7f2ecc3c71c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:29:24.412 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:23 smithi121 conmon[100681]: 2023-09-23T20:29:23.928+0000 7fb5522561c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:29:24.491 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:24.716 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:24 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3097254369' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:24.717 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:24 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3097254369' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:24.725 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:24 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3097254369' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:24.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:24 smithi123 conmon[108184]: 2023-09-23T20:29:24.725+0000 7f2ecc3c71c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:29:24.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:24 smithi123 conmon[108184]: [23/Sep/2023:20:29:24] ENGINE Bus STARTING 2023-09-23T20:29:24.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:24 smithi123 conmon[108184]: CherryPy Checker: 2023-09-23T20:29:24.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:24 smithi123 conmon[108184]: The Application mounted at '' has an empty config. 2023-09-23T20:29:24.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:24 smithi123 conmon[108184]: 2023-09-23T20:29:24.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:24 smithi123 conmon[108184]: [23/Sep/2023:20:29:24] ENGINE Serving on http://:::9283 2023-09-23T20:29:24.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:24 smithi123 conmon[108184]: [23/Sep/2023:20:29:24] ENGINE Bus STARTED 2023-09-23T20:29:24.992 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:24 smithi121 conmon[100681]: 2023-09-23T20:29:24.848+0000 7fb5522561c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:29:24.993 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:24 smithi121 conmon[100681]: [23/Sep/2023:20:29:24] ENGINE Bus STARTING 2023-09-23T20:29:25.412 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:25 smithi121 conmon[100681]: CherryPy Checker: 2023-09-23T20:29:25.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:25 smithi121 conmon[100681]: The Application mounted at '' has an empty config. 2023-09-23T20:29:25.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:25 smithi121 conmon[100681]: 2023-09-23T20:29:25.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:25 smithi121 conmon[100681]: [23/Sep/2023:20:29:25] ENGINE Serving on http://:::9283 2023-09-23T20:29:25.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:25 smithi121 conmon[100681]: [23/Sep/2023:20:29:25] ENGINE Bus STARTED 2023-09-23T20:29:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: Standby manager daemon x restarted 2023-09-23T20:29:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: Standby manager daemon x started 2023-09-23T20:29:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:29:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:29:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:29:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:29:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: Active manager daemon y restarted 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: Activating manager daemon y 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: mgrmap e17: y(active, starting, since 0.00710006s), standbys: x 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:29:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: Manager daemon y is now available 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:25.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:29:25.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:29:25.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:29:25.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:25 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: Standby manager daemon x restarted 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: Standby manager daemon x started 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: Active manager daemon y restarted 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: Activating manager daemon y 2023-09-23T20:29:25.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: mgrmap e17: y(active, starting, since 0.00710006s), standbys: x 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:29:25.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: Manager daemon y is now available 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:29:25.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:29:25.854 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:29:25.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: Standby manager daemon x restarted 2023-09-23T20:29:25.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: Standby manager daemon x started 2023-09-23T20:29:25.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:29:25.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:29:25.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:29:25.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/1692115914' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: Active manager daemon y restarted 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: Activating manager daemon y 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: mgrmap e17: y(active, starting, since 0.00710006s), standbys: x 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:29:25.855 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: Manager daemon y is now available 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:25.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:25.857 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:25.857 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:29:25.857 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:29:25.857 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:29:25.857 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:25 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:29:27.050 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'rbd' created 2023-09-23T20:29:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: mgrmap e18: y(active, since 1.01148s), standbys: x 2023-09-23T20:29:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: [23/Sep/2023:20:29:25] ENGINE Bus STARTING 2023-09-23T20:29:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: [23/Sep/2023:20:29:26] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:29:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: [23/Sep/2023:20:29:26] ENGINE Bus STARTED 2023-09-23T20:29:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2045991914' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:29:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: mgrmap e18: y(active, since 1.01148s), standbys: x 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: [23/Sep/2023:20:29:25] ENGINE Bus STARTING 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: [23/Sep/2023:20:29:26] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: [23/Sep/2023:20:29:26] ENGINE Bus STARTED 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2045991914' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:29:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: mgrmap e18: y(active, since 1.01148s), standbys: x 2023-09-23T20:29:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: [23/Sep/2023:20:29:25] ENGINE Bus STARTING 2023-09-23T20:29:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: [23/Sep/2023:20:29:26] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:29:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: [23/Sep/2023:20:29:26] ENGINE Bus STARTED 2023-09-23T20:29:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2045991914' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:29:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:27.765 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T20:29:27.981 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:27 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:29:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2045991914' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:28.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[100258]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T20:29:28.134 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:28.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2045991914' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:28.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:28 smithi121 ceph-mon[118228]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T20:29:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2045991914' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:28 smithi123 ceph-mon[104968]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T20:29:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi123 to 4036M 2023-09-23T20:29:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:29:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:29:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:29:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:29:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi123 to 4036M 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:29:29.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:29:29.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:29:29.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:29 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi123 to 4036M 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:29 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:30.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:29:30.971 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: Deploying daemon alertmanager.a on smithi121 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3352966468' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[100258]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:29:30.972 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: Deploying daemon alertmanager.a on smithi121 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3352966468' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:29:30.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:30 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:29:30.973 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: Deploying daemon alertmanager.a on smithi121 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:29:30.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3352966468' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:29:30.975 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:30 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:29:31.505 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:29:31.505 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:29:31.505 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:29:31.763 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:31 smithi121 ceph-mon[100258]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:29:31.763 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:31 smithi121 ceph-mon[100258]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:29:31.764 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:31 smithi121 ceph-mon[100258]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:31.854 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:32.040 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:31 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:29:32.040 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:31 smithi121 ceph-mon[118228]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:29:32.040 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:31 smithi121 ceph-mon[118228]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:31 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:29:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:31 smithi123 ceph-mon[104968]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:29:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:31 smithi123 ceph-mon[104968]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:33.611 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:29:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:33 smithi123 ceph-mon[104968]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:33.806 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:33 smithi121 ceph-mon[118228]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:33.807 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:33 smithi121 ceph-mon[100258]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:34.445 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":19,"active_gid":24356,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":1448037439},{"type":"v1","addr":"172.21.15.121:6801","nonce":1448037439}]},"active_addr":"172.21.15.121:6801/1448037439","active_change":"2023-09-23T20:29:24.862916+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24370,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":156999919}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":2000763757}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":167205692}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":3933818352}]}]} 2023-09-23T20:29:34.450 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:29:34.450 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:29:34.450 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:29:34.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:34 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1126546159' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:29:34.794 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:34.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:34 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1126546159' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:29:34.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:34 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1126546159' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:29:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:35 smithi121 ceph-mon[100258]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:35.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:35 smithi121 ceph-mon[118228]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:35.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:35 smithi123 ceph-mon[104968]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:29:36.595 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":64,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:29:30.752734+0000","last_up_change":"2023-09-23T20:28:51.017490+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6803","nonce":801467255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6805","nonce":801467255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6809","nonce":801467255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6807","nonce":801467255}]},"public_addr":"172.21.15.121:6803/801467255","cluster_addr":"172.21.15.121:6805/801467255","heartbeat_back_addr":"172.21.15.121:6809/801467255","heartbeat_front_addr":"172.21.15.121:6807/801467255","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6811","nonce":3606674368}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6813","nonce":3606674368}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6817","nonce":3606674368}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6815","nonce":3606674368}]},"public_addr":"172.21.15.121:6811/3606674368","cluster_addr":"172.21.15.121:6813/3606674368","heartbeat_back_addr":"172.21.15.121:6817/3606674368","heartbeat_front_addr":"172.21.15.121:6815/3606674368","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6819","nonce":4028229487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6821","nonce":4028229487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6825","nonce":4028229487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6823","nonce":4028229487}]},"public_addr":"172.21.15.121:6819/4028229487","cluster_addr":"172.21.15.121:6821/4028229487","heartbeat_back_addr":"172.21.15.121:6825/4028229487","heartbeat_front_addr":"172.21.15.121:6823/4028229487","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6827","nonce":2305478780}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6829","nonce":2305478780}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6833","nonce":2305478780}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6831","nonce":2305478780}]},"public_addr":"172.21.15.121:6827/2305478780","cluster_addr":"172.21.15.121:6829/2305478780","heartbeat_back_addr":"172.21.15.121:6833/2305478780","heartbeat_front_addr":"172.21.15.121:6831/2305478780","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:29:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:36 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2223925550' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:36 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2223925550' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:36 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2223925550' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:37.490 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:29:37.490 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:29:37.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:37 smithi121 ceph-mon[100258]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:37.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:37 smithi121 ceph-mon[118228]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:37.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:37 smithi123 ceph-mon[104968]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:37.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:37 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:29:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:29:39.006 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:40.067 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:39 smithi121 ceph-mon[118228]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:40.067 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:39 smithi121 ceph-mon[100258]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:40.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:39 smithi123 ceph-mon[104968]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:41.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:40 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:41.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:40 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:40 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:40 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:40 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:40 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:42.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:41 smithi121 ceph-mon[100258]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:41 smithi121 ceph-mon[118228]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:42.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:41 smithi123 ceph-mon[104968]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:42.404 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:29:42.405 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":64,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:29:30.752734+0000","last_up_change":"2023-09-23T20:28:51.017490+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6803","nonce":801467255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6805","nonce":801467255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6809","nonce":801467255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6807","nonce":801467255}]},"public_addr":"172.21.15.121:6803/801467255","cluster_addr":"172.21.15.121:6805/801467255","heartbeat_back_addr":"172.21.15.121:6809/801467255","heartbeat_front_addr":"172.21.15.121:6807/801467255","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6811","nonce":3606674368}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6813","nonce":3606674368}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6817","nonce":3606674368}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6815","nonce":3606674368}]},"public_addr":"172.21.15.121:6811/3606674368","cluster_addr":"172.21.15.121:6813/3606674368","heartbeat_back_addr":"172.21.15.121:6817/3606674368","heartbeat_front_addr":"172.21.15.121:6815/3606674368","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6819","nonce":4028229487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6821","nonce":4028229487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6825","nonce":4028229487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6823","nonce":4028229487}]},"public_addr":"172.21.15.121:6819/4028229487","cluster_addr":"172.21.15.121:6821/4028229487","heartbeat_back_addr":"172.21.15.121:6825/4028229487","heartbeat_front_addr":"172.21.15.121:6823/4028229487","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6827","nonce":2305478780}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6829","nonce":2305478780}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6833","nonce":2305478780}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6831","nonce":2305478780}]},"public_addr":"172.21.15.121:6827/2305478780","cluster_addr":"172.21.15.121:6829/2305478780","heartbeat_back_addr":"172.21.15.121:6833/2305478780","heartbeat_front_addr":"172.21.15.121:6831/2305478780","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:29:43.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:42 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/999593645' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:43.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:42 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/999593645' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:43.164 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:43 smithi121 systemd[1]: Starting Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:29:43.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:42 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/999593645' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:29:43.385 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:29:43.386 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:29:43.386 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:29:43.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:29:43.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:29:43.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:29:43.388 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:29:43.388 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:29:44.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:43 smithi123 ceph-mon[104968]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:44.250 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:43 smithi121 ceph-mon[100258]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:44.250 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:43 smithi121 ceph-mon[118228]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:44.547 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.249Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.249Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.250Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.121 port=9094 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.251Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.275Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.275Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 bash[164039]: 8f2c3b9a61abd9f6ce0e4c3b18cd100cde4c076aacba716517e1ecedc67dad02 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.278Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[164481]: ts=2023-09-23T20:29:44.278Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:29:44.548 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:44 smithi121 systemd[1]: Started Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:29:44.711 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:44.765 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[100681]: [23/Sep/2023:20:29:44] ENGINE Bus STOPPING 2023-09-23T20:29:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[100681]: [23/Sep/2023:20:29:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:29:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[100681]: [23/Sep/2023:20:29:44] ENGINE Bus STOPPED 2023-09-23T20:29:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[100681]: [23/Sep/2023:20:29:44] ENGINE Bus STARTING 2023-09-23T20:29:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[100681]: [23/Sep/2023:20:29:44] ENGINE Serving on http://:::9283 2023-09-23T20:29:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:29:44 smithi121 conmon[100681]: [23/Sep/2023:20:29:44] ENGINE Bus STARTED 2023-09-23T20:29:45.172 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:45.220 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:45.269 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:45.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: Deploying daemon grafana.a on smithi123 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: Deploying daemon grafana.a on smithi123 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:45.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: Deploying daemon grafana.a on smithi123 2023-09-23T20:29:45.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:45.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:29:46.663 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:46 smithi121 conmon[164481]: ts=2023-09-23T20:29:46.252Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000221947s 2023-09-23T20:29:46.730 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:47.003 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:47.187 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:47.933 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:47 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:29:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:29:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:47 smithi123 ceph-mon[104968]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:47 smithi121 ceph-mon[100258]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:47 smithi121 ceph-mon[118228]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:49.858 INFO:teuthology.orchestra.run.smithi121.stdout:167503724567 2023-09-23T20:29:49.858 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:29:50.111 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:49 smithi121 ceph-mon[118228]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:50.111 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:49 smithi121 ceph-mon[100258]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:50.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:49 smithi123 ceph-mon[104968]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:51.062 INFO:teuthology.orchestra.run.smithi121.stdout:201863462930 2023-09-23T20:29:51.062 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:29:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:51 smithi123 ceph-mon[104968]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:52.229 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:51 smithi121 ceph-mon[100258]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:52.229 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:51 smithi121 ceph-mon[118228]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:53.250 INFO:teuthology.orchestra.run.smithi121.stdout:34359738415 2023-09-23T20:29:53.250 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:29:53.371 INFO:teuthology.orchestra.run.smithi121.stdout:111669149728 2023-09-23T20:29:53.371 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:29:53.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:53 smithi123 ceph-mon[104968]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:53 smithi121 ceph-mon[100258]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:53 smithi121 ceph-mon[118228]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:53.934 INFO:teuthology.orchestra.run.smithi121.stdout:55834574889 2023-09-23T20:29:53.935 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:29:54.083 INFO:teuthology.orchestra.run.smithi121.stdout:77309411365 2023-09-23T20:29:54.083 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:29:54.535 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:29:54 smithi121 conmon[164481]: ts=2023-09-23T20:29:54.254Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002354941s 2023-09-23T20:29:56.090 INFO:teuthology.orchestra.run.smithi121.stdout:133143986204 2023-09-23T20:29:56.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:29:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:55 smithi121 ceph-mon[100258]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:55 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:55 smithi121 ceph-mon[118228]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:55 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:55 smithi123 ceph-mon[104968]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:55 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:56.570 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:56.575 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:56.575 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:56.744 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:56.766 INFO:teuthology.orchestra.run.smithi121.stdout:236223201294 2023-09-23T20:29:56.767 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:29:56.844 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:57.000 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:57.937 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:57 smithi123 ceph-mon[104968]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:57.938 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:29:57 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:29:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:29:57.969 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:58.012 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:57 smithi121 ceph-mon[118228]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:29:58.012 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:57 smithi121 ceph-mon[100258]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:00.135 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:29:59 smithi121 ceph-mon[100258]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:00.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:29:59 smithi121 ceph-mon[118228]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:00.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:29:59 smithi123 ceph-mon[104968]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:01.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:00 smithi121 ceph-mon[100258]: overall HEALTH_OK 2023-09-23T20:30:01.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:00 smithi121 ceph-mon[118228]: overall HEALTH_OK 2023-09-23T20:30:01.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:00 smithi123 ceph-mon[104968]: overall HEALTH_OK 2023-09-23T20:30:02.093 INFO:teuthology.orchestra.run.smithi121.stdout:111669149729 2023-09-23T20:30:02.103 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:01 smithi121 ceph-mon[118228]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:02.103 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:01 smithi121 ceph-mon[100258]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:01 smithi123 ceph-mon[104968]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:02.409 INFO:teuthology.orchestra.run.smithi121.stdout:167503724570 2023-09-23T20:30:02.868 INFO:teuthology.orchestra.run.smithi121.stdout:77309411366 2023-09-23T20:30:03.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:02 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1957451766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:30:03.202 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:02 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/779993328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:30:03.202 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:02 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3937693167' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:30:03.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1957451766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:30:03.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/779993328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:30:03.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3937693167' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:30:03.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:02 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1957451766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:30:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:02 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/779993328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:30:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:02 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3937693167' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:30:03.288 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149729 for osd.3 2023-09-23T20:30:03.288 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:03.530 INFO:teuthology.orchestra.run.smithi121.stdout:201863462933 2023-09-23T20:30:03.669 INFO:teuthology.orchestra.run.smithi121.stdout:133143986206 2023-09-23T20:30:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:03 smithi123 ceph-mon[104968]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:03 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1567304390' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:30:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:03 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/62641226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:30:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:03 smithi121 ceph-mon[100258]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1567304390' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:30:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/62641226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:30:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:03 smithi121 ceph-mon[118228]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:03 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1567304390' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:30:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:03 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/62641226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:30:04.559 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724570 for osd.5 2023-09-23T20:30:04.559 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:04.824 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986206 for osd.4 2023-09-23T20:30:04.824 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:05.010 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411366 for osd.2 2023-09-23T20:30:05.011 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:05.076 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3928611675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:30:05.076 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:04 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3928611675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:30:05.095 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462933 for osd.6 2023-09-23T20:30:05.096 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:04 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3928611675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:30:05.239 INFO:teuthology.orchestra.run.smithi121.stdout:34359738417 2023-09-23T20:30:05.263 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:05.465 INFO:teuthology.orchestra.run.smithi121.stdout:55834574891 2023-09-23T20:30:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:05 smithi123 ceph-mon[104968]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:05 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1306795975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:30:06.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:05 smithi121 ceph-mon[100258]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:06.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:05 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1306795975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:30:06.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:05 smithi121 ceph-mon[118228]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:06.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:05 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1306795975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:30:06.335 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738415 got 34359738417 for osd.0 2023-09-23T20:30:06.335 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:06.554 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574891 for osd.1 2023-09-23T20:30:06.554 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:07.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 systemd[1]: Starting Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:30:07.831 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:30:07.831 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 bash[152592]: b89ae83624e0ecca795b15ebf3fd6363e72fd5fbd793c253a3c5fbab9cd8b004 2023-09-23T20:30:07.831 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 systemd[1]: Started Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733702519Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733875208Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733892266Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733899128Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.7339049Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733910756Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733916259Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733921695Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733927255Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733932643Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733937814Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T20:30:07.832 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733945142Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733950594Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=settings t=2023-09-23T20:30:07.733956939Z level=info msg="App mode production" 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=sqlstore t=2023-09-23T20:30:07.734014341Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=sqlstore t=2023-09-23T20:30:07.734029762Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.734895364Z level=info msg="Starting DB migrations" 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.734967029Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.76432766Z level=info msg="Executing migration" id="create user table" 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.789426933Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T20:30:07.833 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.831335752Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T20:30:07.834 INFO:teuthology.orchestra.run.smithi121.stdout:236223201297 2023-09-23T20:30:08.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:07 smithi123 ceph-mon[104968]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:08.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:07 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:07 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:07 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:07 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:07 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1006926753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:30:08.093 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.882051983Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T20:30:08.093 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.90711067Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T20:30:08.093 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.932281512Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T20:30:08.093 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:07 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:07.957761209Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T20:30:08.093 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.024646989Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T20:30:08.093 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.049586353Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T20:30:08.093 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.091654525Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T20:30:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[100258]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1006926753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:30:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[118228]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:07 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1006926753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:30:08.376 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.133746561Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T20:30:08.377 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.175551626Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T20:30:08.377 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.200805147Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T20:30:08.377 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.242671278Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T20:30:08.377 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.267803794Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T20:30:08.377 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.309748443Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T20:30:08.377 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.335131732Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T20:30:08.377 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.376936575Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T20:30:08.459 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201297 for osd.7 2023-09-23T20:30:08.459 DEBUG:teuthology.parallel:result is None 2023-09-23T20:30:08.459 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:30:08.460 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:30:08.645 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.402365675Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T20:30:08.645 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.427596798Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T20:30:08.645 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.452600427Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T20:30:08.645 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.477756931Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T20:30:08.646 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.503154096Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T20:30:08.646 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.544829231Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T20:30:08.646 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.570083518Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T20:30:08.646 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.595318719Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T20:30:08.646 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.620561849Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T20:30:08.646 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.64575467Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.68806135Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.704879312Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.721585467Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.738418399Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.755272016Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.77207564Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.805830911Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.822735147Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.839479952Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T20:30:08.907 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.872996551Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T20:30:08.908 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.90650241Z level=info msg="Executing migration" id="create star table" 2023-09-23T20:30:08.922 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:08.967 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:08 smithi121 systemd[1]: Starting Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:30:09.158 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:08 smithi123 ceph-mon[104968]: Deploying daemon node-exporter.a on smithi121 2023-09-23T20:30:09.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.94866096Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T20:30:09.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:08 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:08.982165053Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T20:30:09.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.024158888Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T20:30:09.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.049386129Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T20:30:09.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.091172318Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T20:30:09.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.13321376Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T20:30:09.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.158371866Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T20:30:09.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:08 smithi121 ceph-mon[100258]: Deploying daemon node-exporter.a on smithi121 2023-09-23T20:30:09.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:08 smithi121 ceph-mon[118228]: Deploying daemon node-exporter.a on smithi121 2023-09-23T20:30:09.335 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:09 smithi121 bash[171017]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T20:30:09.423 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.183824089Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T20:30:09.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.259010041Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T20:30:09.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.284378016Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T20:30:09.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.309484322Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T20:30:09.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.351296132Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T20:30:09.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.376653445Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T20:30:09.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.424052002Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T20:30:09.678 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.460661633Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T20:30:09.678 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.485786942Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T20:30:09.678 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.510923645Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T20:30:09.679 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.552670229Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T20:30:09.679 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.577949287Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T20:30:09.679 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.603086806Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T20:30:09.679 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.628345294Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T20:30:09.679 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.653485252Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T20:30:09.679 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.678769356Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T20:30:09.939 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.721118715Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T20:30:09.939 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.746447479Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T20:30:09.939 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.771460951Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T20:30:09.939 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.796610904Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T20:30:09.939 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.838760298Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T20:30:09.939 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.863865172Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T20:30:09.940 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.889034237Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T20:30:09.940 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.914021366Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T20:30:09.940 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.939325184Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T20:30:10.191 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:09 smithi123 ceph-mon[104968]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:10.191 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:09 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:10.191 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:09 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:10.191 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:09 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:09.981346886Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T20:30:10.191 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.006553268Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T20:30:10.191 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.031856422Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T20:30:10.191 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.056892536Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T20:30:10.191 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.082048817Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T20:30:10.192 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.124090959Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T20:30:10.192 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.165862562Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T20:30:10.192 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.190989325Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T20:30:10.213 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:10 smithi121 bash[171017]: Getting image source signatures 2023-09-23T20:30:10.213 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:09 smithi121 ceph-mon[118228]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:10.213 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:09 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:10.213 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:09 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:10.214 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:09 smithi121 ceph-mon[100258]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:10.214 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:09 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:10.214 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:09 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:10.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.233342734Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T20:30:10.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.258753513Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T20:30:10.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.283758204Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T20:30:10.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.325716528Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T20:30:10.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.350822214Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T20:30:10.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.376076455Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T20:30:10.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.438312514Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T20:30:10.596 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:30:10.597 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:30:10.620 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:10 smithi121 bash[171017]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:30:10.620 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:10 smithi121 bash[171017]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:30:10.620 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:10 smithi121 bash[171017]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:30:10.753 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.484765216Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.509989183Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.551982377Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.577181841Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.602283047Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.627537682Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.652726123Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.677798852Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.703054582Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T20:30:10.754 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.728230414Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T20:30:11.030 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.753408736Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.795453703Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.829338597Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.8545459Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.879782212Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.913326758Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.930302697Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.963743963Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:10 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:10.997417196Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T20:30:11.031 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.030962575Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T20:30:11.299 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.073094716Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T20:30:11.299 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.115048107Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T20:30:11.299 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.14034024Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T20:30:11.299 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.165402275Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T20:30:11.299 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.207226868Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T20:30:11.299 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.249349676Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T20:30:11.300 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.274316493Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T20:30:11.300 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.299529728Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T20:30:11.329 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":29,"stamp":"2023-09-23T20:30:08.884391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50368,"kb_used_data":6720,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749910848,"statfs":{"total":767960285184,"available":767908708352,"internally_reserved":0,"allocated":6881280,"data_stored":3898587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002234"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:30.264469+0000","last_change":"2023-09-23T20:29:28.052971+0000","last_active":"2023-09-23T20:29:30.264469+0000","last_peered":"2023-09-23T20:29:30.264469+0000","last_clean":"2023-09-23T20:29:30.264469+0000","last_became_active":"2023-09-23T20:29:28.052681+0000","last_became_peered":"2023-09-23T20:29:28.052681+0000","last_unstale":"2023-09-23T20:29:30.264469+0000","last_undegraded":"2023-09-23T20:29:30.264469+0000","last_fullsized":"2023-09-23T20:29:30.264469+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:17.819345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.057539+0000","last_change":"2023-09-23T20:29:28.208396+0000","last_active":"2023-09-23T20:29:29.057539+0000","last_peered":"2023-09-23T20:29:29.057539+0000","last_clean":"2023-09-23T20:29:29.057539+0000","last_became_active":"2023-09-23T20:29:28.208035+0000","last_became_peered":"2023-09-23T20:29:28.208035+0000","last_unstale":"2023-09-23T20:29:29.057539+0000","last_undegraded":"2023-09-23T20:29:29.057539+0000","last_fullsized":"2023-09-23T20:29:29.057539+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:11.504285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.056349+0000","last_change":"2023-09-23T20:29:28.208399+0000","last_active":"2023-09-23T20:29:29.056349+0000","last_peered":"2023-09-23T20:29:29.056349+0000","last_clean":"2023-09-23T20:29:29.056349+0000","last_became_active":"2023-09-23T20:29:28.208044+0000","last_became_peered":"2023-09-23T20:29:28.208044+0000","last_unstale":"2023-09-23T20:29:29.056349+0000","last_undegraded":"2023-09-23T20:29:29.056349+0000","last_fullsized":"2023-09-23T20:29:29.056349+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:04.477375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.056312+0000","last_change":"2023-09-23T20:29:28.055354+0000","last_active":"2023-09-23T20:29:29.056312+0000","last_peered":"2023-09-23T20:29:29.056312+0000","last_clean":"2023-09-23T20:29:29.056312+0000","last_became_active":"2023-09-23T20:29:28.055064+0000","last_became_peered":"2023-09-23T20:29:28.055064+0000","last_unstale":"2023-09-23T20:29:29.056312+0000","last_undegraded":"2023-09-23T20:29:29.056312+0000","last_fullsized":"2023-09-23T20:29:29.056312+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:55.115453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.056359+0000","last_change":"2023-09-23T20:28:55.048383+0000","last_active":"2023-09-23T20:29:29.056359+0000","last_peered":"2023-09-23T20:29:29.056359+0000","last_clean":"2023-09-23T20:29:29.056359+0000","last_became_active":"2023-09-23T20:28:55.047951+0000","last_became_peered":"2023-09-23T20:28:55.047951+0000","last_unstale":"2023-09-23T20:29:29.056359+0000","last_undegraded":"2023-09-23T20:29:29.056359+0000","last_fullsized":"2023-09-23T20:29:29.056359+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:54.426500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.058099+0000","last_change":"2023-09-23T20:29:28.055170+0000","last_active":"2023-09-23T20:29:29.058099+0000","last_peered":"2023-09-23T20:29:29.058099+0000","last_clean":"2023-09-23T20:29:29.058099+0000","last_became_active":"2023-09-23T20:29:28.054870+0000","last_became_peered":"2023-09-23T20:29:28.054870+0000","last_unstale":"2023-09-23T20:29:29.058099+0000","last_undegraded":"2023-09-23T20:29:29.058099+0000","last_fullsized":"2023-09-23T20:29:29.058099+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.348967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.579155+0000","last_change":"2023-09-23T20:29:28.486180+0000","last_active":"2023-09-23T20:29:29.579155+0000","last_peered":"2023-09-23T20:29:29.579155+0000","last_clean":"2023-09-23T20:29:29.579155+0000","last_became_active":"2023-09-23T20:29:28.485933+0000","last_became_peered":"2023-09-23T20:29:28.485933+0000","last_unstale":"2023-09-23T20:29:29.579155+0000","last_undegraded":"2023-09-23T20:29:29.579155+0000","last_fullsized":"2023-09-23T20:29:29.579155+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:45.386860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.058099+0000","last_change":"2023-09-23T20:29:28.052272+0000","last_active":"2023-09-23T20:29:29.058099+0000","last_peered":"2023-09-23T20:29:29.058099+0000","last_clean":"2023-09-23T20:29:29.058099+0000","last_became_active":"2023-09-23T20:29:28.051940+0000","last_became_peered":"2023-09-23T20:29:28.051940+0000","last_unstale":"2023-09-23T20:29:29.058099+0000","last_undegraded":"2023-09-23T20:29:29.058099+0000","last_fullsized":"2023-09-23T20:29:29.058099+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:45.231496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.057507+0000","last_change":"2023-09-23T20:29:28.051973+0000","last_active":"2023-09-23T20:29:29.057507+0000","last_peered":"2023-09-23T20:29:29.057507+0000","last_clean":"2023-09-23T20:29:29.057507+0000","last_became_active":"2023-09-23T20:29:28.051665+0000","last_became_peered":"2023-09-23T20:29:28.051665+0000","last_unstale":"2023-09-23T20:29:29.057507+0000","last_undegraded":"2023-09-23T20:29:29.057507+0000","last_fullsized":"2023-09-23T20:29:29.057507+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:00.653668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201297,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6328,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.58199999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.61399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.54700000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.436},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.46800000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462934,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6388,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738764,"statfs":{"total":95995035648,"available":95988494336,"internally_reserved":0,"allocated":1224704,"data_stored":855880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.374}]},{"osd":1,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.48899999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.439}]},{"osd":3,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.46400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.57399999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6520,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738632,"statfs":{"total":95995035648,"available":95988359168,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.33600000000000002,"15min":0.30599999999999999},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.35799999999999998,"15min":0.32700000000000001},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.51600000000000001,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.24399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39900000000000002,"15min":0.38800000000000001},"min":{"1min":0.28100000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56299999999999994,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42199999999999999,"15min":0.41499999999999998},"min":{"1min":0.29699999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.48399999999999999,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.308}]},{"osd":4,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59399999999999997,"15min":0.59299999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.628,"15min":0.628},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80500000000000005,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.53600000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:29:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.443}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7164,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737988,"statfs":{"total":95995035648,"available":95987699712,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:29:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.35499999999999998,"15min":0.34000000000000002},"min":{"1min":0.221,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.57899999999999996,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.34599999999999997},"min":{"1min":0.251,"5min":0.25,"15min":0.25},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.72799999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41499999999999998,"15min":0.40899999999999997},"min":{"1min":0.28399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:29:22 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.46500000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.376},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.46899999999999997,"15min":0.46700000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57899999999999996,"15min":0.57999999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:29:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.9740000000000002,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.58299999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6392,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738760,"statfs":{"total":95995035648,"available":95988490240,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":0.32500000000000001,"5min":0.36099999999999999,"15min":0.36799999999999999},"min":{"1min":0.20999999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.50900000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.371,"15min":0.36899999999999999},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.56499999999999995,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.375,"15min":0.36899999999999999},"min":{"1min":0.26100000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.52200000000000002,"5min":0.53200000000000003,"15min":0.53200000000000003},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.37,"5min":0.38,"15min":0.373},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54300000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45700000000000002,"15min":0.45800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.60699999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54800000000000004,"15min":0.55300000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.64300000000000002,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.78700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.69199999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.64100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:29:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.76100000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149731,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.378},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.44500000000000001,"15min":0.44400000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.61099999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.24399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.46300000000000002,"15min":0.46800000000000003},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.56299999999999994,"5min":0.623,"15min":0.623},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.46999999999999997,"15min":0.47699999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.59899999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.495,"15min":0.495},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.371},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48699999999999999,"15min":0.49199999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.63600000000000001,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.75900000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 20:29:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":2.903,"5min":2.903,"15min":2.903},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.57399999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986207,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.68400000000000005,"15min":0.70599999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79900000000000004,"5min":4.6500000000000004,"15min":4.6500000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.67400000000000004,"15min":0.69799999999999995},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.76800000000000002,"5min":3.9489999999999998,"15min":3.9489999999999998},"last":0.53900000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.69599999999999995,"15min":0.72199999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.72199999999999998,"5min":4.6749999999999998,"15min":4.6749999999999998},"last":0.501},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.77800000000000002,"5min":1.425,"15min":1.425},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56100000000000005,"15min":0.56999999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.68100000000000005,"5min":1.393,"15min":1.393},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.68000000000000005,"15min":0.70699999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70199999999999996,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.72399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.69699999999999995,"15min":0.72199999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71599999999999997,"5min":4.0010000000000003,"15min":4.0010000000000003},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.69699999999999995,"15min":0.72099999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.83599999999999997,"5min":4.6120000000000001,"15min":4.6120000000000001},"last":0.78000000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:29:06 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.3129999999999999,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.887,"5min":3.887,"15min":3.887},"last":0.86699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.83599999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.88200000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724571,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5816,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739336,"statfs":{"total":95995035648,"available":95989080064,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.372}]},{"osd":2,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.54200000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:29:31 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.51600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:30:11.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:30:11.559 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.341707652Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T20:30:11.559 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.366927968Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T20:30:11.560 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.392064971Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T20:30:11.560 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.417224473Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T20:30:11.560 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.442393362Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T20:30:11.560 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.46765904Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T20:30:11.560 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.492772673Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T20:30:11.560 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.51802607Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T20:30:11.560 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.559872697Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T20:30:11.663 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:11 smithi121 bash[171017]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T20:30:11.663 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:11 smithi121 bash[171017]: Writing manifest to image destination 2023-09-23T20:30:11.663 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:11 smithi121 bash[171017]: Storing signatures 2023-09-23T20:30:11.836 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.610380039Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T20:30:11.837 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.693904995Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T20:30:11.837 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.718976462Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T20:30:11.837 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.74420658Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T20:30:11.837 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.769338436Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T20:30:11.837 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.794595552Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T20:30:11.837 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.836505419Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T20:30:12.088 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:11 smithi123 ceph-mon[104968]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:30:12.088 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:11 smithi123 ceph-mon[104968]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:12.088 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.870120111Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T20:30:12.088 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.895369361Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T20:30:12.088 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.953814965Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T20:30:12.089 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:11 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:11.995971439Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T20:30:12.089 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.021329807Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T20:30:12.089 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.046355044Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T20:30:12.089 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.088380585Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T20:30:12.237 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:12.373 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.130419725Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T20:30:12.373 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.172308899Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T20:30:12.373 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.214348171Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T20:30:12.373 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.256145821Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T20:30:12.374 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.281423528Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T20:30:12.374 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.306573578Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T20:30:12.374 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.331622711Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T20:30:12.374 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.373653835Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T20:30:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:11 smithi121 ceph-mon[100258]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:30:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:11 smithi121 ceph-mon[100258]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:11 smithi121 ceph-mon[118228]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:30:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:11 smithi121 ceph-mon[118228]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:12.625 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.415482039Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T20:30:12.625 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.440690064Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.457478778Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.474476451Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.507969789Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.524813542Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.558469716Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.575339944Z level=info msg="Executing migration" id="create session table" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.591977182Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.608852337Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T20:30:12.626 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.625706324Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.651102217Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.668047582Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.684658586Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.701723243Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.718577829Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.735395462Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.752121406Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.768984826Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T20:30:12.886 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.794134991Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T20:30:12.887 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.819435746Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T20:30:12.887 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.886163678Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T20:30:13.146 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.945208611Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T20:30:13.146 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:12 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:12.987308095Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T20:30:13.147 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.029590732Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T20:30:13.147 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.054835437Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T20:30:13.147 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.07999903Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T20:30:13.147 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.12180234Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T20:30:13.147 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.146946847Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T20:30:13.163 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.819Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:30:13.163 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.819Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:30:13.163 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.820Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:30:13.163 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.820Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:30:13.163 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.820Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:30:13.163 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.820Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:30:13.164 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:30:13.165 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:30:13.166 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 conmon[171510]: ts=2023-09-23T20:30:12.821Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 bash[171017]: 0e30e04cd0298e970d300db756595b4ff0e00def904f3faf3ac220fdc5e768fe 2023-09-23T20:30:13.167 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:30:12 smithi121 systemd[1]: Started Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:30:13.423 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.172130897Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T20:30:13.423 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.222375698Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T20:30:13.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.264611651Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T20:30:13.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.289455993Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T20:30:13.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.331432877Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T20:30:13.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.373624361Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T20:30:13.424 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.423830481Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T20:30:13.675 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:13 smithi123 systemd[1]: Starting Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.474132267Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.516306216Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.558169292Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.583330697Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.608620313Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.633726232Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.658926882Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T20:30:13.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.675787161Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T20:30:13.936 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:13 smithi123 bash[153056]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T20:30:13.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:13 smithi123 ceph-mon[104968]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:13.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:13 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:13.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:13 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:13.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:13 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:13.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:13 smithi123 ceph-mon[104968]: Deploying daemon node-exporter.b on smithi123 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.717955045Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.734817101Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.75169934Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.777021272Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.802147223Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.82750402Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.8524199Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.894474398Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T20:30:13.937 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.936406829Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T20:30:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[118228]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[118228]: Deploying daemon node-exporter.b on smithi123 2023-09-23T20:30:14.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[100258]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:14.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:14.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:14.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:14.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:13 smithi121 ceph-mon[100258]: Deploying daemon node-exporter.b on smithi123 2023-09-23T20:30:14.188 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:13 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:13.97911597Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T20:30:14.188 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.003985971Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T20:30:14.188 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.054326714Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T20:30:14.188 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.121159257Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T20:30:14.188 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.163186826Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T20:30:14.188 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.188354681Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T20:30:14.441 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.213527334Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T20:30:14.441 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.255399617Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T20:30:14.442 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.280723745Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T20:30:14.442 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.322660118Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T20:30:14.442 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.347550847Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T20:30:14.442 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.372699135Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T20:30:14.442 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.398094102Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T20:30:14.442 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.439850905Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T20:30:14.473 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:30:14.474 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:30:14.692 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.481938104Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T20:30:14.692 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.498943515Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T20:30:14.692 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.532365115Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T20:30:14.692 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.549231762Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T20:30:14.692 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.566079935Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T20:30:14.692 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.583091989Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T20:30:14.692 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.599686751Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T20:30:14.693 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.616547568Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T20:30:14.693 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.633447625Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T20:30:14.693 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.667210374Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T20:30:14.693 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.692372168Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T20:30:14.969 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.734564498Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T20:30:14.969 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.759508425Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T20:30:14.969 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.784684788Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T20:30:14.969 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.810079443Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T20:30:14.969 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.835018998Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T20:30:14.969 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.860207909Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T20:30:14.969 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.885400094Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T20:30:14.970 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.92728068Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T20:30:14.970 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:14 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:14.969320889Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T20:30:15.087 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":31,"stamp":"2023-09-23T20:30:12.884981+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50368,"kb_used_data":6720,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749910848,"statfs":{"total":767960285184,"available":767908708352,"internally_reserved":0,"allocated":6881280,"data_stored":3898587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002170"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:30.264469+0000","last_change":"2023-09-23T20:29:28.052971+0000","last_active":"2023-09-23T20:29:30.264469+0000","last_peered":"2023-09-23T20:29:30.264469+0000","last_clean":"2023-09-23T20:29:30.264469+0000","last_became_active":"2023-09-23T20:29:28.052681+0000","last_became_peered":"2023-09-23T20:29:28.052681+0000","last_unstale":"2023-09-23T20:29:30.264469+0000","last_undegraded":"2023-09-23T20:29:30.264469+0000","last_fullsized":"2023-09-23T20:29:30.264469+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:17.819345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.057539+0000","last_change":"2023-09-23T20:29:28.208396+0000","last_active":"2023-09-23T20:29:29.057539+0000","last_peered":"2023-09-23T20:29:29.057539+0000","last_clean":"2023-09-23T20:29:29.057539+0000","last_became_active":"2023-09-23T20:29:28.208035+0000","last_became_peered":"2023-09-23T20:29:28.208035+0000","last_unstale":"2023-09-23T20:29:29.057539+0000","last_undegraded":"2023-09-23T20:29:29.057539+0000","last_fullsized":"2023-09-23T20:29:29.057539+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:11.504285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.056349+0000","last_change":"2023-09-23T20:29:28.208399+0000","last_active":"2023-09-23T20:29:29.056349+0000","last_peered":"2023-09-23T20:29:29.056349+0000","last_clean":"2023-09-23T20:29:29.056349+0000","last_became_active":"2023-09-23T20:29:28.208044+0000","last_became_peered":"2023-09-23T20:29:28.208044+0000","last_unstale":"2023-09-23T20:29:29.056349+0000","last_undegraded":"2023-09-23T20:29:29.056349+0000","last_fullsized":"2023-09-23T20:29:29.056349+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:04.477375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.056312+0000","last_change":"2023-09-23T20:29:28.055354+0000","last_active":"2023-09-23T20:29:29.056312+0000","last_peered":"2023-09-23T20:29:29.056312+0000","last_clean":"2023-09-23T20:29:29.056312+0000","last_became_active":"2023-09-23T20:29:28.055064+0000","last_became_peered":"2023-09-23T20:29:28.055064+0000","last_unstale":"2023-09-23T20:29:29.056312+0000","last_undegraded":"2023-09-23T20:29:29.056312+0000","last_fullsized":"2023-09-23T20:29:29.056312+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:55.115453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.056359+0000","last_change":"2023-09-23T20:28:55.048383+0000","last_active":"2023-09-23T20:29:29.056359+0000","last_peered":"2023-09-23T20:29:29.056359+0000","last_clean":"2023-09-23T20:29:29.056359+0000","last_became_active":"2023-09-23T20:28:55.047951+0000","last_became_peered":"2023-09-23T20:28:55.047951+0000","last_unstale":"2023-09-23T20:29:29.056359+0000","last_undegraded":"2023-09-23T20:29:29.056359+0000","last_fullsized":"2023-09-23T20:29:29.056359+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:54.426500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.058099+0000","last_change":"2023-09-23T20:29:28.055170+0000","last_active":"2023-09-23T20:29:29.058099+0000","last_peered":"2023-09-23T20:29:29.058099+0000","last_clean":"2023-09-23T20:29:29.058099+0000","last_became_active":"2023-09-23T20:29:28.054870+0000","last_became_peered":"2023-09-23T20:29:28.054870+0000","last_unstale":"2023-09-23T20:29:29.058099+0000","last_undegraded":"2023-09-23T20:29:29.058099+0000","last_fullsized":"2023-09-23T20:29:29.058099+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.348967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.579155+0000","last_change":"2023-09-23T20:29:28.486180+0000","last_active":"2023-09-23T20:29:29.579155+0000","last_peered":"2023-09-23T20:29:29.579155+0000","last_clean":"2023-09-23T20:29:29.579155+0000","last_became_active":"2023-09-23T20:29:28.485933+0000","last_became_peered":"2023-09-23T20:29:28.485933+0000","last_unstale":"2023-09-23T20:29:29.579155+0000","last_undegraded":"2023-09-23T20:29:29.579155+0000","last_fullsized":"2023-09-23T20:29:29.579155+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:45.386860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.058099+0000","last_change":"2023-09-23T20:29:28.052272+0000","last_active":"2023-09-23T20:29:29.058099+0000","last_peered":"2023-09-23T20:29:29.058099+0000","last_clean":"2023-09-23T20:29:29.058099+0000","last_became_active":"2023-09-23T20:29:28.051940+0000","last_became_peered":"2023-09-23T20:29:28.051940+0000","last_unstale":"2023-09-23T20:29:29.058099+0000","last_undegraded":"2023-09-23T20:29:29.058099+0000","last_fullsized":"2023-09-23T20:29:29.058099+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:45.231496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:29:29.057507+0000","last_change":"2023-09-23T20:29:28.051973+0000","last_active":"2023-09-23T20:29:29.057507+0000","last_peered":"2023-09-23T20:29:29.057507+0000","last_clean":"2023-09-23T20:29:29.057507+0000","last_became_active":"2023-09-23T20:29:28.051665+0000","last_became_peered":"2023-09-23T20:29:28.051665+0000","last_unstale":"2023-09-23T20:29:29.057507+0000","last_undegraded":"2023-09-23T20:29:29.057507+0000","last_fullsized":"2023-09-23T20:29:29.057507+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:00.653668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201298,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6328,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.58899999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.441}]},{"osd":3,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.502},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.48999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.54200000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6388,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738764,"statfs":{"total":95995035648,"available":95988494336,"internally_reserved":0,"allocated":1224704,"data_stored":855880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.55600000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.47099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.60399999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6520,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738632,"statfs":{"total":95995035648,"available":95988359168,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.33600000000000002,"15min":0.30599999999999999},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.311},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.35799999999999998,"15min":0.32700000000000001},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.51600000000000001,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39900000000000002,"15min":0.38800000000000001},"min":{"1min":0.28100000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56299999999999994,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42199999999999999,"15min":0.41499999999999998},"min":{"1min":0.29699999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.35999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.374},{"interface":"front","average":{"1min":0.503,"5min":0.48399999999999999,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59399999999999997,"15min":0.59299999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.628,"15min":0.628},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80500000000000005,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:29:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.433}]},{"osd":6,"last update":"Sat Sep 23 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:29:54 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.44800000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7164,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737988,"statfs":{"total":95995035648,"available":95987699712,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:29:43 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.35499999999999998,"15min":0.34000000000000002},"min":{"1min":0.221,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.57899999999999996,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.19800000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.34599999999999997},"min":{"1min":0.251,"5min":0.25,"15min":0.25},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.255}]},{"osd":2,"last update":"Sat Sep 23 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.31},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41499999999999998,"15min":0.40899999999999997},"min":{"1min":0.28399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.23100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:29:22 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.46500000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.46899999999999997,"15min":0.46700000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.443},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57899999999999996,"15min":0.57999999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.58199999999999996,"15min":0.58699999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.40999999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.83799999999999997,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.68100000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:29:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.625},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.66900000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.9740000000000002,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.61499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6392,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738760,"statfs":{"total":95995035648,"available":95988490240,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":0.32500000000000001,"5min":0.36099999999999999,"15min":0.36799999999999999},"min":{"1min":0.20999999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.50900000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.371,"15min":0.36899999999999999},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.56499999999999995,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.375,"15min":0.36899999999999999},"min":{"1min":0.26100000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.52200000000000002,"5min":0.53200000000000003,"15min":0.53200000000000003},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.38,"15min":0.373},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54300000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45700000000000002,"15min":0.45800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54800000000000004,"15min":0.55300000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.64300000000000002,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.76900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58499999999999996,"15min":0.58699999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.73899999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.59199999999999997,"15min":0.59699999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.749,"5min":0.752,"15min":0.752},"last":0.749}]},{"osd":6,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.72199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:29:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.71199999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149732,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.378},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.44500000000000001,"15min":0.44400000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.61099999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.46300000000000002,"15min":0.46800000000000003},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.56299999999999994,"5min":0.623,"15min":0.623},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.46999999999999997,"15min":0.47699999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.59899999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.495,"15min":0.495},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48699999999999999,"15min":0.49199999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.63600000000000001,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 20:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.75900000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.68700000000000006}]},{"osd":5,"last update":"Sat Sep 23 20:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.76200000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.76200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":2.903,"5min":2.903,"15min":2.903},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.70699999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.68400000000000005,"15min":0.70599999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79900000000000004,"5min":4.6500000000000004,"15min":4.6500000000000004},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.67400000000000004,"15min":0.69799999999999995},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.76800000000000002,"5min":3.9489999999999998,"15min":3.9489999999999998},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.69599999999999995,"15min":0.72199999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.72199999999999998,"5min":4.6749999999999998,"15min":4.6749999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54700000000000004,"15min":0.55200000000000005},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.77800000000000002,"5min":1.425,"15min":1.425},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56100000000000005,"15min":0.56999999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.68100000000000005,"5min":1.393,"15min":1.393},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.68000000000000005,"15min":0.70699999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70199999999999996,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.504}]},{"osd":3,"last update":"Sat Sep 23 20:29:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.69699999999999995,"15min":0.72199999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71599999999999997,"5min":4.0010000000000003,"15min":4.0010000000000003},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.69699999999999995,"15min":0.72099999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.83599999999999997,"5min":4.6120000000000001,"15min":4.6120000000000001},"last":0.435}]},{"osd":5,"last update":"Sat Sep 23 20:30:06 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70299999999999996,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.58899999999999997,"15min":0.60399999999999998},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.86699999999999999,"5min":3.887,"15min":3.887},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.55100000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5816,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739336,"statfs":{"total":95995035648,"available":95989080064,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.439}]},{"osd":2,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.40699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:29:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.307},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.379}]},{"osd":6,"last update":"Sat Sep 23 20:29:31 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.34899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.39000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:30:15.090 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:30:15.090 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:30:15.090 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:30:15.090 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:30:15.228 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:15 smithi123 bash[153056]: Getting image source signatures 2023-09-23T20:30:15.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.019748904Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T20:30:15.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.061646716Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T20:30:15.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.087056728Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T20:30:15.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.112136371Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T20:30:15.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.137230599Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T20:30:15.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.162378042Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T20:30:15.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.204497473Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T20:30:15.431 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:15.519 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:15 smithi123 bash[153056]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:30:15.519 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:15 smithi123 bash[153056]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:30:15.519 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.25479532Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T20:30:15.519 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.280171398Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T20:30:15.519 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.322326748Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T20:30:15.520 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.3807793Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T20:30:15.520 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.439193405Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T20:30:15.793 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:15 smithi123 bash[153056]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:30:15.794 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.519400835Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T20:30:15.794 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.591399387Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T20:30:15.794 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.657401571Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T20:30:15.794 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.724334128Z level=info msg="Executing migration" id="create team table" 2023-09-23T20:30:16.086 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:15 smithi123 ceph-mon[104968]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:30:16.086 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:15 smithi123 ceph-mon[104968]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:16.086 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:15 smithi123 bash[153056]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T20:30:16.086 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.793933627Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T20:30:16.086 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.866859144Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T20:30:16.089 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:15 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:15.933737137Z level=info msg="Executing migration" id="create team member table" 2023-09-23T20:30:16.089 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.010644295Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T20:30:16.089 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.086036455Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T20:30:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:15 smithi121 ceph-mon[118228]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:30:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:15 smithi121 ceph-mon[118228]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:15 smithi121 ceph-mon[100258]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:30:16.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:15 smithi121 ceph-mon[100258]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:16.345 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:16 smithi123 bash[153056]: Writing manifest to image destination 2023-09-23T20:30:16.345 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:16 smithi123 bash[153056]: Storing signatures 2023-09-23T20:30:16.346 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.151897613Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T20:30:16.346 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.17730904Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T20:30:16.346 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.219289217Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T20:30:16.346 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.270161785Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T20:30:16.346 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.345738016Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T20:30:16.612 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.396771385Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T20:30:16.612 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.438983061Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T20:30:16.612 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.482824052Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T20:30:16.612 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.527966261Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T20:30:16.865 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.612631379Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T20:30:16.866 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.688771507Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T20:30:16.866 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.731102169Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T20:30:16.866 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.764893859Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T20:30:16.866 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.798548873Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T20:30:16.866 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.832184682Z level=info msg="Executing migration" id="create tag table" 2023-09-23T20:30:16.866 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.86575817Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T20:30:17.038 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:30:17.038 INFO:teuthology.orchestra.run.smithi121.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:30:17.118 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.916650735Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T20:30:17.118 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:16 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:16.992142386Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T20:30:17.118 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.051155925Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T20:30:17.118 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.118384783Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T20:30:17.370 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.194128125Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T20:30:17.370 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.253174639Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T20:30:17.370 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.320116811Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T20:30:17.644 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:30:17.644 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T20:30:17.645 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T20:30:17.654 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi121.front.sepia.ceph.com 2023-09-23T20:30:17.655 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T20:30:17.676 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:30:17.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.370566638Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T20:30:17.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.440243179Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T20:30:17.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.486218633Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T20:30:17.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.54166226Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T20:30:17.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.603797205Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T20:30:17.676 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.676351089Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T20:30:17.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:17 smithi123 ceph-mon[104968]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:17.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:17 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1926072097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:30:17.936 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.73897623Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T20:30:17.936 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.802541706Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T20:30:17.936 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.836066022Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T20:30:17.936 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.861237226Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T20:30:17.936 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.919951726Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T20:30:17.996 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:18.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:17 smithi121 ceph-mon[100258]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:17 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1926072097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:30:18.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:17 smithi121 ceph-mon[118228]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:18.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:17 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1926072097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:30:18.196 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:17.988904473Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T20:30:18.196 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.029333375Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T20:30:18.196 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.071074807Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T20:30:18.196 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.11297492Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T20:30:18.197 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.155112476Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.934Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.935Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.935Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.935Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.935Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.935Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:30:18.197 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:30:18.198 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:30:18.199 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:30:18.200 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.936Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.937Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 conmon[153131]: ts=2023-09-23T20:30:17.937Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 bash[153056]: cedf94396cbb3b1fc8b8fe9ed377ac9daf9d712c710768b5bec11d233cd9f223 2023-09-23T20:30:18.201 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:30:17 smithi123 systemd[1]: Started Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.196849352Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.256018723Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.281281017Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.306422487Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.331589788Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.365396197Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.407311667Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T20:30:18.449 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.449349389Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T20:30:18.728 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.47459346Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.491492483Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.508345014Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.550249235Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.575509989Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.60066621Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.625861903Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.651078071Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T20:30:18.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.676317154Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T20:30:19.051 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.731441548Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T20:30:19.051 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.810197832Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T20:30:19.051 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.860901503Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T20:30:19.051 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:18 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:18.969830945Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T20:30:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:19.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:19 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:19.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:19 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:19 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:19 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:19 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:19 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:19.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:19 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:19.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.063490806Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T20:30:19.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.13869634Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T20:30:19.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.204629219Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T20:30:19.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.229824574Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T20:30:19.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.271732552Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T20:30:19.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.297003238Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T20:30:19.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.322361951Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T20:30:19.584 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.364303819Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T20:30:19.584 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.381154023Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T20:30:19.584 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.423078218Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T20:30:19.584 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.448295897Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T20:30:19.584 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.465393726Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T20:30:19.584 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.482234228Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T20:30:19.585 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.499270778Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T20:30:19.585 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.516227606Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T20:30:19.585 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.549623812Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T20:30:19.585 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.566458907Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T20:30:19.585 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.583425257Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T20:30:19.869 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.617196668Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T20:30:19.869 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.642399265Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T20:30:19.869 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.659277861Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T20:30:19.869 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.71793624Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T20:30:19.869 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.776439768Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T20:30:19.869 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.818564239Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T20:30:19.869 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.869002391Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T20:30:20.145 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:20 smithi123 ceph-mon[104968]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:20.145 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:20 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:20.146 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:19 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:19.952848012Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T20:30:20.146 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.013612411Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T20:30:20.146 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.097099924Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T20:30:20.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:20 smithi121 ceph-mon[118228]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:20.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:20 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:20 smithi121 ceph-mon[100258]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:20 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:20.425 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.145587139Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T20:30:20.425 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.187683153Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T20:30:20.425 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.262862978Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T20:30:20.425 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.338427885Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T20:30:20.728 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.425696618Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T20:30:20.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.472436168Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T20:30:20.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.56470468Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T20:30:20.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.589910425Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T20:30:20.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.665594935Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T20:30:21.050 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.766016136Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T20:30:21.050 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.799449927Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T20:30:21.050 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.824598708Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T20:30:21.050 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.84984921Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T20:30:21.050 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.874945737Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T20:30:21.050 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.916884135Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T20:30:21.050 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:20 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:20.983824268Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T20:30:21.304 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:21 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:30:21.305 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:21 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:30:21.305 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.050161517Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T20:30:21.305 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.093005491Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T20:30:21.305 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.177111305Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T20:30:21.305 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.23063691Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T20:30:21.305 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.305028855Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T20:30:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:21 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:30:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:21 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:30:21.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:21 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:30:21.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:21 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:30:21.570 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.369907445Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T20:30:21.571 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.412020814Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T20:30:21.571 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.478850893Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T20:30:21.571 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.504137531Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T20:30:21.571 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.546086297Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T20:30:21.571 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.571153857Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T20:30:21.822 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.596332724Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.596439543Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.621660047Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.646691242Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.671905808Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.672046322Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.672181991Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.69712451Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.72237118Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.74747572Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T20:30:21.823 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.772586247Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T20:30:21.824 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.797975661Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T20:30:21.824 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.82312213Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T20:30:22.075 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.882427851Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T20:30:22.075 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.92429775Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T20:30:22.075 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:21 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:21.974396157Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T20:30:22.075 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.007977073Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T20:30:22.075 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.058165891Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T20:30:22.075 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.075198151Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T20:30:22.335 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.092007743Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T20:30:22.335 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.12607295Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T20:30:22.335 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.176091747Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T20:30:22.335 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.201164147Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T20:30:22.335 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.243117666Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T20:30:22.335 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.285020041Z level=info msg="Executing migration" id="create permission table" 2023-09-23T20:30:22.336 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.310204878Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T20:30:22.336 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.335453195Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.337 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.337 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.337 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:22.337 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:22.337 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:22 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:30:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:22 smithi121 conmon[100234]: 2023-09-23T20:30:22.103+0000 7f1128c4f700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:30:22.587 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.386071605Z level=info msg="Executing migration" id="create role table" 2023-09-23T20:30:22.587 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.411307719Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T20:30:22.587 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.436686611Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T20:30:22.588 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.461819932Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T20:30:22.588 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.512125857Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T20:30:22.588 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.537225074Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T20:30:22.588 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.562637014Z level=info msg="Executing migration" id="create team role table" 2023-09-23T20:30:22.588 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.587693176Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T20:30:22.849 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.613132087Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T20:30:22.849 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.679952323Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T20:30:22.849 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.705133106Z level=info msg="Executing migration" id="create user role table" 2023-09-23T20:30:22.849 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.730307614Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T20:30:22.850 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.755532883Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T20:30:22.850 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.797505003Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T20:30:22.850 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.822617462Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T20:30:22.850 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.847853809Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T20:30:23.107 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.881552973Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T20:30:23.107 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.923592955Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T20:30:23.107 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.957336862Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T20:30:23.107 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:22 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:22.999326317Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T20:30:23.107 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.041390847Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T20:30:23.107 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.08327573Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T20:30:23.291 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:30:23.291 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: Reconfiguring daemon alertmanager.a on smithi121 2023-09-23T20:30:23.291 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:30:23.291 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: fsmap foo:0 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: Saving service mds.foo spec with placement count:2 2023-09-23T20:30:23.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: Reconfiguring daemon alertmanager.a on smithi121 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:30:23.293 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:30:23.294 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: fsmap foo:0 2023-09-23T20:30:23.294 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: Saving service mds.foo spec with placement count:2 2023-09-23T20:30:23.294 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:23 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:23.311 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T20:30:23.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:30:23.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: Reconfiguring daemon alertmanager.a on smithi121 2023-09-23T20:30:23.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:30:23.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:30:23.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:23.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:30:23.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:30:23.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:30:23.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:30:23.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:30:23.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: fsmap foo:0 2023-09-23T20:30:23.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: Saving service mds.foo spec with placement count:2 2023-09-23T20:30:23.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:23 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:23.378 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.142263667Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T20:30:23.378 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.167455177Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T20:30:23.378 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.192637721Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T20:30:23.379 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.234547973Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T20:30:23.379 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.259869869Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T20:30:23.379 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.28491836Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T20:30:23.379 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.310242654Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T20:30:23.379 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.335492063Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T20:30:23.379 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.377371994Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T20:30:23.637 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.402676581Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T20:30:23.637 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.444667397Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T20:30:23.637 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.486642489Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T20:30:23.637 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.511803794Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T20:30:23.638 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.537163221Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T20:30:23.638 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.562197167Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T20:30:23.638 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.587345078Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T20:30:23.638 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.612636932Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T20:30:23.638 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.63766021Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.679781525Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.705081826Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.730243621Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.755453631Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.755588617Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.780520422Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.780634787Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:30:23.898 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.805822342Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T20:30:23.899 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.831032719Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T20:30:23.899 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.898152953Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:30:24.158 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:24 smithi123 ceph-mon[104968]: pgmap v39: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:24.158 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:24 smithi123 ceph-mon[104968]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:30:24.158 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.940077335Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:30:24.158 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:23 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:23.982184224Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T20:30:24.158 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.032712909Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T20:30:24.158 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.074614504Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T20:30:24.159 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.116274155Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T20:30:24.410 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.158279484Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T20:30:24.410 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.183698917Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T20:30:24.410 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.208647196Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T20:30:24.410 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.250619729Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T20:30:24.410 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.275728314Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T20:30:24.410 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.309395318Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T20:30:24.410 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.334626629Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T20:30:24.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.359716527Z level=info msg="Executing migration" id="add share column" 2023-09-23T20:30:24.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.385006161Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T20:30:24.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.410229422Z level=info msg="Executing migration" id="create file table" 2023-09-23T20:30:24.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:24 smithi121 ceph-mon[100258]: pgmap v39: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:24 smithi121 ceph-mon[100258]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:30:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:24 smithi121 ceph-mon[118228]: pgmap v39: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:24 smithi121 ceph-mon[118228]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:30:24.526 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:24.670 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.460628012Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T20:30:24.670 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.485778029Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T20:30:24.670 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.527924416Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T20:30:24.670 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.569718447Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T20:30:24.671 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.594934354Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T20:30:24.671 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.620139834Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T20:30:24.671 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.645321277Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T20:30:24.671 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.670481954Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T20:30:24.901 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:24 smithi121 systemd[1]: Stopping Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:30:24.922 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.704180903Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T20:30:24.922 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.729492137Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T20:30:24.922 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.75471811Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T20:30:24.922 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.779726102Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T20:30:24.922 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.805087745Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T20:30:24.923 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.830243914Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T20:30:24.923 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.855404511Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T20:30:24.923 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.880546151Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T20:30:24.923 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.922473796Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T20:30:25.163 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:24 smithi121 conmon[164481]: ts=2023-09-23T20:30:24.900Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:30:25.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:25 smithi121 ceph-mon[100258]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:30:25.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:25 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:25.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:25 smithi121 ceph-mon[118228]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:30:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:25 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:25.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:25 smithi123 ceph-mon[104968]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:30:25.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:25 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:25.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:25.174 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:24 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:24.973090874Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T20:30:25.174 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.031564714Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T20:30:25.174 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.100411937Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T20:30:25.174 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.140636813Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T20:30:25.174 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.17423064Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T20:30:25.478 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.233356895Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T20:30:25.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.258426315Z level=info msg="Executing migration" id="create folder table" 2023-09-23T20:30:25.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.300357246Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T20:30:25.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.32552459Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T20:30:25.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.350766388Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T20:30:25.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.375815828Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T20:30:25.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=migrator t=2023-09-23T20:30:25.40116813Z level=info msg="migrations completed" performed=485 skipped=0 duration=17.666219464s 2023-09-23T20:30:25.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=sqlstore t=2023-09-23T20:30:25.401751943Z level=info msg="Created default organization" 2023-09-23T20:30:25.978 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=plugin.loader t=2023-09-23T20:30:25.483596044Z level=info msg="Plugin registered" pluginID=input 2023-09-23T20:30:25.979 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=plugin.loader t=2023-09-23T20:30:25.510638305Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T20:30:25.979 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=plugin.loader t=2023-09-23T20:30:25.510658053Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T20:30:25.979 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=secrets t=2023-09-23T20:30:25.510690737Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T20:30:25.979 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=query_data t=2023-09-23T20:30:25.512414616Z level=info msg="Query Service initialization" 2023-09-23T20:30:25.979 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:25 smithi123 conmon[152634]: logger=live.push_http t=2023-09-23T20:30:25.516999291Z level=info msg="Live Push Gateway initialization" 2023-09-23T20:30:26.128 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:25 smithi121 bash[174190]: 8f2c3b9a61abd9f6ce0e4c3b18cd100cde4c076aacba716517e1ecedc67dad02 2023-09-23T20:30:26.410 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:26 smithi123 ceph-mon[104968]: pgmap v42: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:26.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=infra.usagestats.collector t=2023-09-23T20:30:26.24167983Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T20:30:26.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=provisioning.datasources t=2023-09-23T20:30:26.242078313Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T20:30:26.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=provisioning.datasources t=2023-09-23T20:30:26.275541126Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T20:30:26.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=provisioning.datasources t=2023-09-23T20:30:26.345651622Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T20:30:26.411 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=provisioning.alerting t=2023-09-23T20:30:26.410950745Z level=info msg="starting to provision alerting" 2023-09-23T20:30:26.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:26 smithi121 ceph-mon[100258]: pgmap v42: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:26.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:26 smithi121 ceph-mon[118228]: pgmap v42: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:26.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:26 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:30:26.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:26 smithi121 systemd[1]: Stopped Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:30:26.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:26 smithi121 systemd[1]: Starting Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:30:26.700 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=provisioning.alerting t=2023-09-23T20:30:26.410971336Z level=info msg="finished to provision alerting" 2023-09-23T20:30:26.700 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=ngalert.state.manager t=2023-09-23T20:30:26.411058275Z level=info msg="Warming state cache for startup" 2023-09-23T20:30:26.700 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=grafanaStorageLogger t=2023-09-23T20:30:26.411149793Z level=info msg="storage starting" 2023-09-23T20:30:26.700 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=http.server t=2023-09-23T20:30:26.412904457Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T20:30:26.700 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=sqlstore.transactions t=2023-09-23T20:30:26.421474398Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:30:26.701 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=ngalert.state.manager t=2023-09-23T20:30:26.525400559Z level=info msg="State cache has been initialized" states=0 duration=114.336723ms 2023-09-23T20:30:26.701 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=ngalert.multiorg.alertmanager t=2023-09-23T20:30:26.525530403Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T20:30:26.701 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=ticker t=2023-09-23T20:30:26.525598258Z level=info msg=starting first_tick=2023-09-23T20:30:30Z 2023-09-23T20:30:26.701 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=sqlstore.transactions t=2023-09-23T20:30:26.616632327Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:30:26.701 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=sqlstore.transactions t=2023-09-23T20:30:26.700501954Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:30:26.978 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:30:26 smithi123 conmon[152634]: logger=sqlstore.transactions t=2023-09-23T20:30:26.710815091Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2023-09-23T20:30:27.459 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.215Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.215Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.216Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.121 port=9094 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.217Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.240Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.241Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.243Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:30:27.487 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 conmon[174726]: ts=2023-09-23T20:30:27.243Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:30:27.488 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 bash[174670]: e6f06c4a552633e14ddeb5e4431a2c2e9fb92605746999bed42e64a52fbccb10 2023-09-23T20:30:27.488 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:27 smithi121 systemd[1]: Started Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:30:27.804 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:27.924 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:30:27 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:30:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:28 smithi123 ceph-mon[104968]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:28 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:28.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:28 smithi121 ceph-mon[100258]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:28.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:28.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:28 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:28.336 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:28 smithi121 ceph-mon[118228]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:28.336 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:28.336 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:28 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:29.344 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:29 smithi123 ceph-mon[104968]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:30:29.344 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:29 smithi123 ceph-mon[104968]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:30:29.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:29 smithi121 conmon[174726]: ts=2023-09-23T20:30:29.218Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000230889s 2023-09-23T20:30:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:29 smithi121 ceph-mon[100258]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:30:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:29 smithi121 ceph-mon[100258]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:30:29.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:29 smithi121 ceph-mon[118228]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:30:29.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:29 smithi121 ceph-mon[118228]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:30:30.110 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T20:30:30.121 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:30:30.121 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:30:30.122 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:30:30.354 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 systemd[1]: Stopping Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:30:30.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:30 smithi123 ceph-mon[104968]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:30.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:30 smithi121 ceph-mon[100258]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:30 smithi121 ceph-mon[118228]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:30.494 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:30.728 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.354Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.355Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.355Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:30:30.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.355Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:30:30.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 conmon[149461]: ts=2023-09-23T20:30:30.355Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:30:31.229 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:30 smithi123 bash[154664]: 700b419679b409a9ed46f8531a5545a0469e29bb269c9688604da6d8f2ae6cdc 2023-09-23T20:30:31.229 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:30:31.229 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 systemd[1]: Stopped Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:30:31.229 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 systemd[1]: Starting Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:30:31.780 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STOPPING 2023-09-23T20:30:31.780 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:30:31.781 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STOPPED 2023-09-23T20:30:31.781 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STARTING 2023-09-23T20:30:31.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.591Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:30:31.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.591Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:30:31.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.591Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi123 (none))" 2023-09-23T20:30:31.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.591Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:30:31.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.591Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:30:31.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.592Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.593Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.594Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.594Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.596Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.596Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.691µs 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.596Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.596Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.596Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.597Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.783µs wal_replay_duration=410.519µs wbl_replay_duration=111ns total_replay_duration=448.697µs 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.597Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.597Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:30:31.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.597Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:30:31.981 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 bash[154994]: cb4aa7cfed817dff391a58679751b3b8e5790bde526c7f7503150e22e57970e5 2023-09-23T20:30:31.981 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 systemd[1]: Started Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:30:31.981 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.613Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.527904ms db_storage=671ns remote_storage=1.046µs web_handler=352ns query_engine=422ns scrape=477.849µs scrape_sd=24.54µs notify=24.589µs notify_sd=6.478µs rules=14.799833ms tracing=6.714µs 2023-09-23T20:30:31.981 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.613Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:30:31.981 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:30:31 smithi123 conmon[155035]: ts=2023-09-23T20:30:31.613Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:30:32.141 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Serving on http://:::9283 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STARTED 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STOPPING 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STOPPED 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STARTING 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Serving on http://:::9283 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STARTED 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STOPPING 2023-09-23T20:30:32.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STOPPED 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:31 smithi121 conmon[100681]: [23/Sep/2023:20:30:31] ENGINE Bus STARTING 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:32 smithi121 conmon[100681]: [23/Sep/2023:20:30:32] ENGINE Serving on http://:::9283 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:32 smithi121 conmon[100681]: [23/Sep/2023:20:30:32] ENGINE Bus STARTED 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:30:32.143 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:32.144 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:32 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:32.260 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:30:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:30:32.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:32.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:32 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:33.212 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":19,"active_gid":24356,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":1448037439},{"type":"v1","addr":"172.21.15.121:6801","nonce":1448037439}]},"active_addr":"172.21.15.121:6801/1448037439","active_change":"2023-09-23T20:29:24.862916+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24370,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":156999919}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":2000763757}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":167205692}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":3933818352}]}]} 2023-09-23T20:30:33.383 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:30:33.384 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:30:33.384 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:30:33.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:30:33.434 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2023-09-23T20:30:33.434 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:30:33.434 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:30:33.434 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:30:33.434 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3612397669' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:30:33.435 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:33 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3612397669' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:30:33.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:33 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:30:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:33 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2023-09-23T20:30:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:33 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:30:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:33 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:30:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:33 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:30:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:33 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:30:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:33 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3612397669' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:30:34.076 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:34 smithi121 ceph-mon[100258]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:34 smithi121 ceph-mon[118228]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:34 smithi123 ceph-mon[104968]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:35.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:35.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.631 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:35.631 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:35.631 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:30:35.631 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:35 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:35.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:30:35.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:35 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:36.028 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:30:36.028 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":69,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:30:24.117527+0000","last_up_change":"2023-09-23T20:28:51.017490+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:30:20.536872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:30:21.339996+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6803","nonce":801467255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6805","nonce":801467255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6809","nonce":801467255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6807","nonce":801467255}]},"public_addr":"172.21.15.121:6803/801467255","cluster_addr":"172.21.15.121:6805/801467255","heartbeat_back_addr":"172.21.15.121:6809/801467255","heartbeat_front_addr":"172.21.15.121:6807/801467255","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6811","nonce":3606674368}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6813","nonce":3606674368}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6817","nonce":3606674368}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6815","nonce":3606674368}]},"public_addr":"172.21.15.121:6811/3606674368","cluster_addr":"172.21.15.121:6813/3606674368","heartbeat_back_addr":"172.21.15.121:6817/3606674368","heartbeat_front_addr":"172.21.15.121:6815/3606674368","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6819","nonce":4028229487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6821","nonce":4028229487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6825","nonce":4028229487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6823","nonce":4028229487}]},"public_addr":"172.21.15.121:6819/4028229487","cluster_addr":"172.21.15.121:6821/4028229487","heartbeat_back_addr":"172.21.15.121:6825/4028229487","heartbeat_front_addr":"172.21.15.121:6823/4028229487","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6827","nonce":2305478780}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6829","nonce":2305478780}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6833","nonce":2305478780}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6831","nonce":2305478780}]},"public_addr":"172.21.15.121:6827/2305478780","cluster_addr":"172.21.15.121:6829/2305478780","heartbeat_back_addr":"172.21.15.121:6833/2305478780","heartbeat_front_addr":"172.21.15.121:6831/2305478780","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:30:37.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:36 smithi121 ceph-mon[100258]: Deploying daemon mds.foo.smithi121.sdlxlb on smithi121 2023-09-23T20:30:37.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:36 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4254082533' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:30:37.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:36 smithi121 ceph-mon[118228]: Deploying daemon mds.foo.smithi121.sdlxlb on smithi121 2023-09-23T20:30:37.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:36 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4254082533' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:30:37.186 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:30:37.186 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:30:37.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:36 smithi123 ceph-mon[104968]: Deploying daemon mds.foo.smithi121.sdlxlb on smithi121 2023-09-23T20:30:37.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:36 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4254082533' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:30:37.390 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:30:37 smithi121 conmon[174726]: ts=2023-09-23T20:30:37.220Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002549825s 2023-09-23T20:30:37.656 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:37.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:37 smithi123 ceph-mon[104968]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:37.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:30:37 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:30:38.085 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:37 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:37] "GET /metrics HTTP/1.1" 200 207931 "" "Prometheus/2.43.0" 2023-09-23T20:30:38.085 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:37 smithi121 ceph-mon[118228]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:38.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:37 smithi121 ceph-mon[100258]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:39.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:39.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:39.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: Deploying daemon mds.foo.smithi123.clbyhs on smithi123 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:boot 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: daemon mds.foo.smithi121.sdlxlb assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: Cluster is now healthy 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: fsmap foo:0 1 up:standby 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:creating} 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[118228]: daemon mds.foo.smithi121.sdlxlb is now active in filesystem foo as rank 0 2023-09-23T20:30:39.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: Deploying daemon mds.foo.smithi123.clbyhs on smithi123 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:boot 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: daemon mds.foo.smithi121.sdlxlb assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: Cluster is now healthy 2023-09-23T20:30:39.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:30:39.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: fsmap foo:0 1 up:standby 2023-09-23T20:30:39.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:creating} 2023-09-23T20:30:39.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:39 smithi121 ceph-mon[100258]: daemon mds.foo.smithi121.sdlxlb is now active in filesystem foo as rank 0 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: Deploying daemon mds.foo.smithi123.clbyhs on smithi123 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:boot 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: daemon mds.foo.smithi121.sdlxlb assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:30:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:30:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:30:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: Cluster is now healthy 2023-09-23T20:30:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:30:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: fsmap foo:0 1 up:standby 2023-09-23T20:30:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:creating} 2023-09-23T20:30:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:39 smithi123 ceph-mon[104968]: daemon mds.foo.smithi121.sdlxlb is now active in filesystem foo as rank 0 2023-09-23T20:30:39.656 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:30:39.656 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":69,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:30:24.117527+0000","last_up_change":"2023-09-23T20:28:51.017490+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:30:20.536872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:30:21.339996+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6803","nonce":801467255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6805","nonce":801467255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6809","nonce":801467255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":801467255},{"type":"v1","addr":"172.21.15.121:6807","nonce":801467255}]},"public_addr":"172.21.15.121:6803/801467255","cluster_addr":"172.21.15.121:6805/801467255","heartbeat_back_addr":"172.21.15.121:6809/801467255","heartbeat_front_addr":"172.21.15.121:6807/801467255","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6811","nonce":3606674368}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6813","nonce":3606674368}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6817","nonce":3606674368}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3606674368},{"type":"v1","addr":"172.21.15.121:6815","nonce":3606674368}]},"public_addr":"172.21.15.121:6811/3606674368","cluster_addr":"172.21.15.121:6813/3606674368","heartbeat_back_addr":"172.21.15.121:6817/3606674368","heartbeat_front_addr":"172.21.15.121:6815/3606674368","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6819","nonce":4028229487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6821","nonce":4028229487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6825","nonce":4028229487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4028229487},{"type":"v1","addr":"172.21.15.121:6823","nonce":4028229487}]},"public_addr":"172.21.15.121:6819/4028229487","cluster_addr":"172.21.15.121:6821/4028229487","heartbeat_back_addr":"172.21.15.121:6825/4028229487","heartbeat_front_addr":"172.21.15.121:6823/4028229487","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6827","nonce":2305478780}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6829","nonce":2305478780}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6833","nonce":2305478780}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":2305478780},{"type":"v1","addr":"172.21.15.121:6831","nonce":2305478780}]},"public_addr":"172.21.15.121:6827/2305478780","cluster_addr":"172.21.15.121:6829/2305478780","heartbeat_back_addr":"172.21.15.121:6833/2305478780","heartbeat_front_addr":"172.21.15.121:6831/2305478780","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:30:40.266 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:30:40.266 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:30:40.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:30:40.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:30:40.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:30:40.268 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:30:40.268 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:30:40.269 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:30:40.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[100258]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:40.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3861619712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:30:40.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[100258]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:active 2023-09-23T20:30:40.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[100258]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 2023-09-23T20:30:40.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:40.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:40.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[118228]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:40.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3861619712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:30:40.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[118228]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:active 2023-09-23T20:30:40.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[118228]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 2023-09-23T20:30:40.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:40.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:40 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:40 smithi123 ceph-mon[104968]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:30:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:40 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3861619712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:30:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:40 smithi123 ceph-mon[104968]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:active 2023-09-23T20:30:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:40 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 2023-09-23T20:30:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:40 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:40 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: mds.? [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] up:boot 2023-09-23T20:30:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[100258]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: mds.? [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] up:boot 2023-09-23T20:30:41.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:41.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:30:41.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:41 smithi121 ceph-mon[118228]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: mds.? [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] up:boot 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:30:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:41 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:41.758 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:41.858 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:41.958 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:42.108 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:42.134 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:42.164 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:42.344 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:42.400 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:43.804 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:43 smithi121 ceph-mon[118228]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:43.804 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:43 smithi121 ceph-mon[100258]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:43.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:43 smithi123 ceph-mon[104968]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:44.774 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[118228]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:active 2023-09-23T20:30:44.774 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[118228]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:44.774 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:44.774 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:44.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[100258]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:active 2023-09-23T20:30:44.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[100258]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:44.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:44.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:44 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:44.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:44 smithi123 ceph-mon[104968]: mds.? [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] up:active 2023-09-23T20:30:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:44 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:44 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:44 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[100258]: mds.? [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] up:standby 2023-09-23T20:30:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[100258]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[100258]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[118228]: mds.? [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] up:standby 2023-09-23T20:30:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[118228]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[118228]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:45 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:45 smithi123 ceph-mon[104968]: mds.? [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] up:standby 2023-09-23T20:30:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:45 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:30:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:45 smithi123 ceph-mon[104968]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:45 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:30:47.946 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:30:47 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:30:48.126 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:47 smithi121 ceph-mon[118228]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:48.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:47 smithi121 ceph-mon[100258]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:48.127 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:47 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:47] "GET /metrics HTTP/1.1" 200 238200 "" "Prometheus/2.43.0" 2023-09-23T20:30:48.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:47 smithi123 ceph-mon[104968]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:49.422 INFO:teuthology.orchestra.run.smithi121.stdout:236223201306 2023-09-23T20:30:49.423 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:30:50.183 INFO:teuthology.orchestra.run.smithi121.stdout:77309411377 2023-09-23T20:30:50.183 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:30:50.208 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:49 smithi121 ceph-mon[100258]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:50.209 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:49 smithi121 ceph-mon[118228]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:50.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:49 smithi123 ceph-mon[104968]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:51.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:51 smithi121 ceph-mon[100258]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:51 smithi123 ceph-mon[104968]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:52.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:51 smithi121 ceph-mon[118228]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T20:30:52.627 INFO:teuthology.orchestra.run.smithi121.stdout:111669149740 2023-09-23T20:30:52.627 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:30:53.060 INFO:teuthology.orchestra.run.smithi121.stdout:167503724580 2023-09-23T20:30:53.060 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:30:53.178 INFO:teuthology.orchestra.run.smithi121.stdout:34359738427 2023-09-23T20:30:53.178 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:30:53.221 INFO:teuthology.orchestra.run.smithi121.stdout:201863462943 2023-09-23T20:30:53.221 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:30:53.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:53 smithi121 ceph-mon[100258]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:53.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:53 smithi121 ceph-mon[118228]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:53 smithi123 ceph-mon[104968]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:54.262 INFO:teuthology.orchestra.run.smithi121.stdout:55834574902 2023-09-23T20:30:54.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:30:54.313 INFO:teuthology.orchestra.run.smithi121.stdout:133143986217 2023-09-23T20:30:54.313 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:30:54.495 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:54.525 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:54.541 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:54.568 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:54.759 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:55.259 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:54 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:55.260 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:54 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:54 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:55.518 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:55.722 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:56.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:55 smithi121 ceph-mon[100258]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:56.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:55 smithi121 ceph-mon[118228]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:56.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:55 smithi123 ceph-mon[104968]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:57.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:30:57 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:30:57.998 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:30:57 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:30:57] "GET /metrics HTTP/1.1" 200 238779 "" "Prometheus/2.43.0" 2023-09-23T20:30:58.354 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:30:57 smithi121 ceph-mon[100258]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:58.354 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:30:57 smithi121 ceph-mon[118228]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:30:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:30:57 smithi123 ceph-mon[104968]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:31:00.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:00 smithi121 ceph-mon[118228]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:00.235 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:00 smithi121 ceph-mon[100258]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:00 smithi123 ceph-mon[104968]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:00.508 INFO:teuthology.orchestra.run.smithi121.stdout:34359738429 2023-09-23T20:31:01.233 INFO:teuthology.orchestra.run.smithi121.stdout:111669149743 2023-09-23T20:31:01.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:01 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1278754204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:31:01.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:01 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1278754204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:31:01.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:01 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1278754204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:31:01.709 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:01.833 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738427 got 34359738429 for osd.0 2023-09-23T20:31:01.834 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:02.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:02.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1896753428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:31:02.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1896753428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:02.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.157 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:02 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:02.226 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149740 got 111669149743 for osd.3 2023-09-23T20:31:02.226 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1896753428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:31:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:02 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:02.728 INFO:teuthology.orchestra.run.smithi121.stdout:236223201309 2023-09-23T20:31:02.889 INFO:teuthology.orchestra.run.smithi121.stdout:167503724583 2023-09-23T20:31:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1928284465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:31:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2354318866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:31:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2186839310' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:31:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[100258]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1928284465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:31:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2354318866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:31:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2186839310' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:31:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:03 smithi121 ceph-mon[118228]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:03.826 INFO:teuthology.orchestra.run.smithi121.stdout:77309411380 2023-09-23T20:31:03.935 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724580 got 167503724583 for osd.5 2023-09-23T20:31:03.936 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:03 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1928284465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:31:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:03 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2354318866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:31:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:03 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2186839310' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:31:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:03 smithi123 ceph-mon[104968]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:04.099 INFO:teuthology.orchestra.run.smithi121.stdout:201863462946 2023-09-23T20:31:04.122 INFO:teuthology.orchestra.run.smithi121.stdout:55834574904 2023-09-23T20:31:04.880 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/314475337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:31:04.880 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3788129323' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:31:04.880 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:04.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:04.881 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/314475337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:31:04.881 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3788129323' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:31:04.882 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:04.882 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:04 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:04 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/314475337' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:31:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:04 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3788129323' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:31:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:04 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:04 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:05.408 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574902 got 55834574904 for osd.1 2023-09-23T20:31:05.408 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:05 smithi121 ceph-mon[100258]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:05.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:05 smithi121 ceph-mon[118228]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:05.820 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201306 got 236223201309 for osd.7 2023-09-23T20:31:05.820 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:05 smithi123 ceph-mon[104968]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:06.579 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411377 got 77309411380 for osd.2 2023-09-23T20:31:06.579 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:06.730 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462943 got 201863462946 for osd.6 2023-09-23T20:31:06.730 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:06.930 INFO:teuthology.orchestra.run.smithi121.stdout:133143986219 2023-09-23T20:31:07.411 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:06 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1984505836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:31:07.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:06 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1984505836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:31:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:06 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1984505836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:31:07.885 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986217 got 133143986219 for osd.4 2023-09-23T20:31:07.885 DEBUG:teuthology.parallel:result is None 2023-09-23T20:31:07.886 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:31:07.886 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:31:07.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:31:07 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:31:08.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:07 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:07] "GET /metrics HTTP/1.1" 200 238779 "" "Prometheus/2.43.0" 2023-09-23T20:31:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:07 smithi121 ceph-mon[100258]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:08.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:07 smithi121 ceph-mon[118228]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:08.270 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:07 smithi123 ceph-mon[104968]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:09 smithi121 ceph-mon[100258]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:09 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:09 smithi121 ceph-mon[118228]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:09 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:10.438 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:31:10.439 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:31:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:09 smithi123 ceph-mon[104968]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:09 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:11.127 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":64,"stamp":"2023-09-23T20:31:08.896427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52352,"kb_used_data":7296,"kb_used_omap":0,"kb_used_meta":44992,"kb_avail":749908864,"statfs":{"total":767960285184,"available":767906676736,"internally_reserved":0,"allocated":7471104,"data_stored":4196177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46071808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002342"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359624+0000","last_change":"2023-09-23T20:30:22.127985+0000","last_active":"2023-09-23T20:30:23.359624+0000","last_peered":"2023-09-23T20:30:23.359624+0000","last_clean":"2023-09-23T20:30:23.359624+0000","last_became_active":"2023-09-23T20:30:22.127651+0000","last_became_peered":"2023-09-23T20:30:22.127651+0000","last_unstale":"2023-09-23T20:30:23.359624+0000","last_undegraded":"2023-09-23T20:30:23.359624+0000","last_fullsized":"2023-09-23T20:30:23.359624+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:45.050942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.809192+0000","last_change":"2023-09-23T20:30:22.117719+0000","last_active":"2023-09-23T20:30:38.809192+0000","last_peered":"2023-09-23T20:30:38.809192+0000","last_clean":"2023-09-23T20:30:38.809192+0000","last_became_active":"2023-09-23T20:30:22.117410+0000","last_became_peered":"2023-09-23T20:30:22.117410+0000","last_unstale":"2023-09-23T20:30:38.809192+0000","last_undegraded":"2023-09-23T20:30:38.809192+0000","last_fullsized":"2023-09-23T20:30:38.809192+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:50.581932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.130627+0000","last_change":"2023-09-23T20:30:23.130236+0000","last_active":"2023-09-23T20:30:23.130627+0000","last_peered":"2023-09-23T20:30:23.130627+0000","last_clean":"2023-09-23T20:30:23.130627+0000","last_became_active":"2023-09-23T20:30:23.129941+0000","last_became_peered":"2023-09-23T20:30:23.129941+0000","last_unstale":"2023-09-23T20:30:23.130627+0000","last_undegraded":"2023-09-23T20:30:23.130627+0000","last_fullsized":"2023-09-23T20:30:23.130627+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:42.583768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.998860+0000","last_change":"2023-09-23T20:30:22.115582+0000","last_active":"2023-09-23T20:30:24.998860+0000","last_peered":"2023-09-23T20:30:24.998860+0000","last_clean":"2023-09-23T20:30:24.998860+0000","last_became_active":"2023-09-23T20:30:22.115345+0000","last_became_peered":"2023-09-23T20:30:22.115345+0000","last_unstale":"2023-09-23T20:30:24.998860+0000","last_undegraded":"2023-09-23T20:30:24.998860+0000","last_fullsized":"2023-09-23T20:30:24.998860+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:23.738240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360138+0000","last_change":"2023-09-23T20:30:22.772298+0000","last_active":"2023-09-23T20:30:23.360138+0000","last_peered":"2023-09-23T20:30:23.360138+0000","last_clean":"2023-09-23T20:30:23.360138+0000","last_became_active":"2023-09-23T20:30:22.772008+0000","last_became_peered":"2023-09-23T20:30:22.772008+0000","last_unstale":"2023-09-23T20:30:23.360138+0000","last_undegraded":"2023-09-23T20:30:23.360138+0000","last_fullsized":"2023-09-23T20:30:23.360138+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:17.639153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808878+0000","last_change":"2023-09-23T20:30:22.104198+0000","last_active":"2023-09-23T20:30:38.808878+0000","last_peered":"2023-09-23T20:30:38.808878+0000","last_clean":"2023-09-23T20:30:38.808878+0000","last_became_active":"2023-09-23T20:30:22.103970+0000","last_became_peered":"2023-09-23T20:30:22.103970+0000","last_unstale":"2023-09-23T20:30:38.808878+0000","last_undegraded":"2023-09-23T20:30:38.808878+0000","last_fullsized":"2023-09-23T20:30:38.808878+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:32.820350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360355+0000","last_change":"2023-09-23T20:30:22.486030+0000","last_active":"2023-09-23T20:30:23.360355+0000","last_peered":"2023-09-23T20:30:23.360355+0000","last_clean":"2023-09-23T20:30:23.360355+0000","last_became_active":"2023-09-23T20:30:22.485521+0000","last_became_peered":"2023-09-23T20:30:22.485521+0000","last_unstale":"2023-09-23T20:30:23.360355+0000","last_undegraded":"2023-09-23T20:30:23.360355+0000","last_fullsized":"2023-09-23T20:30:23.360355+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:35.764712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.809129+0000","last_change":"2023-09-23T20:30:22.110159+0000","last_active":"2023-09-23T20:30:38.809129+0000","last_peered":"2023-09-23T20:30:38.809129+0000","last_clean":"2023-09-23T20:30:38.809129+0000","last_became_active":"2023-09-23T20:30:22.109484+0000","last_became_peered":"2023-09-23T20:30:22.109484+0000","last_unstale":"2023-09-23T20:30:38.809129+0000","last_undegraded":"2023-09-23T20:30:38.809129+0000","last_fullsized":"2023-09-23T20:30:38.809129+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:58.417615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.129563+0000","last_change":"2023-09-23T20:30:23.127591+0000","last_active":"2023-09-23T20:30:23.129563+0000","last_peered":"2023-09-23T20:30:23.129563+0000","last_clean":"2023-09-23T20:30:23.129563+0000","last_became_active":"2023-09-23T20:30:23.127292+0000","last_became_peered":"2023-09-23T20:30:23.127292+0000","last_unstale":"2023-09-23T20:30:23.129563+0000","last_undegraded":"2023-09-23T20:30:23.129563+0000","last_fullsized":"2023-09-23T20:30:23.129563+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:23.845610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.498161+0000","last_change":"2023-09-23T20:30:22.111101+0000","last_active":"2023-09-23T20:30:24.498161+0000","last_peered":"2023-09-23T20:30:24.498161+0000","last_clean":"2023-09-23T20:30:24.498161+0000","last_became_active":"2023-09-23T20:30:22.110836+0000","last_became_peered":"2023-09-23T20:30:22.110836+0000","last_unstale":"2023-09-23T20:30:24.498161+0000","last_undegraded":"2023-09-23T20:30:24.498161+0000","last_fullsized":"2023-09-23T20:30:24.498161+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:21.805629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359990+0000","last_change":"2023-09-23T20:30:22.131210+0000","last_active":"2023-09-23T20:30:23.359990+0000","last_peered":"2023-09-23T20:30:23.359990+0000","last_clean":"2023-09-23T20:30:23.359990+0000","last_became_active":"2023-09-23T20:30:22.130879+0000","last_became_peered":"2023-09-23T20:30:22.130879+0000","last_unstale":"2023-09-23T20:30:23.359990+0000","last_undegraded":"2023-09-23T20:30:23.359990+0000","last_fullsized":"2023-09-23T20:30:23.359990+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:10.733014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.498209+0000","last_change":"2023-09-23T20:30:22.108849+0000","last_active":"2023-09-23T20:30:24.498209+0000","last_peered":"2023-09-23T20:30:24.498209+0000","last_clean":"2023-09-23T20:30:24.498209+0000","last_became_active":"2023-09-23T20:30:22.108610+0000","last_became_peered":"2023-09-23T20:30:22.108610+0000","last_unstale":"2023-09-23T20:30:24.498209+0000","last_undegraded":"2023-09-23T20:30:24.498209+0000","last_fullsized":"2023-09-23T20:30:24.498209+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:36.799199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119235+0000","last_change":"2023-09-23T20:29:28.052971+0000","last_active":"2023-09-23T20:30:23.119235+0000","last_peered":"2023-09-23T20:30:23.119235+0000","last_clean":"2023-09-23T20:30:23.119235+0000","last_became_active":"2023-09-23T20:29:28.052681+0000","last_became_peered":"2023-09-23T20:29:28.052681+0000","last_unstale":"2023-09-23T20:30:23.119235+0000","last_undegraded":"2023-09-23T20:30:23.119235+0000","last_fullsized":"2023-09-23T20:30:23.119235+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:17.819345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.363850+0000","last_change":"2023-09-23T20:30:23.363445+0000","last_active":"2023-09-23T20:30:23.363850+0000","last_peered":"2023-09-23T20:30:23.363850+0000","last_clean":"2023-09-23T20:30:23.363850+0000","last_became_active":"2023-09-23T20:30:23.363112+0000","last_became_peered":"2023-09-23T20:30:23.363112+0000","last_unstale":"2023-09-23T20:30:23.363850+0000","last_undegraded":"2023-09-23T20:30:23.363850+0000","last_fullsized":"2023-09-23T20:30:23.363850+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:54.947384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.120263+0000","last_change":"2023-09-23T20:29:28.208396+0000","last_active":"2023-09-23T20:30:23.120263+0000","last_peered":"2023-09-23T20:30:23.120263+0000","last_clean":"2023-09-23T20:30:23.120263+0000","last_became_active":"2023-09-23T20:29:28.208035+0000","last_became_peered":"2023-09-23T20:29:28.208035+0000","last_unstale":"2023-09-23T20:30:23.120263+0000","last_undegraded":"2023-09-23T20:30:23.120263+0000","last_fullsized":"2023-09-23T20:30:23.120263+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:11.504285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.805725+0000","last_change":"2023-09-23T20:30:22.119412+0000","last_active":"2023-09-23T20:30:38.805725+0000","last_peered":"2023-09-23T20:30:38.805725+0000","last_clean":"2023-09-23T20:30:38.805725+0000","last_became_active":"2023-09-23T20:30:22.118963+0000","last_became_peered":"2023-09-23T20:30:22.118963+0000","last_unstale":"2023-09-23T20:30:38.805725+0000","last_undegraded":"2023-09-23T20:30:38.805725+0000","last_fullsized":"2023-09-23T20:30:38.805725+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:44.340631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.361653+0000","last_change":"2023-09-23T20:30:23.360781+0000","last_active":"2023-09-23T20:30:23.361653+0000","last_peered":"2023-09-23T20:30:23.361653+0000","last_clean":"2023-09-23T20:30:23.361653+0000","last_became_active":"2023-09-23T20:30:23.360474+0000","last_became_peered":"2023-09-23T20:30:23.360474+0000","last_unstale":"2023-09-23T20:30:23.361653+0000","last_undegraded":"2023-09-23T20:30:23.361653+0000","last_fullsized":"2023-09-23T20:30:23.361653+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:39.665426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.120383+0000","last_change":"2023-09-23T20:30:22.118001+0000","last_active":"2023-09-23T20:30:23.120383+0000","last_peered":"2023-09-23T20:30:23.120383+0000","last_clean":"2023-09-23T20:30:23.120383+0000","last_became_active":"2023-09-23T20:30:22.117728+0000","last_became_peered":"2023-09-23T20:30:22.117728+0000","last_unstale":"2023-09-23T20:30:23.120383+0000","last_undegraded":"2023-09-23T20:30:23.120383+0000","last_fullsized":"2023-09-23T20:30:23.120383+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:55.249207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.361193+0000","last_change":"2023-09-23T20:29:28.208399+0000","last_active":"2023-09-23T20:30:23.361193+0000","last_peered":"2023-09-23T20:30:23.361193+0000","last_clean":"2023-09-23T20:30:23.361193+0000","last_became_active":"2023-09-23T20:29:28.208044+0000","last_became_peered":"2023-09-23T20:29:28.208044+0000","last_unstale":"2023-09-23T20:30:23.361193+0000","last_undegraded":"2023-09-23T20:30:23.361193+0000","last_fullsized":"2023-09-23T20:30:23.361193+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:04.477375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.130504+0000","last_change":"2023-09-23T20:30:23.130060+0000","last_active":"2023-09-23T20:30:23.130504+0000","last_peered":"2023-09-23T20:30:23.130504+0000","last_clean":"2023-09-23T20:30:23.130504+0000","last_became_active":"2023-09-23T20:30:23.129768+0000","last_became_peered":"2023-09-23T20:30:23.129768+0000","last_unstale":"2023-09-23T20:30:23.130504+0000","last_undegraded":"2023-09-23T20:30:23.130504+0000","last_fullsized":"2023-09-23T20:30:23.130504+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:04.441375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122455+0000","last_change":"2023-09-23T20:30:22.107067+0000","last_active":"2023-09-23T20:30:23.122455+0000","last_peered":"2023-09-23T20:30:23.122455+0000","last_clean":"2023-09-23T20:30:23.122455+0000","last_became_active":"2023-09-23T20:30:22.106739+0000","last_became_peered":"2023-09-23T20:30:22.106739+0000","last_unstale":"2023-09-23T20:30:23.122455+0000","last_undegraded":"2023-09-23T20:30:23.122455+0000","last_fullsized":"2023-09-23T20:30:23.122455+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:01.307848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360275+0000","last_change":"2023-09-23T20:29:28.055354+0000","last_active":"2023-09-23T20:30:23.360275+0000","last_peered":"2023-09-23T20:30:23.360275+0000","last_clean":"2023-09-23T20:30:23.360275+0000","last_became_active":"2023-09-23T20:29:28.055064+0000","last_became_peered":"2023-09-23T20:29:28.055064+0000","last_unstale":"2023-09-23T20:30:23.360275+0000","last_undegraded":"2023-09-23T20:30:23.360275+0000","last_fullsized":"2023-09-23T20:30:23.360275+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:55.115453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119197+0000","last_change":"2023-09-23T20:30:22.772777+0000","last_active":"2023-09-23T20:30:23.119197+0000","last_peered":"2023-09-23T20:30:23.119197+0000","last_clean":"2023-09-23T20:30:23.119197+0000","last_became_active":"2023-09-23T20:30:22.772481+0000","last_became_peered":"2023-09-23T20:30:22.772481+0000","last_unstale":"2023-09-23T20:30:23.119197+0000","last_undegraded":"2023-09-23T20:30:23.119197+0000","last_fullsized":"2023-09-23T20:30:23.119197+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:53.724607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.124499+0000","last_change":"2023-09-23T20:30:22.109986+0000","last_active":"2023-09-23T20:30:23.124499+0000","last_peered":"2023-09-23T20:30:23.124499+0000","last_clean":"2023-09-23T20:30:23.124499+0000","last_became_active":"2023-09-23T20:30:22.109562+0000","last_became_peered":"2023-09-23T20:30:22.109562+0000","last_unstale":"2023-09-23T20:30:23.124499+0000","last_undegraded":"2023-09-23T20:30:23.124499+0000","last_fullsized":"2023-09-23T20:30:23.124499+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:58.350399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":275,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360181+0000","last_change":"2023-09-23T20:28:55.048383+0000","last_active":"2023-09-23T20:30:23.360181+0000","last_peered":"2023-09-23T20:30:23.360181+0000","last_clean":"2023-09-23T20:30:23.360181+0000","last_became_active":"2023-09-23T20:28:55.047951+0000","last_became_peered":"2023-09-23T20:28:55.047951+0000","last_unstale":"2023-09-23T20:30:23.360181+0000","last_undegraded":"2023-09-23T20:30:23.360181+0000","last_fullsized":"2023-09-23T20:30:23.360181+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:54.426500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119798+0000","last_change":"2023-09-23T20:29:28.055170+0000","last_active":"2023-09-23T20:30:23.119798+0000","last_peered":"2023-09-23T20:30:23.119798+0000","last_clean":"2023-09-23T20:30:23.119798+0000","last_became_active":"2023-09-23T20:29:28.054870+0000","last_became_peered":"2023-09-23T20:29:28.054870+0000","last_unstale":"2023-09-23T20:30:23.119798+0000","last_undegraded":"2023-09-23T20:30:23.119798+0000","last_fullsized":"2023-09-23T20:30:23.119798+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.348967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.129405+0000","last_change":"2023-09-23T20:30:23.128748+0000","last_active":"2023-09-23T20:30:23.129405+0000","last_peered":"2023-09-23T20:30:23.129405+0000","last_clean":"2023-09-23T20:30:23.129405+0000","last_became_active":"2023-09-23T20:30:23.128452+0000","last_became_peered":"2023-09-23T20:30:23.128452+0000","last_unstale":"2023-09-23T20:30:23.129405+0000","last_undegraded":"2023-09-23T20:30:23.129405+0000","last_fullsized":"2023-09-23T20:30:23.129405+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:59.675267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119785+0000","last_change":"2023-09-23T20:30:22.118011+0000","last_active":"2023-09-23T20:30:23.119785+0000","last_peered":"2023-09-23T20:30:23.119785+0000","last_clean":"2023-09-23T20:30:23.119785+0000","last_became_active":"2023-09-23T20:30:22.117756+0000","last_became_peered":"2023-09-23T20:30:22.117756+0000","last_unstale":"2023-09-23T20:30:23.119785+0000","last_undegraded":"2023-09-23T20:30:23.119785+0000","last_fullsized":"2023-09-23T20:30:23.119785+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:35.841713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.123385+0000","last_change":"2023-09-23T20:29:28.486180+0000","last_active":"2023-09-23T20:30:23.123385+0000","last_peered":"2023-09-23T20:30:23.123385+0000","last_clean":"2023-09-23T20:30:23.123385+0000","last_became_active":"2023-09-23T20:29:28.485933+0000","last_became_peered":"2023-09-23T20:29:28.485933+0000","last_unstale":"2023-09-23T20:30:23.123385+0000","last_undegraded":"2023-09-23T20:30:23.123385+0000","last_fullsized":"2023-09-23T20:30:23.123385+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:45.386860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.129481+0000","last_change":"2023-09-23T20:30:23.127379+0000","last_active":"2023-09-23T20:30:23.129481+0000","last_peered":"2023-09-23T20:30:23.129481+0000","last_clean":"2023-09-23T20:30:23.129481+0000","last_became_active":"2023-09-23T20:30:23.127074+0000","last_became_peered":"2023-09-23T20:30:23.127074+0000","last_unstale":"2023-09-23T20:30:23.129481+0000","last_undegraded":"2023-09-23T20:30:23.129481+0000","last_fullsized":"2023-09-23T20:30:23.129481+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.579566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360251+0000","last_change":"2023-09-23T20:30:22.106704+0000","last_active":"2023-09-23T20:30:23.360251+0000","last_peered":"2023-09-23T20:30:23.360251+0000","last_clean":"2023-09-23T20:30:23.360251+0000","last_became_active":"2023-09-23T20:30:22.106295+0000","last_became_peered":"2023-09-23T20:30:22.106295+0000","last_unstale":"2023-09-23T20:30:23.360251+0000","last_undegraded":"2023-09-23T20:30:23.360251+0000","last_fullsized":"2023-09-23T20:30:23.360251+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:18.269323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119674+0000","last_change":"2023-09-23T20:29:28.052272+0000","last_active":"2023-09-23T20:30:23.119674+0000","last_peered":"2023-09-23T20:30:23.119674+0000","last_clean":"2023-09-23T20:30:23.119674+0000","last_became_active":"2023-09-23T20:29:28.051940+0000","last_became_peered":"2023-09-23T20:29:28.051940+0000","last_unstale":"2023-09-23T20:30:23.119674+0000","last_undegraded":"2023-09-23T20:30:23.119674+0000","last_fullsized":"2023-09-23T20:30:23.119674+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:45.231496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362369+0000","last_change":"2023-09-23T20:30:23.360887+0000","last_active":"2023-09-23T20:30:23.362369+0000","last_peered":"2023-09-23T20:30:23.362369+0000","last_clean":"2023-09-23T20:30:23.362369+0000","last_became_active":"2023-09-23T20:30:23.360610+0000","last_became_peered":"2023-09-23T20:30:23.360610+0000","last_unstale":"2023-09-23T20:30:23.362369+0000","last_undegraded":"2023-09-23T20:30:23.362369+0000","last_fullsized":"2023-09-23T20:30:23.362369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:04.437152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119929+0000","last_change":"2023-09-23T20:30:22.117832+0000","last_active":"2023-09-23T20:30:23.119929+0000","last_peered":"2023-09-23T20:30:23.119929+0000","last_clean":"2023-09-23T20:30:23.119929+0000","last_became_active":"2023-09-23T20:30:22.117504+0000","last_became_peered":"2023-09-23T20:30:22.117504+0000","last_unstale":"2023-09-23T20:30:23.119929+0000","last_undegraded":"2023-09-23T20:30:23.119929+0000","last_fullsized":"2023-09-23T20:30:23.119929+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:00.872807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.120466+0000","last_change":"2023-09-23T20:29:28.051973+0000","last_active":"2023-09-23T20:30:23.120466+0000","last_peered":"2023-09-23T20:30:23.120466+0000","last_clean":"2023-09-23T20:30:23.120466+0000","last_became_active":"2023-09-23T20:29:28.051665+0000","last_became_peered":"2023-09-23T20:29:28.051665+0000","last_unstale":"2023-09-23T20:30:23.120466+0000","last_undegraded":"2023-09-23T20:30:23.120466+0000","last_fullsized":"2023-09-23T20:30:23.120466+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:00.653668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362550+0000","last_change":"2023-09-23T20:30:23.362020+0000","last_active":"2023-09-23T20:30:23.362550+0000","last_peered":"2023-09-23T20:30:23.362550+0000","last_clean":"2023-09-23T20:30:23.362550+0000","last_became_active":"2023-09-23T20:30:23.361758+0000","last_became_peered":"2023-09-23T20:30:23.361758+0000","last_unstale":"2023-09-23T20:30:23.362550+0000","last_undegraded":"2023-09-23T20:30:23.362550+0000","last_fullsized":"2023-09-23T20:30:23.362550+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:10.340348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122704+0000","last_change":"2023-09-23T20:30:22.115650+0000","last_active":"2023-09-23T20:30:23.122704+0000","last_peered":"2023-09-23T20:30:23.122704+0000","last_clean":"2023-09-23T20:30:23.122704+0000","last_became_active":"2023-09-23T20:30:22.115439+0000","last_became_peered":"2023-09-23T20:30:22.115439+0000","last_unstale":"2023-09-23T20:30:23.122704+0000","last_undegraded":"2023-09-23T20:30:23.122704+0000","last_fullsized":"2023-09-23T20:30:23.122704+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:44.442798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360138+0000","last_change":"2023-09-23T20:30:22.128161+0000","last_active":"2023-09-23T20:30:23.360138+0000","last_peered":"2023-09-23T20:30:23.360138+0000","last_clean":"2023-09-23T20:30:23.360138+0000","last_became_active":"2023-09-23T20:30:22.127867+0000","last_became_peered":"2023-09-23T20:30:22.127867+0000","last_unstale":"2023-09-23T20:30:23.360138+0000","last_undegraded":"2023-09-23T20:30:23.360138+0000","last_fullsized":"2023-09-23T20:30:23.360138+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:54.545151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359969+0000","last_change":"2023-09-23T20:30:22.112377+0000","last_active":"2023-09-23T20:30:23.359969+0000","last_peered":"2023-09-23T20:30:23.359969+0000","last_clean":"2023-09-23T20:30:23.359969+0000","last_became_active":"2023-09-23T20:30:22.111971+0000","last_became_peered":"2023-09-23T20:30:22.111971+0000","last_unstale":"2023-09-23T20:30:23.359969+0000","last_undegraded":"2023-09-23T20:30:23.359969+0000","last_fullsized":"2023-09-23T20:30:23.359969+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:35.815727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360451+0000","last_change":"2023-09-23T20:30:22.257981+0000","last_active":"2023-09-23T20:30:23.360451+0000","last_peered":"2023-09-23T20:30:23.360451+0000","last_clean":"2023-09-23T20:30:23.360451+0000","last_became_active":"2023-09-23T20:30:22.257723+0000","last_became_peered":"2023-09-23T20:30:22.257723+0000","last_unstale":"2023-09-23T20:30:23.360451+0000","last_undegraded":"2023-09-23T20:30:23.360451+0000","last_fullsized":"2023-09-23T20:30:23.360451+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:21.453174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119025+0000","last_change":"2023-09-23T20:30:22.113897+0000","last_active":"2023-09-23T20:30:23.119025+0000","last_peered":"2023-09-23T20:30:23.119025+0000","last_clean":"2023-09-23T20:30:23.119025+0000","last_became_active":"2023-09-23T20:30:22.113622+0000","last_became_peered":"2023-09-23T20:30:22.113622+0000","last_unstale":"2023-09-23T20:30:23.119025+0000","last_undegraded":"2023-09-23T20:30:23.119025+0000","last_fullsized":"2023-09-23T20:30:23.119025+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:25.076236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360369+0000","last_change":"2023-09-23T20:30:22.485943+0000","last_active":"2023-09-23T20:30:23.360369+0000","last_peered":"2023-09-23T20:30:23.360369+0000","last_clean":"2023-09-23T20:30:23.360369+0000","last_became_active":"2023-09-23T20:30:22.485598+0000","last_became_peered":"2023-09-23T20:30:22.485598+0000","last_unstale":"2023-09-23T20:30:23.360369+0000","last_undegraded":"2023-09-23T20:30:23.360369+0000","last_fullsized":"2023-09-23T20:30:23.360369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:11.546769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.805898+0000","last_change":"2023-09-23T20:30:22.117748+0000","last_active":"2023-09-23T20:30:38.805898+0000","last_peered":"2023-09-23T20:30:38.805898+0000","last_clean":"2023-09-23T20:30:38.805898+0000","last_became_active":"2023-09-23T20:30:22.117467+0000","last_became_peered":"2023-09-23T20:30:22.117467+0000","last_unstale":"2023-09-23T20:30:38.805898+0000","last_undegraded":"2023-09-23T20:30:38.805898+0000","last_fullsized":"2023-09-23T20:30:38.805898+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:08.111582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362586+0000","last_change":"2023-09-23T20:30:23.361522+0000","last_active":"2023-09-23T20:30:23.362586+0000","last_peered":"2023-09-23T20:30:23.362586+0000","last_clean":"2023-09-23T20:30:23.362586+0000","last_became_active":"2023-09-23T20:30:23.361210+0000","last_became_peered":"2023-09-23T20:30:23.361210+0000","last_unstale":"2023-09-23T20:30:23.362586+0000","last_undegraded":"2023-09-23T20:30:23.362586+0000","last_fullsized":"2023-09-23T20:30:23.362586+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:25.949716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119853+0000","last_change":"2023-09-23T20:30:22.102832+0000","last_active":"2023-09-23T20:30:23.119853+0000","last_peered":"2023-09-23T20:30:23.119853+0000","last_clean":"2023-09-23T20:30:23.119853+0000","last_became_active":"2023-09-23T20:30:22.102546+0000","last_became_peered":"2023-09-23T20:30:22.102546+0000","last_unstale":"2023-09-23T20:30:23.119853+0000","last_undegraded":"2023-09-23T20:30:23.119853+0000","last_fullsized":"2023-09-23T20:30:23.119853+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:11.730297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.118930+0000","last_change":"2023-09-23T20:30:22.772628+0000","last_active":"2023-09-23T20:30:23.118930+0000","last_peered":"2023-09-23T20:30:23.118930+0000","last_clean":"2023-09-23T20:30:23.118930+0000","last_became_active":"2023-09-23T20:30:22.772350+0000","last_became_peered":"2023-09-23T20:30:22.772350+0000","last_unstale":"2023-09-23T20:30:23.118930+0000","last_undegraded":"2023-09-23T20:30:23.118930+0000","last_fullsized":"2023-09-23T20:30:23.118930+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:03.794857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.999552+0000","last_change":"2023-09-23T20:30:22.113582+0000","last_active":"2023-09-23T20:30:24.999552+0000","last_peered":"2023-09-23T20:30:24.999552+0000","last_clean":"2023-09-23T20:30:24.999552+0000","last_became_active":"2023-09-23T20:30:22.113291+0000","last_became_peered":"2023-09-23T20:30:22.113291+0000","last_unstale":"2023-09-23T20:30:24.999552+0000","last_undegraded":"2023-09-23T20:30:24.999552+0000","last_fullsized":"2023-09-23T20:30:24.999552+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:19.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359852+0000","last_change":"2023-09-23T20:30:22.485806+0000","last_active":"2023-09-23T20:30:23.359852+0000","last_peered":"2023-09-23T20:30:23.359852+0000","last_clean":"2023-09-23T20:30:23.359852+0000","last_became_active":"2023-09-23T20:30:22.485502+0000","last_became_peered":"2023-09-23T20:30:22.485502+0000","last_unstale":"2023-09-23T20:30:23.359852+0000","last_undegraded":"2023-09-23T20:30:23.359852+0000","last_fullsized":"2023-09-23T20:30:23.359852+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:26.590680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.807986+0000","last_change":"2023-09-23T20:30:22.114034+0000","last_active":"2023-09-23T20:30:38.807986+0000","last_peered":"2023-09-23T20:30:38.807986+0000","last_clean":"2023-09-23T20:30:38.807986+0000","last_became_active":"2023-09-23T20:30:22.113740+0000","last_became_peered":"2023-09-23T20:30:22.113740+0000","last_unstale":"2023-09-23T20:30:38.807986+0000","last_undegraded":"2023-09-23T20:30:38.807986+0000","last_fullsized":"2023-09-23T20:30:38.807986+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:30.265420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119660+0000","last_change":"2023-09-23T20:30:22.772475+0000","last_active":"2023-09-23T20:30:23.119660+0000","last_peered":"2023-09-23T20:30:23.119660+0000","last_clean":"2023-09-23T20:30:23.119660+0000","last_became_active":"2023-09-23T20:30:22.772159+0000","last_became_peered":"2023-09-23T20:30:22.772159+0000","last_unstale":"2023-09-23T20:30:23.119660+0000","last_undegraded":"2023-09-23T20:30:23.119660+0000","last_fullsized":"2023-09-23T20:30:23.119660+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:08.373446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.999550+0000","last_change":"2023-09-23T20:30:22.116902+0000","last_active":"2023-09-23T20:30:24.999550+0000","last_peered":"2023-09-23T20:30:24.999550+0000","last_clean":"2023-09-23T20:30:24.999550+0000","last_became_active":"2023-09-23T20:30:22.116587+0000","last_became_peered":"2023-09-23T20:30:22.116587+0000","last_unstale":"2023-09-23T20:30:24.999550+0000","last_undegraded":"2023-09-23T20:30:24.999550+0000","last_fullsized":"2023-09-23T20:30:24.999550+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:20.938440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.364067+0000","last_change":"2023-09-23T20:30:23.363473+0000","last_active":"2023-09-23T20:30:23.364067+0000","last_peered":"2023-09-23T20:30:23.364067+0000","last_clean":"2023-09-23T20:30:23.364067+0000","last_became_active":"2023-09-23T20:30:23.363192+0000","last_became_peered":"2023-09-23T20:30:23.363192+0000","last_unstale":"2023-09-23T20:30:23.364067+0000","last_undegraded":"2023-09-23T20:30:23.364067+0000","last_fullsized":"2023-09-23T20:30:23.364067+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:03.722816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.810011+0000","last_change":"2023-09-23T20:30:22.103090+0000","last_active":"2023-09-23T20:30:38.810011+0000","last_peered":"2023-09-23T20:30:38.810011+0000","last_clean":"2023-09-23T20:30:38.810011+0000","last_became_active":"2023-09-23T20:30:22.102807+0000","last_became_peered":"2023-09-23T20:30:22.102807+0000","last_unstale":"2023-09-23T20:30:38.810011+0000","last_undegraded":"2023-09-23T20:30:38.810011+0000","last_fullsized":"2023-09-23T20:30:38.810011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:56.602788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362690+0000","last_change":"2023-09-23T20:30:23.362282+0000","last_active":"2023-09-23T20:30:23.362690+0000","last_peered":"2023-09-23T20:30:23.362690+0000","last_clean":"2023-09-23T20:30:23.362690+0000","last_became_active":"2023-09-23T20:30:23.361623+0000","last_became_peered":"2023-09-23T20:30:23.361623+0000","last_unstale":"2023-09-23T20:30:23.362690+0000","last_undegraded":"2023-09-23T20:30:23.362690+0000","last_fullsized":"2023-09-23T20:30:23.362690+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:22.136043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360475+0000","last_change":"2023-09-23T20:30:22.110586+0000","last_active":"2023-09-23T20:30:23.360475+0000","last_peered":"2023-09-23T20:30:23.360475+0000","last_clean":"2023-09-23T20:30:23.360475+0000","last_became_active":"2023-09-23T20:30:22.110248+0000","last_became_peered":"2023-09-23T20:30:22.110248+0000","last_unstale":"2023-09-23T20:30:23.360475+0000","last_undegraded":"2023-09-23T20:30:23.360475+0000","last_fullsized":"2023-09-23T20:30:23.360475+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:29.867944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119858+0000","last_change":"2023-09-23T20:30:22.773252+0000","last_active":"2023-09-23T20:30:23.119858+0000","last_peered":"2023-09-23T20:30:23.119858+0000","last_clean":"2023-09-23T20:30:23.119858+0000","last_became_active":"2023-09-23T20:30:22.772972+0000","last_became_peered":"2023-09-23T20:30:22.772972+0000","last_unstale":"2023-09-23T20:30:23.119858+0000","last_undegraded":"2023-09-23T20:30:23.119858+0000","last_fullsized":"2023-09-23T20:30:23.119858+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:57.509255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122835+0000","last_change":"2023-09-23T20:30:22.109250+0000","last_active":"2023-09-23T20:30:23.122835+0000","last_peered":"2023-09-23T20:30:23.122835+0000","last_clean":"2023-09-23T20:30:23.122835+0000","last_became_active":"2023-09-23T20:30:22.109024+0000","last_became_peered":"2023-09-23T20:30:22.109024+0000","last_unstale":"2023-09-23T20:30:23.122835+0000","last_undegraded":"2023-09-23T20:30:23.122835+0000","last_fullsized":"2023-09-23T20:30:23.122835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.856224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.363344+0000","last_change":"2023-09-23T20:30:23.360789+0000","last_active":"2023-09-23T20:30:23.363344+0000","last_peered":"2023-09-23T20:30:23.363344+0000","last_clean":"2023-09-23T20:30:23.363344+0000","last_became_active":"2023-09-23T20:30:23.360518+0000","last_became_peered":"2023-09-23T20:30:23.360518+0000","last_unstale":"2023-09-23T20:30:23.363344+0000","last_undegraded":"2023-09-23T20:30:23.363344+0000","last_fullsized":"2023-09-23T20:30:23.363344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:34.473352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808098+0000","last_change":"2023-09-23T20:30:22.114940+0000","last_active":"2023-09-23T20:30:38.808098+0000","last_peered":"2023-09-23T20:30:38.808098+0000","last_clean":"2023-09-23T20:30:38.808098+0000","last_became_active":"2023-09-23T20:30:22.114679+0000","last_became_peered":"2023-09-23T20:30:22.114679+0000","last_unstale":"2023-09-23T20:30:38.808098+0000","last_undegraded":"2023-09-23T20:30:38.808098+0000","last_fullsized":"2023-09-23T20:30:38.808098+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:12.493294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360462+0000","last_change":"2023-09-23T20:30:22.128593+0000","last_active":"2023-09-23T20:30:23.360462+0000","last_peered":"2023-09-23T20:30:23.360462+0000","last_clean":"2023-09-23T20:30:23.360462+0000","last_became_active":"2023-09-23T20:30:22.128282+0000","last_became_peered":"2023-09-23T20:30:22.128282+0000","last_unstale":"2023-09-23T20:30:23.360462+0000","last_undegraded":"2023-09-23T20:30:23.360462+0000","last_fullsized":"2023-09-23T20:30:23.360462+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:35.401761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808983+0000","last_change":"2023-09-23T20:30:22.107118+0000","last_active":"2023-09-23T20:30:38.808983+0000","last_peered":"2023-09-23T20:30:38.808983+0000","last_clean":"2023-09-23T20:30:38.808983+0000","last_became_active":"2023-09-23T20:30:22.106839+0000","last_became_peered":"2023-09-23T20:30:22.106839+0000","last_unstale":"2023-09-23T20:30:38.808983+0000","last_undegraded":"2023-09-23T20:30:38.808983+0000","last_fullsized":"2023-09-23T20:30:38.808983+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:02.981333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.125858+0000","last_change":"2023-09-23T20:30:23.124437+0000","last_active":"2023-09-23T20:30:23.125858+0000","last_peered":"2023-09-23T20:30:23.125858+0000","last_clean":"2023-09-23T20:30:23.125858+0000","last_became_active":"2023-09-23T20:30:23.124078+0000","last_became_peered":"2023-09-23T20:30:23.124078+0000","last_unstale":"2023-09-23T20:30:23.125858+0000","last_undegraded":"2023-09-23T20:30:23.125858+0000","last_fullsized":"2023-09-23T20:30:23.125858+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:58.955735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808208+0000","last_change":"2023-09-23T20:30:22.117956+0000","last_active":"2023-09-23T20:30:38.808208+0000","last_peered":"2023-09-23T20:30:38.808208+0000","last_clean":"2023-09-23T20:30:38.808208+0000","last_became_active":"2023-09-23T20:30:22.117681+0000","last_became_peered":"2023-09-23T20:30:22.117681+0000","last_unstale":"2023-09-23T20:30:38.808208+0000","last_undegraded":"2023-09-23T20:30:38.808208+0000","last_fullsized":"2023-09-23T20:30:38.808208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:13.129112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.363132+0000","last_change":"2023-09-23T20:30:23.360927+0000","last_active":"2023-09-23T20:30:23.363132+0000","last_peered":"2023-09-23T20:30:23.363132+0000","last_clean":"2023-09-23T20:30:23.363132+0000","last_became_active":"2023-09-23T20:30:23.360591+0000","last_became_peered":"2023-09-23T20:30:23.360591+0000","last_unstale":"2023-09-23T20:30:23.363132+0000","last_undegraded":"2023-09-23T20:30:23.363132+0000","last_fullsized":"2023-09-23T20:30:23.363132+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:56.124709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119761+0000","last_change":"2023-09-23T20:30:22.109674+0000","last_active":"2023-09-23T20:30:23.119761+0000","last_peered":"2023-09-23T20:30:23.119761+0000","last_clean":"2023-09-23T20:30:23.119761+0000","last_became_active":"2023-09-23T20:30:22.109370+0000","last_became_peered":"2023-09-23T20:30:22.109370+0000","last_unstale":"2023-09-23T20:30:23.119761+0000","last_undegraded":"2023-09-23T20:30:23.119761+0000","last_fullsized":"2023-09-23T20:30:23.119761+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:03.191494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.128055+0000","last_change":"2023-09-23T20:30:23.127296+0000","last_active":"2023-09-23T20:30:23.128055+0000","last_peered":"2023-09-23T20:30:23.128055+0000","last_clean":"2023-09-23T20:30:23.128055+0000","last_became_active":"2023-09-23T20:30:23.126893+0000","last_became_peered":"2023-09-23T20:30:23.126893+0000","last_unstale":"2023-09-23T20:30:23.128055+0000","last_undegraded":"2023-09-23T20:30:23.128055+0000","last_fullsized":"2023-09-23T20:30:23.128055+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:31.768430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.125184+0000","last_change":"2023-09-23T20:30:22.116853+0000","last_active":"2023-09-23T20:30:23.125184+0000","last_peered":"2023-09-23T20:30:23.125184+0000","last_clean":"2023-09-23T20:30:23.125184+0000","last_became_active":"2023-09-23T20:30:22.116609+0000","last_became_peered":"2023-09-23T20:30:22.116609+0000","last_unstale":"2023-09-23T20:30:23.125184+0000","last_undegraded":"2023-09-23T20:30:23.125184+0000","last_fullsized":"2023-09-23T20:30:23.125184+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:14.249091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119234+0000","last_change":"2023-09-23T20:30:22.131125+0000","last_active":"2023-09-23T20:30:23.119234+0000","last_peered":"2023-09-23T20:30:23.119234+0000","last_clean":"2023-09-23T20:30:23.119234+0000","last_became_active":"2023-09-23T20:30:22.130818+0000","last_became_peered":"2023-09-23T20:30:22.130818+0000","last_unstale":"2023-09-23T20:30:23.119234+0000","last_undegraded":"2023-09-23T20:30:23.119234+0000","last_fullsized":"2023-09-23T20:30:23.119234+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:55.432616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122808+0000","last_change":"2023-09-23T20:30:22.119946+0000","last_active":"2023-09-23T20:30:23.122808+0000","last_peered":"2023-09-23T20:30:23.122808+0000","last_clean":"2023-09-23T20:30:23.122808+0000","last_became_active":"2023-09-23T20:30:22.119643+0000","last_became_peered":"2023-09-23T20:30:22.119643+0000","last_unstale":"2023-09-23T20:30:23.122808+0000","last_undegraded":"2023-09-23T20:30:23.122808+0000","last_fullsized":"2023-09-23T20:30:23.122808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:30.646487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362707+0000","last_change":"2023-09-23T20:30:23.362393+0000","last_active":"2023-09-23T20:30:23.362707+0000","last_peered":"2023-09-23T20:30:23.362707+0000","last_clean":"2023-09-23T20:30:23.362707+0000","last_became_active":"2023-09-23T20:30:23.362005+0000","last_became_peered":"2023-09-23T20:30:23.362005+0000","last_unstale":"2023-09-23T20:30:23.362707+0000","last_undegraded":"2023-09-23T20:30:23.362707+0000","last_fullsized":"2023-09-23T20:30:23.362707+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:26.361518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119771+0000","last_change":"2023-09-23T20:30:22.110079+0000","last_active":"2023-09-23T20:30:23.119771+0000","last_peered":"2023-09-23T20:30:23.119771+0000","last_clean":"2023-09-23T20:30:23.119771+0000","last_became_active":"2023-09-23T20:30:22.109745+0000","last_became_peered":"2023-09-23T20:30:22.109745+0000","last_unstale":"2023-09-23T20:30:23.119771+0000","last_undegraded":"2023-09-23T20:30:23.119771+0000","last_fullsized":"2023-09-23T20:30:23.119771+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:37.628634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362217+0000","last_change":"2023-09-23T20:30:23.360193+0000","last_active":"2023-09-23T20:30:23.362217+0000","last_peered":"2023-09-23T20:30:23.362217+0000","last_clean":"2023-09-23T20:30:23.362217+0000","last_became_active":"2023-09-23T20:30:23.359847+0000","last_became_peered":"2023-09-23T20:30:23.359847+0000","last_unstale":"2023-09-23T20:30:23.362217+0000","last_undegraded":"2023-09-23T20:30:23.362217+0000","last_fullsized":"2023-09-23T20:30:23.362217+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:06.567927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.902511+0000","last_change":"2023-09-23T20:30:22.112277+0000","last_active":"2023-09-23T20:30:24.902511+0000","last_peered":"2023-09-23T20:30:24.902511+0000","last_clean":"2023-09-23T20:30:24.902511+0000","last_became_active":"2023-09-23T20:30:22.111979+0000","last_became_peered":"2023-09-23T20:30:22.111979+0000","last_unstale":"2023-09-23T20:30:24.902511+0000","last_undegraded":"2023-09-23T20:30:24.902511+0000","last_fullsized":"2023-09-23T20:30:24.902511+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:33.786341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201310,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6608,"kb_used_data":1288,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738544,"statfs":{"total":95995035648,"available":95988269056,"internally_reserved":0,"allocated":1318912,"data_stored":894173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.47899999999999998,"15min":0.48299999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.59299999999999997,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.47099999999999997,"15min":0.46500000000000002},"min":{"1min":0.32100000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.432}]},{"osd":1,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.50900000000000001,"15min":0.51300000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.32600000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.61699999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.47299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.52200000000000002,"15min":0.53400000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67100000000000004,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.35099999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.51200000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.48999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.43099999999999999,"15min":0.434},"min":{"1min":0.309,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60999999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47899999999999998,"15min":0.47999999999999998},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.65100000000000002,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.746}]},{"osd":5,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.46999999999999997,"15min":0.46600000000000003},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.501,"15min":0.501},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.40300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.69999999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.39000000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.68700000000000006,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.59199999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462947,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6592,"kb_used_data":1272,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738560,"statfs":{"total":95995035648,"available":95988285440,"internally_reserved":0,"allocated":1302528,"data_stored":893820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.48199999999999998,"15min":0.48799999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.59999999999999998,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70999999999999996,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.436}]},{"osd":1,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.53900000000000003,"15min":0.54600000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.65300000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.54000000000000004,"15min":0.54500000000000004},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67700000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.60399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.48999999999999999,"15min":0.49199999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.61699999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.47799999999999998,"15min":0.47599999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.67500000000000004,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34200000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76700000000000002,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70899999999999996,"5min":0.877,"15min":0.877},"last":0.629}]},{"osd":4,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.40300000000000002,"15min":0.39300000000000002},"min":{"1min":0.33100000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.434,"15min":0.42399999999999999},"min":{"1min":0.30299999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.438,"15min":0.43099999999999999},"min":{"1min":0.378,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.46800000000000003,"15min":0.45800000000000002},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.5,"15min":0.48999999999999999},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.53500000000000003,"15min":0.52500000000000002},"min":{"1min":0.44600000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.56399999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574905,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6772,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738380,"statfs":{"total":95995035648,"available":95988101120,"internally_reserved":0,"allocated":700416,"data_stored":302376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.35199999999999998,"15min":0.311},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.59099999999999997,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.374,"15min":0.33200000000000002},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:30:04 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39900000000000002,"15min":0.38800000000000001},"min":{"1min":0.28100000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56299999999999994,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42199999999999999,"15min":0.41499999999999998},"min":{"1min":0.29699999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.371}]},{"osd":3,"last update":"Sat Sep 23 20:30:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.45400000000000001,"15min":0.45800000000000002},"min":{"1min":0.30299999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.502,"5min":0.621,"15min":0.621},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.46800000000000003,"15min":0.47599999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.56899999999999995,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.505}]},{"osd":4,"last update":"Sat Sep 23 20:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.622,"15min":0.626},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:30:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.58999999999999997,"15min":0.59599999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:30:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.62,"15min":0.63200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57999999999999996,"15min":0.58299999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:30:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60399999999999998,"15min":0.60799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.42699999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7360,"kb_used_data":1272,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93737792,"statfs":{"total":95995035648,"available":95987499008,"internally_reserved":0,"allocated":1302528,"data_stored":893320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.35199999999999998,"15min":0.33900000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.40000000000000002,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.40300000000000002,"15min":0.35899999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.9969999999999999,"5min":3.9969999999999999,"15min":3.9969999999999999},"last":0.27300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":0.38400000000000001,"15min":0.39600000000000002},"min":{"1min":0.24399999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.46700000000000003,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40699999999999997,"15min":0.40600000000000003},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.497,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.375}]},{"osd":3,"last update":"Sat Sep 23 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.44800000000000001,"15min":0.45900000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.61699999999999999,"5min":0.75,"15min":0.75},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.60599999999999998,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.56299999999999994,"15min":0.54500000000000004},"min":{"1min":0.39400000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.9409999999999998,"5min":3.9409999999999998,"15min":3.9409999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56699999999999995,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.72099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.58199999999999996,"15min":0.58699999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.40999999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.83799999999999997,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:30:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.44400000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.61799999999999999,"15min":0.60199999999999998},"min":{"1min":0.38,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.8900000000000001,"5min":3.8900000000000001,"15min":3.8900000000000001},"last":0.67400000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.70399999999999996,"15min":0.72799999999999998},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.314},"max":{"1min":0.77000000000000002,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55600000000000005,"15min":0.55500000000000005},"min":{"1min":0.438,"5min":0.379,"15min":0.379},"max":{"1min":0.69299999999999995,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.66300000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411381,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6588,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738564,"statfs":{"total":95995035648,"available":95988289536,"internally_reserved":0,"allocated":708608,"data_stored":302964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.36599999999999999,"15min":0.37},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.374,"15min":0.37},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38,"15min":0.371},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.379,"15min":0.373},"min":{"1min":0.247,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54300000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.27400000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.147,"5min":1.147,"15min":1.147},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.44,"15min":0.44800000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68200000000000005,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.308}]},{"osd":4,"last update":"Sat Sep 23 20:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.58599999999999997,"15min":0.56999999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":3.3119999999999998,"5min":3.3119999999999998,"15min":3.3119999999999998},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.57499999999999996,"15min":0.56200000000000006},"min":{"1min":0.42199999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.2229999999999999,"5min":3.2229999999999999,"15min":3.2229999999999999},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58499999999999996,"15min":0.58699999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.73899999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.59199999999999997,"15min":0.59699999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.749,"5min":0.752,"15min":0.752},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.58399999999999996,"15min":0.57199999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.2909999999999999,"5min":3.2909999999999999,"15min":3.2909999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.58599999999999997,"15min":0.57299999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":3.3849999999999998},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:30:57 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.58199999999999996,"15min":0.56299999999999994},"min":{"1min":0.39200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.3479999999999999,"5min":3.3479999999999999,"15min":3.3479999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.60899999999999999,"15min":0.59099999999999997},"min":{"1min":0.41299999999999998,"5min":0.376,"15min":0.376},"max":{"1min":3.258,"5min":3.258,"15min":3.258},"last":0.53000000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149744,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6136,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739016,"statfs":{"total":95995035648,"available":95988752384,"internally_reserved":0,"allocated":704512,"data_stored":302466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":0.36799999999999999,"15min":0.374},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.45900000000000002,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.255},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.42399999999999999,"15min":0.437},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.48099999999999998,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.41399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.44,"15min":0.46000000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.47599999999999998,"5min":0.623,"15min":0.623},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.376,"5min":0.44900000000000001,"15min":0.46999999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.59899999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.47199999999999998,"15min":0.48699999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.52000000000000002,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.46700000000000003,"15min":0.48499999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.61199999999999999,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.39100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.55700000000000005,"15min":0.57499999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73799999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.56100000000000005,"15min":0.57499999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68700000000000006,"5min":0.751,"15min":0.751},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.76200000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:30:30 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.65800000000000003,"15min":0.68400000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72599999999999998,"5min":2.903,"15min":2.903},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57599999999999996,"15min":0.58299999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.68200000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:30:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.59699999999999998,"15min":0.60299999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77300000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.54800000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986220,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6220,"kb_used_data":708,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738932,"statfs":{"total":95995035648,"available":95988666368,"internally_reserved":0,"allocated":724992,"data_stored":304058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.68999999999999995,"15min":0.70799999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.7759999999999998,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.63200000000000001,"15min":0.68400000000000005},"min":{"1min":0.34200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78000000000000003,"5min":3.9489999999999998,"15min":3.9489999999999998},"last":0.70099999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.72899999999999998,"15min":0.73299999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.4960000000000004,"5min":9.4960000000000004,"15min":9.4960000000000004},"last":0.629},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.53100000000000003,"15min":0.54700000000000004},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.74099999999999999,"5min":1.425,"15min":1.425},"last":0.64100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.59799999999999998,"15min":0.58199999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":5.9160000000000004,"5min":5.9160000000000004,"15min":5.9160000000000004},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.68999999999999995,"15min":0.70999999999999996},"min":{"1min":0.34399999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.8799999999999999,"5min":5.8799999999999999,"15min":5.8799999999999999},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.73899999999999999,"15min":0.73599999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":9.5630000000000006,"5min":9.5630000000000006,"15min":9.5630000000000006},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.74199999999999999,"15min":0.73599999999999999},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.6170000000000009,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:06 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.70299999999999996,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.58899999999999997,"15min":0.60399999999999998},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.86699999999999999,"5min":3.887,"15min":3.887},"last":0.40600000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.438,"15min":0.43099999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.45100000000000001,"15min":0.44400000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.55600000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724584,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6076,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739076,"statfs":{"total":95995035648,"available":95988813824,"internally_reserved":0,"allocated":708608,"data_stored":303000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.53900000000000003,"15min":0.54800000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67500000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.433},{"interface":"front","average":{"1min":0.434,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67800000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.51000000000000001,"15min":0.51500000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.495},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.51700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.623,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.505,"15min":0.51000000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.373},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.1539999999999999,"5min":2.1539999999999999,"15min":2.1539999999999999},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.497,"15min":0.499},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.35299999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.40000000000000002,"15min":0.39600000000000002},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.628,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.41999999999999998,"15min":0.41299999999999998},"min":{"1min":0.34599999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:30:33 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.65100000000000002,"5min":1.54,"15min":1.54},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.47399999999999998,"15min":0.46899999999999997},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66000000000000003,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68700000000000006,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.53500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:11.132 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:31:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:11.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:11.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:11 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:11 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:11 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:11 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:11.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:11 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:11.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:11 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:11.493 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:12 smithi121 ceph-mon[100258]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:12 smithi121 ceph-mon[100258]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:12 smithi121 ceph-mon[118228]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:12 smithi121 ceph-mon[118228]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:12 smithi123 ceph-mon[104968]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:12 smithi123 ceph-mon[104968]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:13.202 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:31:13.204 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:31:13.496 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:13 smithi121 ceph-mon[118228]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:13.496 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:13 smithi121 ceph-mon[118228]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:13.497 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:13 smithi121 ceph-mon[100258]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:13.497 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:13 smithi121 ceph-mon[100258]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:13.790 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":66,"stamp":"2023-09-23T20:31:12.897346+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52352,"kb_used_data":7296,"kb_used_omap":0,"kb_used_meta":44992,"kb_avail":749908864,"statfs":{"total":767960285184,"available":767906676736,"internally_reserved":0,"allocated":7471104,"data_stored":4196177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46071808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002470"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359624+0000","last_change":"2023-09-23T20:30:22.127985+0000","last_active":"2023-09-23T20:30:23.359624+0000","last_peered":"2023-09-23T20:30:23.359624+0000","last_clean":"2023-09-23T20:30:23.359624+0000","last_became_active":"2023-09-23T20:30:22.127651+0000","last_became_peered":"2023-09-23T20:30:22.127651+0000","last_unstale":"2023-09-23T20:30:23.359624+0000","last_undegraded":"2023-09-23T20:30:23.359624+0000","last_fullsized":"2023-09-23T20:30:23.359624+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:45.050942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.809192+0000","last_change":"2023-09-23T20:30:22.117719+0000","last_active":"2023-09-23T20:30:38.809192+0000","last_peered":"2023-09-23T20:30:38.809192+0000","last_clean":"2023-09-23T20:30:38.809192+0000","last_became_active":"2023-09-23T20:30:22.117410+0000","last_became_peered":"2023-09-23T20:30:22.117410+0000","last_unstale":"2023-09-23T20:30:38.809192+0000","last_undegraded":"2023-09-23T20:30:38.809192+0000","last_fullsized":"2023-09-23T20:30:38.809192+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:50.581932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.130627+0000","last_change":"2023-09-23T20:30:23.130236+0000","last_active":"2023-09-23T20:30:23.130627+0000","last_peered":"2023-09-23T20:30:23.130627+0000","last_clean":"2023-09-23T20:30:23.130627+0000","last_became_active":"2023-09-23T20:30:23.129941+0000","last_became_peered":"2023-09-23T20:30:23.129941+0000","last_unstale":"2023-09-23T20:30:23.130627+0000","last_undegraded":"2023-09-23T20:30:23.130627+0000","last_fullsized":"2023-09-23T20:30:23.130627+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:42.583768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.998860+0000","last_change":"2023-09-23T20:30:22.115582+0000","last_active":"2023-09-23T20:30:24.998860+0000","last_peered":"2023-09-23T20:30:24.998860+0000","last_clean":"2023-09-23T20:30:24.998860+0000","last_became_active":"2023-09-23T20:30:22.115345+0000","last_became_peered":"2023-09-23T20:30:22.115345+0000","last_unstale":"2023-09-23T20:30:24.998860+0000","last_undegraded":"2023-09-23T20:30:24.998860+0000","last_fullsized":"2023-09-23T20:30:24.998860+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:23.738240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360138+0000","last_change":"2023-09-23T20:30:22.772298+0000","last_active":"2023-09-23T20:30:23.360138+0000","last_peered":"2023-09-23T20:30:23.360138+0000","last_clean":"2023-09-23T20:30:23.360138+0000","last_became_active":"2023-09-23T20:30:22.772008+0000","last_became_peered":"2023-09-23T20:30:22.772008+0000","last_unstale":"2023-09-23T20:30:23.360138+0000","last_undegraded":"2023-09-23T20:30:23.360138+0000","last_fullsized":"2023-09-23T20:30:23.360138+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:17.639153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808878+0000","last_change":"2023-09-23T20:30:22.104198+0000","last_active":"2023-09-23T20:30:38.808878+0000","last_peered":"2023-09-23T20:30:38.808878+0000","last_clean":"2023-09-23T20:30:38.808878+0000","last_became_active":"2023-09-23T20:30:22.103970+0000","last_became_peered":"2023-09-23T20:30:22.103970+0000","last_unstale":"2023-09-23T20:30:38.808878+0000","last_undegraded":"2023-09-23T20:30:38.808878+0000","last_fullsized":"2023-09-23T20:30:38.808878+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:32.820350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360355+0000","last_change":"2023-09-23T20:30:22.486030+0000","last_active":"2023-09-23T20:30:23.360355+0000","last_peered":"2023-09-23T20:30:23.360355+0000","last_clean":"2023-09-23T20:30:23.360355+0000","last_became_active":"2023-09-23T20:30:22.485521+0000","last_became_peered":"2023-09-23T20:30:22.485521+0000","last_unstale":"2023-09-23T20:30:23.360355+0000","last_undegraded":"2023-09-23T20:30:23.360355+0000","last_fullsized":"2023-09-23T20:30:23.360355+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:35.764712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.809129+0000","last_change":"2023-09-23T20:30:22.110159+0000","last_active":"2023-09-23T20:30:38.809129+0000","last_peered":"2023-09-23T20:30:38.809129+0000","last_clean":"2023-09-23T20:30:38.809129+0000","last_became_active":"2023-09-23T20:30:22.109484+0000","last_became_peered":"2023-09-23T20:30:22.109484+0000","last_unstale":"2023-09-23T20:30:38.809129+0000","last_undegraded":"2023-09-23T20:30:38.809129+0000","last_fullsized":"2023-09-23T20:30:38.809129+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:58.417615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.129563+0000","last_change":"2023-09-23T20:30:23.127591+0000","last_active":"2023-09-23T20:30:23.129563+0000","last_peered":"2023-09-23T20:30:23.129563+0000","last_clean":"2023-09-23T20:30:23.129563+0000","last_became_active":"2023-09-23T20:30:23.127292+0000","last_became_peered":"2023-09-23T20:30:23.127292+0000","last_unstale":"2023-09-23T20:30:23.129563+0000","last_undegraded":"2023-09-23T20:30:23.129563+0000","last_fullsized":"2023-09-23T20:30:23.129563+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:23.845610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.498161+0000","last_change":"2023-09-23T20:30:22.111101+0000","last_active":"2023-09-23T20:30:24.498161+0000","last_peered":"2023-09-23T20:30:24.498161+0000","last_clean":"2023-09-23T20:30:24.498161+0000","last_became_active":"2023-09-23T20:30:22.110836+0000","last_became_peered":"2023-09-23T20:30:22.110836+0000","last_unstale":"2023-09-23T20:30:24.498161+0000","last_undegraded":"2023-09-23T20:30:24.498161+0000","last_fullsized":"2023-09-23T20:30:24.498161+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:21.805629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359990+0000","last_change":"2023-09-23T20:30:22.131210+0000","last_active":"2023-09-23T20:30:23.359990+0000","last_peered":"2023-09-23T20:30:23.359990+0000","last_clean":"2023-09-23T20:30:23.359990+0000","last_became_active":"2023-09-23T20:30:22.130879+0000","last_became_peered":"2023-09-23T20:30:22.130879+0000","last_unstale":"2023-09-23T20:30:23.359990+0000","last_undegraded":"2023-09-23T20:30:23.359990+0000","last_fullsized":"2023-09-23T20:30:23.359990+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:10.733014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.498209+0000","last_change":"2023-09-23T20:30:22.108849+0000","last_active":"2023-09-23T20:30:24.498209+0000","last_peered":"2023-09-23T20:30:24.498209+0000","last_clean":"2023-09-23T20:30:24.498209+0000","last_became_active":"2023-09-23T20:30:22.108610+0000","last_became_peered":"2023-09-23T20:30:22.108610+0000","last_unstale":"2023-09-23T20:30:24.498209+0000","last_undegraded":"2023-09-23T20:30:24.498209+0000","last_fullsized":"2023-09-23T20:30:24.498209+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:36.799199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119235+0000","last_change":"2023-09-23T20:29:28.052971+0000","last_active":"2023-09-23T20:30:23.119235+0000","last_peered":"2023-09-23T20:30:23.119235+0000","last_clean":"2023-09-23T20:30:23.119235+0000","last_became_active":"2023-09-23T20:29:28.052681+0000","last_became_peered":"2023-09-23T20:29:28.052681+0000","last_unstale":"2023-09-23T20:30:23.119235+0000","last_undegraded":"2023-09-23T20:30:23.119235+0000","last_fullsized":"2023-09-23T20:30:23.119235+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:17.819345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.363850+0000","last_change":"2023-09-23T20:30:23.363445+0000","last_active":"2023-09-23T20:30:23.363850+0000","last_peered":"2023-09-23T20:30:23.363850+0000","last_clean":"2023-09-23T20:30:23.363850+0000","last_became_active":"2023-09-23T20:30:23.363112+0000","last_became_peered":"2023-09-23T20:30:23.363112+0000","last_unstale":"2023-09-23T20:30:23.363850+0000","last_undegraded":"2023-09-23T20:30:23.363850+0000","last_fullsized":"2023-09-23T20:30:23.363850+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:54.947384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.120263+0000","last_change":"2023-09-23T20:29:28.208396+0000","last_active":"2023-09-23T20:30:23.120263+0000","last_peered":"2023-09-23T20:30:23.120263+0000","last_clean":"2023-09-23T20:30:23.120263+0000","last_became_active":"2023-09-23T20:29:28.208035+0000","last_became_peered":"2023-09-23T20:29:28.208035+0000","last_unstale":"2023-09-23T20:30:23.120263+0000","last_undegraded":"2023-09-23T20:30:23.120263+0000","last_fullsized":"2023-09-23T20:30:23.120263+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:11.504285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.805725+0000","last_change":"2023-09-23T20:30:22.119412+0000","last_active":"2023-09-23T20:30:38.805725+0000","last_peered":"2023-09-23T20:30:38.805725+0000","last_clean":"2023-09-23T20:30:38.805725+0000","last_became_active":"2023-09-23T20:30:22.118963+0000","last_became_peered":"2023-09-23T20:30:22.118963+0000","last_unstale":"2023-09-23T20:30:38.805725+0000","last_undegraded":"2023-09-23T20:30:38.805725+0000","last_fullsized":"2023-09-23T20:30:38.805725+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:44.340631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.361653+0000","last_change":"2023-09-23T20:30:23.360781+0000","last_active":"2023-09-23T20:30:23.361653+0000","last_peered":"2023-09-23T20:30:23.361653+0000","last_clean":"2023-09-23T20:30:23.361653+0000","last_became_active":"2023-09-23T20:30:23.360474+0000","last_became_peered":"2023-09-23T20:30:23.360474+0000","last_unstale":"2023-09-23T20:30:23.361653+0000","last_undegraded":"2023-09-23T20:30:23.361653+0000","last_fullsized":"2023-09-23T20:30:23.361653+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:39.665426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.120383+0000","last_change":"2023-09-23T20:30:22.118001+0000","last_active":"2023-09-23T20:30:23.120383+0000","last_peered":"2023-09-23T20:30:23.120383+0000","last_clean":"2023-09-23T20:30:23.120383+0000","last_became_active":"2023-09-23T20:30:22.117728+0000","last_became_peered":"2023-09-23T20:30:22.117728+0000","last_unstale":"2023-09-23T20:30:23.120383+0000","last_undegraded":"2023-09-23T20:30:23.120383+0000","last_fullsized":"2023-09-23T20:30:23.120383+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:55.249207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.361193+0000","last_change":"2023-09-23T20:29:28.208399+0000","last_active":"2023-09-23T20:30:23.361193+0000","last_peered":"2023-09-23T20:30:23.361193+0000","last_clean":"2023-09-23T20:30:23.361193+0000","last_became_active":"2023-09-23T20:29:28.208044+0000","last_became_peered":"2023-09-23T20:29:28.208044+0000","last_unstale":"2023-09-23T20:30:23.361193+0000","last_undegraded":"2023-09-23T20:30:23.361193+0000","last_fullsized":"2023-09-23T20:30:23.361193+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:04.477375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.130504+0000","last_change":"2023-09-23T20:30:23.130060+0000","last_active":"2023-09-23T20:30:23.130504+0000","last_peered":"2023-09-23T20:30:23.130504+0000","last_clean":"2023-09-23T20:30:23.130504+0000","last_became_active":"2023-09-23T20:30:23.129768+0000","last_became_peered":"2023-09-23T20:30:23.129768+0000","last_unstale":"2023-09-23T20:30:23.130504+0000","last_undegraded":"2023-09-23T20:30:23.130504+0000","last_fullsized":"2023-09-23T20:30:23.130504+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:04.441375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122455+0000","last_change":"2023-09-23T20:30:22.107067+0000","last_active":"2023-09-23T20:30:23.122455+0000","last_peered":"2023-09-23T20:30:23.122455+0000","last_clean":"2023-09-23T20:30:23.122455+0000","last_became_active":"2023-09-23T20:30:22.106739+0000","last_became_peered":"2023-09-23T20:30:22.106739+0000","last_unstale":"2023-09-23T20:30:23.122455+0000","last_undegraded":"2023-09-23T20:30:23.122455+0000","last_fullsized":"2023-09-23T20:30:23.122455+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:01.307848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360275+0000","last_change":"2023-09-23T20:29:28.055354+0000","last_active":"2023-09-23T20:30:23.360275+0000","last_peered":"2023-09-23T20:30:23.360275+0000","last_clean":"2023-09-23T20:30:23.360275+0000","last_became_active":"2023-09-23T20:29:28.055064+0000","last_became_peered":"2023-09-23T20:29:28.055064+0000","last_unstale":"2023-09-23T20:30:23.360275+0000","last_undegraded":"2023-09-23T20:30:23.360275+0000","last_fullsized":"2023-09-23T20:30:23.360275+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:55.115453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119197+0000","last_change":"2023-09-23T20:30:22.772777+0000","last_active":"2023-09-23T20:30:23.119197+0000","last_peered":"2023-09-23T20:30:23.119197+0000","last_clean":"2023-09-23T20:30:23.119197+0000","last_became_active":"2023-09-23T20:30:22.772481+0000","last_became_peered":"2023-09-23T20:30:22.772481+0000","last_unstale":"2023-09-23T20:30:23.119197+0000","last_undegraded":"2023-09-23T20:30:23.119197+0000","last_fullsized":"2023-09-23T20:30:23.119197+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:53.724607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.124499+0000","last_change":"2023-09-23T20:30:22.109986+0000","last_active":"2023-09-23T20:30:23.124499+0000","last_peered":"2023-09-23T20:30:23.124499+0000","last_clean":"2023-09-23T20:30:23.124499+0000","last_became_active":"2023-09-23T20:30:22.109562+0000","last_became_peered":"2023-09-23T20:30:22.109562+0000","last_unstale":"2023-09-23T20:30:23.124499+0000","last_undegraded":"2023-09-23T20:30:23.124499+0000","last_fullsized":"2023-09-23T20:30:23.124499+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:58.350399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":275,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360181+0000","last_change":"2023-09-23T20:28:55.048383+0000","last_active":"2023-09-23T20:30:23.360181+0000","last_peered":"2023-09-23T20:30:23.360181+0000","last_clean":"2023-09-23T20:30:23.360181+0000","last_became_active":"2023-09-23T20:28:55.047951+0000","last_became_peered":"2023-09-23T20:28:55.047951+0000","last_unstale":"2023-09-23T20:30:23.360181+0000","last_undegraded":"2023-09-23T20:30:23.360181+0000","last_fullsized":"2023-09-23T20:30:23.360181+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:54.426500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119798+0000","last_change":"2023-09-23T20:29:28.055170+0000","last_active":"2023-09-23T20:30:23.119798+0000","last_peered":"2023-09-23T20:30:23.119798+0000","last_clean":"2023-09-23T20:30:23.119798+0000","last_became_active":"2023-09-23T20:29:28.054870+0000","last_became_peered":"2023-09-23T20:29:28.054870+0000","last_unstale":"2023-09-23T20:30:23.119798+0000","last_undegraded":"2023-09-23T20:30:23.119798+0000","last_fullsized":"2023-09-23T20:30:23.119798+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:53.348967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.129405+0000","last_change":"2023-09-23T20:30:23.128748+0000","last_active":"2023-09-23T20:30:23.129405+0000","last_peered":"2023-09-23T20:30:23.129405+0000","last_clean":"2023-09-23T20:30:23.129405+0000","last_became_active":"2023-09-23T20:30:23.128452+0000","last_became_peered":"2023-09-23T20:30:23.128452+0000","last_unstale":"2023-09-23T20:30:23.129405+0000","last_undegraded":"2023-09-23T20:30:23.129405+0000","last_fullsized":"2023-09-23T20:30:23.129405+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:59.675267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119785+0000","last_change":"2023-09-23T20:30:22.118011+0000","last_active":"2023-09-23T20:30:23.119785+0000","last_peered":"2023-09-23T20:30:23.119785+0000","last_clean":"2023-09-23T20:30:23.119785+0000","last_became_active":"2023-09-23T20:30:22.117756+0000","last_became_peered":"2023-09-23T20:30:22.117756+0000","last_unstale":"2023-09-23T20:30:23.119785+0000","last_undegraded":"2023-09-23T20:30:23.119785+0000","last_fullsized":"2023-09-23T20:30:23.119785+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:35.841713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.123385+0000","last_change":"2023-09-23T20:29:28.486180+0000","last_active":"2023-09-23T20:30:23.123385+0000","last_peered":"2023-09-23T20:30:23.123385+0000","last_clean":"2023-09-23T20:30:23.123385+0000","last_became_active":"2023-09-23T20:29:28.485933+0000","last_became_peered":"2023-09-23T20:29:28.485933+0000","last_unstale":"2023-09-23T20:30:23.123385+0000","last_undegraded":"2023-09-23T20:30:23.123385+0000","last_fullsized":"2023-09-23T20:30:23.123385+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:45.386860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.129481+0000","last_change":"2023-09-23T20:30:23.127379+0000","last_active":"2023-09-23T20:30:23.129481+0000","last_peered":"2023-09-23T20:30:23.129481+0000","last_clean":"2023-09-23T20:30:23.129481+0000","last_became_active":"2023-09-23T20:30:23.127074+0000","last_became_peered":"2023-09-23T20:30:23.127074+0000","last_unstale":"2023-09-23T20:30:23.129481+0000","last_undegraded":"2023-09-23T20:30:23.129481+0000","last_fullsized":"2023-09-23T20:30:23.129481+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.579566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360251+0000","last_change":"2023-09-23T20:30:22.106704+0000","last_active":"2023-09-23T20:30:23.360251+0000","last_peered":"2023-09-23T20:30:23.360251+0000","last_clean":"2023-09-23T20:30:23.360251+0000","last_became_active":"2023-09-23T20:30:22.106295+0000","last_became_peered":"2023-09-23T20:30:22.106295+0000","last_unstale":"2023-09-23T20:30:23.360251+0000","last_undegraded":"2023-09-23T20:30:23.360251+0000","last_fullsized":"2023-09-23T20:30:23.360251+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:18.269323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119674+0000","last_change":"2023-09-23T20:29:28.052272+0000","last_active":"2023-09-23T20:30:23.119674+0000","last_peered":"2023-09-23T20:30:23.119674+0000","last_clean":"2023-09-23T20:30:23.119674+0000","last_became_active":"2023-09-23T20:29:28.051940+0000","last_became_peered":"2023-09-23T20:29:28.051940+0000","last_unstale":"2023-09-23T20:30:23.119674+0000","last_undegraded":"2023-09-23T20:30:23.119674+0000","last_fullsized":"2023-09-23T20:30:23.119674+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:45.231496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362369+0000","last_change":"2023-09-23T20:30:23.360887+0000","last_active":"2023-09-23T20:30:23.362369+0000","last_peered":"2023-09-23T20:30:23.362369+0000","last_clean":"2023-09-23T20:30:23.362369+0000","last_became_active":"2023-09-23T20:30:23.360610+0000","last_became_peered":"2023-09-23T20:30:23.360610+0000","last_unstale":"2023-09-23T20:30:23.362369+0000","last_undegraded":"2023-09-23T20:30:23.362369+0000","last_fullsized":"2023-09-23T20:30:23.362369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:04.437152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119929+0000","last_change":"2023-09-23T20:30:22.117832+0000","last_active":"2023-09-23T20:30:23.119929+0000","last_peered":"2023-09-23T20:30:23.119929+0000","last_clean":"2023-09-23T20:30:23.119929+0000","last_became_active":"2023-09-23T20:30:22.117504+0000","last_became_peered":"2023-09-23T20:30:22.117504+0000","last_unstale":"2023-09-23T20:30:23.119929+0000","last_undegraded":"2023-09-23T20:30:23.119929+0000","last_fullsized":"2023-09-23T20:30:23.119929+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:00.872807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.120466+0000","last_change":"2023-09-23T20:29:28.051973+0000","last_active":"2023-09-23T20:30:23.120466+0000","last_peered":"2023-09-23T20:30:23.120466+0000","last_clean":"2023-09-23T20:30:23.120466+0000","last_became_active":"2023-09-23T20:29:28.051665+0000","last_became_peered":"2023-09-23T20:29:28.051665+0000","last_unstale":"2023-09-23T20:30:23.120466+0000","last_undegraded":"2023-09-23T20:30:23.120466+0000","last_fullsized":"2023-09-23T20:30:23.120466+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:00.653668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362550+0000","last_change":"2023-09-23T20:30:23.362020+0000","last_active":"2023-09-23T20:30:23.362550+0000","last_peered":"2023-09-23T20:30:23.362550+0000","last_clean":"2023-09-23T20:30:23.362550+0000","last_became_active":"2023-09-23T20:30:23.361758+0000","last_became_peered":"2023-09-23T20:30:23.361758+0000","last_unstale":"2023-09-23T20:30:23.362550+0000","last_undegraded":"2023-09-23T20:30:23.362550+0000","last_fullsized":"2023-09-23T20:30:23.362550+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:10.340348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122704+0000","last_change":"2023-09-23T20:30:22.115650+0000","last_active":"2023-09-23T20:30:23.122704+0000","last_peered":"2023-09-23T20:30:23.122704+0000","last_clean":"2023-09-23T20:30:23.122704+0000","last_became_active":"2023-09-23T20:30:22.115439+0000","last_became_peered":"2023-09-23T20:30:22.115439+0000","last_unstale":"2023-09-23T20:30:23.122704+0000","last_undegraded":"2023-09-23T20:30:23.122704+0000","last_fullsized":"2023-09-23T20:30:23.122704+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:44.442798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360138+0000","last_change":"2023-09-23T20:30:22.128161+0000","last_active":"2023-09-23T20:30:23.360138+0000","last_peered":"2023-09-23T20:30:23.360138+0000","last_clean":"2023-09-23T20:30:23.360138+0000","last_became_active":"2023-09-23T20:30:22.127867+0000","last_became_peered":"2023-09-23T20:30:22.127867+0000","last_unstale":"2023-09-23T20:30:23.360138+0000","last_undegraded":"2023-09-23T20:30:23.360138+0000","last_fullsized":"2023-09-23T20:30:23.360138+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:54.545151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359969+0000","last_change":"2023-09-23T20:30:22.112377+0000","last_active":"2023-09-23T20:30:23.359969+0000","last_peered":"2023-09-23T20:30:23.359969+0000","last_clean":"2023-09-23T20:30:23.359969+0000","last_became_active":"2023-09-23T20:30:22.111971+0000","last_became_peered":"2023-09-23T20:30:22.111971+0000","last_unstale":"2023-09-23T20:30:23.359969+0000","last_undegraded":"2023-09-23T20:30:23.359969+0000","last_fullsized":"2023-09-23T20:30:23.359969+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:35.815727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360451+0000","last_change":"2023-09-23T20:30:22.257981+0000","last_active":"2023-09-23T20:30:23.360451+0000","last_peered":"2023-09-23T20:30:23.360451+0000","last_clean":"2023-09-23T20:30:23.360451+0000","last_became_active":"2023-09-23T20:30:22.257723+0000","last_became_peered":"2023-09-23T20:30:22.257723+0000","last_unstale":"2023-09-23T20:30:23.360451+0000","last_undegraded":"2023-09-23T20:30:23.360451+0000","last_fullsized":"2023-09-23T20:30:23.360451+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:21.453174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119025+0000","last_change":"2023-09-23T20:30:22.113897+0000","last_active":"2023-09-23T20:30:23.119025+0000","last_peered":"2023-09-23T20:30:23.119025+0000","last_clean":"2023-09-23T20:30:23.119025+0000","last_became_active":"2023-09-23T20:30:22.113622+0000","last_became_peered":"2023-09-23T20:30:22.113622+0000","last_unstale":"2023-09-23T20:30:23.119025+0000","last_undegraded":"2023-09-23T20:30:23.119025+0000","last_fullsized":"2023-09-23T20:30:23.119025+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:25.076236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360369+0000","last_change":"2023-09-23T20:30:22.485943+0000","last_active":"2023-09-23T20:30:23.360369+0000","last_peered":"2023-09-23T20:30:23.360369+0000","last_clean":"2023-09-23T20:30:23.360369+0000","last_became_active":"2023-09-23T20:30:22.485598+0000","last_became_peered":"2023-09-23T20:30:22.485598+0000","last_unstale":"2023-09-23T20:30:23.360369+0000","last_undegraded":"2023-09-23T20:30:23.360369+0000","last_fullsized":"2023-09-23T20:30:23.360369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:11.546769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.805898+0000","last_change":"2023-09-23T20:30:22.117748+0000","last_active":"2023-09-23T20:30:38.805898+0000","last_peered":"2023-09-23T20:30:38.805898+0000","last_clean":"2023-09-23T20:30:38.805898+0000","last_became_active":"2023-09-23T20:30:22.117467+0000","last_became_peered":"2023-09-23T20:30:22.117467+0000","last_unstale":"2023-09-23T20:30:38.805898+0000","last_undegraded":"2023-09-23T20:30:38.805898+0000","last_fullsized":"2023-09-23T20:30:38.805898+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:08.111582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362586+0000","last_change":"2023-09-23T20:30:23.361522+0000","last_active":"2023-09-23T20:30:23.362586+0000","last_peered":"2023-09-23T20:30:23.362586+0000","last_clean":"2023-09-23T20:30:23.362586+0000","last_became_active":"2023-09-23T20:30:23.361210+0000","last_became_peered":"2023-09-23T20:30:23.361210+0000","last_unstale":"2023-09-23T20:30:23.362586+0000","last_undegraded":"2023-09-23T20:30:23.362586+0000","last_fullsized":"2023-09-23T20:30:23.362586+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:25.949716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119853+0000","last_change":"2023-09-23T20:30:22.102832+0000","last_active":"2023-09-23T20:30:23.119853+0000","last_peered":"2023-09-23T20:30:23.119853+0000","last_clean":"2023-09-23T20:30:23.119853+0000","last_became_active":"2023-09-23T20:30:22.102546+0000","last_became_peered":"2023-09-23T20:30:22.102546+0000","last_unstale":"2023-09-23T20:30:23.119853+0000","last_undegraded":"2023-09-23T20:30:23.119853+0000","last_fullsized":"2023-09-23T20:30:23.119853+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:11.730297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.118930+0000","last_change":"2023-09-23T20:30:22.772628+0000","last_active":"2023-09-23T20:30:23.118930+0000","last_peered":"2023-09-23T20:30:23.118930+0000","last_clean":"2023-09-23T20:30:23.118930+0000","last_became_active":"2023-09-23T20:30:22.772350+0000","last_became_peered":"2023-09-23T20:30:22.772350+0000","last_unstale":"2023-09-23T20:30:23.118930+0000","last_undegraded":"2023-09-23T20:30:23.118930+0000","last_fullsized":"2023-09-23T20:30:23.118930+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:03.794857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.999552+0000","last_change":"2023-09-23T20:30:22.113582+0000","last_active":"2023-09-23T20:30:24.999552+0000","last_peered":"2023-09-23T20:30:24.999552+0000","last_clean":"2023-09-23T20:30:24.999552+0000","last_became_active":"2023-09-23T20:30:22.113291+0000","last_became_peered":"2023-09-23T20:30:22.113291+0000","last_unstale":"2023-09-23T20:30:24.999552+0000","last_undegraded":"2023-09-23T20:30:24.999552+0000","last_fullsized":"2023-09-23T20:30:24.999552+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:19.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.359852+0000","last_change":"2023-09-23T20:30:22.485806+0000","last_active":"2023-09-23T20:30:23.359852+0000","last_peered":"2023-09-23T20:30:23.359852+0000","last_clean":"2023-09-23T20:30:23.359852+0000","last_became_active":"2023-09-23T20:30:22.485502+0000","last_became_peered":"2023-09-23T20:30:22.485502+0000","last_unstale":"2023-09-23T20:30:23.359852+0000","last_undegraded":"2023-09-23T20:30:23.359852+0000","last_fullsized":"2023-09-23T20:30:23.359852+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:26.590680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.807986+0000","last_change":"2023-09-23T20:30:22.114034+0000","last_active":"2023-09-23T20:30:38.807986+0000","last_peered":"2023-09-23T20:30:38.807986+0000","last_clean":"2023-09-23T20:30:38.807986+0000","last_became_active":"2023-09-23T20:30:22.113740+0000","last_became_peered":"2023-09-23T20:30:22.113740+0000","last_unstale":"2023-09-23T20:30:38.807986+0000","last_undegraded":"2023-09-23T20:30:38.807986+0000","last_fullsized":"2023-09-23T20:30:38.807986+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:30.265420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119660+0000","last_change":"2023-09-23T20:30:22.772475+0000","last_active":"2023-09-23T20:30:23.119660+0000","last_peered":"2023-09-23T20:30:23.119660+0000","last_clean":"2023-09-23T20:30:23.119660+0000","last_became_active":"2023-09-23T20:30:22.772159+0000","last_became_peered":"2023-09-23T20:30:22.772159+0000","last_unstale":"2023-09-23T20:30:23.119660+0000","last_undegraded":"2023-09-23T20:30:23.119660+0000","last_fullsized":"2023-09-23T20:30:23.119660+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:08.373446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.999550+0000","last_change":"2023-09-23T20:30:22.116902+0000","last_active":"2023-09-23T20:30:24.999550+0000","last_peered":"2023-09-23T20:30:24.999550+0000","last_clean":"2023-09-23T20:30:24.999550+0000","last_became_active":"2023-09-23T20:30:22.116587+0000","last_became_peered":"2023-09-23T20:30:22.116587+0000","last_unstale":"2023-09-23T20:30:24.999550+0000","last_undegraded":"2023-09-23T20:30:24.999550+0000","last_fullsized":"2023-09-23T20:30:24.999550+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:20.938440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.364067+0000","last_change":"2023-09-23T20:30:23.363473+0000","last_active":"2023-09-23T20:30:23.364067+0000","last_peered":"2023-09-23T20:30:23.364067+0000","last_clean":"2023-09-23T20:30:23.364067+0000","last_became_active":"2023-09-23T20:30:23.363192+0000","last_became_peered":"2023-09-23T20:30:23.363192+0000","last_unstale":"2023-09-23T20:30:23.364067+0000","last_undegraded":"2023-09-23T20:30:23.364067+0000","last_fullsized":"2023-09-23T20:30:23.364067+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:03.722816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.810011+0000","last_change":"2023-09-23T20:30:22.103090+0000","last_active":"2023-09-23T20:30:38.810011+0000","last_peered":"2023-09-23T20:30:38.810011+0000","last_clean":"2023-09-23T20:30:38.810011+0000","last_became_active":"2023-09-23T20:30:22.102807+0000","last_became_peered":"2023-09-23T20:30:22.102807+0000","last_unstale":"2023-09-23T20:30:38.810011+0000","last_undegraded":"2023-09-23T20:30:38.810011+0000","last_fullsized":"2023-09-23T20:30:38.810011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:56.602788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362690+0000","last_change":"2023-09-23T20:30:23.362282+0000","last_active":"2023-09-23T20:30:23.362690+0000","last_peered":"2023-09-23T20:30:23.362690+0000","last_clean":"2023-09-23T20:30:23.362690+0000","last_became_active":"2023-09-23T20:30:23.361623+0000","last_became_peered":"2023-09-23T20:30:23.361623+0000","last_unstale":"2023-09-23T20:30:23.362690+0000","last_undegraded":"2023-09-23T20:30:23.362690+0000","last_fullsized":"2023-09-23T20:30:23.362690+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:22.136043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360475+0000","last_change":"2023-09-23T20:30:22.110586+0000","last_active":"2023-09-23T20:30:23.360475+0000","last_peered":"2023-09-23T20:30:23.360475+0000","last_clean":"2023-09-23T20:30:23.360475+0000","last_became_active":"2023-09-23T20:30:22.110248+0000","last_became_peered":"2023-09-23T20:30:22.110248+0000","last_unstale":"2023-09-23T20:30:23.360475+0000","last_undegraded":"2023-09-23T20:30:23.360475+0000","last_fullsized":"2023-09-23T20:30:23.360475+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:29.867944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119858+0000","last_change":"2023-09-23T20:30:22.773252+0000","last_active":"2023-09-23T20:30:23.119858+0000","last_peered":"2023-09-23T20:30:23.119858+0000","last_clean":"2023-09-23T20:30:23.119858+0000","last_became_active":"2023-09-23T20:30:22.772972+0000","last_became_peered":"2023-09-23T20:30:22.772972+0000","last_unstale":"2023-09-23T20:30:23.119858+0000","last_undegraded":"2023-09-23T20:30:23.119858+0000","last_fullsized":"2023-09-23T20:30:23.119858+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:57.509255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122835+0000","last_change":"2023-09-23T20:30:22.109250+0000","last_active":"2023-09-23T20:30:23.122835+0000","last_peered":"2023-09-23T20:30:23.122835+0000","last_clean":"2023-09-23T20:30:23.122835+0000","last_became_active":"2023-09-23T20:30:22.109024+0000","last_became_peered":"2023-09-23T20:30:22.109024+0000","last_unstale":"2023-09-23T20:30:23.122835+0000","last_undegraded":"2023-09-23T20:30:23.122835+0000","last_fullsized":"2023-09-23T20:30:23.122835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.856224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.363344+0000","last_change":"2023-09-23T20:30:23.360789+0000","last_active":"2023-09-23T20:30:23.363344+0000","last_peered":"2023-09-23T20:30:23.363344+0000","last_clean":"2023-09-23T20:30:23.363344+0000","last_became_active":"2023-09-23T20:30:23.360518+0000","last_became_peered":"2023-09-23T20:30:23.360518+0000","last_unstale":"2023-09-23T20:30:23.363344+0000","last_undegraded":"2023-09-23T20:30:23.363344+0000","last_fullsized":"2023-09-23T20:30:23.363344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:34.473352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808098+0000","last_change":"2023-09-23T20:30:22.114940+0000","last_active":"2023-09-23T20:30:38.808098+0000","last_peered":"2023-09-23T20:30:38.808098+0000","last_clean":"2023-09-23T20:30:38.808098+0000","last_became_active":"2023-09-23T20:30:22.114679+0000","last_became_peered":"2023-09-23T20:30:22.114679+0000","last_unstale":"2023-09-23T20:30:38.808098+0000","last_undegraded":"2023-09-23T20:30:38.808098+0000","last_fullsized":"2023-09-23T20:30:38.808098+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:12.493294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.360462+0000","last_change":"2023-09-23T20:30:22.128593+0000","last_active":"2023-09-23T20:30:23.360462+0000","last_peered":"2023-09-23T20:30:23.360462+0000","last_clean":"2023-09-23T20:30:23.360462+0000","last_became_active":"2023-09-23T20:30:22.128282+0000","last_became_peered":"2023-09-23T20:30:22.128282+0000","last_unstale":"2023-09-23T20:30:23.360462+0000","last_undegraded":"2023-09-23T20:30:23.360462+0000","last_fullsized":"2023-09-23T20:30:23.360462+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:35.401761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808983+0000","last_change":"2023-09-23T20:30:22.107118+0000","last_active":"2023-09-23T20:30:38.808983+0000","last_peered":"2023-09-23T20:30:38.808983+0000","last_clean":"2023-09-23T20:30:38.808983+0000","last_became_active":"2023-09-23T20:30:22.106839+0000","last_became_peered":"2023-09-23T20:30:22.106839+0000","last_unstale":"2023-09-23T20:30:38.808983+0000","last_undegraded":"2023-09-23T20:30:38.808983+0000","last_fullsized":"2023-09-23T20:30:38.808983+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:02.981333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.125858+0000","last_change":"2023-09-23T20:30:23.124437+0000","last_active":"2023-09-23T20:30:23.125858+0000","last_peered":"2023-09-23T20:30:23.125858+0000","last_clean":"2023-09-23T20:30:23.125858+0000","last_became_active":"2023-09-23T20:30:23.124078+0000","last_became_peered":"2023-09-23T20:30:23.124078+0000","last_unstale":"2023-09-23T20:30:23.125858+0000","last_undegraded":"2023-09-23T20:30:23.125858+0000","last_fullsized":"2023-09-23T20:30:23.125858+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:58.955735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:38.808208+0000","last_change":"2023-09-23T20:30:22.117956+0000","last_active":"2023-09-23T20:30:38.808208+0000","last_peered":"2023-09-23T20:30:38.808208+0000","last_clean":"2023-09-23T20:30:38.808208+0000","last_became_active":"2023-09-23T20:30:22.117681+0000","last_became_peered":"2023-09-23T20:30:22.117681+0000","last_unstale":"2023-09-23T20:30:38.808208+0000","last_undegraded":"2023-09-23T20:30:38.808208+0000","last_fullsized":"2023-09-23T20:30:38.808208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:13.129112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.363132+0000","last_change":"2023-09-23T20:30:23.360927+0000","last_active":"2023-09-23T20:30:23.363132+0000","last_peered":"2023-09-23T20:30:23.363132+0000","last_clean":"2023-09-23T20:30:23.363132+0000","last_became_active":"2023-09-23T20:30:23.360591+0000","last_became_peered":"2023-09-23T20:30:23.360591+0000","last_unstale":"2023-09-23T20:30:23.363132+0000","last_undegraded":"2023-09-23T20:30:23.363132+0000","last_fullsized":"2023-09-23T20:30:23.363132+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:56.124709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119761+0000","last_change":"2023-09-23T20:30:22.109674+0000","last_active":"2023-09-23T20:30:23.119761+0000","last_peered":"2023-09-23T20:30:23.119761+0000","last_clean":"2023-09-23T20:30:23.119761+0000","last_became_active":"2023-09-23T20:30:22.109370+0000","last_became_peered":"2023-09-23T20:30:22.109370+0000","last_unstale":"2023-09-23T20:30:23.119761+0000","last_undegraded":"2023-09-23T20:30:23.119761+0000","last_fullsized":"2023-09-23T20:30:23.119761+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:03.191494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.128055+0000","last_change":"2023-09-23T20:30:23.127296+0000","last_active":"2023-09-23T20:30:23.128055+0000","last_peered":"2023-09-23T20:30:23.128055+0000","last_clean":"2023-09-23T20:30:23.128055+0000","last_became_active":"2023-09-23T20:30:23.126893+0000","last_became_peered":"2023-09-23T20:30:23.126893+0000","last_unstale":"2023-09-23T20:30:23.128055+0000","last_undegraded":"2023-09-23T20:30:23.128055+0000","last_fullsized":"2023-09-23T20:30:23.128055+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:31.768430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.125184+0000","last_change":"2023-09-23T20:30:22.116853+0000","last_active":"2023-09-23T20:30:23.125184+0000","last_peered":"2023-09-23T20:30:23.125184+0000","last_clean":"2023-09-23T20:30:23.125184+0000","last_became_active":"2023-09-23T20:30:22.116609+0000","last_became_peered":"2023-09-23T20:30:22.116609+0000","last_unstale":"2023-09-23T20:30:23.125184+0000","last_undegraded":"2023-09-23T20:30:23.125184+0000","last_fullsized":"2023-09-23T20:30:23.125184+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:14.249091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119234+0000","last_change":"2023-09-23T20:30:22.131125+0000","last_active":"2023-09-23T20:30:23.119234+0000","last_peered":"2023-09-23T20:30:23.119234+0000","last_clean":"2023-09-23T20:30:23.119234+0000","last_became_active":"2023-09-23T20:30:22.130818+0000","last_became_peered":"2023-09-23T20:30:22.130818+0000","last_unstale":"2023-09-23T20:30:23.119234+0000","last_undegraded":"2023-09-23T20:30:23.119234+0000","last_fullsized":"2023-09-23T20:30:23.119234+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:55.432616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.122808+0000","last_change":"2023-09-23T20:30:22.119946+0000","last_active":"2023-09-23T20:30:23.122808+0000","last_peered":"2023-09-23T20:30:23.122808+0000","last_clean":"2023-09-23T20:30:23.122808+0000","last_became_active":"2023-09-23T20:30:22.119643+0000","last_became_peered":"2023-09-23T20:30:22.119643+0000","last_unstale":"2023-09-23T20:30:23.122808+0000","last_undegraded":"2023-09-23T20:30:23.122808+0000","last_fullsized":"2023-09-23T20:30:23.122808+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:30.646487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362707+0000","last_change":"2023-09-23T20:30:23.362393+0000","last_active":"2023-09-23T20:30:23.362707+0000","last_peered":"2023-09-23T20:30:23.362707+0000","last_clean":"2023-09-23T20:30:23.362707+0000","last_became_active":"2023-09-23T20:30:23.362005+0000","last_became_peered":"2023-09-23T20:30:23.362005+0000","last_unstale":"2023-09-23T20:30:23.362707+0000","last_undegraded":"2023-09-23T20:30:23.362707+0000","last_fullsized":"2023-09-23T20:30:23.362707+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:26.361518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.119771+0000","last_change":"2023-09-23T20:30:22.110079+0000","last_active":"2023-09-23T20:30:23.119771+0000","last_peered":"2023-09-23T20:30:23.119771+0000","last_clean":"2023-09-23T20:30:23.119771+0000","last_became_active":"2023-09-23T20:30:22.109745+0000","last_became_peered":"2023-09-23T20:30:22.109745+0000","last_unstale":"2023-09-23T20:30:23.119771+0000","last_undegraded":"2023-09-23T20:30:23.119771+0000","last_fullsized":"2023-09-23T20:30:23.119771+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:37.628634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T20:30:23.362217+0000","last_change":"2023-09-23T20:30:23.360193+0000","last_active":"2023-09-23T20:30:23.362217+0000","last_peered":"2023-09-23T20:30:23.362217+0000","last_clean":"2023-09-23T20:30:23.362217+0000","last_became_active":"2023-09-23T20:30:23.359847+0000","last_became_peered":"2023-09-23T20:30:23.359847+0000","last_unstale":"2023-09-23T20:30:23.362217+0000","last_undegraded":"2023-09-23T20:30:23.362217+0000","last_fullsized":"2023-09-23T20:30:23.362217+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:06.567927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T20:30:24.902511+0000","last_change":"2023-09-23T20:30:22.112277+0000","last_active":"2023-09-23T20:30:24.902511+0000","last_peered":"2023-09-23T20:30:24.902511+0000","last_clean":"2023-09-23T20:30:24.902511+0000","last_became_active":"2023-09-23T20:30:22.111979+0000","last_became_peered":"2023-09-23T20:30:22.111979+0000","last_unstale":"2023-09-23T20:30:24.902511+0000","last_undegraded":"2023-09-23T20:30:24.902511+0000","last_fullsized":"2023-09-23T20:30:24.902511+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:33.786341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201311,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6608,"kb_used_data":1288,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738544,"statfs":{"total":95995035648,"available":95988269056,"internally_reserved":0,"allocated":1318912,"data_stored":894173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.47899999999999998,"15min":0.48299999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.59299999999999997,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.47099999999999997,"15min":0.46500000000000002},"min":{"1min":0.32100000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.59499999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.50900000000000001,"15min":0.51300000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.32600000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.61699999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.52200000000000002,"15min":0.53400000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67100000000000004,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.35099999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.64700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.51200000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.68300000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.43099999999999999,"15min":0.434},"min":{"1min":0.309,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60999999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47899999999999998,"15min":0.47999999999999998},"min":{"1min":0.26500000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.65100000000000002,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.69699999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.46999999999999997,"15min":0.46600000000000003},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.503,"5min":0.501,"15min":0.501},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.57399999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:30:52 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.40300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.69999999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.39000000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.68700000000000006,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.73999999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462948,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6592,"kb_used_data":1272,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738560,"statfs":{"total":95995035648,"available":95988285440,"internally_reserved":0,"allocated":1302528,"data_stored":893820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.48199999999999998,"15min":0.48799999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.59999999999999998,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.70999999999999996,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.41299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.53900000000000003,"15min":0.54600000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.65300000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.622},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.54000000000000004,"15min":0.54500000000000004},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.67700000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.55600000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.48999999999999999,"15min":0.49199999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.61699999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.498},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.47799999999999998,"15min":0.47599999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.67500000000000004,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34200000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76700000000000002,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70899999999999996,"5min":0.877,"15min":0.877},"last":0.59899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.40300000000000002,"15min":0.39300000000000002},"min":{"1min":0.33100000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.434,"15min":0.42399999999999999},"min":{"1min":0.30299999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.71899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.438,"15min":0.43099999999999999},"min":{"1min":0.378,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.46800000000000003,"15min":0.45800000000000002},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.5,"15min":0.48999999999999999},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.53500000000000003,"15min":0.52500000000000002},"min":{"1min":0.44600000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.75600000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574906,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6772,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738380,"statfs":{"total":95995035648,"available":95988101120,"internally_reserved":0,"allocated":700416,"data_stored":302376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.35199999999999998,"15min":0.311},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.59099999999999997,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.374,"15min":0.33200000000000002},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.36799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:31:08 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.40200000000000002,"15min":0.38900000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.42699999999999999,"15min":0.41699999999999998},"min":{"1min":0.28899999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.28699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:30:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.45400000000000001,"15min":0.45800000000000002},"min":{"1min":0.30299999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.502,"5min":0.621,"15min":0.621},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.46800000000000003,"15min":0.47599999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.56899999999999995,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.622,"15min":0.626},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.65600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:30:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.58999999999999997,"15min":0.59599999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.626}]},{"osd":6,"last update":"Sat Sep 23 20:30:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.62,"15min":0.63200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57999999999999996,"15min":0.58299999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.67400000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:30:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60399999999999998,"15min":0.60799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.53700000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738432,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7360,"kb_used_data":1272,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93737792,"statfs":{"total":95995035648,"available":95987499008,"internally_reserved":0,"allocated":1302528,"data_stored":893320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.35199999999999998,"15min":0.33900000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.40000000000000002,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.40300000000000002,"15min":0.35899999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":3.9969999999999999,"5min":3.9969999999999999,"15min":3.9969999999999999},"last":0.32200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":0.38400000000000001,"15min":0.39600000000000002},"min":{"1min":0.24399999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.46700000000000003,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40699999999999997,"15min":0.40600000000000003},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.497,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.44800000000000001,"15min":0.45900000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.61699999999999999,"5min":0.75,"15min":0.75},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.60599999999999998,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.56299999999999994,"15min":0.54500000000000004},"min":{"1min":0.39400000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.9409999999999998,"5min":3.9409999999999998,"15min":3.9409999999999998},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.56699999999999995,"15min":0.57599999999999996},"min":{"1min":0.38800000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:31:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.57499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40999999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.67500000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.625},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.58099999999999996,"15min":0.59499999999999997},"min":{"1min":0.41699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81799999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:30:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.44400000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.61799999999999999,"15min":0.60199999999999998},"min":{"1min":0.38,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.8900000000000001,"5min":3.8900000000000001,"15min":3.8900000000000001},"last":0.73699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.70399999999999996,"15min":0.72799999999999998},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.314},"max":{"1min":0.77000000000000002,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55600000000000005,"15min":0.55500000000000005},"min":{"1min":0.438,"5min":0.379,"15min":0.379},"max":{"1min":0.69299999999999995,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.64200000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411382,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6588,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738564,"statfs":{"total":95995035648,"available":95988289536,"internally_reserved":0,"allocated":708608,"data_stored":302964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.36599999999999999,"15min":0.37},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.374,"15min":0.37},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.27700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:31:04 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38,"15min":0.371},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.379,"15min":0.373},"min":{"1min":0.247,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54300000000000004,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.147,"5min":1.147,"15min":1.147},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.44,"15min":0.44800000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68200000000000005,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.312}]},{"osd":4,"last update":"Sat Sep 23 20:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.58599999999999997,"15min":0.56999999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":3.3119999999999998,"5min":3.3119999999999998,"15min":3.3119999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.57499999999999996,"15min":0.56200000000000006},"min":{"1min":0.42199999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.2229999999999999,"5min":3.2229999999999999,"15min":3.2229999999999999},"last":0.56399999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:30:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58499999999999996,"15min":0.58699999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.73899999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.59199999999999997,"15min":0.59699999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.749,"5min":0.752,"15min":0.752},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.58399999999999996,"15min":0.57199999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.2909999999999999,"5min":3.2909999999999999,"15min":3.2909999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.58599999999999997,"15min":0.57299999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":3.3849999999999998},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:30:57 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.58199999999999996,"15min":0.56299999999999994},"min":{"1min":0.39200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.3479999999999999,"5min":3.3479999999999999,"15min":3.3479999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.60899999999999999,"15min":0.59099999999999997},"min":{"1min":0.41299999999999998,"5min":0.376,"15min":0.376},"max":{"1min":3.258,"5min":3.258,"15min":3.258},"last":0.41399999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149745,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6136,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739016,"statfs":{"total":95995035648,"available":95988752384,"internally_reserved":0,"allocated":704512,"data_stored":302466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":0.36799999999999999,"15min":0.374},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.45900000000000002,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.42399999999999999,"15min":0.437},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.48099999999999998,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.44,"15min":0.46000000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.47599999999999998,"5min":0.623,"15min":0.623},"last":0.318},{"interface":"front","average":{"1min":0.376,"5min":0.44900000000000001,"15min":0.46999999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.59899999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.36499999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.47199999999999998,"15min":0.48699999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.52000000000000002,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.46700000000000003,"15min":0.48499999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.61199999999999999,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.30199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.55700000000000005,"15min":0.57499999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73799999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.56100000000000005,"15min":0.57499999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68700000000000006,"5min":0.751,"15min":0.751},"last":0.39800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.76200000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:30:30 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.65800000000000003,"15min":0.68400000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72599999999999998,"5min":2.903,"15min":2.903},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.57599999999999996,"15min":0.58299999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.68200000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:30:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.59699999999999998,"15min":0.60299999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77300000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.42899999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986221,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6220,"kb_used_data":708,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738932,"statfs":{"total":95995035648,"available":95988666368,"internally_reserved":0,"allocated":724992,"data_stored":304058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.68999999999999995,"15min":0.70799999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.7759999999999998,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":0.754},{"interface":"front","average":{"1min":0.501,"5min":0.63200000000000001,"15min":0.68400000000000005},"min":{"1min":0.34200000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78000000000000003,"5min":3.9489999999999998,"15min":3.9489999999999998},"last":0.65900000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.72899999999999998,"15min":0.73299999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.4960000000000004,"5min":9.4960000000000004,"15min":9.4960000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.53100000000000003,"15min":0.54700000000000004},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.74099999999999999,"5min":1.425,"15min":1.425},"last":0.70299999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.59799999999999998,"15min":0.58199999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":5.9160000000000004,"5min":5.9160000000000004,"15min":5.9160000000000004},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.68999999999999995,"15min":0.70999999999999996},"min":{"1min":0.34399999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.8799999999999999,"5min":5.8799999999999999,"15min":5.8799999999999999},"last":0.68899999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.73899999999999999,"15min":0.73599999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":9.5630000000000006,"5min":9.5630000000000006,"15min":9.5630000000000006},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.74199999999999999,"15min":0.73599999999999999},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.6170000000000009,"5min":9.6170000000000009,"15min":9.6170000000000009},"last":0.64300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:31:09 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.42599999999999999,"15min":0.41799999999999998},"min":{"1min":0.317,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.58699999999999997,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.56399999999999995,"15min":0.59599999999999997},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.66000000000000003,"5min":3.887,"15min":3.887},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:30:29 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.438,"15min":0.43099999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.45100000000000001,"15min":0.44400000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.55700000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.38900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.71899999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724585,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6076,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739076,"statfs":{"total":95995035648,"available":95988813824,"internally_reserved":0,"allocated":708608,"data_stored":303000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.53900000000000003,"15min":0.54800000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67500000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.434,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67800000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.51000000000000001,"15min":0.51500000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.62},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.51700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.623,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.505,"15min":0.51000000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.1539999999999999,"5min":2.1539999999999999,"15min":2.1539999999999999},"last":0.54500000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.497,"15min":0.499},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.35299999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":0.434}]},{"osd":4,"last update":"Sat Sep 23 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.40000000000000002,"15min":0.39600000000000002},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.628,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.41999999999999998,"15min":0.41299999999999998},"min":{"1min":0.34599999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.60099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:30:33 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.65100000000000002,"5min":1.54,"15min":1.54},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.504,"5min":0.47399999999999998,"15min":0.46899999999999997},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:30:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66000000000000003,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68700000000000006,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.57399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:13.796 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:31:13.796 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:31:13.796 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:31:13.796 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:31:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:13 smithi123 ceph-mon[104968]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:13 smithi123 ceph-mon[104968]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:14.143 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi121.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:31:16.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:15 smithi123 ceph-mon[104968]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:15 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/32628116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:31:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:15 smithi121 ceph-mon[100258]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:15 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/32628116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:31:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:15 smithi121 ceph-mon[118228]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:15 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/32628116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:31:16.641 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:31:16.642 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:31:16.650 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T20:31:16.650 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T20:31:16.661 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T20:31:16.662 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:31:16.662 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T20:31:16.663 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:31:16.664 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:31:16.664 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T20:31:16.664 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T20:31:16.665 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:31:16.665 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T20:31:16.665 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi121.front.sepia.ceph.com 2023-09-23T20:31:16.665 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T20:31:17.030 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:17.074 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:17.954 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:31:17 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:31:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:17 smithi121 ceph-mon[100258]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:18.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:17 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:17] "GET /metrics HTTP/1.1" 200 238752 "" "Prometheus/2.43.0" 2023-09-23T20:31:18.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:17 smithi121 ceph-mon[118228]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:17 smithi123 ceph-mon[104968]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:19.965 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_0' created 2023-09-23T20:31:19.968 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:19 smithi121 conmon[100681]: [23/Sep/2023:20:31:19] ENGINE Bus STOPPING 2023-09-23T20:31:19.968 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:19 smithi121 conmon[100681]: [23/Sep/2023:20:31:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:31:19.968 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:19 smithi121 conmon[100681]: [23/Sep/2023:20:31:19] ENGINE Bus STOPPED 2023-09-23T20:31:19.968 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:19 smithi121 conmon[100681]: [23/Sep/2023:20:31:19] ENGINE Bus STARTING 2023-09-23T20:31:19.968 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:19 smithi121 conmon[100681]: [23/Sep/2023:20:31:19] ENGINE Serving on http://:::9283 2023-09-23T20:31:19.968 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:19 smithi121 conmon[100681]: [23/Sep/2023:20:31:19] ENGINE Bus STARTED 2023-09-23T20:31:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:19 smithi123 ceph-mon[104968]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:19 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/584860869' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:31:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:19 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' 2023-09-23T20:31:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:19 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:19 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:20.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:19 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:20.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:19 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[100258]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/584860869' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[100258]: from='client.? ' entity='client.admin' 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:20.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[118228]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/584860869' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:31:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' 2023-09-23T20:31:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:19 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:20.529 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T20:31:21.035 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T20:31:21.102 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:21.349 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/584860869' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:31:21.349 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:20 smithi121 ceph-mon[100258]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:31:21.349 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:20 smithi121 ceph-mon[100258]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:31:21.349 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:20 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/584860869' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:31:21.349 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:20 smithi121 ceph-mon[118228]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:31:21.349 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:20 smithi121 ceph-mon[118228]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:31:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:20 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/584860869' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:31:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:20 smithi123 ceph-mon[104968]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:31:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:20 smithi123 ceph-mon[104968]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:31:22.196 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:22.249 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:21 smithi121 ceph-mon[118228]: pgmap v71: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:22.249 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:21 smithi121 ceph-mon[118228]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:31:22.249 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:21 smithi121 ceph-mon[100258]: pgmap v71: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:22.249 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:21 smithi121 ceph-mon[100258]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:31:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:21 smithi123 ceph-mon[104968]: pgmap v71: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:22.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:21 smithi123 ceph-mon[104968]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:31:23.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:23 smithi121 ceph-mon[118228]: pgmap v74: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:23 smithi121 ceph-mon[100258]: pgmap v74: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:23.935 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T20:31:23.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:23 smithi123 ceph-mon[104968]: pgmap v74: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:24.287 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:24.518 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T20:31:24.807 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:24 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/729019108' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:31:24.807 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:24 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/729019108' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:31:24.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:24 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/729019108' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:31:25.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/729019108' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:31:25.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[100258]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:31:25.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[100258]: pgmap v76: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:25.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:25.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/729019108' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:31:25.906 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[118228]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:31:25.906 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[118228]: pgmap v76: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:25.906 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:25 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:25 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/729019108' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:31:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:25 smithi123 ceph-mon[104968]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:31:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:25 smithi123 ceph-mon[104968]: pgmap v76: 89 pgs: 16 creating+peering, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:25 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:26.612 INFO:teuthology.orchestra.run.smithi121.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:31:26.698 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T20:31:26.700 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:31:26.701 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:31:26.746 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T20:31:26.746 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:31:26.746 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_189847 2023-09-23T20:31:26.747 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:31:26.747 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:31:27.743 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:31:27.747 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 3989 3973 248.104 248.312 0.00620964 0.00401731 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:27.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[100258]: pgmap v77: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:27.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:31:27.865 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:27 smithi121 ceph-mon[118228]: pgmap v77: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:27.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:31:27 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:31:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:27 smithi123 ceph-mon[104968]: pgmap v77: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:28.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:27 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:27] "GET /metrics HTTP/1.1" 200 241036 "" "Prometheus/2.43.0" 2023-09-23T20:31:28.415 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:28.747 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 7187 7171 223.989 199.875 0.00646906 0.00445131 2023-09-23T20:31:29.748 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 10509 10493 218.53 207.625 0.00264384 0.00457013 2023-09-23T20:31:30.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:29 smithi123 ceph-mon[104968]: pgmap v78: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:30.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:29 smithi121 ceph-mon[100258]: pgmap v78: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:29 smithi121 ceph-mon[118228]: pgmap v78: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:31:30.747 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 13705 13689 213.83 199.75 0.00246435 0.00467154 2023-09-23T20:31:31.748 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 16688 16672 208.349 186.438 0.00438237 0.00479374 2023-09-23T20:31:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:31 smithi123 ceph-mon[104968]: pgmap v79: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:31:32.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:31 smithi121 ceph-mon[118228]: pgmap v79: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:31:32.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:31 smithi121 ceph-mon[100258]: pgmap v79: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:31:32.748 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 19254 19238 200.352 160.375 0.00575989 0.00498546 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.a smithi121 *:9093,9094 running (66s) 22s ago 108s 20.5M - 0.25.0 c8568f914cd2 e6f06c4a5526 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.a smithi123 *:3000 running (85s) 28s ago 85s 88.4M - 9.4.7 2c41d148cca3 b89ae83624e0 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi121.sdlxlb smithi121 running (55s) 22s ago 54s 14.6M - 17.2.6-1123-g61431f6b 0db14aa01094 c7389c893171 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi123.clbyhs smithi123 running (52s) 28s ago 52s 12.4M - 17.2.6-1123-g61431f6b 0db14aa01094 1704af1c5a0c 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.x smithi123 *:8443 running (5m) 28s ago 5m 439M - 17.2.6-1123-g61431f6b 0db14aa01094 24010296b9ff 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.y smithi121 *:9283 running (7m) 22s ago 7m 495M - 17.2.6-1123-g61431f6b 0db14aa01094 666708c47463 2023-09-23T20:31:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a smithi121 running (7m) 22s ago 7m 85.6M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 081ecc327acf 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b smithi123 running (6m) 28s ago 6m 59.4M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 6a36337c715a 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c smithi121 running (6m) 22s ago 6m 55.4M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 a0406cc28771 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.a smithi121 *:9100 running (80s) 22s ago 79s 16.8M - 1.5.0 0da6a335fe13 0e30e04cd029 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.b smithi123 *:9100 running (75s) 28s ago 74s 9609k - 1.5.0 0da6a335fe13 cedf94396cbb 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (5m) 22s ago 5m 57.5M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 ae38a40da032 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (5m) 22s ago 5m 62.5M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 b5379433baef 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (4m) 22s ago 4m 50.6M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 2c9b742b8c06 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (4m) 22s ago 4m 51.5M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 dcc588b3144b 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi123 running (3m) 28s ago 3m 51.5M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 470c2a4d010b 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi123 running (3m) 28s ago 3m 49.3M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 ab39a1466c34 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi123 running (3m) 28s ago 3m 49.0M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 3f6aba9e3a6b 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi123 running (2m) 28s ago 2m 51.6M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 514f59904aaa 2023-09-23T20:31:32.779 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.a smithi123 *:9095 running (61s) 28s ago 2m 39.0M - 2.43.0 a07b618ecd1d cb4aa7cfed81 2023-09-23T20:31:33.748 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 21888 21872 195.247 164.625 0.00510145 0.00511667 2023-09-23T20:31:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:33 smithi121 ceph-mon[100258]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:31:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:33 smithi121 ceph-mon[100258]: pgmap v80: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:31:33.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:33 smithi121 ceph-mon[118228]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:31:33.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:33 smithi121 ceph-mon[118228]: pgmap v80: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:31:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:33 smithi123 ceph-mon[104968]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:31:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:33 smithi123 ceph-mon[104968]: pgmap v80: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:31:34.748 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 24121 24105 188.285 139.562 0.00146551 0.00524038 2023-09-23T20:31:35.330 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T20:31:35.751 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 26254 26238 182.143 133.312 0.00638593 0.00548531 2023-09-23T20:31:36.001 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:36.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:35 smithi123 ceph-mon[104968]: pgmap v81: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:31:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:35 smithi121 ceph-mon[100258]: pgmap v81: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:31:36.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:35 smithi121 ceph-mon[118228]: pgmap v81: 89 pgs: 89 active+clean; 883 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-09-23T20:31:36.750 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 28348 28332 177.016 130.875 0.010542 0.00564492 2023-09-23T20:31:37.750 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 30427 30411 172.736 129.938 0.00248232 0.00578284 2023-09-23T20:31:37.973 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:31:37 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:31:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:37 smithi121 ceph-mon[100258]: pgmap v82: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:31:38.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:37 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:37] "GET /metrics HTTP/1.1" 200 241036 "" "Prometheus/2.43.0" 2023-09-23T20:31:38.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:37 smithi121 ceph-mon[118228]: pgmap v82: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:31:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:37 smithi123 ceph-mon[104968]: pgmap v82: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:31:38.750 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 32173 32157 167.435 109.125 0.00823216 0.00596777 2023-09-23T20:31:39.302 INFO:teuthology.orchestra.run.smithi121.stdout:wait for minority of mons to upgrade 2023-09-23T20:31:39.750 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 33905 33889 162.882 108.25 0.00863109 0.00613493 2023-09-23T20:31:40.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:39 smithi121 ceph-mon[118228]: pgmap v83: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:31:40.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:39 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:40.419 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:39 smithi121 ceph-mon[100258]: pgmap v83: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:31:40.419 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:39 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:39 smithi123 ceph-mon[104968]: pgmap v83: 89 pgs: 89 active+clean; 1.6 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T20:31:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:39 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:40.750 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 35267 35251 157.329 85.125 0.0136424 0.00635019 2023-09-23T20:31:40.937 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T20:31:41.650 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:41.751 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 37013 36997 154.103 109.125 0.00945294 0.00648415 2023-09-23T20:31:42.178 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:41 smithi121 ceph-mon[118228]: pgmap v84: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 185 MiB/s wr, 2.96k op/s 2023-09-23T20:31:42.179 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:41 smithi121 ceph-mon[100258]: pgmap v84: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 185 MiB/s wr, 2.96k op/s 2023-09-23T20:31:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:41 smithi123 ceph-mon[104968]: pgmap v84: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 185 MiB/s wr, 2.96k op/s 2023-09-23T20:31:42.751 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 38784 38768 151.39 110.688 0.0136979 0.00660128 2023-09-23T20:31:43.751 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 40022 40006 147.036 77.375 0.00206629 0.0067814 2023-09-23T20:31:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:43 smithi121 ceph-mon[100258]: pgmap v85: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:31:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:43 smithi121 ceph-mon[118228]: pgmap v85: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:31:43.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:43 smithi123 ceph-mon[104968]: pgmap v85: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:31:44.752 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 41564 41548 144.222 96.375 0.0109498 0.00692649 2023-09-23T20:31:45.753 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 43182 43166 141.954 101.125 0.00790769 0.00703978 2023-09-23T20:31:46.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:45 smithi121 ceph-mon[100258]: pgmap v86: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:31:46.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:45 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2112268111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:46.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:45 smithi121 ceph-mon[118228]: pgmap v86: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:31:46.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:45 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2112268111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:46.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:45 smithi123 ceph-mon[104968]: pgmap v86: 89 pgs: 89 active+clean; 2.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:31:46.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:45 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2112268111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:46.752 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:31:46.752464+0000 min lat: 0.00113584 max lat: 0.336687 avg lat: 0.00711402 2023-09-23T20:31:46.752 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:31:46.752 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 44976 44960 140.462 112.125 0.00949497 0.00711402 2023-09-23T20:31:47.752 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 15 46557 46542 138.482 98.875 0.0076193 0.00721548 2023-09-23T20:31:47.978 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:31:47 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:31:48.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:48 smithi121 ceph-mon[100258]: pgmap v87: 89 pgs: 89 active+clean; 2.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2023-09-23T20:31:48.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:48 smithi121 ceph-mon[118228]: pgmap v87: 89 pgs: 89 active+clean; 2.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2023-09-23T20:31:48.166 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:47 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:47] "GET /metrics HTTP/1.1" 200 242080 "" "Prometheus/2.43.0" 2023-09-23T20:31:48.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:48 smithi123 ceph-mon[104968]: pgmap v87: 89 pgs: 89 active+clean; 2.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2023-09-23T20:31:48.755 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 48106 48090 136.584 96.75 0.0105581 0.00731712 2023-09-23T20:31:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:49 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2037239881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:49 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2037239881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:49 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2037239881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:49.752 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 49679 49663 134.92 98.3125 0.0133902 0.0074068 2023-09-23T20:31:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:50 smithi121 ceph-mon[100258]: pgmap v88: 89 pgs: 89 active+clean; 2.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:31:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:50 smithi121 ceph-mon[118228]: pgmap v88: 89 pgs: 89 active+clean; 2.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:31:50.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:50 smithi123 ceph-mon[104968]: pgmap v88: 89 pgs: 89 active+clean; 2.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:31:50.762 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 51041 51025 132.828 85.125 0.00334666 0.00752287 2023-09-23T20:31:51.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:51 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3668632768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:51.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:51 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3668632768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:51.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:51 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3668632768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:51.755 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 52481 52465 131.115 90 0.013098 0.00762207 2023-09-23T20:31:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:52 smithi121 ceph-mon[100258]: pgmap v89: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T20:31:52.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:52 smithi121 ceph-mon[118228]: pgmap v89: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T20:31:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:52 smithi123 ceph-mon[104968]: pgmap v89: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T20:31:52.755 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 53951 53935 129.606 91.875 0.0112122 0.00771018 2023-09-23T20:31:53.756 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 55282 55266 127.887 83.1875 0.00345386 0.00781393 2023-09-23T20:31:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:53 smithi121 ceph-mon[118228]: pgmap v90: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:31:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:53 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3055650287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:53.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:53 smithi121 ceph-mon[100258]: pgmap v90: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:31:53.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:53 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3055650287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:53 smithi123 ceph-mon[104968]: pgmap v90: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:31:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:53 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3055650287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:54.756 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 56720 56704 126.53 89.875 0.0180424 0.00789845 2023-09-23T20:31:55.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:55 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:55.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:55 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:55.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:55 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:55.757 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 57987 57971 124.893 79.1875 0.00720709 0.00800044 2023-09-23T20:31:56.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:56 smithi121 ceph-mon[118228]: pgmap v91: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:31:56.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:56 smithi121 ceph-mon[100258]: pgmap v91: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:31:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:56 smithi123 ceph-mon[104968]: pgmap v91: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:31:56.757 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 59299 59283 123.46 82 0.0105989 0.00809507 2023-09-23T20:31:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:57 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1170860682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:57.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:57 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1170860682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:57.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:57 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1170860682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:57.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 60688 60672 122.278 86.8125 0.0120471 0.00817374 2023-09-23T20:31:57.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:31:57 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:31:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:58 smithi121 ceph-mon[100258]: pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:31:58.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:31:57 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:31:57] "GET /metrics HTTP/1.1" 200 242214 "" "Prometheus/2.43.0" 2023-09-23T20:31:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:58 smithi121 ceph-mon[118228]: pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:31:58.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:58 smithi123 ceph-mon[104968]: pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-09-23T20:31:58.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 61936 61920 120.894 78 0.00938884 0.0082665 2023-09-23T20:31:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:31:59 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2498737068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:59.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:31:59 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2498737068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:31:59 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2498737068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:31:59.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 63377 63361 119.96 90.0625 0.0128782 0.00833206 2023-09-23T20:32:00.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:00 smithi121 ceph-mon[118228]: pgmap v93: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:32:00.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:00 smithi121 ceph-mon[100258]: pgmap v93: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:32:00.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:00 smithi123 ceph-mon[104968]: pgmap v93: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T20:32:00.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 64882 64866 119.198 94.0625 0.0101611 0.00838511 2023-09-23T20:32:01.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 66331 66315 118.38 90.5625 0.00557989 0.00844273 2023-09-23T20:32:02.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:02 smithi121 ceph-mon[118228]: pgmap v94: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:32:02.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:02 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2801847756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:02 smithi121 ceph-mon[100258]: pgmap v94: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:32:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2801847756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:02 smithi123 ceph-mon[104968]: pgmap v94: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:32:02.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:02 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2801847756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:02.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 67823 67807 117.682 93.25 0.00683228 0.00849274 2023-09-23T20:32:03.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 69259 69243 116.927 89.75 0.00661464 0.00854734 2023-09-23T20:32:03.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:03 smithi121 ceph-mon[118228]: pgmap v95: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:03.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:03 smithi121 ceph-mon[100258]: pgmap v95: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:03.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:03 smithi123 ceph-mon[104968]: pgmap v95: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:04.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 70674 70658 116.177 88.4375 0.00640948 0.0086032 2023-09-23T20:32:04.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1155110395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:04.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:04 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1155110395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:04.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:04 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1155110395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:05.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 72104 72088 115.49 89.375 0.0127137 0.0086542 2023-09-23T20:32:05.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:05 smithi121 ceph-mon[100258]: pgmap v96: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:05 smithi121 ceph-mon[118228]: pgmap v96: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:05 smithi123 ceph-mon[104968]: pgmap v96: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:06.758 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:32:06.759150+0000 min lat: 0.00113584 max lat: 0.336687 avg lat: 0.00871495 2023-09-23T20:32:06.759 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:32:06.759 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 73437 73421 114.686 83.3125 0.0125154 0.00871495 2023-09-23T20:32:07.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:06 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4155318018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:06 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4155318018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:06 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4155318018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:07.761 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 74807 74791 113.974 85.625 0.0171726 0.0087703 2023-09-23T20:32:07.889 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:32:07 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:32:08.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:32:07 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:07] "GET /metrics HTTP/1.1" 200 242214 "" "Prometheus/2.43.0" 2023-09-23T20:32:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:07 smithi121 ceph-mon[100258]: pgmap v97: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:32:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:07 smithi121 ceph-mon[118228]: pgmap v97: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:32:08.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:07 smithi123 ceph-mon[104968]: pgmap v97: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:32:08.760 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 76169 76153 113.287 85.125 0.0130444 0.00882266 2023-09-23T20:32:09.762 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 77458 77442 112.52 80.5625 0.00743656 0.00888322 2023-09-23T20:32:10.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:09 smithi121 ceph-mon[100258]: pgmap v98: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:32:10.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:09 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3605574476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:10.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:09 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:10.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:09 smithi121 ceph-mon[118228]: pgmap v98: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:32:10.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:09 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3605574476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:10.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:09 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:09 smithi123 ceph-mon[104968]: pgmap v98: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:32:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:09 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3605574476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:09 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:10.762 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 78765 78749 111.819 81.6875 0.0127336 0.00893858 2023-09-23T20:32:11.762 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 79986 79970 111.03 76.3125 0.0149561 0.00900161 2023-09-23T20:32:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:11 smithi121 ceph-mon[100258]: pgmap v99: 89 pgs: 89 active+clean; 4.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:32:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:11 smithi121 ceph-mon[118228]: pgmap v99: 89 pgs: 89 active+clean; 4.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:32:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:12 smithi123 ceph-mon[104968]: pgmap v99: 89 pgs: 89 active+clean; 4.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:32:12.762 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 81271 81255 110.362 80.3125 0.0119039 0.00905689 2023-09-23T20:32:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:12 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1644831033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:12 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1644831033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:12 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1644831033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:13.762 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 82572 82556 109.744 81.3125 0.0157587 0.0091075 2023-09-23T20:32:14.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:14 smithi121 ceph-mon[100258]: pgmap v100: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:14.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:14 smithi121 ceph-mon[118228]: pgmap v100: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:14 smithi123 ceph-mon[104968]: pgmap v100: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:14.762 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 83766 83750 109.013 74.625 0.0164484 0.00916874 2023-09-23T20:32:15.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:15 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1777596783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:15 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1777596783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:15.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:15 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1777596783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:15.763 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 84955 84939 108.305 74.3125 0.00953879 0.00922921 2023-09-23T20:32:16.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:16 smithi123 ceph-mon[104968]: pgmap v101: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:16.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:16 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:16.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:16 smithi123 ceph-mon[104968]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:32:16.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:16 smithi123 ceph-mon[104968]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:32:16.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:16 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:32:16.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:16 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:16.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:16 smithi123 ceph-mon[104968]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:32:16.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[100258]: pgmap v101: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:16.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:16.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[100258]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:32:16.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[100258]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[100258]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[118228]: pgmap v101: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[118228]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[118228]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:16.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:16 smithi121 ceph-mon[118228]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:32:16.765 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 86125 86109 107.596 73.125 0.0168902 0.00929021 2023-09-23T20:32:17.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:17 smithi123 ceph-mon[104968]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi123 2023-09-23T20:32:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:17 smithi121 ceph-mon[100258]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi123 2023-09-23T20:32:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:17 smithi121 ceph-mon[118228]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi123 2023-09-23T20:32:17.765 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 87204 87188 106.808 67.4375 0.00783082 0.00935809 2023-09-23T20:32:17.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:32:17 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:32:18.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:32:17 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:17] "GET /metrics HTTP/1.1" 200 242286 "" "Prometheus/2.43.0" 2023-09-23T20:32:18.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:18 smithi123 ceph-mon[104968]: pgmap v102: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T20:32:18.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:18 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/596432777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:18.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:18 smithi121 ceph-mon[100258]: pgmap v102: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T20:32:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:18 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/596432777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:18 smithi121 ceph-mon[118228]: pgmap v102: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2023-09-23T20:32:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:18 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/596432777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:18.765 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 88269 88253 106.035 66.5625 0.0210172 0.00942635 2023-09-23T20:32:19.768 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 89334 89318 105.285 66.5625 0.0109214 0.00949408 2023-09-23T20:32:20.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:20 smithi123 ceph-mon[104968]: pgmap v103: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:32:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:20 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1674339473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:20.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:20 smithi121 ceph-mon[118228]: pgmap v103: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:32:20.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:20 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1674339473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:20.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:20 smithi121 ceph-mon[100258]: pgmap v103: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:32:20.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1674339473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:20.768 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 90396 90380 104.564 66.375 0.0218727 0.00955933 2023-09-23T20:32:21.768 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 91462 91446 103.875 66.625 0.025763 0.00962289 2023-09-23T20:32:22.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:22 smithi123 ceph-mon[104968]: pgmap v104: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:32:22.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:22 smithi121 ceph-mon[100258]: pgmap v104: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:32:22.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:22 smithi121 ceph-mon[118228]: pgmap v104: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:32:22.769 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 92465 92449 103.138 62.6875 0.0317583 0.00969162 2023-09-23T20:32:23.769 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 93464 93448 102.424 62.4375 0.0155073 0.00975857 2023-09-23T20:32:23.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:23 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1504409270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:23.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:23 smithi121 ceph-mon[100258]: pgmap v105: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:32:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:23 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1504409270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:23 smithi121 ceph-mon[118228]: pgmap v105: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:32:23.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:23 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1504409270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:23.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:23 smithi123 ceph-mon[104968]: pgmap v105: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:32:24.769 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 94472 94456 101.744 63 0.0187721 0.0098236 2023-09-23T20:32:25.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T20:32:25.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T20:32:25.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T20:32:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T20:32:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:24 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:24 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T20:32:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:24 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:24 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:24 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T20:32:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:24 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:24 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:32:25.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:24 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:25.769 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 95455 95439 101.061 61.4375 0.0143741 0.00989047 2023-09-23T20:32:26.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[100258]: pgmap v106: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-09-23T20:32:26.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T20:32:26.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T20:32:26.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[100258]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4210737685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[118228]: pgmap v106: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[118228]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:32:26.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:26 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4210737685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:26.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:26 smithi123 ceph-mon[104968]: pgmap v106: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-09-23T20:32:26.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T20:32:26.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T20:32:26.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:26 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T20:32:26.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:26 smithi123 ceph-mon[104968]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:32:26.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:26 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4210737685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:26.769 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:32:26.770188+0000 min lat: 0.00113584 max lat: 0.336687 avg lat: 0.00995466 2023-09-23T20:32:26.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:32:26.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 96446 96430 100.409 61.9375 0.00896183 0.00995466 2023-09-23T20:32:27.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:27 smithi121 ceph-mon[118228]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:32:27.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:27 smithi121 ceph-mon[100258]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:32:27.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:27 smithi123 ceph-mon[104968]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:32:27.769 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 97424 97408 99.7653 61.125 0.0200793 0.0100192 2023-09-23T20:32:27.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:32:27 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:32:28.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:32:27 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:27] "GET /metrics HTTP/1.1" 200 242359 "" "Prometheus/2.43.0" 2023-09-23T20:32:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:28 smithi121 ceph-mon[118228]: pgmap v109: 89 pgs: 3 peering, 86 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:32:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:28 smithi121 ceph-mon[118228]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:32:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:28 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4217660881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:28 smithi121 ceph-mon[100258]: pgmap v109: 89 pgs: 3 peering, 86 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:32:28.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:28 smithi121 ceph-mon[100258]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:32:28.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:28 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4217660881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:28 smithi123 ceph-mon[104968]: pgmap v109: 89 pgs: 3 peering, 86 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T20:32:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:28 smithi123 ceph-mon[104968]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:32:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:28 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4217660881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:28.769 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 98405 98389 99.1452 61.3125 0.0106595 0.0100814 2023-09-23T20:32:29.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 99390 99374 98.5487 61.5625 0.00689029 0.0101427 2023-09-23T20:32:30.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:30 smithi121 ceph-mon[100258]: pgmap v110: 89 pgs: 3 peering, 86 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2023-09-23T20:32:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:30 smithi121 ceph-mon[118228]: pgmap v110: 89 pgs: 3 peering, 86 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2023-09-23T20:32:30.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:30 smithi123 ceph-mon[104968]: pgmap v110: 89 pgs: 3 peering, 86 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2023-09-23T20:32:30.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 100348 100332 97.9446 59.875 0.0191934 0.0102054 2023-09-23T20:32:31.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:31 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3116771003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:31 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3116771003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:31 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3116771003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:31.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 101325 101309 97.3772 61.0625 0.0221299 0.0102648 2023-09-23T20:32:32.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:32 smithi121 ceph-mon[100258]: pgmap v111: 89 pgs: 1 peering, 88 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T20:32:32.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:32 smithi121 ceph-mon[100258]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:32:32.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:32 smithi121 ceph-mon[100258]: Cluster is now healthy 2023-09-23T20:32:32.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:32 smithi121 ceph-mon[118228]: pgmap v111: 89 pgs: 1 peering, 88 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T20:32:32.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:32 smithi121 ceph-mon[118228]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:32:32.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:32 smithi121 ceph-mon[118228]: Cluster is now healthy 2023-09-23T20:32:32.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:32 smithi123 ceph-mon[104968]: pgmap v111: 89 pgs: 1 peering, 88 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T20:32:32.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:32 smithi123 ceph-mon[104968]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:32:32.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:32 smithi123 ceph-mon[104968]: Cluster is now healthy 2023-09-23T20:32:32.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 102261 102245 96.7882 58.5 0.0162086 0.0103274 2023-09-23T20:32:33.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 103228 103212 96.2457 60.4375 0.00806239 0.0103866 2023-09-23T20:32:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:33 smithi121 ceph-mon[100258]: pgmap v112: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T20:32:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:33 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2784395992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:33.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:33 smithi121 ceph-mon[118228]: pgmap v112: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T20:32:33.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:33 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2784395992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:33 smithi123 ceph-mon[104968]: pgmap v112: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T20:32:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:33 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2784395992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:34.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 104109 104093 95.6402 55.0625 0.0198511 0.0104505 2023-09-23T20:32:35.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 105004 104988 95.0649 55.9375 0.0211552 0.0105145 2023-09-23T20:32:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:35 smithi121 ceph-mon[100258]: pgmap v113: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T20:32:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:35 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3584620178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:36.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:35 smithi121 ceph-mon[118228]: pgmap v113: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T20:32:36.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:35 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3584620178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:36.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:35 smithi123 ceph-mon[104968]: pgmap v113: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T20:32:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:35 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3584620178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:36.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 105888 105872 94.4962 55.25 0.0178713 0.0105779 2023-09-23T20:32:37.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 106748 106732 93.9224 53.75 0.0175752 0.0106428 2023-09-23T20:32:37.980 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:32:37 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:32:38.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:32:37 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:37] "GET /metrics HTTP/1.1" 200 242359 "" "Prometheus/2.43.0" 2023-09-23T20:32:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:38 smithi121 ceph-mon[118228]: pgmap v114: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2023-09-23T20:32:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:38 smithi121 ceph-mon[100258]: pgmap v114: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2023-09-23T20:32:38.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:38 smithi123 ceph-mon[104968]: pgmap v114: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2023-09-23T20:32:38.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 107606 107590 93.3628 53.625 0.0341728 0.0107057 2023-09-23T20:32:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:39 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1379630364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:39 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1379630364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:39.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:39 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1379630364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:39.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 15 108464 108449 92.8194 53.6875 0.024479 0.0107685 2023-09-23T20:32:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:40 smithi121 ceph-mon[118228]: pgmap v115: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-23T20:32:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:40 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:40 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:40.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:40 smithi121 ceph-mon[100258]: pgmap v115: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-23T20:32:40.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:40 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:40.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:40 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:40 smithi123 ceph-mon[104968]: pgmap v115: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-09-23T20:32:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:40 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:40 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:32:40.770 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 109304 109288 92.2737 52.4375 0.0261287 0.0108322 2023-09-23T20:32:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:41 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1679123349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:41.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:41 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1679123349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:41 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1679123349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:41.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 110178 110162 91.7718 54.625 0.0258976 0.0108915 2023-09-23T20:32:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:42 smithi121 ceph-mon[118228]: pgmap v116: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-09-23T20:32:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:42 smithi121 ceph-mon[100258]: pgmap v116: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-09-23T20:32:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:42 smithi123 ceph-mon[104968]: pgmap v116: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-09-23T20:32:42.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 110926 110910 91.1795 46.75 0.00699782 0.0109626 2023-09-23T20:32:43.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 111787 111771 90.6943 53.8125 0.02651 0.0110216 2023-09-23T20:32:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:43 smithi121 ceph-mon[118228]: pgmap v117: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-09-23T20:32:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:43 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1975113185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:43 smithi121 ceph-mon[100258]: pgmap v117: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-09-23T20:32:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:43 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1975113185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:43.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:43 smithi123 ceph-mon[104968]: pgmap v117: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-09-23T20:32:43.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:43 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1975113185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:44.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 112615 112599 90.1951 51.75 0.00965511 0.0110822 2023-09-23T20:32:45.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 113473 113457 89.7322 53.625 0.0257861 0.0111396 2023-09-23T20:32:46.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:45 smithi121 ceph-mon[100258]: pgmap v118: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2023-09-23T20:32:46.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:45 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/745751725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:46.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:45 smithi121 ceph-mon[118228]: pgmap v118: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2023-09-23T20:32:46.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:45 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/745751725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:46.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:45 smithi123 ceph-mon[104968]: pgmap v118: 89 pgs: 89 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2023-09-23T20:32:46.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:45 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/745751725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:46.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:32:46.771848+0000 min lat: 0.00113584 max lat: 0.336687 avg lat: 0.0111986 2023-09-23T20:32:46.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:32:46.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 114308 114292 89.263 52.1875 0.0156022 0.0111986 2023-09-23T20:32:47.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 115278 115262 88.9094 60.625 0.0098544 0.0112428 2023-09-23T20:32:47.867 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:32:47 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:32:48.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:48 smithi121 ceph-mon[100258]: pgmap v119: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:32:48.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:32:47 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:47] "GET /metrics HTTP/1.1" 200 242427 "" "Prometheus/2.43.0" 2023-09-23T20:32:48.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:48 smithi121 ceph-mon[118228]: pgmap v119: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:32:48.171 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:48 smithi123 ceph-mon[104968]: pgmap v119: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:32:48.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 116198 116182 88.5264 57.5 0.0119929 0.0112913 2023-09-23T20:32:49.419 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:49 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1480305125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:49.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:49 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1480305125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:49.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:49 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1480305125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:49.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 117153 117137 88.1789 59.6875 0.00372339 0.0113322 2023-09-23T20:32:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:50 smithi121 ceph-mon[100258]: pgmap v120: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s 2023-09-23T20:32:50.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:50 smithi121 ceph-mon[118228]: pgmap v120: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s 2023-09-23T20:32:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:50 smithi123 ceph-mon[104968]: pgmap v120: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s 2023-09-23T20:32:50.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 118033 118017 87.784 55 0.0160864 0.0113877 2023-09-23T20:32:51.771 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 118982 118966 87.4491 59.3125 0.0158878 0.0114313 2023-09-23T20:32:52.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:52 smithi121 ceph-mon[100258]: pgmap v121: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T20:32:52.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:52 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2369263222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:52 smithi121 ceph-mon[118228]: pgmap v121: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T20:32:52.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:52 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2369263222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:52 smithi123 ceph-mon[104968]: pgmap v121: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T20:32:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:52 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2369263222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:52.772 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 15 119953 119938 87.1386 60.75 0.016011 0.0114716 2023-09-23T20:32:53.772 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 120870 120854 86.7951 57.25 0.00454891 0.0115173 2023-09-23T20:32:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:53 smithi121 ceph-mon[100258]: pgmap v122: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T20:32:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:53 smithi121 ceph-mon[118228]: pgmap v122: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T20:32:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:53 smithi123 ceph-mon[104968]: pgmap v122: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T20:32:54.772 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 121815 121799 86.4799 59.0625 0.017088 0.0115582 2023-09-23T20:32:54.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:54 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/742181255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:54 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/742181255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:54.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:54 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/742181255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:55.779 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 122764 122748 86.1681 59.3125 0.0280467 0.0116004 2023-09-23T20:32:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:55 smithi121 ceph-mon[118228]: pgmap v123: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-09-23T20:32:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:55 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:55 smithi121 ceph-mon[100258]: pgmap v123: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-09-23T20:32:55.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:55 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:55 smithi123 ceph-mon[104968]: pgmap v123: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-09-23T20:32:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:55 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:56.779 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.0086 2023-09-23T20:32:56.779 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 123644 2023-09-23T20:32:56.779 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T20:32:56.779 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T20:32:56.780 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 85.8556 2023-09-23T20:32:56.780 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 38.8095 2023-09-23T20:32:56.780 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 248.312 2023-09-23T20:32:56.780 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 46.75 2023-09-23T20:32:56.780 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 1373 2023-09-23T20:32:56.780 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 620.951 2023-09-23T20:32:56.780 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 3973 2023-09-23T20:32:56.781 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 748 2023-09-23T20:32:56.781 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0116441 2023-09-23T20:32:56.781 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.00874329 2023-09-23T20:32:56.781 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 0.336687 2023-09-23T20:32:56.781 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00113584 2023-09-23T20:32:56.785 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:32:56.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:56 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3549863024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:56 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3549863024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:56.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:56 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3549863024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:57.859 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:57 smithi121 ceph-mon[100258]: pgmap v124: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:32:57.860 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:57 smithi121 ceph-mon[118228]: pgmap v124: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:32:57.862 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:32:57 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:57] "GET /metrics HTTP/1.1" 200 242430 "" "Prometheus/2.43.0" 2023-09-23T20:32:57.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:32:57 smithi123 conmon[108184]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:32:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:32:57.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:57 smithi123 ceph-mon[104968]: pgmap v124: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:32:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:32:59 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/637124135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:32:59 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/637124135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:32:59.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:32:59 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/637124135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: pgmap v125: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T20:33:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: Upgrade: Updating mgr.x 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[100258]: Deploying daemon mgr.x on smithi123 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: pgmap v125: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: Upgrade: Updating mgr.x 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:00.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:33:00.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:33:00.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:33:00.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:00.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:00 smithi121 ceph-mon[118228]: Deploying daemon mgr.x on smithi123 2023-09-23T20:33:00.459 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: pgmap v125: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T20:33:00.459 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: Upgrade: Updating mgr.x 2023-09-23T20:33:00.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:00.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:33:00.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:33:00.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:33:00.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:00.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:00 smithi123 ceph-mon[104968]: Deploying daemon mgr.x on smithi123 2023-09-23T20:33:01.731 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:01 smithi123 systemd[1]: Stopping Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:33:02.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:02 smithi121 ceph-mon[100258]: pgmap v126: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 824 op/s 2023-09-23T20:33:02.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4268526577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:02.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:02 smithi121 ceph-mon[118228]: pgmap v126: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 824 op/s 2023-09-23T20:33:02.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:02 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4268526577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:02 smithi123 ceph-mon[104968]: pgmap v126: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 824 op/s 2023-09-23T20:33:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:02 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4268526577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:02.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:02 smithi123 bash[159804]: 24010296b9ffab77d8e6deec78d3cb5c33aa3d5f96c4c9fb66d48069cecaee08 2023-09-23T20:33:03.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:02 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:33:03.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:02 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T20:33:03.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:02 smithi123 systemd[1]: Stopped Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:33:03.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:03 smithi123 systemd[1]: Starting Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:33:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:03 smithi121 ceph-mon[100258]: pgmap v127: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 1.11k op/s 2023-09-23T20:33:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:03 smithi121 ceph-mon[118228]: pgmap v127: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 1.11k op/s 2023-09-23T20:33:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:03 smithi123 ceph-mon[104968]: pgmap v127: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 1.11k op/s 2023-09-23T20:33:04.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:04 smithi123 bash[160134]: 45884a500e333dfb091c4a15838a1a51b701cc81c321c9f9f280ded0736d5972 2023-09-23T20:33:04.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:04 smithi123 ceph-mgr[160198]: -- 172.21.15.123:0/389125046 <== mon.2 v2:172.21.15.121:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559a783931e0 con 0x559a7836a800 2023-09-23T20:33:04.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:04 smithi123 systemd[1]: Started Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:33:04.483 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:04 smithi123 conmon[160171]: 2023-09-23T20:33:04.377+0000 7f5a39240200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:33:04.803 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:04 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2345833590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:04.803 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:04 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:04.803 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:04 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:04.803 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:04 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:04.805 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:04 smithi123 conmon[160171]: 2023-09-23T20:33:04.503+0000 7f5a39240200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:33:04.805 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:04 smithi123 conmon[160171]: 2023-09-23T20:33:04.644+0000 7f5a39240200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:33:04.805 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:04 smithi123 conmon[160171]: 2023-09-23T20:33:04.798+0000 7f5a39240200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:33:04.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2345833590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:04.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:04.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:04.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:04.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2345833590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:04.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:04.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:04.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:04 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:05.088 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:05 smithi123 conmon[160171]: 2023-09-23T20:33:05.086+0000 7f5a39240200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:33:05.894 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:05 smithi123 ceph-mon[104968]: pgmap v128: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 781 op/s 2023-09-23T20:33:05.895 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:05 smithi123 conmon[160171]: 2023-09-23T20:33:05.686+0000 7f5a39240200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:33:05.895 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:05 smithi123 conmon[160171]: 2023-09-23T20:33:05.891+0000 7f5a39240200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:33:05.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:05 smithi121 ceph-mon[118228]: pgmap v128: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 781 op/s 2023-09-23T20:33:05.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:05 smithi121 ceph-mon[100258]: pgmap v128: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 781 op/s 2023-09-23T20:33:06.191 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:06 smithi123 conmon[160171]: 2023-09-23T20:33:06.031+0000 7f5a39240200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:33:06.191 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:06 smithi123 conmon[160171]: 2023-09-23T20:33:06.187+0000 7f5a39240200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:33:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:06 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2377428627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:07.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:06 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2377428627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:06 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2377428627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:07.231 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:06 smithi123 conmon[160171]: 2023-09-23T20:33:06.893+0000 7f5a39240200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:33:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:07 smithi121 ceph-mon[100258]: pgmap v129: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 1.13k op/s 2023-09-23T20:33:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:07 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:07 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:08.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:07 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:07] "GET /metrics HTTP/1.1" 200 242430 "" "Prometheus/2.43.0" 2023-09-23T20:33:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:07 smithi121 ceph-mon[118228]: pgmap v129: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 1.13k op/s 2023-09-23T20:33:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:07 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:08.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:07 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:08.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:07 smithi123 ceph-mon[104968]: pgmap v129: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 1.13k op/s 2023-09-23T20:33:08.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:07 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:08.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:07 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:08.178 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:08 smithi123 conmon[160171]: 2023-09-23T20:33:08.032+0000 7f5a39240200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:33:08.178 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:08 smithi123 conmon[160171]: 2023-09-23T20:33:08.177+0000 7f5a39240200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:33:09.731 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:09 smithi123 conmon[160171]: 2023-09-23T20:33:09.454+0000 7f5a39240200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:33:09.731 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:09 smithi123 conmon[160171]: 2023-09-23T20:33:09.602+0000 7f5a39240200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:33:10.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:10 smithi123 ceph-mon[104968]: pgmap v130: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.9 MiB/s wr, 735 op/s 2023-09-23T20:33:10.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:10 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2621354291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:10.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:10 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:10.236 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:09 smithi123 conmon[160171]: 2023-09-23T20:33:09.888+0000 7f5a39240200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:33:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:10 smithi121 ceph-mon[100258]: pgmap v130: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.9 MiB/s wr, 735 op/s 2023-09-23T20:33:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:10 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2621354291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:10 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:10 smithi121 ceph-mon[118228]: pgmap v130: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.9 MiB/s wr, 735 op/s 2023-09-23T20:33:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:10 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2621354291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:10 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:10.686 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:10 smithi123 conmon[160171]: 2023-09-23T20:33:10.355+0000 7f5a39240200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:33:10.686 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:10 smithi123 conmon[160171]: 2023-09-23T20:33:10.678+0000 7f5a39240200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:33:10.945 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:10 smithi123 conmon[160171]: 2023-09-23T20:33:10.930+0000 7f5a39240200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:33:11.730 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:11 smithi123 conmon[160171]: 2023-09-23T20:33:11.371+0000 7f5a39240200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:33:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[100258]: pgmap v131: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.9 MiB/s wr, 776 op/s 2023-09-23T20:33:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1020356418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[118228]: pgmap v131: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.9 MiB/s wr, 776 op/s 2023-09-23T20:33:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:11 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1020356418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:12.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:11 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:12.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:11 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:12.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:11 smithi123 ceph-mon[104968]: pgmap v131: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.9 MiB/s wr, 776 op/s 2023-09-23T20:33:12.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:11 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1020356418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:12.732 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:12 smithi123 conmon[160171]: 2023-09-23T20:33:12.458+0000 7f5a39240200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:33:12.732 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:12 smithi123 conmon[160171]: 2023-09-23T20:33:12.673+0000 7f5a39240200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:33:13.402 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:13 smithi123 conmon[160171]: 2023-09-23T20:33:13.024+0000 7f5a39240200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:33:13.402 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:13 smithi123 conmon[160171]: 2023-09-23T20:33:13.215+0000 7f5a39240200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:33:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:13 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:13 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:13 smithi121 ceph-mon[118228]: pgmap v132: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.4 MiB/s wr, 999 op/s 2023-09-23T20:33:13.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:13 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:13.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:13 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:13.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:13 smithi121 ceph-mon[100258]: pgmap v132: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.4 MiB/s wr, 999 op/s 2023-09-23T20:33:13.689 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:13 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:13.689 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:13 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:13.689 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:13 smithi123 ceph-mon[104968]: pgmap v132: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 3.4 MiB/s wr, 999 op/s 2023-09-23T20:33:13.689 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:13 smithi123 conmon[160171]: 2023-09-23T20:33:13.603+0000 7f5a39240200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:33:14.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:14 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2618557620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:14.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:14 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2618557620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:14.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:14 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2618557620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:15.478 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:15 smithi123 conmon[160171]: 2023-09-23T20:33:14.998+0000 7f5a39240200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:33:15.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:15 smithi123 conmon[160171]: [23/Sep/2023:20:33:15] ENGINE Bus STARTING 2023-09-23T20:33:15.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:15 smithi123 conmon[160171]: CherryPy Checker: 2023-09-23T20:33:15.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:15 smithi123 conmon[160171]: The Application mounted at '' has an empty config. 2023-09-23T20:33:15.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:15 smithi123 conmon[160171]: 2023-09-23T20:33:15.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:15 smithi123 conmon[160171]: [23/Sep/2023:20:33:15] ENGINE Serving on http://:::9283 2023-09-23T20:33:15.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:15 smithi123 conmon[160171]: [23/Sep/2023:20:33:15] ENGINE Bus STARTED 2023-09-23T20:33:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[100258]: pgmap v133: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[100258]: Standby manager daemon x restarted 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[100258]: Standby manager daemon x started 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[118228]: pgmap v133: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T20:33:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[118228]: Standby manager daemon x restarted 2023-09-23T20:33:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[118228]: Standby manager daemon x started 2023-09-23T20:33:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:33:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:33:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:33:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:15 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:33:15.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:15 smithi123 ceph-mon[104968]: pgmap v133: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T20:33:15.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:15 smithi123 ceph-mon[104968]: Standby manager daemon x restarted 2023-09-23T20:33:15.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:15 smithi123 ceph-mon[104968]: Standby manager daemon x started 2023-09-23T20:33:15.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:15 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:33:15.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:15 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:33:15.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:15 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:33:15.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:15 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:33:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:16 smithi121 ceph-mon[100258]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T20:33:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:16 smithi121 ceph-mon[118228]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T20:33:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:16 smithi123 ceph-mon[104968]: mgrmap e20: y(active, since 3m), standbys: x 2023-09-23T20:33:17.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:17 smithi121 ceph-mon[100258]: pgmap v134: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-09-23T20:33:17.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:17 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2658729792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:17.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:17 smithi121 ceph-mon[118228]: pgmap v134: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-09-23T20:33:17.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:17 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2658729792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:17.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:17 smithi123 ceph-mon[104968]: pgmap v134: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-09-23T20:33:17.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:17 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2658729792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:17.984 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:17 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:33:18.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:17 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:17] "GET /metrics HTTP/1.1" 200 242505 "" "Prometheus/2.43.0" 2023-09-23T20:33:20.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:20 smithi121 ceph-mon[100258]: pgmap v135: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T20:33:20.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2324273177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:20.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:20 smithi121 ceph-mon[118228]: pgmap v135: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T20:33:20.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:20 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2324273177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:20 smithi123 ceph-mon[104968]: pgmap v135: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T20:33:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:20 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2324273177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:20.858 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:20 smithi123 conmon[160171]: [23/Sep/2023:20:33:20] ENGINE Bus STOPPING 2023-09-23T20:33:21.152 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:20 smithi123 conmon[160171]: [23/Sep/2023:20:33:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:33:21.152 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:20 smithi123 conmon[160171]: [23/Sep/2023:20:33:20] ENGINE Bus STOPPED 2023-09-23T20:33:21.152 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: [23/Sep/2023:20:33:21] ENGINE Bus STARTING 2023-09-23T20:33:21.152 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: [23/Sep/2023:20:33:21] ENGINE Serving on http://:::9283 2023-09-23T20:33:21.152 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: [23/Sep/2023:20:33:21] ENGINE Bus STARTED 2023-09-23T20:33:21.167 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:20 smithi121 conmon[100681]: ignoring --setuser ceph since I am not root 2023-09-23T20:33:21.167 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:20 smithi121 conmon[100681]: ignoring --setgroup ceph since I am not root 2023-09-23T20:33:21.168 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:21 smithi121 conmon[100681]: 2023-09-23T20:33:21.069+0000 7fd2ef00d1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:33:21.442 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:21 smithi121 conmon[100681]: 2023-09-23T20:33:21.184+0000 7fd2ef00d1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:33:21.442 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:21 smithi121 conmon[100681]: 2023-09-23T20:33:21.311+0000 7fd2ef00d1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:33:21.443 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:21 smithi121 conmon[100681]: 2023-09-23T20:33:21.435+0000 7fd2ef00d1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:33:21.655 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.550+0000 7f5996672700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.655 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.550+0000 7f5996672700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.655 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.550+0000 7f5996672700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.655 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.550+0000 7f5996672700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.655 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.550+0000 7f5996672700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.655 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.563+0000 7f5998676700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.655 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.563+0000 7f5998676700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.656 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.563+0000 7f5998676700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.656 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.563+0000 7f5998676700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.656 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:21 smithi123 conmon[160171]: 2023-09-23T20:33:21.563+0000 7f5998676700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:33:21.660 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: mgrmap e21: x(active, starting, since 0.0106766s) 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:33:21.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: Manager daemon x is now available 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:21.663 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:33:21.663 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:33:21.663 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:33:21.663 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:21 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:33:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: mgrmap e21: x(active, starting, since 0.0106766s) 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:33:21.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: Manager daemon x is now available 2023-09-23T20:33:21.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:21.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:21.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:33:21.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:33:21.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:33:21.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 172.21.15.121:0/3697598858' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24356 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:33:21.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: mgrmap e21: x(active, starting, since 0.0106766s) 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:33:21.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: Manager daemon x is now available 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:21.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:33:21.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:33:21.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:33:21.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:21 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:33:21.921 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:21 smithi121 conmon[100681]: 2023-09-23T20:33:21.683+0000 7fd2ef00d1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:33:22.527 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:22 smithi121 conmon[100681]: 2023-09-23T20:33:22.183+0000 7fd2ef00d1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:33:22.528 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:22 smithi121 conmon[100681]: 2023-09-23T20:33:22.324+0000 7fd2ef00d1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:33:22.528 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:22 smithi121 conmon[100681]: 2023-09-23T20:33:22.518+0000 7fd2ef00d1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[100258]: Deploying cephadm binary to smithi123 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[100258]: Deploying cephadm binary to smithi121 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[100258]: mgrmap e22: x(active, since 1.03156s) 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[100258]: pgmap v3: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/887468538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:22.815 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:22 smithi121 conmon[100681]: 2023-09-23T20:33:22.660+0000 7fd2ef00d1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:33:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[118228]: Deploying cephadm binary to smithi123 2023-09-23T20:33:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[118228]: Deploying cephadm binary to smithi121 2023-09-23T20:33:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[118228]: mgrmap e22: x(active, since 1.03156s) 2023-09-23T20:33:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[118228]: pgmap v3: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:23.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:22 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/887468538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:22 smithi123 ceph-mon[104968]: Deploying cephadm binary to smithi123 2023-09-23T20:33:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:22 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:22 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:22 smithi123 ceph-mon[104968]: Deploying cephadm binary to smithi121 2023-09-23T20:33:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:22 smithi123 ceph-mon[104968]: mgrmap e22: x(active, since 1.03156s) 2023-09-23T20:33:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:22 smithi123 ceph-mon[104968]: pgmap v3: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:22 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/887468538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:23.666 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:23 smithi121 conmon[100681]: 2023-09-23T20:33:23.280+0000 7fd2ef00d1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:33:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[118228]: [23/Sep/2023:20:33:22] ENGINE Bus STARTING 2023-09-23T20:33:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[118228]: [23/Sep/2023:20:33:22] ENGINE Serving on http://172.21.15.123:8765 2023-09-23T20:33:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[118228]: [23/Sep/2023:20:33:22] ENGINE Serving on https://172.21.15.123:7150 2023-09-23T20:33:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[118228]: [23/Sep/2023:20:33:22] ENGINE Bus STARTED 2023-09-23T20:33:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[118228]: pgmap v4: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:24.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[100258]: [23/Sep/2023:20:33:22] ENGINE Bus STARTING 2023-09-23T20:33:24.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[100258]: [23/Sep/2023:20:33:22] ENGINE Serving on http://172.21.15.123:8765 2023-09-23T20:33:24.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[100258]: [23/Sep/2023:20:33:22] ENGINE Serving on https://172.21.15.123:7150 2023-09-23T20:33:24.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[100258]: [23/Sep/2023:20:33:22] ENGINE Bus STARTED 2023-09-23T20:33:24.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:23 smithi121 ceph-mon[100258]: pgmap v4: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:24.166 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:24 smithi121 conmon[100681]: 2023-09-23T20:33:24.090+0000 7fd2ef00d1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:33:24.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:23 smithi123 ceph-mon[104968]: [23/Sep/2023:20:33:22] ENGINE Bus STARTING 2023-09-23T20:33:24.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:23 smithi123 ceph-mon[104968]: [23/Sep/2023:20:33:22] ENGINE Serving on http://172.21.15.123:8765 2023-09-23T20:33:24.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:23 smithi123 ceph-mon[104968]: [23/Sep/2023:20:33:22] ENGINE Serving on https://172.21.15.123:7150 2023-09-23T20:33:24.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:23 smithi123 ceph-mon[104968]: [23/Sep/2023:20:33:22] ENGINE Bus STARTED 2023-09-23T20:33:24.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:23 smithi123 ceph-mon[104968]: pgmap v4: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:24.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:24 smithi121 conmon[100681]: 2023-09-23T20:33:24.340+0000 7fd2ef00d1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:33:25.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:24 smithi121 ceph-mon[100258]: mgrmap e23: x(active, since 3s) 2023-09-23T20:33:25.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:24 smithi121 ceph-mon[118228]: mgrmap e23: x(active, since 3s) 2023-09-23T20:33:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:24 smithi123 ceph-mon[104968]: mgrmap e23: x(active, since 3s) 2023-09-23T20:33:25.888 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:25 smithi121 conmon[100681]: 2023-09-23T20:33:25.547+0000 7fd2ef00d1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:33:25.888 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:25 smithi121 conmon[100681]: 2023-09-23T20:33:25.702+0000 7fd2ef00d1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:33:25.892 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:25 smithi121 ceph-mon[118228]: pgmap v5: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:25.892 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:25 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4059254376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:26.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:25 smithi121 ceph-mon[100258]: pgmap v5: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:26.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:25 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4059254376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:26.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:25 smithi121 conmon[100681]: 2023-09-23T20:33:25.972+0000 7fd2ef00d1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:33:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:25 smithi123 ceph-mon[104968]: pgmap v5: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail 2023-09-23T20:33:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:25 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4059254376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:26.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:26 smithi121 conmon[100681]: 2023-09-23T20:33:26.398+0000 7fd2ef00d1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:33:26.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:26 smithi121 conmon[100681]: 2023-09-23T20:33:26.532+0000 7fd2ef00d1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:33:27.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:26 smithi121 conmon[100681]: 2023-09-23T20:33:26.970+0000 7fd2ef00d1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:33:27.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:27 smithi121 ceph-mon[100258]: pgmap v6: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 93 op/s 2023-09-23T20:33:27.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:27 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4221570051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:27.905 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:27 smithi121 conmon[100681]: 2023-09-23T20:33:27.416+0000 7fd2ef00d1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:33:27.906 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:27 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:27] "GET /metrics HTTP/1.1" 200 33295 "" "Prometheus/2.43.0" 2023-09-23T20:33:28.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:27 smithi121 ceph-mon[118228]: pgmap v6: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 93 op/s 2023-09-23T20:33:28.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:27 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4221570051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:27 smithi123 ceph-mon[104968]: pgmap v6: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 93 op/s 2023-09-23T20:33:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:27 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4221570051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:28.811 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:28 smithi121 conmon[100681]: 2023-09-23T20:33:28.551+0000 7fd2ef00d1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:33:28.812 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:28 smithi121 conmon[100681]: 2023-09-23T20:33:28.759+0000 7fd2ef00d1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:33:29.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:29 smithi121 conmon[100681]: 2023-09-23T20:33:29.061+0000 7fd2ef00d1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:33:29.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:29 smithi121 conmon[100681]: 2023-09-23T20:33:29.243+0000 7fd2ef00d1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:33:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[100258]: pgmap v7: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-09-23T20:33:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:29.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:29 smithi121 conmon[100681]: 2023-09-23T20:33:29.588+0000 7fd2ef00d1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:33:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[118228]: pgmap v7: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-09-23T20:33:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:29 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:29 smithi123 ceph-mon[104968]: pgmap v7: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-09-23T20:33:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:29 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:29 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:29 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:29 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1919448169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:31.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:30 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:30 smithi121 conmon[100681]: 2023-09-23T20:33:30.956+0000 7fd2ef00d1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:33:31.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:30 smithi121 conmon[100681]: [23/Sep/2023:20:33:30] ENGINE Bus STARTING 2023-09-23T20:33:31.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:31 smithi121 conmon[100681]: CherryPy Checker: 2023-09-23T20:33:31.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:31 smithi121 conmon[100681]: The Application mounted at '' has an empty config. 2023-09-23T20:33:31.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:31 smithi121 conmon[100681]: 2023-09-23T20:33:31.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:31 smithi121 conmon[100681]: [23/Sep/2023:20:33:31] ENGINE Serving on http://:::9283 2023-09-23T20:33:31.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:31 smithi121 conmon[100681]: [23/Sep/2023:20:33:31] ENGINE Bus STARTED 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1919448169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1919448169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:31.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:30 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:32.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:31 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi121 to 2500M 2023-09-23T20:33:32.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:31 smithi123 ceph-mon[104968]: pgmap v8: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-09-23T20:33:32.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:31 smithi123 ceph-mon[104968]: Standby manager daemon y started 2023-09-23T20:33:32.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:33:32.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:33:32.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:33:32.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:33:32.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi121 to 2500M 2023-09-23T20:33:32.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[100258]: pgmap v8: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-09-23T20:33:32.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[100258]: Standby manager daemon y started 2023-09-23T20:33:32.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:33:32.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:33:32.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi121 to 2500M 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[118228]: pgmap v8: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 414 op/s 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[118228]: Standby manager daemon y started 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:33:32.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2123120596' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:33:33.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:32 smithi123 ceph-mon[104968]: mgrmap e24: x(active, since 11s), standbys: y 2023-09-23T20:33:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:32 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:33:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:32 smithi121 ceph-mon[100258]: mgrmap e24: x(active, since 11s), standbys: y 2023-09-23T20:33:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:32 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:33:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:32 smithi121 ceph-mon[118228]: mgrmap e24: x(active, since 11s), standbys: y 2023-09-23T20:33:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:32 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:33:34.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:33 smithi123 ceph-mon[104968]: pgmap v9: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-09-23T20:33:34.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:33 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2062597530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:33 smithi121 ceph-mon[100258]: pgmap v9: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-09-23T20:33:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:33 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2062597530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:33 smithi121 ceph-mon[118228]: pgmap v9: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-09-23T20:33:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:33 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2062597530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:36.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:35 smithi123 ceph-mon[104968]: pgmap v10: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T20:33:36.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:35 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2050834988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:35 smithi121 ceph-mon[100258]: pgmap v10: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T20:33:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:35 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2050834988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:36.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:35 smithi121 ceph-mon[118228]: pgmap v10: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-09-23T20:33:36.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:35 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2050834988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:37.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:36 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:36 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:36 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:37.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:37 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:37] "GET /metrics HTTP/1.1" 200 36230 "" "Prometheus/2.43.0" 2023-09-23T20:33:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:37 smithi121 ceph-mon[100258]: pgmap v11: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:33:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:37 smithi121 ceph-mon[118228]: pgmap v11: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:33:38.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:37 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:33:38.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:37 smithi123 ceph-mon[104968]: pgmap v11: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:33:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:38 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/379316260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:38 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/379316260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:38 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/379316260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-09-23T20:33:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: Adjusting osd_memory_target on smithi123 to 2500M 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: Adjusting osd_memory_target on smithi123 to 2500M 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:33:40.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:40 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: Adjusting osd_memory_target on smithi123 to 2500M 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:33:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:40 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:33:41.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:33:41.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:33:41.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3791710534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3791710534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:41 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:41 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:33:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:41 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:33:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:41 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:33:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:41 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3791710534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:41 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:41 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:42 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:42 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:43.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:43 smithi121 ceph-mon[100258]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:33:43.437 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:43 smithi121 ceph-mon[100258]: Reconfiguring daemon alertmanager.a on smithi121 2023-09-23T20:33:43.437 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:43 smithi121 ceph-mon[118228]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:33:43.437 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:43 smithi121 ceph-mon[118228]: Reconfiguring daemon alertmanager.a on smithi121 2023-09-23T20:33:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:43 smithi123 ceph-mon[104968]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:33:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:43 smithi123 ceph-mon[104968]: Reconfiguring daemon alertmanager.a on smithi121 2023-09-23T20:33:44.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:44 smithi121 ceph-mon[100258]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-09-23T20:33:44.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:44 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4152420644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:44.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:44 smithi121 ceph-mon[118228]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-09-23T20:33:44.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:44 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4152420644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:44.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:44 smithi123 ceph-mon[104968]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-09-23T20:33:44.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:44 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4152420644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:46.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:46 smithi121 ceph-mon[100258]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T20:33:46.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:46 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/728531001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:46.380 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:46 smithi121 ceph-mon[118228]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T20:33:46.380 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:46 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/728531001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:46 smithi123 ceph-mon[104968]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T20:33:46.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:46 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/728531001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:47.914 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:47 smithi121 systemd[1]: Stopping Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:33:47.916 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:47 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:33:47.982 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:47 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:47] "GET /metrics HTTP/1.1" 200 36230 "" "Prometheus/2.43.0" 2023-09-23T20:33:48.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:48 smithi121 ceph-mon[100258]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T20:33:48.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:48 smithi121 ceph-mon[118228]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T20:33:48.418 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:48 smithi121 conmon[174726]: ts=2023-09-23T20:33:48.127Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:33:48.418 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:48 smithi121 bash[196170]: e6f06c4a552633e14ddeb5e4431a2c2e9fb92605746999bed42e64a52fbccb10 2023-09-23T20:33:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:48 smithi123 ceph-mon[104968]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-09-23T20:33:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:49 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/11869910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:49.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:49 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:33:49.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:49 smithi121 systemd[1]: Stopped Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:33:49.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:49 smithi121 systemd[1]: Starting Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:33:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:49 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/11869910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:49 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/11869910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:50 smithi121 ceph-mon[100258]: pgmap v17: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-09-23T20:33:50.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 bash[196496]: a7c6818f0f7f6b1dd9d44f9b3c144a7d7af5bcf0baf84225bcd716477a27ae66 2023-09-23T20:33:50.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 systemd[1]: Started Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:33:50.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.133Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:33:50.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.133Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:33:50.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.134Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.121 port=9094 2023-09-23T20:33:50.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.140Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:33:50.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.174Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:33:50.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.174Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:33:50.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.180Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:33:50.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:50 smithi121 conmon[196532]: ts=2023-09-23T20:33:50.181Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:33:50.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:50 smithi121 ceph-mon[118228]: pgmap v17: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-09-23T20:33:50.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:50 smithi123 ceph-mon[104968]: pgmap v17: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-09-23T20:33:51.163 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:50 smithi121 systemd[1]: Stopping Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:33:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:51 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:51 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:51 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:51.555 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:51 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:51.555 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:51 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:51.555 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:51 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:51.555 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:51 smithi121 bash[196767]: 0e30e04cd0298e970d300db756595b4ff0e00def904f3faf3ac220fdc5e768fe 2023-09-23T20:33:51.557 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:51 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:51.557 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:51 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:51.557 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:51 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[100258]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:33:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[100258]: Reconfiguring daemon node-exporter.a on smithi121 2023-09-23T20:33:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[100258]: pgmap v18: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:33:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2756802199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:52.415 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:33:52 smithi121 conmon[196532]: ts=2023-09-23T20:33:52.141Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000917149s 2023-09-23T20:33:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[118228]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:33:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[118228]: Reconfiguring daemon node-exporter.a on smithi121 2023-09-23T20:33:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[118228]: pgmap v18: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:33:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:52 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2756802199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:52 smithi123 ceph-mon[104968]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:33:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:52 smithi123 ceph-mon[104968]: Reconfiguring daemon node-exporter.a on smithi121 2023-09-23T20:33:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:52 smithi123 ceph-mon[104968]: pgmap v18: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:33:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:52 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2756802199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:52.913 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:52 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:33:52.913 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:52 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T20:33:52.913 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:52 smithi121 systemd[1]: Stopped Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:33:52.913 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:52 smithi121 systemd[1]: Starting Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:33:53.598 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:53 smithi121 ceph-mon[100258]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-09-23T20:33:53.598 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:53 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:53.598 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:53 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:53.598 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.248Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:33:53.598 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.248Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:33:53.598 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.254Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:33:53.598 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.254Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:33:53.598 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.254Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.254Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:33:53.599 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:33:53.600 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:33:53.601 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.255Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.256Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.256Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.256Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 conmon[197118]: ts=2023-09-23T20:33:53.256Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 bash[197080]: ab86679c1d7db3f48d201b2b719844462ea4c029f50b48e2f33e1f236b2e832f 2023-09-23T20:33:53.602 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 20:33:53 smithi121 systemd[1]: Started Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:33:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:53 smithi121 ceph-mon[118228]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-09-23T20:33:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:53 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:53 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:53.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:53 smithi123 ceph-mon[104968]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-09-23T20:33:53.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:53 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:53.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:53 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:33:54.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:54 smithi123 ceph-mon[104968]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:33:54.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:54 smithi123 ceph-mon[104968]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:33:54.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:54 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3292634165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:54 smithi121 ceph-mon[100258]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:33:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:54 smithi121 ceph-mon[100258]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:33:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:54 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3292634165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:54.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:54 smithi121 ceph-mon[118228]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:33:54.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:54 smithi121 ceph-mon[118228]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:33:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:54 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3292634165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:55 smithi121 ceph-mon[100258]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:33:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:55 smithi121 ceph-mon[118228]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:33:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:55 smithi123 ceph-mon[104968]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:33:56.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:56 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1508619172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:56 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1508619172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:56.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:56 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1508619172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:33:57.852 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:57 smithi123 ceph-mon[104968]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:33:57.853 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:33:57 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:57] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-09-23T20:33:57.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:33:57 smithi121 conmon[100681]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:33:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:33:57.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:57 smithi121 ceph-mon[100258]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:33:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:57 smithi121 ceph-mon[118228]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T20:33:58.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:58 smithi123 systemd[1]: Stopping Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:33:59.612 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.264Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:33:59.612 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.264Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:33:59.612 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.264Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:33:59.612 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.264Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.264Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.264Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.265Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.265Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.265Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.266Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.266Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 conmon[155035]: ts=2023-09-23T20:33:59.266Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:33:59.613 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:33:59 smithi123 bash[171687]: cb4aa7cfed817dff391a58679751b3b8e5790bde526c7f7503150e22e57970e5 2023-09-23T20:33:59.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:59 smithi123 ceph-mon[104968]: pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:33:59.937 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:33:59 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1009079486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:00.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:59 smithi121 ceph-mon[100258]: pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:34:00.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:33:59 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1009079486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:59 smithi121 ceph-mon[118228]: pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:34:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:33:59 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1009079486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:00.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 20:34:00 smithi121 conmon[196532]: ts=2023-09-23T20:34:00.147Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.007148665s 2023-09-23T20:34:00.478 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:00 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:34:00.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:00 smithi123 systemd[1]: Stopped Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:00.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:00 smithi123 systemd[1]: Starting Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.204Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.204Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.204Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi123 (none))" 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.204Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.204Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.206Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.209Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:34:01.479 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.210Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.210Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 bash[171999]: bb270ef7db9a8c620cb56b60d7f94c94511f0bb88372f3266d90439ee6ed177c 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.215Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.215Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.693µs 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.215Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.215Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 systemd[1]: Started Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.253Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.254Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.254Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.612µs wal_replay_duration=38.876353ms wbl_replay_duration=107ns total_replay_duration=38.949435ms 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.258Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:34:01.480 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.258Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:34:01.481 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.258Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:34:01.481 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.278Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.573026ms db_storage=1.094µs remote_storage=1.378µs web_handler=888ns query_engine=1.135µs scrape=587.278µs scrape_sd=104.194µs notify=14.59µs notify_sd=15.034µs rules=19.334548ms tracing=7.544µs 2023-09-23T20:34:01.481 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.278Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:34:01.481 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:01 smithi123 conmon[172037]: ts=2023-09-23T20:34:01.278Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:34:02.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:01 smithi123 ceph-mon[104968]: pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T20:34:02.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:01 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:02.219 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:01 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:02.219 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:01 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1931693300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:02.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[100258]: pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T20:34:02.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:02.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:02.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1931693300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:02.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[118228]: pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-09-23T20:34:02.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:02.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:02.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:01 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1931693300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:02 smithi123 ceph-mon[104968]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:34:03.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:02 smithi123 ceph-mon[104968]: Reconfiguring daemon grafana.a on smithi123 2023-09-23T20:34:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:02 smithi121 ceph-mon[100258]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:34:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:02 smithi121 ceph-mon[100258]: Reconfiguring daemon grafana.a on smithi123 2023-09-23T20:34:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:02 smithi121 ceph-mon[118228]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:34:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:02 smithi121 ceph-mon[118228]: Reconfiguring daemon grafana.a on smithi123 2023-09-23T20:34:04.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:03 smithi123 ceph-mon[104968]: pgmap v24: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-09-23T20:34:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:03 smithi121 ceph-mon[100258]: pgmap v24: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-09-23T20:34:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:03 smithi121 ceph-mon[118228]: pgmap v24: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-09-23T20:34:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:04 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1816596947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:05.229 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:04 smithi123 systemd[1]: Stopping Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:04 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1816596947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:05.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:04 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1816596947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:06.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:05 smithi121 ceph-mon[118228]: pgmap v25: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T20:34:06.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:05 smithi121 ceph-mon[100258]: pgmap v25: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T20:34:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:05 smithi123 ceph-mon[104968]: pgmap v25: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-09-23T20:34:06.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:06 smithi123 bash[172500]: b89ae83624e0ecca795b15ebf3fd6363e72fd5fbd793c253a3c5fbab9cd8b004 2023-09-23T20:34:06.479 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:06 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:34:06.982 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:06 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T20:34:06.982 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:06 smithi123 systemd[1]: Stopped Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:06.982 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:06 smithi123 systemd[1]: Starting Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:07.352 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:07 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:07.352 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:07 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4053527945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:07 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:07 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4053527945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:07 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:07 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4053527945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:07.770 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 bash[172809]: c2e1f7abecffcaa31d9bcfca24798c3ef57f315e904eb8eb508d6fe508ec66cf 2023-09-23T20:34:07.770 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 systemd[1]: Started Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:07.770 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765146606Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T20:34:07.770 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765408206Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765437869Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765449519Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765458925Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765469262Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765479003Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765488307Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765498518Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765507554Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765516194Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765524985Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765533786Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T20:34:07.771 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=settings t=2023-09-23T20:34:07.765542602Z level=info msg="App mode production" 2023-09-23T20:34:07.772 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=sqlstore t=2023-09-23T20:34:07.765612752Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T20:34:07.772 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=sqlstore t=2023-09-23T20:34:07.765635819Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T20:34:08.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:08 smithi123 ceph-mon[104968]: pgmap v26: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T20:34:08.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:08 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:08.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:08 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:08.230 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=migrator t=2023-09-23T20:34:07.794885047Z level=info msg="Starting DB migrations" 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=migrator t=2023-09-23T20:34:07.80753475Z level=info msg="migrations completed" performed=0 skipped=481 duration=942.006µs 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=sqlstore t=2023-09-23T20:34:07.808134438Z level=info msg="Created default organization" 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=plugin.loader t=2023-09-23T20:34:07.892840788Z level=info msg="Plugin registered" pluginID=input 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=plugin.loader t=2023-09-23T20:34:07.936232131Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=plugin.loader t=2023-09-23T20:34:07.936276911Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=secrets t=2023-09-23T20:34:07.936331607Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=query_data t=2023-09-23T20:34:07.938895462Z level=info msg="Query Service initialization" 2023-09-23T20:34:08.231 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:07 smithi123 conmon[172845]: logger=live.push_http t=2023-09-23T20:34:07.952973063Z level=info msg="Live Push Gateway initialization" 2023-09-23T20:34:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:08 smithi121 ceph-mon[100258]: pgmap v26: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T20:34:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:08 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:08 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:08 smithi121 ceph-mon[118228]: pgmap v26: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T20:34:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:08 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:08 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:08.737 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:08 smithi123 systemd[1]: Stopping Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:09.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:09 smithi123 ceph-mon[104968]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:34:09.322 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:09 smithi123 ceph-mon[104968]: Reconfiguring daemon node-exporter.b on smithi123 2023-09-23T20:34:09.322 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:09 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:34:09] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-09-23T20:34:09.322 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:09 smithi123 bash[173062]: cedf94396cbb3b1fc8b8fe9ed377ac9daf9d712c710768b5bec11d233cd9f223 2023-09-23T20:34:09.323 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:09 smithi123 conmon[172845]: logger=infra.usagestats.collector t=2023-09-23T20:34:09.200181205Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T20:34:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:09 smithi121 ceph-mon[118228]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:34:09.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:09 smithi121 ceph-mon[118228]: Reconfiguring daemon node-exporter.b on smithi123 2023-09-23T20:34:09.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:09 smithi121 ceph-mon[100258]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:34:09.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:09 smithi121 ceph-mon[100258]: Reconfiguring daemon node-exporter.b on smithi123 2023-09-23T20:34:09.734 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:09 smithi123 conmon[172845]: logger=provisioning.datasources t=2023-09-23T20:34:09.376188163Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T20:34:09.734 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:09 smithi123 conmon[172845]: logger=provisioning.datasources t=2023-09-23T20:34:09.376237613Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T20:34:09.734 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:09 smithi123 conmon[172845]: logger=provisioning.datasources t=2023-09-23T20:34:09.387088832Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=provisioning.alerting t=2023-09-23T20:34:10.244271988Z level=info msg="starting to provision alerting" 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=provisioning.alerting t=2023-09-23T20:34:10.244314987Z level=info msg="finished to provision alerting" 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=ngalert.state.manager t=2023-09-23T20:34:10.247692479Z level=info msg="Warming state cache for startup" 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=http.server t=2023-09-23T20:34:10.248778279Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=grafanaStorageLogger t=2023-09-23T20:34:10.249348165Z level=info msg="storage starting" 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=ngalert.state.manager t=2023-09-23T20:34:10.326873452Z level=info msg="State cache has been initialized" states=0 duration=79.177819ms 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=ticker t=2023-09-23T20:34:10.326980286Z level=info msg=starting first_tick=2023-09-23T20:34:20Z 2023-09-23T20:34:10.328 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:34:10 smithi123 conmon[172845]: logger=ngalert.multiorg.alertmanager t=2023-09-23T20:34:10.327002339Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T20:34:10.329 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:10 smithi123 ceph-mon[104968]: pgmap v27: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-09-23T20:34:10.329 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:10 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2735905018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:10 smithi121 ceph-mon[118228]: pgmap v27: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-09-23T20:34:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:10 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2735905018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:10 smithi121 ceph-mon[100258]: pgmap v27: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-09-23T20:34:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:10 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2735905018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:10.732 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:10 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:34:10.732 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:10 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T20:34:10.732 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:10 smithi123 systemd[1]: Stopped Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:10.732 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:10 smithi123 systemd[1]: Starting Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:11.571 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.265Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:34:11.571 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.266Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:34:11.571 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.273Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:34:11.571 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.273Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.273Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:34:11.572 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:34:11.573 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:34:11.574 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[173409]: ts=2023-09-23T20:34:11.274Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 bash[173375]: 5f079d4704ad7de30fdb44216bb0dc2adad076f55bde93a50b85a6e9af0caf00 2023-09-23T20:34:11.575 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 20:34:11 smithi123 systemd[1]: Started Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:11.576 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STOPPING 2023-09-23T20:34:11.576 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:34:11.576 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STOPPED 2023-09-23T20:34:11.576 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STARTING 2023-09-23T20:34:11.576 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Serving on http://:::9283 2023-09-23T20:34:11.576 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STARTED 2023-09-23T20:34:11.576 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STOPPING 2023-09-23T20:34:11.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:34:11.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STOPPED 2023-09-23T20:34:11.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STARTING 2023-09-23T20:34:11.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Serving on http://:::9283 2023-09-23T20:34:11.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STARTED 2023-09-23T20:34:11.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STOPPING 2023-09-23T20:34:11.980 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:34:11.980 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STOPPED 2023-09-23T20:34:11.980 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STARTING 2023-09-23T20:34:11.980 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Serving on http://:::9283 2023-09-23T20:34:11.980 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:11 smithi123 conmon[160171]: [23/Sep/2023:20:34:11] ENGINE Bus STARTED 2023-09-23T20:34:12.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: pgmap v28: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T20:34:12.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.402 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:34:12.402 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:34:12.402 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.402 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:12.403 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123:9095"}]: dispatch 2023-09-23T20:34:12.403 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.403 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:34:12.403 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123:3000"}]: dispatch 2023-09-23T20:34:12.403 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.403 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:12.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: pgmap v28: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T20:34:12.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123:9095"}]: dispatch 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123:3000"}]: dispatch 2023-09-23T20:34:12.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.406 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:12 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:12.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: pgmap v28: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-09-23T20:34:12.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123:9095"}]: dispatch 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123:3000"}]: dispatch 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:12.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:12 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:13.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:34:13.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:34:13.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:13.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123:9095"}]: dispatch 2023-09-23T20:34:13.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123:3000"}]: dispatch 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2042066540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123:9095"}]: dispatch 2023-09-23T20:34:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:34:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123:3000"}]: dispatch 2023-09-23T20:34:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:34:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:34:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:13 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2042066540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:13.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123:9095"}]: dispatch 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123:3000"}]: dispatch 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:34:13.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:13 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2042066540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:14.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:14 smithi121 ceph-mon[118228]: pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-09-23T20:34:14.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:14 smithi121 ceph-mon[100258]: pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-09-23T20:34:14.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:14 smithi123 ceph-mon[104968]: pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-09-23T20:34:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:15 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1789780182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:15.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:15 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1789780182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:15 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1789780182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:16 smithi121 ceph-mon[100258]: pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:34:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:16 smithi121 ceph-mon[118228]: pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:34:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:16 smithi123 ceph-mon[104968]: pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:34:17.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[100258]: Upgrade: Updating mgr.y 2023-09-23T20:34:17.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:17.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:34:17.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:34:17.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:34:17.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:17.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[100258]: Deploying daemon mgr.y on smithi121 2023-09-23T20:34:17.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[118228]: Upgrade: Updating mgr.y 2023-09-23T20:34:17.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:17.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:34:17.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:34:17.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:34:17.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:17.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:17 smithi121 ceph-mon[118228]: Deploying daemon mgr.y on smithi121 2023-09-23T20:34:17.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:17 smithi123 ceph-mon[104968]: Upgrade: Updating mgr.y 2023-09-23T20:34:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:17 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:17 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:34:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:17 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:34:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:17 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:34:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:17 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:17 smithi123 ceph-mon[104968]: Deploying daemon mgr.y on smithi121 2023-09-23T20:34:18.397 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:18 smithi121 ceph-mon[118228]: pgmap v31: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-09-23T20:34:18.397 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:18 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1293328673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:18.400 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:18 smithi121 ceph-mon[100258]: pgmap v31: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-09-23T20:34:18.401 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:18 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1293328673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:18.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:18 smithi123 ceph-mon[104968]: pgmap v31: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-09-23T20:34:18.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:18 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1293328673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:18.666 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:18 smithi121 systemd[1]: Stopping Ceph mgr.y for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:18.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:18 smithi123 conmon[172037]: ts=2023-09-23T20:34:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:34:19.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:19 smithi121 bash[197977]: 666708c47463a034a01e0f6dbf6473f8a19bb7a5e56032e8c503c568ba5e4977 2023-09-23T20:34:19.728 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:19 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:34:19] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-09-23T20:34:20.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:19 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:34:20.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:19 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T20:34:20.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:19 smithi121 systemd[1]: Stopped Ceph mgr.y for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:20.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:20 smithi121 systemd[1]: Starting Ceph mgr.y for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:20.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:20 smithi121 ceph-mon[100258]: pgmap v32: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T20:34:20.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/811508944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:20.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:20 smithi121 ceph-mon[118228]: pgmap v32: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T20:34:20.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:20 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/811508944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:20.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:20 smithi123 ceph-mon[104968]: pgmap v32: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T20:34:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:20 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/811508944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:21.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:21 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:21 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:21.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:21 smithi121 bash[198338]: adf60a30334d79ca6365d082a0f5dbb423766d2a717923b38ff216fbd5f5cb0a 2023-09-23T20:34:21.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:21 smithi121 systemd[1]: Started Ceph mgr.y for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:21.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:21 smithi121 conmon[198376]: 2023-09-23T20:34:21.295+0000 7f2423e61200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:34:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:21 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:21.722 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:21 smithi121 conmon[198376]: 2023-09-23T20:34:21.421+0000 7f2423e61200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:34:21.723 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:21 smithi121 conmon[198376]: 2023-09-23T20:34:21.561+0000 7f2423e61200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:34:21.723 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:21 smithi121 conmon[198376]: 2023-09-23T20:34:21.719+0000 7f2423e61200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:34:21.993 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:21 smithi121 conmon[198376]: 2023-09-23T20:34:21.989+0000 7f2423e61200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:34:22.257 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[100258]: pgmap v33: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:34:22.258 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:22.258 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:22.258 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:22.258 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[118228]: pgmap v33: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:34:22.258 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:22.259 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:22.259 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:22 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:22.605 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:22 smithi121 conmon[198376]: 2023-09-23T20:34:22.602+0000 7f2423e61200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:34:22.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:22 smithi123 ceph-mon[104968]: pgmap v33: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-09-23T20:34:22.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:22 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:22.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:22 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:22.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:22 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:22.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:22 smithi121 conmon[198376]: 2023-09-23T20:34:22.795+0000 7f2423e61200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:34:23.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:22 smithi121 conmon[198376]: 2023-09-23T20:34:22.932+0000 7f2423e61200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:34:23.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:23 smithi121 conmon[198376]: 2023-09-23T20:34:23.074+0000 7f2423e61200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:34:23.591 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:23 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1056923391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:23.591 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:23 smithi123 ceph-mon[104968]: pgmap v34: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2023-09-23T20:34:23.921 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:23 smithi121 conmon[198376]: 2023-09-23T20:34:23.752+0000 7f2423e61200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:34:23.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:23 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1056923391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:23.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:23 smithi121 ceph-mon[100258]: pgmap v34: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2023-09-23T20:34:23.922 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:23 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1056923391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:23.922 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:23 smithi121 ceph-mon[118228]: pgmap v34: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2023-09-23T20:34:25.168 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:24 smithi121 conmon[198376]: 2023-09-23T20:34:24.770+0000 7f2423e61200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:34:25.169 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:24 smithi121 conmon[198376]: 2023-09-23T20:34:24.915+0000 7f2423e61200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:34:26.228 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:25 smithi121 ceph-mon[100258]: pgmap v35: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T20:34:26.229 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:25 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2275627360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:26.229 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:25 smithi121 ceph-mon[118228]: pgmap v35: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T20:34:26.229 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:25 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2275627360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:26.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:25 smithi123 ceph-mon[104968]: pgmap v35: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-09-23T20:34:26.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:25 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2275627360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:26.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:26 smithi121 conmon[198376]: 2023-09-23T20:34:26.226+0000 7f2423e61200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:34:26.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:26 smithi121 conmon[198376]: 2023-09-23T20:34:26.378+0000 7f2423e61200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:34:26.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:26 smithi121 conmon[198376]: 2023-09-23T20:34:26.655+0000 7f2423e61200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:34:27.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:27 smithi121 conmon[198376]: 2023-09-23T20:34:27.111+0000 7f2423e61200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:34:27.713 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.714 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.714 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.714 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.714 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[100258]: pgmap v36: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T20:34:27.714 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:27 smithi121 conmon[198376]: 2023-09-23T20:34:27.451+0000 7f2423e61200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:34:27.714 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:27 smithi121 conmon[198376]: 2023-09-23T20:34:27.706+0000 7f2423e61200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:34:27.714 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.715 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.715 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.715 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.715 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:27 smithi121 ceph-mon[118228]: pgmap v36: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T20:34:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:27 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:27 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:27 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:27 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:27 smithi123 ceph-mon[104968]: pgmap v36: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-09-23T20:34:28.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:28 smithi121 conmon[198376]: 2023-09-23T20:34:28.165+0000 7f2423e61200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:34:28.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:28 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:28.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:28 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:28.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:28 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2476996386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:28.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:28 smithi123 conmon[172037]: ts=2023-09-23T20:34:28.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:34:29.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:28 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:29.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:28 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:29.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:28 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2476996386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:29.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:28 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:29.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:28 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:29.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:28 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2476996386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:29.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:29 smithi121 conmon[198376]: 2023-09-23T20:34:29.243+0000 7f2423e61200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:34:29.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:29 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:34:29] "GET /metrics HTTP/1.1" 200 36229 "" "Prometheus/2.43.0" 2023-09-23T20:34:29.786 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:29 smithi121 conmon[198376]: 2023-09-23T20:34:29.471+0000 7f2423e61200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:34:29.786 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:29 smithi121 conmon[198376]: 2023-09-23T20:34:29.783+0000 7f2423e61200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:34:30.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:29 smithi121 ceph-mon[100258]: pgmap v37: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-09-23T20:34:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:29 smithi121 ceph-mon[118228]: pgmap v37: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-09-23T20:34:30.165 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:29 smithi121 conmon[198376]: 2023-09-23T20:34:29.958+0000 7f2423e61200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:34:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:29 smithi123 ceph-mon[104968]: pgmap v37: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-09-23T20:34:30.662 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:30 smithi121 conmon[198376]: 2023-09-23T20:34:30.299+0000 7f2423e61200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:34:31.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:30 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4260748260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:31.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:30 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4260748260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:31.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:30 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4260748260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:31.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:31 smithi121 conmon[198376]: 2023-09-23T20:34:31.516+0000 7f2423e61200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:34:31.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:31 smithi121 conmon[198376]: [23/Sep/2023:20:34:31] ENGINE Bus STARTING 2023-09-23T20:34:31.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:31 smithi121 conmon[198376]: CherryPy Checker: 2023-09-23T20:34:31.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:31 smithi121 conmon[198376]: The Application mounted at '' has an empty config. 2023-09-23T20:34:31.838 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:31 smithi121 conmon[198376]: 2023-09-23T20:34:31.838 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:31 smithi121 conmon[198376]: [23/Sep/2023:20:34:31] ENGINE Serving on http://:::9283 2023-09-23T20:34:31.838 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:31 smithi121 conmon[198376]: [23/Sep/2023:20:34:31] ENGINE Bus STARTED 2023-09-23T20:34:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[118228]: pgmap v38: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T20:34:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[118228]: Standby manager daemon y restarted 2023-09-23T20:34:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[118228]: Standby manager daemon y started 2023-09-23T20:34:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:34:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:34:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:34:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:34:32.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[100258]: pgmap v38: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T20:34:32.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[100258]: Standby manager daemon y restarted 2023-09-23T20:34:32.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[100258]: Standby manager daemon y started 2023-09-23T20:34:32.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:34:32.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:34:32.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:34:32.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:31 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:34:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:31 smithi123 ceph-mon[104968]: pgmap v38: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-09-23T20:34:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:31 smithi123 ceph-mon[104968]: Standby manager daemon y restarted 2023-09-23T20:34:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:31 smithi123 ceph-mon[104968]: Standby manager daemon y started 2023-09-23T20:34:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:34:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:34:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:34:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:31 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:34:33.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:32 smithi123 ceph-mon[104968]: mgrmap e25: x(active, since 71s), standbys: y 2023-09-23T20:34:33.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:32 smithi121 ceph-mon[100258]: mgrmap e25: x(active, since 71s), standbys: y 2023-09-23T20:34:33.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:32 smithi121 ceph-mon[118228]: mgrmap e25: x(active, since 71s), standbys: y 2023-09-23T20:34:34.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:33 smithi123 ceph-mon[104968]: pgmap v39: 89 pgs: 89 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-09-23T20:34:34.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:33 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3631705813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:34.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:33 smithi121 ceph-mon[100258]: pgmap v39: 89 pgs: 89 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-09-23T20:34:34.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:33 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3631705813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:34.421 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:33 smithi121 ceph-mon[118228]: pgmap v39: 89 pgs: 89 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-09-23T20:34:34.421 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:33 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3631705813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:36.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:35 smithi123 ceph-mon[104968]: pgmap v40: 89 pgs: 89 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T20:34:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:35 smithi121 ceph-mon[100258]: pgmap v40: 89 pgs: 89 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T20:34:36.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:35 smithi121 ceph-mon[118228]: pgmap v40: 89 pgs: 89 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T20:34:36.807 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:36 smithi123 conmon[160171]: ignoring --setuser ceph since I am not root 2023-09-23T20:34:36.808 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:36 smithi123 conmon[160171]: ignoring --setgroup ceph since I am not root 2023-09-23T20:34:36.808 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:36 smithi123 conmon[160171]: 2023-09-23T20:34:36.714+0000 7f2ee3ffb200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: [23/Sep/2023:20:34:36] ENGINE Bus STOPPING 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: [23/Sep/2023:20:34:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: [23/Sep/2023:20:34:36] ENGINE Bus STOPPED 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: [23/Sep/2023:20:34:36] ENGINE Bus STARTING 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: [23/Sep/2023:20:34:36] ENGINE Serving on http://:::9283 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: [23/Sep/2023:20:34:36] ENGINE Bus STARTED 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.864+0000 7f23832d7700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:36.866 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.864+0000 7f23832d7700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:36.867 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.864+0000 7f23832d7700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:36.867 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.864+0000 7f23832d7700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:36.867 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.864+0000 7f23832d7700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:37.123 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:37.123 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/788391467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:37.123 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.123 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.24875 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: mgrmap e26: y(active, starting, since 0.0218945s) 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:34:37.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:34:37.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: Manager daemon y is now available 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:34:37.126 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:36 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:34:37.127 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:36 smithi123 conmon[160171]: 2023-09-23T20:34:36.827+0000 7f2ee3ffb200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:34:37.127 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:36 smithi123 conmon[160171]: 2023-09-23T20:34:36.968+0000 7f2ee3ffb200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:34:37.127 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:37 smithi123 conmon[160171]: 2023-09-23T20:34:37.121+0000 7f2ee3ffb200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:34:37.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:37.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/788391467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:37.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.24875 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: mgrmap e26: y(active, starting, since 0.0218945s) 2023-09-23T20:34:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:34:37.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: Manager daemon y is now available 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:34:37.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:34:37.170 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.869+0000 7f2381ad4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:37.170 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.869+0000 7f2381ad4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:37.170 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.869+0000 7f2381ad4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:37.170 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.869+0000 7f2381ad4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:37.170 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:36 smithi121 conmon[198376]: 2023-09-23T20:34:36.869+0000 7f2381ad4700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:34:37.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:37.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/788391467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 172.21.15.123:0/2675284637' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.24875 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: mgrmap e26: y(active, starting, since 0.0218945s) 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:34:37.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:34:37.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: Manager daemon y is now available 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:34:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:36 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:34:37.425 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:37 smithi123 conmon[160171]: 2023-09-23T20:34:37.422+0000 7f2ee3ffb200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:34:38.367 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:38 smithi123 conmon[160171]: 2023-09-23T20:34:38.024+0000 7f2ee3ffb200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:34:38.367 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:38 smithi123 conmon[160171]: 2023-09-23T20:34:38.222+0000 7f2ee3ffb200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:34:38.367 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:38 smithi123 conmon[160171]: 2023-09-23T20:34:38.365+0000 7f2ee3ffb200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:34:38.367 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:37 smithi123 ceph-mon[104968]: [23/Sep/2023:20:34:37] ENGINE Bus STARTING 2023-09-23T20:34:38.367 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:37 smithi123 ceph-mon[104968]: mgrmap e27: y(active, since 1.02483s) 2023-09-23T20:34:38.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:37 smithi121 ceph-mon[100258]: [23/Sep/2023:20:34:37] ENGINE Bus STARTING 2023-09-23T20:34:38.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:37 smithi121 ceph-mon[100258]: mgrmap e27: y(active, since 1.02483s) 2023-09-23T20:34:38.503 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:37 smithi121 ceph-mon[118228]: [23/Sep/2023:20:34:37] ENGINE Bus STARTING 2023-09-23T20:34:38.503 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:37 smithi121 ceph-mon[118228]: mgrmap e27: y(active, since 1.02483s) 2023-09-23T20:34:38.662 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:38 smithi123 conmon[160171]: 2023-09-23T20:34:38.512+0000 7f2ee3ffb200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:34:38.982 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:38 smithi123 conmon[172037]: ts=2023-09-23T20:34:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:34:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[118228]: [23/Sep/2023:20:34:37] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:34:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[118228]: [23/Sep/2023:20:34:37] ENGINE Serving on http://172.21.15.121:8765 2023-09-23T20:34:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[118228]: [23/Sep/2023:20:34:37] ENGINE Bus STARTED 2023-09-23T20:34:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[118228]: pgmap v3: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[118228]: mgrmap e28: y(active, since 2s) 2023-09-23T20:34:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2294366777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:39.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[100258]: [23/Sep/2023:20:34:37] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:34:39.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[100258]: [23/Sep/2023:20:34:37] ENGINE Serving on http://172.21.15.121:8765 2023-09-23T20:34:39.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[100258]: [23/Sep/2023:20:34:37] ENGINE Bus STARTED 2023-09-23T20:34:39.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[100258]: pgmap v3: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:39.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[100258]: mgrmap e28: y(active, since 2s) 2023-09-23T20:34:39.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:38 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2294366777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:38 smithi123 ceph-mon[104968]: [23/Sep/2023:20:34:37] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:34:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:38 smithi123 ceph-mon[104968]: [23/Sep/2023:20:34:37] ENGINE Serving on http://172.21.15.121:8765 2023-09-23T20:34:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:38 smithi123 ceph-mon[104968]: [23/Sep/2023:20:34:37] ENGINE Bus STARTED 2023-09-23T20:34:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:38 smithi123 ceph-mon[104968]: pgmap v3: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:38 smithi123 ceph-mon[104968]: mgrmap e28: y(active, since 2s) 2023-09-23T20:34:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:38 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2294366777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:39.480 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:39 smithi123 conmon[160171]: 2023-09-23T20:34:39.183+0000 7f2ee3ffb200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:34:40.363 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:39 smithi123 ceph-mon[104968]: pgmap v4: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:40.364 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:40 smithi123 conmon[160171]: 2023-09-23T20:34:40.221+0000 7f2ee3ffb200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:34:40.364 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:40 smithi123 conmon[160171]: 2023-09-23T20:34:40.359+0000 7f2ee3ffb200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:34:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:39 smithi121 ceph-mon[118228]: pgmap v4: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:39 smithi121 ceph-mon[100258]: pgmap v4: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:41.981 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:41 smithi123 conmon[160171]: 2023-09-23T20:34:41.647+0000 7f2ee3ffb200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:34:41.981 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:41 smithi123 conmon[160171]: 2023-09-23T20:34:41.798+0000 7f2ee3ffb200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:34:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[118228]: pgmap v5: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2389701871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:42.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[100258]: pgmap v5: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:42.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:42.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:42.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:42 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2389701871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:42.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:42 smithi123 conmon[160171]: 2023-09-23T20:34:42.083+0000 7f2ee3ffb200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:34:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:42 smithi123 ceph-mon[104968]: pgmap v5: 89 pgs: 89 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:34:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:42 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:42 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:42 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2389701871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:42.900 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:42 smithi123 conmon[160171]: 2023-09-23T20:34:42.560+0000 7f2ee3ffb200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:34:42.900 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:42 smithi123 conmon[160171]: 2023-09-23T20:34:42.897+0000 7f2ee3ffb200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:34:43.209 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:43 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:43.209 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:43 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:43.210 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:43 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:43.210 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:43 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:43.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:43 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:43.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:43 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:43.234 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:43 smithi123 conmon[160171]: 2023-09-23T20:34:43.138+0000 7f2ee3ffb200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:34:43.990 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:43 smithi123 conmon[160171]: 2023-09-23T20:34:43.620+0000 7f2ee3ffb200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:34:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:44 smithi121 ceph-mon[118228]: pgmap v6: 89 pgs: 89 active+clean; 2.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 163 op/s 2023-09-23T20:34:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:44 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1419013468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:44 smithi121 ceph-mon[100258]: pgmap v6: 89 pgs: 89 active+clean; 2.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 163 op/s 2023-09-23T20:34:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:44 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1419013468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:44.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:44 smithi123 ceph-mon[104968]: pgmap v6: 89 pgs: 89 active+clean; 2.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 163 op/s 2023-09-23T20:34:44.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:44 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1419013468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:44.983 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:44 smithi123 conmon[160171]: 2023-09-23T20:34:44.710+0000 7f2ee3ffb200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:34:44.983 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:44 smithi123 conmon[160171]: 2023-09-23T20:34:44.926+0000 7f2ee3ffb200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:34:45.305 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:45 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:45.305 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:45 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:45.306 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:45 smithi121 ceph-mon[118228]: pgmap v7: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s 2023-09-23T20:34:45.306 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:45 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:45.306 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:45 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:45.306 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:45 smithi121 ceph-mon[100258]: pgmap v7: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s 2023-09-23T20:34:45.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:45 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:45.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:45 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:45.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:45 smithi123 ceph-mon[104968]: pgmap v7: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s 2023-09-23T20:34:45.480 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:45 smithi123 conmon[160171]: 2023-09-23T20:34:45.237+0000 7f2ee3ffb200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:34:45.480 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:45 smithi123 conmon[160171]: 2023-09-23T20:34:45.427+0000 7f2ee3ffb200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:34:46.228 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:45 smithi123 conmon[160171]: 2023-09-23T20:34:45.788+0000 7f2ee3ffb200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:34:46.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:46 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:46.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:46 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:46.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:46 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/349948889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:47.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:46 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:47.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:46 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:47.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:46 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/349948889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:47.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:46 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:47.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:46 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:47.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:46 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/349948889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:47.482 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:47 smithi123 conmon[160171]: 2023-09-23T20:34:47.134+0000 7f2ee3ffb200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:34:47.482 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:47 smithi123 conmon[160171]: [23/Sep/2023:20:34:47] ENGINE Bus STARTING 2023-09-23T20:34:47.482 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:47 smithi123 conmon[160171]: CherryPy Checker: 2023-09-23T20:34:47.482 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:47 smithi123 conmon[160171]: The Application mounted at '' has an empty config. 2023-09-23T20:34:47.482 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:47 smithi123 conmon[160171]: 2023-09-23T20:34:47.482 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:47 smithi123 conmon[160171]: [23/Sep/2023:20:34:47] ENGINE Serving on http://:::9283 2023-09-23T20:34:47.482 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:47 smithi123 conmon[160171]: [23/Sep/2023:20:34:47] ENGINE Bus STARTED 2023-09-23T20:34:48.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[118228]: pgmap v8: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 375 op/s 2023-09-23T20:34:48.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[118228]: Standby manager daemon x started 2023-09-23T20:34:48.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:34:48.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:34:48.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:34:48.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:34:48.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[100258]: pgmap v8: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 375 op/s 2023-09-23T20:34:48.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[100258]: Standby manager daemon x started 2023-09-23T20:34:48.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:34:48.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:34:48.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:34:48.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:47 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:34:48.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:47 smithi123 ceph-mon[104968]: pgmap v8: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 375 op/s 2023-09-23T20:34:48.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:47 smithi123 ceph-mon[104968]: Standby manager daemon x started 2023-09-23T20:34:48.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:47 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:34:48.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:47 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:34:48.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:47 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:34:48.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:47 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/230603579' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:34:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[118228]: mgrmap e29: y(active, since 11s), standbys: x 2023-09-23T20:34:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:34:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:49.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[100258]: mgrmap e29: y(active, since 11s), standbys: x 2023-09-23T20:34:49.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:34:49.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:49.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:49.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:49.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:48 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:49.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:48 smithi123 ceph-mon[104968]: mgrmap e29: y(active, since 11s), standbys: x 2023-09-23T20:34:49.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:48 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:34:49.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:48 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:49.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:48 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:49.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:48 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:49.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:48 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:49.733 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:34:49 smithi123 conmon[160171]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:34:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:34:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:34:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:34:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[100258]: pgmap v9: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T20:34:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:34:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:34:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2163872649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:34:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:34:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[118228]: pgmap v9: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T20:34:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:34:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:34:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:49 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2163872649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:49 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:34:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:49 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:34:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:49 smithi123 ceph-mon[104968]: pgmap v9: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T20:34:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:49 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:34:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:49 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:34:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:49 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2163872649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:50 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:34:51.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:34:51.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:34:51.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:34:51.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:50 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:51.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:51 smithi123 ceph-mon[104968]: pgmap v10: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T20:34:51.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:51 smithi123 ceph-mon[104968]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:34:51.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:51 smithi123 ceph-mon[104968]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:34:51.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:51 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[100258]: pgmap v10: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[100258]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[100258]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[118228]: pgmap v10: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[118228]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[118228]: Reconfiguring daemon prometheus.a on smithi123 2023-09-23T20:34:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:51 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:52 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3420334831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:52 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3420334831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:53.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:52 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3420334831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:53 smithi121 ceph-mon[118228]: pgmap v11: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:34:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:53 smithi121 ceph-mon[100258]: pgmap v11: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:34:54.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:53 smithi123 ceph-mon[104968]: pgmap v11: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T20:34:54.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:54 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/68632805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:54 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/68632805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:54 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/68632805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:55.981 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:55 smithi123 systemd[1]: Stopping Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:55.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:55 smithi123 ceph-mon[104968]: pgmap v12: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-09-23T20:34:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:55 smithi121 ceph-mon[100258]: pgmap v12: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-09-23T20:34:56.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:55 smithi121 ceph-mon[118228]: pgmap v12: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-09-23T20:34:56.676 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.329Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:34:56.677 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.329Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:34:56.677 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.329Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:34:56.677 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.329Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:34:56.677 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.329Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:34:56.677 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.329Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:34:56.677 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.330Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:34:56.677 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.330Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:34:56.678 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.330Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:34:56.678 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.330Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:34:56.678 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.330Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:34:56.678 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 conmon[172037]: ts=2023-09-23T20:34:56.330Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:34:56.678 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:56 smithi123 bash[177194]: bb270ef7db9a8c620cb56b60d7f94c94511f0bb88372f3266d90439ee6ed177c 2023-09-23T20:34:57.732 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:57 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:34:57.732 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:57 smithi123 systemd[1]: Stopped Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:57.732 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:57 smithi123 systemd[1]: Starting Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:34:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:57 smithi121 ceph-mon[100258]: pgmap v13: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:34:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:57 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/382765291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:57 smithi121 ceph-mon[118228]: pgmap v13: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:34:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:57 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/382765291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:58.181 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:57 smithi123 ceph-mon[104968]: pgmap v13: 89 pgs: 89 active+clean; 2.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:34:58.181 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:57 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/382765291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:34:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 bash[177499]: 9b8ac663186b06114df476628d4b78cdfcc5a1ddd2554bd47c4be2bbaafa0bd6 2023-09-23T20:34:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 systemd[1]: Started Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:34:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.367Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:34:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.367Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:34:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.367Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi123 (none))" 2023-09-23T20:34:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.367Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.367Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.383Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.384Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.385Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.385Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.387Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.387Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.66µs 2023-09-23T20:34:58.730 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.387Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.388Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.429Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.433Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.434Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.434Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.694µs wal_replay_duration=46.472418ms wbl_replay_duration=119ns total_replay_duration=46.531894ms 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.438Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.438Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:34:58.731 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.438Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:34:58.732 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.458Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.67478ms db_storage=1.225µs remote_storage=1.851µs web_handler=456ns query_engine=984ns scrape=627.336µs scrape_sd=80.38µs notify=17.253µs notify_sd=15.408µs rules=19.400015ms tracing=7.999µs 2023-09-23T20:34:58.732 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.459Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:34:58.732 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:34:58 smithi123 conmon[177538]: ts=2023-09-23T20:34:58.459Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:34:58.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:58 smithi121 conmon[198376]: [23/Sep/2023:20:34:58] ENGINE Bus STOPPING 2023-09-23T20:34:58.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:58 smithi121 conmon[198376]: [23/Sep/2023:20:34:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:34:58.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:58 smithi121 conmon[198376]: [23/Sep/2023:20:34:58] ENGINE Bus STOPPED 2023-09-23T20:34:58.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:58 smithi121 conmon[198376]: [23/Sep/2023:20:34:58] ENGINE Bus STARTING 2023-09-23T20:34:58.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:58 smithi121 conmon[198376]: [23/Sep/2023:20:34:58] ENGINE Serving on http://:::9283 2023-09-23T20:34:58.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:34:58 smithi121 conmon[198376]: [23/Sep/2023:20:34:58] ENGINE Bus STARTED 2023-09-23T20:34:59.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:59.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:59.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:34:59.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:34:59.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:59.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[100258]: pgmap v14: 89 pgs: 89 active+clean; 1.9 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-09-23T20:34:59.675 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:59.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:34:59 smithi121 ceph-mon[118228]: pgmap v14: 89 pgs: 89 active+clean; 1.9 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-09-23T20:34:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:34:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:34:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:34:59.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:34:59.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:59.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:34:59 smithi123 ceph-mon[104968]: pgmap v14: 89 pgs: 89 active+clean; 1.9 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-09-23T20:35:00.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:00 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1221080678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:00 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1221080678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:00 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1221080678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:01.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:01 smithi123 ceph-mon[104968]: pgmap v15: 89 pgs: 89 active+clean; 1.9 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T20:35:01.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:01 smithi123 ceph-mon[104968]: Upgrade: Updating mgr.x 2023-09-23T20:35:01.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:01 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:35:01.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:01 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:35:01.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:01 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:35:01.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:01 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:01.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:01 smithi123 ceph-mon[104968]: Deploying daemon mgr.x on smithi123 2023-09-23T20:35:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[100258]: pgmap v15: 89 pgs: 89 active+clean; 1.9 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T20:35:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[100258]: Upgrade: Updating mgr.x 2023-09-23T20:35:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:35:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:35:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:35:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[100258]: Deploying daemon mgr.x on smithi123 2023-09-23T20:35:01.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[118228]: pgmap v15: 89 pgs: 89 active+clean; 1.9 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-09-23T20:35:01.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[118228]: Upgrade: Updating mgr.x 2023-09-23T20:35:01.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:35:01.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:35:01.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:35:01.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:01.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:01 smithi121 ceph-mon[118228]: Deploying daemon mgr.x on smithi123 2023-09-23T20:35:02.454 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:02 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4216210991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:02 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4216210991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:02 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/4216210991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:03.114 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:03 smithi123 systemd[1]: Stopping Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:35:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:03 smithi121 ceph-mon[100258]: pgmap v16: 89 pgs: 89 active+clean; 1.9 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T20:35:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:03 smithi121 ceph-mon[118228]: pgmap v16: 89 pgs: 89 active+clean; 1.9 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T20:35:03.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:03 smithi123 ceph-mon[104968]: pgmap v16: 89 pgs: 89 active+clean; 1.9 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T20:35:03.984 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:03 smithi123 bash[178145]: 45884a500e333dfb091c4a15838a1a51b701cc81c321c9f9f280ded0736d5972 2023-09-23T20:35:04.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:04 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:35:04.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:04 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T20:35:04.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:04 smithi123 systemd[1]: Stopped Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:35:04.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:04 smithi123 systemd[1]: Starting Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:35:05.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:35:04] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2023-09-23T20:35:05.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:05 smithi121 ceph-mon[118228]: pgmap v17: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T20:35:05.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:05 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1543643161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:05 smithi121 ceph-mon[100258]: pgmap v17: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T20:35:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:05 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1543643161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:05.967 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:05 smithi123 ceph-mon[104968]: pgmap v17: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 910 op/s 2023-09-23T20:35:05.967 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:05 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1543643161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:05.967 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:05 smithi123 bash[178488]: df813ef15dd6e0eb4d965a9a9bd502c29c2ad0b9afeab1ef69181b2c07528131 2023-09-23T20:35:06.230 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:05 smithi123 systemd[1]: Started Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:35:06.230 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:06 smithi123 conmon[178523]: 2023-09-23T20:35:06.188+0000 7fb316128200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:35:06.637 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:06 smithi123 conmon[178523]: 2023-09-23T20:35:06.323+0000 7fb316128200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:35:06.637 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:06 smithi123 conmon[178523]: 2023-09-23T20:35:06.470+0000 7fb316128200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:35:06.638 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:06 smithi123 conmon[178523]: 2023-09-23T20:35:06.626+0000 7fb316128200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:35:06.909 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:06 smithi123 conmon[178523]: 2023-09-23T20:35:06.901+0000 7fb316128200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:35:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:07 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:07 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:07 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:07 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:07 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:07 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:07 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:07.874 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:07 smithi123 conmon[178523]: 2023-09-23T20:35:07.519+0000 7fb316128200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:35:07.874 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:07 smithi123 conmon[178523]: 2023-09-23T20:35:07.718+0000 7fb316128200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:35:07.874 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:07 smithi123 conmon[178523]: 2023-09-23T20:35:07.865+0000 7fb316128200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:35:08.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:08 smithi123 ceph-mon[104968]: pgmap v18: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:35:08.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:08 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/157008718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:08.233 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:08 smithi123 conmon[178523]: 2023-09-23T20:35:08.025+0000 7fb316128200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:35:08.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:08 smithi121 ceph-mon[100258]: pgmap v18: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:35:08.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:08 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/157008718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:08.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:08 smithi121 ceph-mon[118228]: pgmap v18: 89 pgs: 89 active+clean; 1.7 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:35:08.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:08 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/157008718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:08.981 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:35:08 smithi123 conmon[177538]: ts=2023-09-23T20:35:08.669Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:35:08.981 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:08 smithi123 conmon[178523]: 2023-09-23T20:35:08.742+0000 7fb316128200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:35:10.121 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:09 smithi123 conmon[178523]: 2023-09-23T20:35:09.842+0000 7fb316128200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:35:10.121 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:09 smithi123 conmon[178523]: 2023-09-23T20:35:09.990+0000 7fb316128200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:35:10.122 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:10 smithi123 ceph-mon[104968]: pgmap v19: 89 pgs: 89 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T20:35:10.122 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:10 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3682989967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:10 smithi121 ceph-mon[100258]: pgmap v19: 89 pgs: 89 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T20:35:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:10 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3682989967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:10 smithi121 ceph-mon[118228]: pgmap v19: 89 pgs: 89 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T20:35:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:10 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3682989967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:11.735 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:11 smithi123 conmon[178523]: 2023-09-23T20:35:11.309+0000 7fb316128200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:35:11.735 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:11 smithi123 conmon[178523]: 2023-09-23T20:35:11.467+0000 7fb316128200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:35:12.125 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:12 smithi123 ceph-mon[104968]: pgmap v20: 89 pgs: 89 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T20:35:12.125 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:11 smithi123 conmon[178523]: 2023-09-23T20:35:11.780+0000 7fb316128200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:35:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:12 smithi121 ceph-mon[118228]: pgmap v20: 89 pgs: 89 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T20:35:12.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:12 smithi121 ceph-mon[100258]: pgmap v20: 89 pgs: 89 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T20:35:12.480 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:12 smithi123 conmon[178523]: 2023-09-23T20:35:12.266+0000 7fb316128200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:35:12.981 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:12 smithi123 conmon[178523]: 2023-09-23T20:35:12.601+0000 7fb316128200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:35:12.982 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:12 smithi123 conmon[178523]: 2023-09-23T20:35:12.843+0000 7fb316128200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:35:13.591 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:13 smithi123 ceph-mon[104968]: pgmap v21: 89 pgs: 89 active+clean; 1.5 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:35:13.591 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:13 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3337206423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:13.591 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:13 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:13.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:13 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:13.592 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:13 smithi123 conmon[178523]: 2023-09-23T20:35:13.291+0000 7fb316128200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:35:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[100258]: pgmap v21: 89 pgs: 89 active+clean; 1.5 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:35:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3337206423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[118228]: pgmap v21: 89 pgs: 89 active+clean; 1.5 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:35:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3337206423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:13 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:14.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:14 smithi123 conmon[178523]: 2023-09-23T20:35:14.396+0000 7fb316128200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:35:14.729 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:14 smithi123 conmon[178523]: 2023-09-23T20:35:14.613+0000 7fb316128200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:35:15.167 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:35:14] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T20:35:15.230 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:14 smithi123 conmon[178523]: 2023-09-23T20:35:14.955+0000 7fb316128200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:35:15.230 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:15 smithi123 conmon[178523]: 2023-09-23T20:35:15.147+0000 7fb316128200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:35:15.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:15 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:15.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:15 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:15.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:15 smithi123 ceph-mon[104968]: pgmap v22: 89 pgs: 89 active+clean; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-09-23T20:35:15.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:15 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/252879712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:15.676 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:15 smithi123 conmon[178523]: 2023-09-23T20:35:15.518+0000 7fb316128200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:35:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[100258]: pgmap v22: 89 pgs: 89 active+clean; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-09-23T20:35:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/252879712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[118228]: pgmap v22: 89 pgs: 89 active+clean; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-09-23T20:35:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:15 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/252879712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:17.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:16 smithi123 conmon[178523]: 2023-09-23T20:35:16.896+0000 7fb316128200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:35:17.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:16 smithi123 conmon[178523]: [23/Sep/2023:20:35:16] ENGINE Bus STARTING 2023-09-23T20:35:17.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:17 smithi123 conmon[178523]: CherryPy Checker: 2023-09-23T20:35:17.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:17 smithi123 conmon[178523]: The Application mounted at '' has an empty config. 2023-09-23T20:35:17.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:17 smithi123 conmon[178523]: 2023-09-23T20:35:17.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:17 smithi123 conmon[178523]: [23/Sep/2023:20:35:17] ENGINE Serving on http://:::9283 2023-09-23T20:35:17.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:17 smithi123 conmon[178523]: [23/Sep/2023:20:35:17] ENGINE Bus STARTED 2023-09-23T20:35:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[100258]: pgmap v23: 89 pgs: 89 active+clean; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T20:35:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[100258]: Standby manager daemon x restarted 2023-09-23T20:35:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[100258]: Standby manager daemon x started 2023-09-23T20:35:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:35:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[100258]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[118228]: pgmap v23: 89 pgs: 89 active+clean; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[118228]: Standby manager daemon x restarted 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[118228]: Standby manager daemon x started 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:35:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:35:17.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:17 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:35:17.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:17 smithi123 ceph-mon[104968]: pgmap v23: 89 pgs: 89 active+clean; 1.3 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T20:35:17.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:17 smithi123 ceph-mon[104968]: Standby manager daemon x restarted 2023-09-23T20:35:17.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:17 smithi123 ceph-mon[104968]: Standby manager daemon x started 2023-09-23T20:35:17.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:17 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:35:17.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:17 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:35:17.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:17 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:35:17.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:17 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/4032868324' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:35:18.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:18 smithi121 ceph-mon[100258]: mgrmap e30: y(active, since 41s), standbys: x 2023-09-23T20:35:18.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:18 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/2495186085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:18.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:18 smithi121 ceph-mon[118228]: mgrmap e30: y(active, since 41s), standbys: x 2023-09-23T20:35:18.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:18 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/2495186085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:18.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:18 smithi123 ceph-mon[104968]: mgrmap e30: y(active, since 41s), standbys: x 2023-09-23T20:35:18.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:18 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2495186085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:18.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:35:18 smithi123 conmon[177538]: ts=2023-09-23T20:35:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:35:19.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:19 smithi121 ceph-mon[100258]: pgmap v24: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-09-23T20:35:19.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:19 smithi121 ceph-mon[118228]: pgmap v24: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-09-23T20:35:19.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:19 smithi123 ceph-mon[104968]: pgmap v24: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2023-09-23T20:35:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:20 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/189746844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:20 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/189746844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:20 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/189746844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:21 smithi121 ceph-mon[100258]: pgmap v25: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T20:35:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:21 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:21 smithi121 ceph-mon[118228]: pgmap v25: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T20:35:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:21 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:21 smithi123 ceph-mon[104968]: pgmap v25: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s 2023-09-23T20:35:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:21 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:23.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:23.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: pgmap v26: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T20:35:23.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:23.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:35:23.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: Upgrade: Setting container_image for all mgr 2023-09-23T20:35:23.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:35:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/3982891597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: pgmap v26: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:23.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: Upgrade: Setting container_image for all mgr 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:35:23.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:23 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3982891597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: pgmap v26: 89 pgs: 89 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:35:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: Upgrade: Setting container_image for all mgr 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:35:23.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:23 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3982891597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:25.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:35:24] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.43.0" 2023-09-23T20:35:25.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:25 smithi121 ceph-mon[100258]: pgmap v27: 89 pgs: 89 active+clean; 868 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-09-23T20:35:25.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:25 smithi121 ceph-mon[118228]: pgmap v27: 89 pgs: 89 active+clean; 868 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-09-23T20:35:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:25 smithi123 ceph-mon[104968]: pgmap v27: 89 pgs: 89 active+clean; 868 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-09-23T20:35:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[100258]: Upgrade: Updating mon.a 2023-09-23T20:35:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[100258]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:35:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:35:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[100258]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[100258]: Deploying daemon mon.a on smithi121 2023-09-23T20:35:26.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[100258]: from='client.? 172.21.15.121:0/1351249912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[118228]: Upgrade: Updating mon.a 2023-09-23T20:35:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:35:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:35:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[118228]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[118228]: Deploying daemon mon.a on smithi121 2023-09-23T20:35:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:26 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1351249912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:26.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:26 smithi123 ceph-mon[104968]: Upgrade: Updating mon.a 2023-09-23T20:35:26.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:26 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='mgr.y' 2023-09-23T20:35:26.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:26 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:35:26.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:26 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:35:26.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:26 smithi123 ceph-mon[104968]: from='mgr.25031 172.21.15.121:0/2951891624' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:26.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:26 smithi123 ceph-mon[104968]: Deploying daemon mon.a on smithi121 2023-09-23T20:35:26.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:26 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1351249912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:27.874 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:27 smithi121 ceph-mon[100258]: pgmap v28: 89 pgs: 89 active+clean; 868 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T20:35:27.874 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:27 smithi121 systemd[1]: Stopping Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:35:27.874 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:27 smithi121 ceph-mon[118228]: pgmap v28: 89 pgs: 89 active+clean; 868 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T20:35:27.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:27 smithi123 ceph-mon[104968]: pgmap v28: 89 pgs: 89 active+clean; 868 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T20:35:28.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:28 smithi121 conmon[100234]: 2023-09-23T20:35:28.369+0000 7f112e45a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:35:28.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:28 smithi121 conmon[100234]: 2023-09-23T20:35:28.369+0000 7f112e45a700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T20:35:28.623 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:28 smithi121 bash[211116]: 081ecc327acfa7958e19348e2e028fe1a2d042dc6edd34bfcea72a95e3fdf195 2023-09-23T20:35:28.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:35:28 smithi123 conmon[177538]: ts=2023-09-23T20:35:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:35:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:29 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T20:35:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:29 smithi121 systemd[1]: Stopped Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:35:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:29 smithi121 systemd[1]: Starting Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:35:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 bash[211459]: 41cbfbe73b6ea1e161be620558526c576fa2e3554d617a5ddcf12c2091ee6c8a 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 9 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: pidfile_write: ignore empty --pid-file 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: load: jerasure load: lrc 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Git sha 0 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: DB SUMMARY 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: DB Session ID: OASLJLW0S1000Y3QFYHL 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:35:30.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1171 Bytes 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 2582115 ; 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.env: 0x563c11e50720 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.info_log: 0x563c13c37420 2023-09-23T20:35:30.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.statistics: (nil) 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.use_fsync: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:35:30.922 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.db_log_dir: 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.wal_dir: 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:35:30.923 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.write_buffer_manager: 0x563c12ec6c80 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.unordered_write: 0 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:35:30.924 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.row_cache: None 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.wal_filter: None 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.wal_compression: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:35:30.925 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:35:30.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_open_files: -1 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Compression algorithms supported: 2023-09-23T20:35:30.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kZSTD supported: 0 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:35:30.928 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.merge_operator: 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_filter: None 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563c13c37520) 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:35:30.929 INFO:journalctl@ceph.mon.a.smithi121.stdout: index_type: 0 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: data_block_index_type: 0 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: index_shortening: 1 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: checksum: 4 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: no_block_cache: 0 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache: 0x563c12f49350 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache_options: 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: capacity : 536870912 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: num_shard_bits : 4 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: strict_capacity_limit : 0 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:35:30.930 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_cache_compressed: (nil) 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: persistent_cache: (nil) 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_size: 4096 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_size_deviation: 10 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_restart_interval: 16 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: index_block_restart_interval: 1 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: metadata_block_size: 4096 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: partition_filters: 0 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: use_delta_encoding: 1 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: filter_policy: bloomfilter 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: whole_key_filtering: 1 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: verify_compression: 0 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:35:30.931 INFO:journalctl@ceph.mon.a.smithi121.stdout: format_version: 5 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout: enable_index_compression: 1 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout: block_align: 0 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout: max_auto_readahead_size: 262144 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout: prepopulate_block_cache: 0 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression: NoCompression 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:35:30.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:35:30.933 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.num_levels: 7 2023-09-23T20:35:30.933 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:35:30.933 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:35:30.933 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:35:30.933 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:35:30.933 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:35:30.933 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:35:30.934 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:35:30.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:35:30.936 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:35:30.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.ttl: 2592000 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:35:30.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-09-23T20:35:30.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 systemd[1]: Started Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12280, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d3cdedbb-126c-42ce-8a13-1eb8400b9e28 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501330645346, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501330655709, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2375545, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12281, "largest_seqno": 13580, "table_properties": {"data_size": 2369420, "index_size": 3698, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 12650, "raw_average_key_size": 22, "raw_value_size": 2357541, "raw_average_value_size": 4278, "num_data_blocks": 170, "num_entries": 551, "num_filter_entries": 551, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695501330, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d3cdedbb-126c-42ce-8a13-1eb8400b9e28", "db_session_id": "OASLJLW0S1000Y3QFYHL", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501330655827, "job": 1, "event": "recovery_finished"} 2023-09-23T20:35:30.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563c12fe4000 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: DB pointer 0x563c12fd2000 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** DB Stats ** 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:35:30.942 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: L0 1/0 2.27 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 472.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: L6 1/0 5.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: Sum 2/0 7.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 472.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 472.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:35:30.943 INFO:journalctl@ceph.mon.a.smithi121.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 472.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: Cumulative compaction: 0.00 GB write, 83.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: Interval compaction: 0.00 GB write, 83.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:35:30.944 INFO:journalctl@ceph.mon.a.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: Block cache BinnedLRUCache@0x563c12f49350#9 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: starting mon.a rank 0 at public addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] at bind addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???) e3 preinit fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???).mds e9 new map 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???).mds e9 print_map 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: e9 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:35:30.945 INFO:journalctl@ceph.mon.a.smithi121.stdout: legacy client fscid: 1 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: Filesystem 'foo' (1) 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: fs_name foo 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: epoch 8 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: created 2023-09-23T20:30:22.104035+0000 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: modified 2023-09-23T20:30:43.483349+0000 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: tableserver 0 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: root 0 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: session_timeout 60 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: session_autoclose 300 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: max_file_size 1099511627776 2023-09-23T20:35:30.946 INFO:journalctl@ceph.mon.a.smithi121.stdout: required_client_features {} 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: last_failure 0 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: last_failure_osd_epoch 0 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: max_mds 1 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: in 0 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: up {0=24578} 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: failed 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: damaged 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: stopped 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: data_pools [4] 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: metadata_pool 3 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: inline_data disabled 2023-09-23T20:35:30.947 INFO:journalctl@ceph.mon.a.smithi121.stdout: balancer 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: bal_rank_mask -1 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: standby_count_wanted 1 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: [mds.foo.smithi121.sdlxlb{0:24578} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: Standby daemons: 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout: [mds.foo.smithi123.clbyhs{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:35:30.948 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expand map: {default=false} 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta from 'false' to 'false' 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expanded map: {default=false} 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expand map: {default=info} 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta from 'info' to 'info' 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expanded map: {default=info} 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expand map: {default=daemon} 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expand map: {default=debug} 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T20:35:30.949 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:30 smithi121 ceph-mon[211526]: expand_channel_meta expanded map: {default=debug} 2023-09-23T20:35:32.479 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:32 smithi123 conmon[178523]: ignoring --setuser ceph since I am not root 2023-09-23T20:35:32.480 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:32 smithi123 conmon[178523]: ignoring --setgroup ceph since I am not root 2023-09-23T20:35:32.480 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mgr[178550]: -- 172.21.15.123:0/2947885161 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558568e0f1e0 con 0x558568de6c00 2023-09-23T20:35:32.480 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:32 smithi123 conmon[178523]: 2023-09-23T20:35:32.409+0000 7f4c195f7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:35:32.491 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/4109475536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:32.491 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: pgmap v29: 89 pgs: 89 active+clean; 646 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: pgmap v30: 89 pgs: 89 active+clean; 646 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3646442314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: mon.a calling monitor election 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: mgrmap e30: y(active, since 55s), standbys: x 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: mgrmap e31: y(active, since 55s), standbys: x 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: overall HEALTH_OK 2023-09-23T20:35:32.492 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:32 smithi123 ceph-mon[104968]: from='mgr.25031 ' entity='' 2023-09-23T20:35:32.522 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/4109475536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:32.522 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: pgmap v29: 89 pgs: 89 active+clean; 646 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-09-23T20:35:32.522 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: pgmap v30: 89 pgs: 89 active+clean; 646 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:35:32.522 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3646442314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: mon.a calling monitor election 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: mgrmap e30: y(active, since 55s), standbys: x 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: mgrmap e31: y(active, since 55s), standbys: x 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: overall HEALTH_OK 2023-09-23T20:35:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[118228]: from='mgr.25031 ' entity='' 2023-09-23T20:35:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4109475536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: pgmap v29: 89 pgs: 89 active+clean; 646 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-09-23T20:35:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: pgmap v30: 89 pgs: 89 active+clean; 646 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:35:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3646442314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: mon.a calling monitor election 2023-09-23T20:35:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: mgrmap e30: y(active, since 55s), standbys: x 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: mgrmap e31: y(active, since 55s), standbys: x 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: overall HEALTH_OK 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:32 smithi121 ceph-mon[211526]: from='mgr.25031 ' entity='' 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:32 smithi121 conmon[198376]: ignoring --setuser ceph since I am not root 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:32 smithi121 conmon[198376]: ignoring --setgroup ceph since I am not root 2023-09-23T20:35:32.525 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:32 smithi121 conmon[198376]: 2023-09-23T20:35:32.403+0000 7f47dc551200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:35:32.526 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:32 smithi121 conmon[198376]: 2023-09-23T20:35:32.518+0000 7f47dc551200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:35:32.803 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:32 smithi121 conmon[198376]: 2023-09-23T20:35:32.646+0000 7f47dc551200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:35:32.803 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:32 smithi121 conmon[198376]: 2023-09-23T20:35:32.798+0000 7f47dc551200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:35:32.816 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:32 smithi123 conmon[178523]: 2023-09-23T20:35:32.534+0000 7f4c195f7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:35:32.817 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:32 smithi123 conmon[178523]: 2023-09-23T20:35:32.671+0000 7f4c195f7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:35:32.817 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:32 smithi123 conmon[178523]: 2023-09-23T20:35:32.816+0000 7f4c195f7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:35:33.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:33 smithi121 conmon[198376]: 2023-09-23T20:35:33.035+0000 7f47dc551200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:35:33.478 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:33 smithi123 conmon[178523]: 2023-09-23T20:35:33.083+0000 7f4c195f7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:35:33.829 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T20:35:33.950 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:33 smithi123 conmon[178523]: 2023-09-23T20:35:33.628+0000 7f4c195f7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:35:33.950 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:33 smithi123 conmon[178523]: 2023-09-23T20:35:33.813+0000 7f4c195f7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:35:33.951 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:33 smithi123 conmon[178523]: 2023-09-23T20:35:33.950+0000 7f4c195f7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:35:33.951 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:33 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1789969567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:34.015 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1789969567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:34.017 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:33 smithi121 conmon[198376]: 2023-09-23T20:35:33.660+0000 7f47dc551200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:35:34.017 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:33 smithi121 conmon[198376]: 2023-09-23T20:35:33.864+0000 7f47dc551200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:35:34.017 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:34 smithi121 conmon[198376]: 2023-09-23T20:35:34.013+0000 7f47dc551200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:35:34.018 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:33 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1789969567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:35:34.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:34 smithi123 conmon[178523]: 2023-09-23T20:35:34.092+0000 7f4c195f7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:35:34.421 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:34 smithi121 conmon[198376]: 2023-09-23T20:35:34.161+0000 7f47dc551200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:35:35.165 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:34 smithi121 conmon[198376]: 2023-09-23T20:35:34.777+0000 7f47dc551200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:35:35.232 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:34 smithi123 conmon[178523]: 2023-09-23T20:35:34.749+0000 7f4c195f7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:35:35.522 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:35:35.753 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:35 smithi121 conmon[198376]: 2023-09-23T20:35:35.750+0000 7f47dc551200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:35:35.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:35 smithi123 conmon[178523]: 2023-09-23T20:35:35.717+0000 7f4c195f7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:35:35.979 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:35 smithi123 conmon[178523]: 2023-09-23T20:35:35.846+0000 7f4c195f7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:35:36.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:35 smithi121 conmon[198376]: 2023-09-23T20:35:35.901+0000 7f47dc551200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:35:36.269 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:37.139 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:37 smithi121 conmon[198376]: 2023-09-23T20:35:37.137+0000 7f47dc551200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:35:37.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:37 smithi121 conmon[198376]: 2023-09-23T20:35:37.279+0000 7f47dc551200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:35:37.478 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:37 smithi123 conmon[178523]: 2023-09-23T20:35:37.039+0000 7f4c195f7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:35:37.478 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:37 smithi123 conmon[178523]: 2023-09-23T20:35:37.179+0000 7f4c195f7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:35:37.478 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:37 smithi123 conmon[178523]: 2023-09-23T20:35:37.475+0000 7f4c195f7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:35:37.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:37 smithi121 conmon[198376]: 2023-09-23T20:35:37.545+0000 7f47dc551200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:35:38.207 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:37 smithi123 conmon[178523]: 2023-09-23T20:35:37.886+0000 7f4c195f7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:35:38.207 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:38 smithi123 conmon[178523]: 2023-09-23T20:35:38.202+0000 7f4c195f7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:35:38.282 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:37 smithi121 conmon[198376]: 2023-09-23T20:35:37.935+0000 7f47dc551200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:35:38.481 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:38 smithi123 conmon[178523]: 2023-09-23T20:35:38.424+0000 7f4c195f7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:35:38.552 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:38 smithi121 conmon[198376]: 2023-09-23T20:35:38.279+0000 7f47dc551200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:35:38.552 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:38 smithi121 conmon[198376]: 2023-09-23T20:35:38.514+0000 7f47dc551200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:35:39.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:38 smithi123 conmon[178523]: 2023-09-23T20:35:38.845+0000 7f4c195f7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:35:39.414 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:38 smithi121 conmon[198376]: 2023-09-23T20:35:38.987+0000 7f47dc551200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:35:40.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:39 smithi123 conmon[178523]: 2023-09-23T20:35:39.901+0000 7f4c195f7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:35:40.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:40 smithi123 conmon[178523]: 2023-09-23T20:35:40.112+0000 7f4c195f7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:35:40.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:40 smithi121 conmon[198376]: 2023-09-23T20:35:40.011+0000 7f47dc551200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:35:40.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:40 smithi121 conmon[198376]: 2023-09-23T20:35:40.211+0000 7f47dc551200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:35:40.738 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:40 smithi123 conmon[178523]: 2023-09-23T20:35:40.418+0000 7f4c195f7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:35:40.739 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:40 smithi123 conmon[178523]: 2023-09-23T20:35:40.593+0000 7f4c195f7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:35:40.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:40 smithi121 conmon[198376]: 2023-09-23T20:35:40.513+0000 7f47dc551200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:35:40.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:40 smithi121 conmon[198376]: 2023-09-23T20:35:40.692+0000 7f47dc551200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:35:41.229 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:40 smithi123 conmon[178523]: 2023-09-23T20:35:40.943+0000 7f4c195f7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:35:41.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:41 smithi121 conmon[198376]: 2023-09-23T20:35:41.024+0000 7f47dc551200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:35:42.129 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 123644 objects 2023-09-23T20:35:42.129 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :165.344 2023-09-23T20:35:42.460 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[178523]: 2023-09-23T20:35:42.206+0000 7f4c195f7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:35:42.461 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[178523]: [23/Sep/2023:20:35:42] ENGINE Bus STARTING 2023-09-23T20:35:42.461 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[178523]: CherryPy Checker: 2023-09-23T20:35:42.461 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[178523]: The Application mounted at '' has an empty config. 2023-09-23T20:35:42.461 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[178523]: 2023-09-23T20:35:42.461 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[178523]: [23/Sep/2023:20:35:42] ENGINE Serving on http://:::9283 2023-09-23T20:35:42.461 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[178523]: [23/Sep/2023:20:35:42] ENGINE Bus STARTED 2023-09-23T20:35:42.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 conmon[104928]: 2023-09-23T20:35:42.241+0000 7f136e659700 -1 mon.b@1(peon).osd e78 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:42.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: Standby manager daemon x restarted 2023-09-23T20:35:42.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: Standby manager daemon x started 2023-09-23T20:35:42.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:35:42.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:35:42.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:35:42.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:35:42.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: Active manager daemon y restarted 2023-09-23T20:35:42.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: Activating manager daemon y 2023-09-23T20:35:42.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:35:42.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: mgrmap e32: y(active, starting, since 0.00762703s), standbys: x 2023-09-23T20:35:42.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:35:42.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:35:42.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:35:42.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:35:42.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:35:42.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:35:42.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:35:42.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:35:42.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:35:42.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:35:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:35:42.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: Standby manager daemon x restarted 2023-09-23T20:35:42.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: Standby manager daemon x started 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: Active manager daemon y restarted 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: Activating manager daemon y 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: mgrmap e32: y(active, starting, since 0.00762703s), standbys: x 2023-09-23T20:35:42.482 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:35:42.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[118204]: 2023-09-23T20:35:42.240+0000 7f4a91a4a700 -1 mon.c@2(peon).osd e78 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: Standby manager daemon x restarted 2023-09-23T20:35:42.484 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: Standby manager daemon x started 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.? 172.21.15.123:0/3301289172' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: Active manager daemon y restarted 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: Activating manager daemon y 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: mgrmap e32: y(active, starting, since 0.00762703s), standbys: x 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:35:42.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi123.clbyhs"}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi121.sdlxlb"}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:35:42.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.222+0000 7f47dc551200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:35:42.487 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: [23/Sep/2023:20:35:42] ENGINE Bus STARTING 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.385+0000 7f4739883700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.385+0000 7f4739883700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.385+0000 7f4739883700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.385+0000 7f4739883700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.385+0000 7f4739883700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.386+0000 7f473707e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.386+0000 7f473707e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.386+0000 7f473707e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.488 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.386+0000 7f473707e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.386+0000 7f473707e700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:35:42.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: CherryPy Checker: 2023-09-23T20:35:42.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: The Application mounted at '' has an empty config. 2023-09-23T20:35:42.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: 2023-09-23T20:35:42.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: [23/Sep/2023:20:35:42] ENGINE Serving on http://:::9283 2023-09-23T20:35:42.489 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:42 smithi121 conmon[198376]: [23/Sep/2023:20:35:42] ENGINE Bus STARTED 2023-09-23T20:35:42.960 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T20:35:42.960 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.a smithi121 *:9093,9094 running (113s) - 5m 18.1M - 0.25.0 c8568f914cd2 a7c6818f0f7f 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.a smithi123 *:3000 running (96s) - 5m 81.0M - 9.4.7 2c41d148cca3 c2e1f7abecff 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi121.sdlxlb smithi121 running (5m) - 5m 16.7M - 17.2.6-1123-g61431f6b 0db14aa01094 c7389c893171 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi123.clbyhs smithi123 running (5m) - 5m 14.2M - 17.2.6-1123-g61431f6b 0db14aa01094 1704af1c5a0c 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.x smithi123 *:8443,9283,8765 running (37s) - 9m 58.7M - 18.2.0-460-gbb0daeff e32c2a64c126 df813ef15dd6 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.y smithi121 *:8443,9283,8765 running (82s) - 11m 481M - 18.2.0-460-gbb0daeff e32c2a64c126 adf60a30334d 2023-09-23T20:35:43.310 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a smithi121 running (11m) - 11m 107M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 081ecc327acf 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b smithi123 running (10m) - 10m 73.4M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 6a36337c715a 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c smithi121 running (10m) - 10m 69.7M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 a0406cc28771 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.a smithi121 *:9100 running (110s) - 5m 11.2M - 1.5.0 0da6a335fe13 ab86679c1d7d 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.b smithi123 *:9100 running (92s) - 5m 9873k - 1.5.0 0da6a335fe13 5f079d4704ad 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (9m) - 9m 1853M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ae38a40da032 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (9m) - 9m 1120M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 b5379433baef 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (8m) - 8m 2410M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 2c9b742b8c06 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (8m) - 8m 792M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcc588b3144b 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi123 running (8m) - 8m 2018M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 470c2a4d010b 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi123 running (7m) - 7m 2148M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ab39a1466c34 2023-09-23T20:35:43.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi123 running (7m) - 7m 1030M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 3f6aba9e3a6b 2023-09-23T20:35:43.312 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi123 running (6m) - 6m 1523M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 514f59904aaa 2023-09-23T20:35:43.312 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.a smithi123 *:9095 running (45s) - 6m 44.5M - 2.43.0 a07b618ecd1d 9b8ac663186b 2023-09-23T20:35:43.601 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:43 smithi121 conmon[198376]: 2023-09-23T20:35:43.287+0000 7f479dcb1700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T20:35:43.601 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.601 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.601 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.601 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: Manager daemon y is now available 2023-09-23T20:35:43.601 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:43.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:43.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:43.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:35:43.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:35:43.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: [23/Sep/2023:20:35:43] ENGINE Bus STARTING 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[118228]: [23/Sep/2023:20:35:43] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: Manager daemon y is now available 2023-09-23T20:35:43.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:43.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:43.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:43.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:35:43.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:35:43.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:35:43.605 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:35:43.605 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:43.605 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:43.605 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: [23/Sep/2023:20:35:43] ENGINE Bus STARTING 2023-09-23T20:35:43.605 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:43 smithi121 ceph-mon[211526]: [23/Sep/2023:20:35:43] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:35:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: Manager daemon y is now available 2023-09-23T20:35:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: failed to encode map e78 with expected crc 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:35:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:43.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:43.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: [23/Sep/2023:20:35:43] ENGINE Bus STARTING 2023-09-23T20:35:43.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:43 smithi123 ceph-mon[104968]: [23/Sep/2023:20:35:43] ENGINE Serving on https://172.21.15.121:7150 2023-09-23T20:35:44.656 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[118228]: mgrmap e33: y(active, since 1.04916s), standbys: x 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[118228]: from='client.34113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[118228]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[118228]: [23/Sep/2023:20:35:43] ENGINE Serving on http://172.21.15.121:8765 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[118228]: [23/Sep/2023:20:35:43] ENGINE Bus STARTED 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[211526]: mgrmap e33: y(active, since 1.04916s), standbys: x 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[211526]: from='client.34113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[211526]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[211526]: [23/Sep/2023:20:35:43] ENGINE Serving on http://172.21.15.121:8765 2023-09-23T20:35:44.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:44 smithi121 ceph-mon[211526]: [23/Sep/2023:20:35:43] ENGINE Bus STARTED 2023-09-23T20:35:44.658 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:44.681 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:35:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:44 smithi123 ceph-mon[104968]: mgrmap e33: y(active, since 1.04916s), standbys: x 2023-09-23T20:35:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:44 smithi123 ceph-mon[104968]: from='client.34113 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:35:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:44 smithi123 ceph-mon[104968]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:44 smithi123 ceph-mon[104968]: [23/Sep/2023:20:35:43] ENGINE Serving on http://172.21.15.121:8765 2023-09-23T20:35:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:44 smithi123 ceph-mon[104968]: [23/Sep/2023:20:35:43] ENGINE Bus STARTED 2023-09-23T20:35:44.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:35:44] "GET /metrics HTTP/1.1" 200 33504 "" "Prometheus/2.43.0" 2023-09-23T20:35:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:45 smithi121 ceph-mon[118228]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:45 smithi121 ceph-mon[118228]: mgrmap e34: y(active, since 2s), standbys: x 2023-09-23T20:35:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:45 smithi121 ceph-mon[211526]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:45 smithi121 ceph-mon[211526]: mgrmap e34: y(active, since 2s), standbys: x 2023-09-23T20:35:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:45 smithi123 ceph-mon[104968]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:45 smithi123 ceph-mon[104968]: mgrmap e34: y(active, since 2s), standbys: x 2023-09-23T20:35:46.199 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:47.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.354 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:46 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:46 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:46 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:47.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:46 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:48.015 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_0' removed 2023-09-23T20:35:48.333 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:47 smithi121 ceph-mon[118228]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:48.333 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:47 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/3283208508' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:35:48.333 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:47 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:35:48.333 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:48 smithi121 conmon[118204]: 2023-09-23T20:35:48.003+0000 7f4a91a4a700 -1 mon.c@2(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:48.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:47 smithi121 ceph-mon[211526]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:48.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3283208508' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:35:48.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:35:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:47 smithi123 ceph-mon[104968]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:47 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3283208508' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:35:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:47 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:35:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:48 smithi123 conmon[104928]: 2023-09-23T20:35:48.003+0000 7f136e659700 -1 mon.b@1(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:48.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:35:48 smithi123 conmon[177538]: ts=2023-09-23T20:35:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:35:49.160 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:35:49.161 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:35:49.161 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T20:35:49.161 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T20:35:49.182 INFO:teuthology.orchestra.run.smithi121.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:35:49.300 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:35:49.300 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[118228]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:35:49.300 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[118228]: failed to encode map e79 with expected crc 2023-09-23T20:35:49.300 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[118228]: failed to encode map e79 with expected crc 2023-09-23T20:35:49.301 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:35:49.301 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[211526]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:35:49.301 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[211526]: failed to encode map e79 with expected crc 2023-09-23T20:35:49.301 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:49 smithi121 ceph-mon[211526]: failed to encode map e79 with expected crc 2023-09-23T20:35:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:49 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:35:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:49 smithi123 ceph-mon[104968]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:35:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:49 smithi123 ceph-mon[104968]: failed to encode map e79 with expected crc 2023-09-23T20:35:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:49 smithi123 ceph-mon[104968]: failed to encode map e79 with expected crc 2023-09-23T20:35:49.920 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:35:49.958 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:50 smithi123 ceph-mon[104968]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:50 smithi123 ceph-mon[104968]: from='client.25220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:35:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:50 smithi123 ceph-mon[104968]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:35:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:50 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:50 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:50 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[118228]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[118228]: from='client.25220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[118228]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[211526]: pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 679 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:50.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[211526]: from='client.25220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:35:50.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[211526]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:35:50.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:50.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:51.120 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:52.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[118228]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 684 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:52.142 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.142 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.142 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.142 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.142 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:52.142 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:52.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[211526]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 684 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:52.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:52.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:52 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 ceph-mon[104968]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 684 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:52.743 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_1' created 2023-09-23T20:35:52.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:52 smithi123 conmon[104928]: 2023-09-23T20:35:52.725+0000 7f136e659700 -1 mon.b@1(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:53.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:52 smithi121 conmon[118204]: 2023-09-23T20:35:52.725+0000 7f4a91a4a700 -1 mon.c@2(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:35:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:35:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:35:53.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:53.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.conf 2023-09-23T20:35:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.conf 2023-09-23T20:35:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:35:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:35:53.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:35:53.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:35:53.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T20:35:53.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:53.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:53.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:53.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:35:53.736 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T20:35:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:53 smithi123 conmon[104928]: 2023-09-23T20:35:53.726+0000 7f136e659700 -1 mon.b@1(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:54.079 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:54.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:53 smithi121 conmon[118204]: 2023-09-23T20:35:53.726+0000 7f4a91a4a700 -1 mon.c@2(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 684 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:35:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:35:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:35:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:35:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: failed to encode map e80 with expected crc 2023-09-23T20:35:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: failed to encode map e80 with expected crc 2023-09-23T20:35:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T20:35:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 684 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:35:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:35:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:35:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:35:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: failed to encode map e80 with expected crc 2023-09-23T20:35:54.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: failed to encode map e80 with expected crc 2023-09-23T20:35:54.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T20:35:54.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 684 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: Updating smithi121:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:35:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:35:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: Updating smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:35:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: Updating smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:35:54.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: failed to encode map e80 with expected crc 2023-09-23T20:35:54.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: failed to encode map e80 with expected crc 2023-09-23T20:35:54.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T20:35:54.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:54.748 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:54 smithi121 conmon[118204]: 2023-09-23T20:35:54.728+0000 7f4a91a4a700 -1 mon.c@2(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:55.155 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:35:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:35:54] "GET /metrics HTTP/1.1" 200 33504 "" "Prometheus/2.43.0" 2023-09-23T20:35:55.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:54 smithi123 conmon[104928]: 2023-09-23T20:35:54.728+0000 7f136e659700 -1 mon.b@1(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:55.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T20:35:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[211526]: failed to encode map e82 with expected crc 2023-09-23T20:35:55.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T20:35:55.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:55 smithi121 ceph-mon[118228]: failed to encode map e82 with expected crc 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: failed to encode map e81 with expected crc 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T20:35:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:55 smithi123 ceph-mon[104968]: failed to encode map e82 with expected crc 2023-09-23T20:35:56.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[211526]: pgmap v12: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:56.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[211526]: failed to encode map e82 with expected crc 2023-09-23T20:35:56.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1233768131' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:35:56.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:35:56.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[118228]: pgmap v12: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:56.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[118228]: failed to encode map e82 with expected crc 2023-09-23T20:35:56.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[118228]: from='client.? 172.21.15.121:0/1233768131' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:35:56.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:56 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:35:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:56 smithi123 ceph-mon[104968]: pgmap v12: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:56 smithi123 ceph-mon[104968]: failed to encode map e82 with expected crc 2023-09-23T20:35:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:56 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/1233768131' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:35:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:56 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:35:56.695 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T20:35:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:56 smithi123 conmon[104928]: 2023-09-23T20:35:56.681+0000 7f136e659700 -1 mon.b@1(peon).osd e83 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:56 smithi121 conmon[118204]: 2023-09-23T20:35:56.681+0000 7f4a91a4a700 -1 mon.c@2(peon).osd e83 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:35:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:57 smithi123 ceph-mon[104968]: pgmap v14: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:57 smithi123 ceph-mon[104968]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:35:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:57 smithi123 ceph-mon[104968]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:35:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:57 smithi123 ceph-mon[104968]: failed to encode map e83 with expected crc 2023-09-23T20:35:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:57 smithi123 ceph-mon[104968]: failed to encode map e83 with expected crc 2023-09-23T20:35:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:57 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[211526]: pgmap v14: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:35:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[211526]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:35:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[211526]: failed to encode map e83 with expected crc 2023-09-23T20:35:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[211526]: failed to encode map e83 with expected crc 2023-09-23T20:35:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:58.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[118228]: pgmap v14: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:35:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[118228]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:35:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[118228]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:35:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[118228]: failed to encode map e83 with expected crc 2023-09-23T20:35:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[118228]: failed to encode map e83 with expected crc 2023-09-23T20:35:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:57 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:58.434 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T20:35:58.437 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:35:58.437 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:35:58.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T20:35:58.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:35:58.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_216437 2023-09-23T20:35:58.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:35:58.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:35:58.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:35:58 smithi123 conmon[177538]: ts=2023-09-23T20:35:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:35:59.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 4214 4198 262.355 262.375 0.00338807 0.00379753 2023-09-23T20:35:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:35:59 smithi123 ceph-mon[104968]: pgmap v16: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:00.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:35:59 smithi121 ceph-mon[211526]: pgmap v16: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:00.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:35:59 smithi121 ceph-mon[118228]: pgmap v16: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 698 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:00.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 8047 8031 250.945 239.562 0.0027408 0.0039764 2023-09-23T20:36:01.493 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 11535 11519 239.957 218 0.00668045 0.00416142 2023-09-23T20:36:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:01 smithi123 ceph-mon[104968]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:01 smithi121 ceph-mon[211526]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:01 smithi121 ceph-mon[118228]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:02.492 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 14740 14724 230.042 200.312 0.00630209 0.00434062 2023-09-23T20:36:03.493 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 17595 17579 219.691 178.438 0.00485522 0.00454682 2023-09-23T20:36:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:03 smithi123 ceph-mon[104968]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:04.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:03 smithi121 ceph-mon[211526]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:03 smithi121 ceph-mon[118228]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:36:04.493 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 20431 20415 212.616 177.25 0.00459124 0.00469808 2023-09-23T20:36:05.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:36:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:36:04] "GET /metrics HTTP/1.1" 200 36161 "" "Prometheus/2.43.0" 2023-09-23T20:36:05.493 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 23126 23110 206.302 168.438 0.00521396 0.00484187 2023-09-23T20:36:05.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:05 smithi123 ceph-mon[104968]: pgmap v19: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:36:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:05 smithi121 ceph-mon[211526]: pgmap v19: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:36:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:05 smithi121 ceph-mon[118228]: pgmap v19: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:36:06.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 25680 25664 200.466 159.625 0.00535789 0.00498256 2023-09-23T20:36:07.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 28103 28087 195.017 151.438 0.00553767 0.00512236 2023-09-23T20:36:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:07 smithi123 ceph-mon[104968]: pgmap v20: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T20:36:08.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:07 smithi121 ceph-mon[118228]: pgmap v20: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T20:36:08.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:07 smithi121 ceph-mon[211526]: pgmap v20: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T20:36:08.493 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 30429 30413 190.052 145.375 0.00878356 0.00525711 2023-09-23T20:36:08.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:36:08 smithi123 conmon[177538]: ts=2023-09-23T20:36:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:36:09.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 32649 32633 185.387 138.75 0.00897111 0.00539043 2023-09-23T20:36:10.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:09 smithi121 ceph-mon[118228]: pgmap v21: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T20:36:10.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:09 smithi121 ceph-mon[211526]: pgmap v21: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T20:36:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:09 smithi123 ceph-mon[104968]: pgmap v21: 89 pgs: 89 active+clean; 1.0 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T20:36:10.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 34780 34764 181.036 133.188 0.00584763 0.0055188 2023-09-23T20:36:11.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 36830 36814 176.966 128.125 0.00367193 0.00564575 2023-09-23T20:36:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:11 smithi121 ceph-mon[118228]: pgmap v22: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-09-23T20:36:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:11 smithi121 ceph-mon[211526]: pgmap v22: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-09-23T20:36:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:11 smithi123 ceph-mon[104968]: pgmap v22: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-09-23T20:36:12.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 38673 38657 172.552 115.188 0.0064891 0.00578933 2023-09-23T20:36:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:12 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:12 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:13.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 40383 40367 168.173 106.875 0.00883728 0.00594122 2023-09-23T20:36:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:13 smithi121 ceph-mon[118228]: pgmap v23: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-09-23T20:36:14.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:13 smithi121 ceph-mon[211526]: pgmap v23: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-09-23T20:36:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:13 smithi123 ceph-mon[104968]: pgmap v23: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-09-23T20:36:14.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 42305 42289 165.17 120.125 0.00564316 0.00604948 2023-09-23T20:36:15.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:36:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:36:14] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-09-23T20:36:15.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 44180 44164 162.347 117.188 0.00952847 0.0061548 2023-09-23T20:36:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:15 smithi121 ceph-mon[118228]: pgmap v24: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 208 MiB/s wr, 3.33k op/s 2023-09-23T20:36:16.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:15 smithi121 ceph-mon[211526]: pgmap v24: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 208 MiB/s wr, 3.33k op/s 2023-09-23T20:36:16.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:15 smithi123 ceph-mon[104968]: pgmap v24: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 208 MiB/s wr, 3.33k op/s 2023-09-23T20:36:16.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 45990 45974 159.612 113.125 0.0056436 0.0062603 2023-09-23T20:36:17.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 47605 47589 156.523 100.938 0.00955826 0.00638478 2023-09-23T20:36:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:17 smithi121 ceph-mon[211526]: pgmap v25: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:36:18.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:17 smithi121 ceph-mon[118228]: pgmap v25: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:36:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:17 smithi123 ceph-mon[104968]: pgmap v25: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:36:18.494 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:36:18.495170+0000 min lat: 0.00147314 max lat: 0.0847276 avg lat: 0.00648682 2023-09-23T20:36:18.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:36:18.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 49319 49303 154.053 107.125 0.010058 0.00648682 2023-09-23T20:36:18.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:36:18 smithi123 conmon[177538]: ts=2023-09-23T20:36:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:36:19.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 51022 51006 151.784 106.438 0.0103399 0.00658343 2023-09-23T20:36:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:19 smithi121 ceph-mon[118228]: pgmap v26: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:36:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:19 smithi121 ceph-mon[211526]: pgmap v26: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:36:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:19 smithi123 ceph-mon[104968]: pgmap v26: 89 pgs: 89 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:36:20.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 52675 52659 149.58 103.312 0.00981144 0.00668034 2023-09-23T20:36:21.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 54310 54294 147.519 102.188 0.0187509 0.00677389 2023-09-23T20:36:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:21 smithi121 ceph-mon[118228]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2023-09-23T20:36:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:21 smithi121 ceph-mon[211526]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2023-09-23T20:36:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:21 smithi123 ceph-mon[104968]: pgmap v27: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2023-09-23T20:36:22.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 55899 55883 145.51 99.3125 0.0147683 0.00686779 2023-09-23T20:36:23.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 57456 57440 143.582 97.3125 0.0149687 0.00696003 2023-09-23T20:36:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:23 smithi121 ceph-mon[211526]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:36:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:23 smithi121 ceph-mon[118228]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:36:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:23 smithi123 ceph-mon[104968]: pgmap v28: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:36:24.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 15 59012 58997 141.802 97.3125 0.00629783 0.00704777 2023-09-23T20:36:25.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:36:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:36:24] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-09-23T20:36:25.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 60540 60524 140.085 95.4375 0.0130955 0.0071338 2023-09-23T20:36:26.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:25 smithi121 ceph-mon[211526]: pgmap v29: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:36:26.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:25 smithi121 ceph-mon[118228]: pgmap v29: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:36:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:25 smithi123 ceph-mon[104968]: pgmap v29: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:36:26.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 62028 62012 138.403 93 0.00716272 0.00722081 2023-09-23T20:36:27.495 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 63468 63452 136.734 90 0.00925086 0.00730923 2023-09-23T20:36:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:27 smithi121 ceph-mon[211526]: pgmap v30: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:36:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:28.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:27 smithi121 ceph-mon[118228]: pgmap v30: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:36:28.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:27 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:27 smithi123 ceph-mon[104968]: pgmap v30: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:36:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:27 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:28.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 64907 64891 135.174 89.9375 0.00563752 0.00739304 2023-09-23T20:36:28.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:36:28 smithi123 conmon[177538]: ts=2023-09-23T20:36:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:36:29.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 66335 66319 133.692 89.25 0.00710532 0.00747541 2023-09-23T20:36:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:29 smithi121 ceph-mon[118228]: pgmap v31: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:36:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:29 smithi121 ceph-mon[211526]: pgmap v31: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:36:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:29 smithi123 ceph-mon[104968]: pgmap v31: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:36:30.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 67733 67717 132.244 87.375 0.0078224 0.00755787 2023-09-23T20:36:31.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 69084 69068 130.796 84.4375 0.0117309 0.00764057 2023-09-23T20:36:32.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:31 smithi121 ceph-mon[211526]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T20:36:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:31 smithi121 ceph-mon[118228]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T20:36:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:31 smithi123 ceph-mon[104968]: pgmap v32: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T20:36:32.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 70429 70413 129.421 84.0625 0.00727743 0.00772186 2023-09-23T20:36:33.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 71768 71752 128.114 83.6875 0.0143556 0.0078014 2023-09-23T20:36:34.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:33 smithi121 ceph-mon[118228]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:36:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:33 smithi121 ceph-mon[211526]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:36:34.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:33 smithi123 ceph-mon[104968]: pgmap v33: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:36:34.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 73072 73056 126.818 81.5 0.0109964 0.00788079 2023-09-23T20:36:35.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:36:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:36:34] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.43.0" 2023-09-23T20:36:35.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 74376 74360 125.593 81.5 0.0101231 0.00795738 2023-09-23T20:36:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:35 smithi121 ceph-mon[211526]: pgmap v34: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T20:36:36.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:35 smithi121 ceph-mon[118228]: pgmap v34: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T20:36:36.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:35 smithi123 ceph-mon[104968]: pgmap v34: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-09-23T20:36:36.497 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 75655 75639 124.391 79.9375 0.0142074 0.00803435 2023-09-23T20:36:37.497 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 76884 76868 123.171 76.8125 0.00724677 0.00811457 2023-09-23T20:36:38.170 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:37 smithi121 ceph-mon[211526]: pgmap v35: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:36:38.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:37 smithi121 ceph-mon[118228]: pgmap v35: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:36:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:37 smithi123 ceph-mon[104968]: pgmap v35: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:36:38.498 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:36:38.497516+0000 min lat: 0.00147314 max lat: 0.0847276 avg lat: 0.00819223 2023-09-23T20:36:38.498 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:36:38.498 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 78110 78094 122.007 76.625 0.0140478 0.00819223 2023-09-23T20:36:38.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:36:38 smithi123 conmon[177538]: ts=2023-09-23T20:36:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:36:39.497 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 79343 79327 120.911 77.0625 0.0100174 0.00826606 2023-09-23T20:36:40.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:39 smithi121 ceph-mon[211526]: pgmap v36: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:36:40.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:39 smithi121 ceph-mon[118228]: pgmap v36: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:36:40.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:39 smithi123 ceph-mon[104968]: pgmap v36: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:36:40.497 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 80572 80556 119.861 76.8125 0.0121916 0.00833841 2023-09-23T20:36:41.497 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 81783 81767 118.833 75.6875 0.0194308 0.0084111 2023-09-23T20:36:42.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:41 smithi121 ceph-mon[211526]: pgmap v37: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T20:36:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:41 smithi121 ceph-mon[118228]: pgmap v37: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T20:36:42.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:41 smithi123 ceph-mon[104968]: pgmap v37: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T20:36:42.497 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 82989 82973 117.846 75.375 0.0179549 0.00848174 2023-09-23T20:36:43.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:43.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:42 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:43.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:42 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:43.497 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 84190 84174 116.895 75.0625 0.030321 0.00855096 2023-09-23T20:36:44.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:43 smithi121 ceph-mon[118228]: pgmap v38: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:36:44.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:43 smithi121 ceph-mon[211526]: pgmap v38: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:36:44.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:43 smithi123 ceph-mon[104968]: pgmap v38: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:36:44.498 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 85383 85367 115.971 74.5625 0.0148192 0.00861864 2023-09-23T20:36:45.165 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:36:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:36:44] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T20:36:45.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 86538 86522 115.04 72.1875 0.0126432 0.00868788 2023-09-23T20:36:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:45 smithi121 ceph-mon[118228]: pgmap v39: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:36:46.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:45 smithi121 ceph-mon[211526]: pgmap v39: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:36:46.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:45 smithi123 ceph-mon[104968]: pgmap v39: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:36:46.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 87676 87660 114.125 71.125 0.0119464 0.00875775 2023-09-23T20:36:47.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 88779 88763 113.202 68.9375 0.0154957 0.00882921 2023-09-23T20:36:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:47 smithi123 ceph-mon[104968]: pgmap v40: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:36:48.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:47 smithi121 ceph-mon[118228]: pgmap v40: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:36:48.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:47 smithi121 ceph-mon[211526]: pgmap v40: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:36:48.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 89897 89881 112.336 69.875 0.0163583 0.00889742 2023-09-23T20:36:48.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:36:48 smithi123 conmon[177538]: ts=2023-09-23T20:36:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:36:49.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 91036 91020 111.529 71.1875 0.0126983 0.00896162 2023-09-23T20:36:50.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:49 smithi123 ceph-mon[104968]: pgmap v41: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:36:50.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:49 smithi121 ceph-mon[211526]: pgmap v41: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:36:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:49 smithi121 ceph-mon[118228]: pgmap v41: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T20:36:50.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 92150 92134 110.723 69.625 0.00976624 0.00902716 2023-09-23T20:36:51.500 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 93259 93243 109.942 69.3125 0.0140946 0.00909105 2023-09-23T20:36:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:51 smithi123 ceph-mon[104968]: pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T20:36:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:51 smithi121 ceph-mon[211526]: pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T20:36:52.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:51 smithi121 ceph-mon[118228]: pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T20:36:52.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 94355 94339 109.174 68.5 0.016335 0.00915485 2023-09-23T20:36:53.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 95360 95344 108.331 62.8125 0.0150075 0.009226 2023-09-23T20:36:54.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:53 smithi123 ceph-mon[104968]: pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T20:36:54.273 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:53 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:54.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:53 smithi121 ceph-mon[211526]: pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T20:36:54.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:53 smithi121 ceph-mon[118228]: pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T20:36:54.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:53 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:54.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 96416 96400 107.575 66 0.02173 0.00929087 2023-09-23T20:36:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:36:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:36:54] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T20:36:55.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 97456 97440 106.828 65 0.0137372 0.00935576 2023-09-23T20:36:56.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:55 smithi123 ceph-mon[104968]: pgmap v44: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:36:56.331 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:55 smithi121 ceph-mon[211526]: pgmap v44: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:36:56.332 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:55 smithi121 ceph-mon[118228]: pgmap v44: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:36:56.359 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:36:56.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 97984 97968 105.555 33 0.00677343 0.0094683 2023-09-23T20:36:57.259 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:57.500 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 98850 98834 104.682 54.125 0.0172665 0.00954823 2023-09-23T20:36:58.010 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[118228]: pgmap v45: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-09-23T20:36:58.010 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:58.010 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:36:58.010 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:36:58.011 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[211526]: pgmap v45: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-09-23T20:36:58.011 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:58.011 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:36:58.011 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:36:58.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:57 smithi123 ceph-mon[104968]: pgmap v45: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-09-23T20:36:58.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:57 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:58.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:57 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:36:58.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:57 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:36:58.501 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:36:58.501366+0000 min lat: 0.00147314 max lat: 0.359194 avg lat: 0.00962279 2023-09-23T20:36:58.501 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:36:58.501 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 99754 99738 103.879 56.5 0.0117293 0.00962279 2023-09-23T20:36:58.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:36:58 smithi123 conmon[177538]: ts=2023-09-23T20:36:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:36:59.503 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 100708 100692 103.151 59.625 0.0175941 0.00968938 2023-09-23T20:37:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:36:59 smithi123 ceph-mon[104968]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-09-23T20:37:00.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:36:59 smithi121 ceph-mon[211526]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-09-23T20:37:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:36:59 smithi121 ceph-mon[118228]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-09-23T20:37:00.502 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 101589 101573 102.376 55.0625 0.021706 0.0097627 2023-09-23T20:37:00.748 INFO:teuthology.orchestra.run.smithi121.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:01.502 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 102373 102357 101.529 49 0.012231 0.00984484 2023-09-23T20:37:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[211526]: pgmap v47: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:37:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[211526]: from='client.25132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[211526]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:02.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[118228]: pgmap v47: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:37:02.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[118228]: from='client.25132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:02.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[118228]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:02.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:01 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:01 smithi123 ceph-mon[104968]: pgmap v47: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:37:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:01 smithi123 ceph-mon[104968]: from='client.25132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:01 smithi123 ceph-mon[104968]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:01 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:02.253 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T20:37:02.502 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 103263 103247 100.811 55.625 0.00617781 0.00991564 2023-09-23T20:37:03.094 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:03.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 104130 104114 100.09 54.1875 0.0193837 0.00998695 2023-09-23T20:37:04.132 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[211526]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T20:37:04.132 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.133 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.133 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:04.133 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:04.133 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[118228]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T20:37:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:04 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:04 smithi123 ceph-mon[104968]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-09-23T20:37:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:04 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:04 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:04 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:04.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:04 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:04.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:04 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:04.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 104992 104976 99.3895 53.875 0.0250883 0.0100559 2023-09-23T20:37:05.146 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:37:04] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-09-23T20:37:05.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:05.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:37:05.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:37:05.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:37:05.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[211526]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:37:05.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:05.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:37:05.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:37:05.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:37:05.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:05 smithi121 ceph-mon[118228]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:37:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:05 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:05 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:37:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:05 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:37:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:05 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:37:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:05 smithi123 ceph-mon[104968]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:37:05.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 105857 105841 98.7109 54.0625 0.0210935 0.0101261 2023-09-23T20:37:05.756 INFO:teuthology.orchestra.run.smithi121.stdout:wait for majority of mons to upgrade 2023-09-23T20:37:06.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:06 smithi121 ceph-mon[211526]: pgmap v49: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T20:37:06.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:06 smithi121 ceph-mon[118228]: pgmap v49: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T20:37:06.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:06 smithi123 ceph-mon[104968]: pgmap v49: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T20:37:06.507 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 15 106555 106540 97.9017 43.6875 0.0150456 0.0102102 2023-09-23T20:37:07.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T20:37:07.509 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 107401 107385 97.2443 52.8125 0.0157643 0.0102785 2023-09-23T20:37:08.119 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[211526]: pgmap v50: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T20:37:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:08.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[118228]: pgmap v50: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T20:37:08.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[118228]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:08.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:08.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:08.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:08 smithi121 ceph-mon[118228]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:08 smithi123 ceph-mon[104968]: pgmap v50: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T20:37:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:08 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:08 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:37:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:08 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:37:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:08 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:08.510 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 108214 108198 96.5811 50.8125 0.0168884 0.0103492 2023-09-23T20:37:08.982 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:37:08 smithi123 conmon[177538]: ts=2023-09-23T20:37:08.665Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:37:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:09 smithi121 ceph-mon[118228]: Upgrade: Updating mon.c 2023-09-23T20:37:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:09 smithi121 ceph-mon[118228]: Deploying daemon mon.c on smithi121 2023-09-23T20:37:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:09 smithi121 ceph-mon[211526]: Upgrade: Updating mon.c 2023-09-23T20:37:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:09 smithi121 ceph-mon[211526]: Deploying daemon mon.c on smithi121 2023-09-23T20:37:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:09 smithi123 ceph-mon[104968]: Upgrade: Updating mon.c 2023-09-23T20:37:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:09 smithi123 ceph-mon[104968]: Deploying daemon mon.c on smithi121 2023-09-23T20:37:09.512 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 109092 109076 95.9921 54.875 0.014072 0.0104126 2023-09-23T20:37:10.445 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:10 smithi121 ceph-mon[211526]: pgmap v51: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T20:37:10.446 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:10 smithi121 ceph-mon[118228]: pgmap v51: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T20:37:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:10 smithi123 ceph-mon[104968]: pgmap v51: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2023-09-23T20:37:10.511 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 109977 109961 95.4272 55.3125 0.0123246 0.0104745 2023-09-23T20:37:10.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:10 smithi121 systemd[1]: Stopping Ceph mon.c for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:37:11.493 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[198376]: [23/Sep/2023:20:37:11] ENGINE Bus STOPPING 2023-09-23T20:37:11.493 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[198376]: [23/Sep/2023:20:37:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:37:11.493 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[198376]: [23/Sep/2023:20:37:11] ENGINE Bus STOPPED 2023-09-23T20:37:11.493 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[198376]: [23/Sep/2023:20:37:11] ENGINE Bus STARTING 2023-09-23T20:37:11.493 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[198376]: [23/Sep/2023:20:37:11] ENGINE Serving on http://:::9283 2023-09-23T20:37:11.493 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[198376]: [23/Sep/2023:20:37:11] ENGINE Bus STARTED 2023-09-23T20:37:11.494 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[118204]: 2023-09-23T20:37:11.231+0000 7f4a97255700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:37:11.494 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:11 smithi121 conmon[118204]: 2023-09-23T20:37:11.231+0000 7f4a97255700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T20:37:11.494 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:11 smithi121 bash[219266]: a0406cc2877167ac48c6de134c35cbaedd111239f5d8ac3109e37b9f5ea0858f 2023-09-23T20:37:11.514 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 110775 110759 94.7986 49.875 0.0253384 0.0105438 2023-09-23T20:37:12.516 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 111464 111448 94.0981 43.0625 0.0186713 0.0106223 2023-09-23T20:37:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:12 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T20:37:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:12 smithi121 systemd[1]: Stopped Ceph mon.c for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:37:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:12 smithi121 systemd[1]: Starting Ceph mon.c for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:37:13.516 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 112316 112300 93.5535 53.25 0.0166199 0.0106842 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 bash[219644]: 828ec0e17fe967965a3e8392decf1d53622aa9149206cf5e071a252eec920aa7 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 8 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: pidfile_write: ignore empty --pid-file 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 systemd[1]: Started Ceph mon.c for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: load: jerasure load: lrc 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Git sha 0 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:37:14.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: DB SUMMARY 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: DB Session ID: Z9BQQWTV9Q3MUZ6W9HK6 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 7415504 ; 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:37:14.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.env: 0x55e668ac0720 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.info_log: 0x55e66b8c9420 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.statistics: (nil) 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.use_fsync: 0 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:37:14.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.db_log_dir: 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.wal_dir: 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:37:14.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.write_buffer_manager: 0x55e66ab58c80 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.unordered_write: 0 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:37:14.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.row_cache: None 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.wal_filter: None 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.wal_compression: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:37:14.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:37:14.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_open_files: -1 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Compression algorithms supported: 2023-09-23T20:37:14.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kZSTD supported: 0 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:37:14.176 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.merge_operator: 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_filter: None 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e66b8c9520) 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout: index_type: 0 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout: data_block_index_type: 0 2023-09-23T20:37:14.177 INFO:journalctl@ceph.mon.c.smithi121.stdout: index_shortening: 1 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: checksum: 4 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: no_block_cache: 0 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache: 0x55e66abdb1f0 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache_options: 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: capacity : 536870912 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: num_shard_bits : 4 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: strict_capacity_limit : 0 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_cache_compressed: (nil) 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: persistent_cache: (nil) 2023-09-23T20:37:14.178 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_size: 4096 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_size_deviation: 10 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_restart_interval: 16 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: index_block_restart_interval: 1 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: metadata_block_size: 4096 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: partition_filters: 0 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: use_delta_encoding: 1 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: filter_policy: bloomfilter 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: whole_key_filtering: 1 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: verify_compression: 0 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: format_version: 5 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: enable_index_compression: 1 2023-09-23T20:37:14.179 INFO:journalctl@ceph.mon.c.smithi121.stdout: block_align: 0 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout: max_auto_readahead_size: 262144 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout: prepopulate_block_cache: 0 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression: NoCompression 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.num_levels: 7 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:37:14.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:37:14.181 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:37:14.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:37:14.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:37:14.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:37:14.185 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.ttl: 2592000 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:37:14.186 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12829, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 71cd5bed-8157-48de-9e44-0f2b4de6a315 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501433691359, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501433738820, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4491748, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12834, "largest_seqno": 16029, "table_properties": {"data_size": 4479904, "index_size": 7495, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3333, "raw_key_size": 28164, "raw_average_key_size": 21, "raw_value_size": 4453956, "raw_average_value_size": 3387, "num_data_blocks": 340, "num_entries": 1315, "num_filter_entries": 1315, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695501433, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "71cd5bed-8157-48de-9e44-0f2b4de6a315", "db_session_id": "Z9BQQWTV9Q3MUZ6W9HK6", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501433738950, "job": 1, "event": "recovery_finished"} 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:37:14.187 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e66ac76000 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: DB pointer 0x55e66ac64000 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** DB Stats ** 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:37:14.188 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: L0 1/0 4.28 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 132.5 0.03 0.00 1 0.032 0 0 0.0 0.0 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: L6 1/0 5.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: Sum 2/0 9.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 132.5 0.03 0.00 1 0.032 0 0 0.0 0.0 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 132.5 0.03 0.00 1 0.032 0 0 0.0 0.0 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** Compaction Stats [default] ** 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:37:14.189 INFO:journalctl@ceph.mon.c.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 132.5 0.03 0.00 1 0.032 0 0 0.0 0.0 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: Flush(GB): cumulative 0.004, interval 0.004 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: Cumulative compaction: 0.00 GB write, 74.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: Interval compaction: 0.00 GB write, 74.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:37:14.190 INFO:journalctl@ceph.mon.c.smithi121.stdout: Block cache BinnedLRUCache@0x55e66abdb1f0#8 capacity: 512.00 MB usage: 56.53 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: Block cache entry stats(count,size,portion): DataBlock(4,23.31 KB,0.00444651%) FilterBlock(2,12.06 KB,0.00230074%) IndexBlock(2,21.16 KB,0.00403523%) Misc(1,0.00 KB,0%) 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: starting mon.c rank 2 at public addrs [v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0] at bind addrs [v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???) e3 preinit fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???).mds e9 new map 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???).mds e9 print_map 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: e9 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: legacy client fscid: 1 2023-09-23T20:37:14.191 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: Filesystem 'foo' (1) 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: fs_name foo 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: epoch 8 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: created 2023-09-23T20:30:22.104035+0000 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: modified 2023-09-23T20:30:43.483349+0000 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: tableserver 0 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: root 0 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: session_timeout 60 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: session_autoclose 300 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: max_file_size 1099511627776 2023-09-23T20:37:14.192 INFO:journalctl@ceph.mon.c.smithi121.stdout: required_client_features {} 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: last_failure 0 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: last_failure_osd_epoch 0 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: max_mds 1 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: in 0 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: up {0=24578} 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: failed 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: damaged 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: stopped 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: data_pools [4] 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: metadata_pool 3 2023-09-23T20:37:14.193 INFO:journalctl@ceph.mon.c.smithi121.stdout: inline_data disabled 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: balancer 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: bal_rank_mask -1 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: standby_count_wanted 1 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: [mds.foo.smithi121.sdlxlb{0:24578} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: Standby daemons: 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout: [mds.foo.smithi123.clbyhs{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:37:14.194 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:37:14.195 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:37:14.195 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:13 smithi121 ceph-mon[219709]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T20:37:14.516 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 113213 113197 93.0603 56.0625 0.0132669 0.0107412 2023-09-23T20:37:15.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:37:14] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-09-23T20:37:15.517 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 114160 114144 92.619 59.1875 0.0164145 0.0107923 2023-09-23T20:37:16.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 115091 115075 92.1726 58.1875 0.0155848 0.0108444 2023-09-23T20:37:17.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 116034 116018 91.752 58.9375 0.0155778 0.0108942 2023-09-23T20:37:18.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:37:18.522509+0000 min lat: 0.00147314 max lat: 0.359194 avg lat: 0.0109456 2023-09-23T20:37:18.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:37:18.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 116963 116947 91.3309 58.0625 0.0121358 0.0109456 2023-09-23T20:37:18.971 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:37:18 smithi123 conmon[177538]: ts=2023-09-23T20:37:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:37:19.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:18 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:18 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:18 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:18 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/3038779924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:37:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:18 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:18 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:18 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3038779924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:37:19.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 117833 117817 90.8747 54.375 0.00934313 0.0109999 2023-09-23T20:37:20.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:37:20.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:37:20.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:37:20.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3038779924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:37:20.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 118613 118597 90.361 48.75 0.0196048 0.011062 2023-09-23T20:37:20.596 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: mon.c calling monitor election 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: mon.b calling monitor election 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: mon.a calling monitor election 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: mgrmap e34: y(active, since 97s), standbys: x 2023-09-23T20:37:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: overall HEALTH_OK 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/105350701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: mon.c calling monitor election 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: mon.b calling monitor election 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: mon.a calling monitor election 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:37:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:37:21.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: mgrmap e34: y(active, since 97s), standbys: x 2023-09-23T20:37:21.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:37:21.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T20:37:21.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: overall HEALTH_OK 2023-09-23T20:37:21.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/105350701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: mon.c calling monitor election 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: mon.b calling monitor election 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: mon.a calling monitor election 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: monmap e3: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: mgrmap e34: y(active, since 97s), standbys: x 2023-09-23T20:37:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:37:21.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: Cluster is now healthy 2023-09-23T20:37:21.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: overall HEALTH_OK 2023-09-23T20:37:21.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:21 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/105350701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:37:21.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 119463 119447 89.9124 53.125 0.0147645 0.011117 2023-09-23T20:37:22.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:22 smithi121 ceph-mon[219709]: pgmap v57: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:37:22.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:22 smithi121 ceph-mon[219709]: mgrmap e35: y(active, since 98s), standbys: x 2023-09-23T20:37:22.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:22 smithi121 ceph-mon[211526]: pgmap v57: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:37:22.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:22 smithi121 ceph-mon[211526]: mgrmap e35: y(active, since 98s), standbys: x 2023-09-23T20:37:22.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:22 smithi123 ceph-mon[104968]: pgmap v57: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T20:37:22.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:22 smithi123 ceph-mon[104968]: mgrmap e35: y(active, since 98s), standbys: x 2023-09-23T20:37:22.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 120036 120020 89.2685 35.8125 0.0130133 0.0111983 2023-09-23T20:37:22.844 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:37:23.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 120829 120813 88.8014 49.5625 0.0175716 0.0112563 2023-09-23T20:37:23.914 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:23 smithi121 conmon[198376]: 2023-09-23T20:37:23.764+0000 7f479dcb1700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T20:37:23.957 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:24.343 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:24 smithi121 ceph-mon[211526]: pgmap v58: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-09-23T20:37:24.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:24 smithi121 ceph-mon[219709]: pgmap v58: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-09-23T20:37:24.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:24 smithi123 ceph-mon[104968]: pgmap v58: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-09-23T20:37:24.522 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 121605 121589 88.3329 48.5 0.0219334 0.0113155 2023-09-23T20:37:25.167 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:37:24] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.43.0" 2023-09-23T20:37:25.523 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 122415 122399 87.8995 50.625 0.0201365 0.0113716 2023-09-23T20:37:26.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:26 smithi121 ceph-mon[219709]: pgmap v59: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T20:37:26.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:26 smithi121 ceph-mon[211526]: pgmap v59: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T20:37:26.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:26 smithi123 ceph-mon[104968]: pgmap v59: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T20:37:26.524 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 123229 123213 87.4776 50.875 0.0132457 0.011426 2023-09-23T20:37:27.105 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.a smithi121 *:9093,9094 running (3m) - 7m 26.6M - 0.25.0 c8568f914cd2 a7c6818f0f7f 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.a smithi123 *:3000 running (3m) - 7m 88.1M - 9.4.7 2c41d148cca3 c2e1f7abecff 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi121.sdlxlb smithi121 running (6m) - 6m 17.4M - 17.2.6-1123-g61431f6b 0db14aa01094 c7389c893171 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi123.clbyhs smithi123 running (6m) - 6m 14.6M - 17.2.6-1123-g61431f6b 0db14aa01094 1704af1c5a0c 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.x smithi123 *:8443,9283,8765 running (2m) - 11m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 df813ef15dd6 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.y smithi121 *:8443,9283,8765 running (3m) - 13m 502M - 18.2.0-460-gbb0daeff e32c2a64c126 adf60a30334d 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a smithi121 running (116s) - 13m 48.8M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 41cbfbe73b6e 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b smithi123 running (12m) - 12m 90.2M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 6a36337c715a 2023-09-23T20:37:27.106 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c smithi121 starting - - - 2048M 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.a smithi121 *:9100 running (3m) - 7m 18.6M - 1.5.0 0da6a335fe13 ab86679c1d7d 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.b smithi123 *:9100 running (3m) - 7m 9.94M - 1.5.0 0da6a335fe13 5f079d4704ad 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (11m) - 11m 2680M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ae38a40da032 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (11m) - 10m 1917M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 b5379433baef 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (10m) - 10m 3558M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 2c9b742b8c06 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (10m) - 10m 2147M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcc588b3144b 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi123 running (9m) - 9m 2751M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 470c2a4d010b 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi123 running (9m) - 9m 3012M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ab39a1466c34 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi123 running (9m) - 9m 1438M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 3f6aba9e3a6b 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi123 running (8m) - 8m 2469M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 514f59904aaa 2023-09-23T20:37:27.107 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.a smithi123 *:9095 running (2m) - 8m 52.1M - 2.43.0 a07b618ecd1d 9b8ac663186b 2023-09-23T20:37:27.524 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 123993 123977 87.0313 47.75 0.021177 0.011486 2023-09-23T20:37:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:28 smithi121 ceph-mon[219709]: pgmap v60: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s 2023-09-23T20:37:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:28 smithi121 ceph-mon[219709]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:28 smithi121 ceph-mon[211526]: pgmap v60: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s 2023-09-23T20:37:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:28 smithi121 ceph-mon[211526]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:28.470 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:37:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:28 smithi123 ceph-mon[104968]: pgmap v60: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s 2023-09-23T20:37:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:28 smithi123 ceph-mon[104968]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:28.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:28 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:28.524 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.0085 2023-09-23T20:37:28.524 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 124753 2023-09-23T20:37:28.524 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T20:37:28.524 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 86.6259 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 44.9785 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 262.375 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 33 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 1386 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 719.656 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 4198 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 528 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0115402 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.00987393 2023-09-23T20:37:28.525 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 0.396053 2023-09-23T20:37:28.526 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00147314 2023-09-23T20:37:28.528 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:37:28.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:37:28 smithi123 conmon[177538]: ts=2023-09-23T20:37:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:37:29.170 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:29.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:29 smithi121 ceph-mon[211526]: pgmap v61: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s 2023-09-23T20:37:29.812 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:29 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:29.812 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:29 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:29.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:29 smithi121 ceph-mon[219709]: pgmap v61: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s 2023-09-23T20:37:29.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:29 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:29.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:29 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:29 smithi123 ceph-mon[104968]: pgmap v61: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s 2023-09-23T20:37:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:29 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:29 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:31.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:30 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:31.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:30 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:31.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:30 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:31.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:30 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:31.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:30 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:30 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:32.055 INFO:teuthology.orchestra.run.smithi121.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:32.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:31 smithi121 ceph-mon[211526]: pgmap v62: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:37:32.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:31 smithi121 ceph-mon[219709]: pgmap v62: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:37:32.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:31 smithi123 ceph-mon[104968]: pgmap v62: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-09-23T20:37:33.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:33 smithi121 ceph-mon[219709]: from='client.25150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:33.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:33 smithi121 ceph-mon[219709]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:33.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:33.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:33 smithi121 ceph-mon[211526]: from='client.25150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:33.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:33 smithi121 ceph-mon[211526]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:33.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:33 smithi123 ceph-mon[104968]: from='client.25150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:37:33.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:33 smithi123 ceph-mon[104968]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:37:33.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:33 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:33.658 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:37:34.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:34 smithi121 ceph-mon[211526]: pgmap v63: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 41 MiB/s wr, 653 op/s 2023-09-23T20:37:34.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:34 smithi121 ceph-mon[219709]: pgmap v63: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 41 MiB/s wr, 653 op/s 2023-09-23T20:37:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:34 smithi123 ceph-mon[104968]: pgmap v63: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 41 MiB/s wr, 653 op/s 2023-09-23T20:37:34.484 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:35.062 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:37:34] "GET /metrics HTTP/1.1" 200 36215 "" "Prometheus/2.43.0" 2023-09-23T20:37:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:36 smithi121 ceph-mon[211526]: pgmap v64: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 978 op/s 2023-09-23T20:37:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:36 smithi121 ceph-mon[219709]: pgmap v64: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 978 op/s 2023-09-23T20:37:36.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:36 smithi123 ceph-mon[104968]: pgmap v64: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 978 op/s 2023-09-23T20:37:38.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: pgmap v65: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 654 op/s 2023-09-23T20:37:38.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:38.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:38.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:38.485 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:38.485 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:38.485 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:38 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: pgmap v65: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 654 op/s 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: pgmap v65: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 654 op/s 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:38.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:37:38.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:37:38.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:37:38.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:37:38 smithi123 conmon[177538]: ts=2023-09-23T20:37:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:37:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:40 smithi123 ceph-mon[104968]: pgmap v66: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 654 op/s 2023-09-23T20:37:40.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:40 smithi121 ceph-mon[211526]: pgmap v66: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 654 op/s 2023-09-23T20:37:40.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:40 smithi121 ceph-mon[219709]: pgmap v66: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 654 op/s 2023-09-23T20:37:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:42 smithi123 ceph-mon[104968]: pgmap v67: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 975 op/s 2023-09-23T20:37:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:42 smithi121 ceph-mon[211526]: pgmap v67: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 975 op/s 2023-09-23T20:37:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:42 smithi121 ceph-mon[219709]: pgmap v67: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 975 op/s 2023-09-23T20:37:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:43.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:43 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:44.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:44 smithi121 ceph-mon[211526]: pgmap v68: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 714 KiB/s wr, 645 op/s 2023-09-23T20:37:44.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:44 smithi121 ceph-mon[219709]: pgmap v68: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 714 KiB/s wr, 645 op/s 2023-09-23T20:37:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:44 smithi123 ceph-mon[104968]: pgmap v68: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 714 KiB/s wr, 645 op/s 2023-09-23T20:37:45.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:37:44] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.43.0" 2023-09-23T20:37:46.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:46 smithi121 ceph-mon[211526]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 714 KiB/s wr, 1.01k op/s 2023-09-23T20:37:46.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:46 smithi121 ceph-mon[219709]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 714 KiB/s wr, 1.01k op/s 2023-09-23T20:37:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:46 smithi123 ceph-mon[104968]: pgmap v69: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 714 KiB/s wr, 1.01k op/s 2023-09-23T20:37:48.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:48 smithi123 ceph-mon[104968]: pgmap v70: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T20:37:48.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:48 smithi121 ceph-mon[211526]: pgmap v70: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T20:37:48.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:48 smithi121 ceph-mon[219709]: pgmap v70: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T20:37:48.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:37:48 smithi123 conmon[177538]: ts=2023-09-23T20:37:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:37:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:50 smithi121 ceph-mon[211526]: pgmap v71: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T20:37:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:50 smithi121 ceph-mon[219709]: pgmap v71: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T20:37:50.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:50 smithi123 ceph-mon[104968]: pgmap v71: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-09-23T20:37:52.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:52 smithi121 ceph-mon[211526]: pgmap v72: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-09-23T20:37:52.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:52 smithi121 ceph-mon[219709]: pgmap v72: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-09-23T20:37:52.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:52 smithi123 ceph-mon[104968]: pgmap v72: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-09-23T20:37:53.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:53 smithi121 ceph-mon[211526]: pgmap v73: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T20:37:53.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:53 smithi121 ceph-mon[219709]: pgmap v73: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T20:37:53.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:53 smithi123 ceph-mon[104968]: pgmap v73: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2023-09-23T20:37:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:37:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:37:54] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.43.0" 2023-09-23T20:37:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:55 smithi121 ceph-mon[211526]: pgmap v74: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2023-09-23T20:37:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:55 smithi121 ceph-mon[219709]: pgmap v74: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2023-09-23T20:37:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:55 smithi123 ceph-mon[104968]: pgmap v74: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2023-09-23T20:37:58.170 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:57 smithi121 ceph-mon[211526]: pgmap v75: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T20:37:58.171 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:58.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:57 smithi121 ceph-mon[219709]: pgmap v75: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T20:37:58.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:57 smithi123 ceph-mon[104968]: pgmap v75: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-09-23T20:37:58.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:57 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:58.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:37:58 smithi123 conmon[177538]: ts=2023-09-23T20:37:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:38:00.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:37:59 smithi121 ceph-mon[211526]: pgmap v76: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-09-23T20:38:00.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:37:59 smithi121 ceph-mon[219709]: pgmap v76: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-09-23T20:38:00.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:37:59 smithi123 ceph-mon[104968]: pgmap v76: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-09-23T20:38:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:01 smithi121 ceph-mon[211526]: pgmap v77: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-09-23T20:38:02.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:01 smithi121 ceph-mon[219709]: pgmap v77: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-09-23T20:38:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:01 smithi123 ceph-mon[104968]: pgmap v77: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-09-23T20:38:04.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:03 smithi121 ceph-mon[211526]: pgmap v78: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T20:38:04.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:03 smithi121 ceph-mon[219709]: pgmap v78: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T20:38:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:03 smithi123 ceph-mon[104968]: pgmap v78: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T20:38:05.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:38:04] "GET /metrics HTTP/1.1" 200 36229 "" "Prometheus/2.43.0" 2023-09-23T20:38:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:05 smithi121 ceph-mon[211526]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2023-09-23T20:38:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:05 smithi121 ceph-mon[219709]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2023-09-23T20:38:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:05 smithi123 ceph-mon[104968]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2023-09-23T20:38:08.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:07 smithi121 ceph-mon[219709]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T20:38:08.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:07 smithi121 ceph-mon[211526]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T20:38:08.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:07 smithi123 ceph-mon[104968]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T20:38:08.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:38:08 smithi123 conmon[177538]: ts=2023-09-23T20:38:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:38:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:09 smithi121 ceph-mon[211526]: pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T20:38:10.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:09 smithi121 ceph-mon[219709]: pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T20:38:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:09 smithi123 ceph-mon[104968]: pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-09-23T20:38:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:11 smithi121 ceph-mon[211526]: pgmap v82: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-09-23T20:38:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:11 smithi121 ceph-mon[219709]: pgmap v82: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-09-23T20:38:12.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:11 smithi123 ceph-mon[104968]: pgmap v82: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-09-23T20:38:13.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:13.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:12 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:13 smithi121 ceph-mon[219709]: pgmap v83: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T20:38:14.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:13 smithi121 ceph-mon[211526]: pgmap v83: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T20:38:14.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:13 smithi123 ceph-mon[104968]: pgmap v83: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T20:38:15.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:38:14] "GET /metrics HTTP/1.1" 200 36230 "" "Prometheus/2.43.0" 2023-09-23T20:38:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:15 smithi121 ceph-mon[211526]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2023-09-23T20:38:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:15 smithi121 ceph-mon[219709]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2023-09-23T20:38:16.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:15 smithi123 ceph-mon[104968]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2023-09-23T20:38:18.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:17 smithi121 ceph-mon[219709]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:38:18.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:17 smithi121 ceph-mon[211526]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:38:18.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:17 smithi123 ceph-mon[104968]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:38:18.978 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:38:18 smithi123 conmon[177538]: ts=2023-09-23T20:38:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:38:20.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:19 smithi121 ceph-mon[211526]: pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:38:20.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:19 smithi121 ceph-mon[219709]: pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:38:20.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:19 smithi123 ceph-mon[104968]: pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:38:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:21 smithi121 ceph-mon[211526]: pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-09-23T20:38:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:21 smithi121 ceph-mon[219709]: pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-09-23T20:38:22.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:21 smithi123 ceph-mon[104968]: pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-09-23T20:38:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:23 smithi123 ceph-mon[104968]: pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:38:24.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:23 smithi121 ceph-mon[219709]: pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:38:24.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:23 smithi121 ceph-mon[211526]: pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:38:25.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:38:24] "GET /metrics HTTP/1.1" 200 36230 "" "Prometheus/2.43.0" 2023-09-23T20:38:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:25 smithi123 ceph-mon[104968]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:26.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:25 smithi121 ceph-mon[211526]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:26.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:25 smithi121 ceph-mon[219709]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:27 smithi123 ceph-mon[104968]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:38:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:27 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:27 smithi121 ceph-mon[219709]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:38:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:27 smithi121 ceph-mon[211526]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:38:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:28.969 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:38:28 smithi123 conmon[177538]: ts=2023-09-23T20:38:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:38:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:28 smithi123 ceph-mon[104968]: pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:38:29.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:28 smithi121 ceph-mon[211526]: pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:38:29.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:28 smithi121 ceph-mon[219709]: pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:38:30.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:30 smithi121 ceph-mon[219709]: pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:30.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:30 smithi121 ceph-mon[211526]: pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:30 smithi123 ceph-mon[104968]: pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:32 smithi123 ceph-mon[104968]: pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T20:38:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:32 smithi121 ceph-mon[211526]: pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T20:38:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:32 smithi121 ceph-mon[219709]: pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T20:38:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:34 smithi121 ceph-mon[211526]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:34 smithi121 ceph-mon[219709]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:34 smithi123 ceph-mon[104968]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T20:38:35.169 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:38:34] "GET /metrics HTTP/1.1" 200 36229 "" "Prometheus/2.43.0" 2023-09-23T20:38:37.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:36 smithi121 ceph-mon[211526]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T20:38:37.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:36 smithi121 ceph-mon[219709]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T20:38:37.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:36 smithi123 ceph-mon[104968]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T20:38:37.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:37 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:38.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:37 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:37 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:38.980 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:38:38 smithi123 conmon[177538]: ts=2023-09-23T20:38:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:38:38.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:38 smithi123 ceph-mon[104968]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T20:38:39.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:38 smithi121 ceph-mon[211526]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T20:38:39.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:38 smithi121 ceph-mon[219709]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-09-23T20:38:39.495 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:38:40.417 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:40.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:40 smithi121 ceph-mon[219709]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-09-23T20:38:40.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:40 smithi121 ceph-mon[211526]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-09-23T20:38:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:40 smithi123 ceph-mon[104968]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-09-23T20:38:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[211526]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T20:38:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[219709]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T20:38:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:43 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:43 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:43 smithi123 ceph-mon[104968]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T20:38:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:43 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:43.973 INFO:teuthology.orchestra.run.smithi121.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:38:45.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:44 smithi121 ceph-mon[211526]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:45.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:44 smithi121 ceph-mon[211526]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:38:45.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:45.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:44 smithi121 ceph-mon[211526]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:38:45.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:38:44] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2023-09-23T20:38:45.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:45 smithi121 ceph-mon[219709]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:45.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:45 smithi121 ceph-mon[219709]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:38:45.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:45 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:45.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:45 smithi121 ceph-mon[219709]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:38:45.251 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T20:38:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:44 smithi123 ceph-mon[104968]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:38:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:44 smithi123 ceph-mon[104968]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:38:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:44 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:44 smithi123 ceph-mon[104968]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:38:46.055 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:46.782 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:46 smithi121 ceph-mon[211526]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:38:47.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:46 smithi121 ceph-mon[219709]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:38:47.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:46 smithi123 ceph-mon[104968]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:38:48.618 INFO:teuthology.orchestra.run.smithi121.stdout:wait for all mons to upgrade 2023-09-23T20:38:48.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 20:38:48 smithi123 conmon[177538]: ts=2023-09-23T20:38:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi123.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.121\", device_class=\"ssd\", hostname=\"smithi121\", instance=\"smithi121:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.121\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:38:49.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[219709]: pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:38:49.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:49.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:49.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[211526]: pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:38:49.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:49.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:49.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:49 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:49 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:49 smithi123 ceph-mon[104968]: pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:38:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:49 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:49 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:49 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:49.534 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T20:38:50.225 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:50.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:50 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:50.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:50 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:38:50.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:50 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:38:50.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:50 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:38:50.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:50 smithi123 ceph-mon[104968]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:38:50.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:50.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:38:50.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:38:50.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:38:50.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[211526]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:38:50.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:38:50.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:38:50.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:38:50.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:38:50.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:50 smithi121 ceph-mon[219709]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:38:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:51 smithi121 ceph-mon[211526]: pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2023-09-23T20:38:51.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:51 smithi121 ceph-mon[219709]: pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2023-09-23T20:38:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:51 smithi123 ceph-mon[104968]: pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2023-09-23T20:38:52.410 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:52 smithi121 ceph-mon[219709]: pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T20:38:52.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:52 smithi121 ceph-mon[211526]: pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T20:38:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:52 smithi123 ceph-mon[104968]: pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T20:38:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[211526]: Upgrade: Updating mon.b 2023-09-23T20:38:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:38:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:38:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[211526]: Deploying daemon mon.b on smithi123 2023-09-23T20:38:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2934872289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:38:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[219709]: Upgrade: Updating mon.b 2023-09-23T20:38:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:38:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:38:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[219709]: Deploying daemon mon.b on smithi123 2023-09-23T20:38:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2934872289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:38:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:53 smithi123 ceph-mon[104968]: Upgrade: Updating mon.b 2023-09-23T20:38:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:53 smithi123 ceph-mon[104968]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:38:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:53 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:38:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:53 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:38:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:53 smithi123 ceph-mon[104968]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:53 smithi123 ceph-mon[104968]: Deploying daemon mon.b on smithi123 2023-09-23T20:38:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:53 smithi123 ceph-mon[104968]: from='client.? 172.21.15.121:0/2934872289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:38:54.880 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:54 smithi123 systemd[1]: Stopping Ceph mon.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:38:55.160 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:54 smithi123 ceph-mon[104968]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T20:38:55.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:38:54 smithi121 ceph-mon[211526]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T20:38:55.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:38:54] "GET /metrics HTTP/1.1" 200 36228 "" "Prometheus/2.43.0" 2023-09-23T20:38:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:38:54 smithi121 ceph-mon[219709]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T20:38:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:55 smithi123 conmon[104928]: 2023-09-23T20:38:55.158+0000 7f1373e64700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:38:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:55 smithi123 conmon[104928]: 2023-09-23T20:38:55.158+0000 7f1373e64700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T20:38:55.667 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:55 smithi121 conmon[198376]: [23/Sep/2023:20:38:55] ENGINE Bus STOPPING 2023-09-23T20:38:55.667 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:55 smithi121 conmon[198376]: [23/Sep/2023:20:38:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:38:55.667 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:55 smithi121 conmon[198376]: [23/Sep/2023:20:38:55] ENGINE Bus STOPPED 2023-09-23T20:38:55.667 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:55 smithi121 conmon[198376]: [23/Sep/2023:20:38:55] ENGINE Bus STARTING 2023-09-23T20:38:55.667 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:55 smithi121 conmon[198376]: [23/Sep/2023:20:38:55] ENGINE Serving on http://:::9283 2023-09-23T20:38:55.667 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:38:55 smithi121 conmon[198376]: [23/Sep/2023:20:38:55] ENGINE Bus STARTED 2023-09-23T20:38:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:55 smithi123 bash[193499]: 6a36337c715a282e8d4ae3540f8ed0f0d11566b814dfb20eea600ec8478bb04f 2023-09-23T20:38:56.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:56 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T20:38:56.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:56 smithi123 systemd[1]: Stopped Ceph mon.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:38:56.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:56 smithi123 systemd[1]: Starting Ceph mon.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:38:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 bash[193841]: 5eccbfe5a4b758c5f36cf1ddc0550ba21a2429fd99cf8820d2cb8acdbb36ea82 2023-09-23T20:38:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:38:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T20:38:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: pidfile_write: ignore empty --pid-file 2023-09-23T20:38:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: load: jerasure load: lrc 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Git sha 0 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: DB SUMMARY 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: DB Session ID: SAGNF2EOHZW1RVR9ALKZ 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9361082 ; 2023-09-23T20:38:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.env: 0x55a98e17a720 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.info_log: 0x55a99125b420 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.statistics: (nil) 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.use_fsync: 0 2023-09-23T20:38:57.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:38:57.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.db_log_dir: 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.wal_dir: 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:38:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.write_buffer_manager: 0x55a9904eac80 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.unordered_write: 0 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:38:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.row_cache: None 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.wal_filter: None 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.wal_compression: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:38:57.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:38:57.737 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_open_files: -1 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:38:57.738 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Compression algorithms supported: 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kZSTD supported: 0 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:38:57.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.merge_operator: 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_filter: None 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:38:57.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a99125b520) 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: index_type: 0 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: data_block_index_type: 0 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: index_shortening: 1 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: checksum: 4 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: no_block_cache: 0 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache: 0x55a99056d350 2023-09-23T20:38:57.741 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache_options: 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: capacity : 536870912 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: num_shard_bits : 4 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: strict_capacity_limit : 0 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_cache_compressed: (nil) 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: persistent_cache: (nil) 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_size: 4096 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_size_deviation: 10 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_restart_interval: 16 2023-09-23T20:38:57.742 INFO:journalctl@ceph.mon.b.smithi123.stdout: index_block_restart_interval: 1 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: metadata_block_size: 4096 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: partition_filters: 0 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: use_delta_encoding: 1 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: filter_policy: bloomfilter 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: whole_key_filtering: 1 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: verify_compression: 0 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: format_version: 5 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: enable_index_compression: 1 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: block_align: 0 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: max_auto_readahead_size: 262144 2023-09-23T20:38:57.743 INFO:journalctl@ceph.mon.b.smithi123.stdout: prepopulate_block_cache: 0 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression: NoCompression 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.num_levels: 7 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:38:57.744 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:38:57.745 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 systemd[1]: Started Ceph mon.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:38:57.746 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:38:57.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:38:57.748 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:38:57.749 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.ttl: 2592000 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:38:57.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:38:57.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12910, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: bfcf2e88-0a7d-40ac-a1ff-f75ece987af9 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501537386874, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501537425504, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5682978, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12915, "largest_seqno": 17440, "table_properties": {"data_size": 5667136, "index_size": 10533, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4293, "raw_key_size": 36237, "raw_average_key_size": 21, "raw_value_size": 5632857, "raw_average_value_size": 3307, "num_data_blocks": 475, "num_entries": 1703, "num_filter_entries": 1703, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695501537, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "bfcf2e88-0a7d-40ac-a1ff-f75ece987af9", "db_session_id": "SAGNF2EOHZW1RVR9ALKZ", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695501537425657, "job": 1, "event": "recovery_finished"} 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a990608000 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: DB pointer 0x55a9905f6000 2023-09-23T20:38:57.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** DB Stats ** 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** Compaction Stats [default] ** 2023-09-23T20:38:57.753 INFO:journalctl@ceph.mon.b.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: L0 1/0 5.42 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 295.0 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: L6 1/0 5.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: Sum 2/0 10.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 295.0 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 295.0 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** Compaction Stats [default] ** 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 295.0 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-09-23T20:38:57.754 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Flush(GB): cumulative 0.005, interval 0.005 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Cumulative compaction: 0.01 GB write, 113.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Interval compaction: 0.01 GB write, 113.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Block cache BinnedLRUCache@0x55a99056d350#7 capacity: 512.00 MB usage: 16.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2023-09-23T20:38:57.755 INFO:journalctl@ceph.mon.b.smithi123.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: starting mon.b rank 1 at public addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] at bind addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???) e3 preinit fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???).mds e9 new map 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???).mds e9 print_map 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: e9 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: legacy client fscid: 1 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.756 INFO:journalctl@ceph.mon.b.smithi123.stdout: Filesystem 'foo' (1) 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: fs_name foo 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: epoch 8 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: created 2023-09-23T20:30:22.104035+0000 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: modified 2023-09-23T20:30:43.483349+0000 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: tableserver 0 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: root 0 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: session_timeout 60 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: session_autoclose 300 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: max_file_size 1099511627776 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: required_client_features {} 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: last_failure 0 2023-09-23T20:38:57.757 INFO:journalctl@ceph.mon.b.smithi123.stdout: last_failure_osd_epoch 0 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: max_mds 1 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: in 0 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: up {0=24578} 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: failed 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: damaged 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: stopped 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: data_pools [4] 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: metadata_pool 3 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: inline_data disabled 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: balancer 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: bal_rank_mask -1 2023-09-23T20:38:57.758 INFO:journalctl@ceph.mon.b.smithi123.stdout: standby_count_wanted 1 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout: [mds.foo.smithi121.sdlxlb{0:24578} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.121:6834/2787400221,v1:172.21.15.121:6835/2787400221] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout: Standby daemons: 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout: 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout: [mds.foo.smithi123.clbyhs{-1:24556} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.123:6832/1811984959,v1:172.21.15.123:6833/1811984959] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:38:57.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:38:57.760 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:38:57.760 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:38:57.760 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:38:57 smithi123 ceph-mon[193903]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-09-23T20:38:58.641 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T20:39:00.481 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:39:01.157 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:02.791 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:39:02 smithi121 conmon[198376]: 2023-09-23T20:39:02.450+0000 7f479dcb1700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T20:39:03.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: overall HEALTH_OK 2023-09-23T20:39:03.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mon.b calling monitor election 2023-09-23T20:39:03.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mon.a calling monitor election 2023-09-23T20:39:03.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mon.c calling monitor election 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3480424184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mon.b calling monitor election 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: overall HEALTH_OK 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mon.a calling monitor election 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:39:03.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: monmap e4: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: overall HEALTH_OK 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='' 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: overall HEALTH_OK 2023-09-23T20:39:03.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mon.b calling monitor election 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mon.a calling monitor election 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mon.c calling monitor election 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3480424184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mon.b calling monitor election 2023-09-23T20:39:03.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: overall HEALTH_OK 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mon.a calling monitor election 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: monmap e4: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: overall HEALTH_OK 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='' 2023-09-23T20:39:03.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:03.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: overall HEALTH_OK 2023-09-23T20:39:03.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mon.b calling monitor election 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mon.a calling monitor election 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mon.c calling monitor election 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3480424184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mon.b calling monitor election 2023-09-23T20:39:03.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: overall HEALTH_OK 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mon.a calling monitor election 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: monmap e4: 3 mons at {a=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],b=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],c=[v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]} removed_ranks: {} 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: fsmap foo:1 {0=foo.smithi121.sdlxlb=up:active} 1 up:standby 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: mgrmap e35: y(active, since 3m), standbys: x 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: overall HEALTH_OK 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='' 2023-09-23T20:39:03.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:03.937 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.a smithi121 *:9093,9094 running (5m) - 9m 26.7M - 0.25.0 c8568f914cd2 a7c6818f0f7f 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.a smithi123 *:3000 running (4m) - 8m 90.8M - 9.4.7 2c41d148cca3 c2e1f7abecff 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi121.sdlxlb smithi121 running (8m) - 8m 18.0M - 17.2.6-1123-g61431f6b 0db14aa01094 c7389c893171 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi123.clbyhs smithi123 running (8m) - 8m 14.9M - 17.2.6-1123-g61431f6b 0db14aa01094 1704af1c5a0c 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.x smithi123 *:8443,9283,8765 running (3m) - 13m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 df813ef15dd6 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.y smithi121 *:8443,9283,8765 running (4m) - 14m 506M - 18.2.0-460-gbb0daeff e32c2a64c126 adf60a30334d 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a smithi121 running (3m) - 15m 60.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 41cbfbe73b6e 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b smithi123 starting - - - 2048M 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c smithi121 running (110s) - 13m 43.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 828ec0e17fe9 2023-09-23T20:39:03.938 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.a smithi121 *:9100 running (5m) - 8m 18.7M - 1.5.0 0da6a335fe13 ab86679c1d7d 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.b smithi123 *:9100 running (4m) - 8m 17.7M - 1.5.0 0da6a335fe13 5f079d4704ad 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (13m) - 13m 3144M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ae38a40da032 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (12m) - 12m 2453M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 b5379433baef 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (12m) - 12m 4136M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 2c9b742b8c06 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (11m) - 11m 3086M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcc588b3144b 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi123 running (11m) - 11m 3290M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 470c2a4d010b 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi123 running (11m) - 11m 3663M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ab39a1466c34 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi123 running (10m) - 10m 1736M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 3f6aba9e3a6b 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi123 running (10m) - 10m 3200M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 514f59904aaa 2023-09-23T20:39:03.939 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.a smithi123 *:9095 running (4m) - 9m 61.9M - 2.43.0 a07b618ecd1d 9b8ac663186b 2023-09-23T20:39:04.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:04 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:04.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:04 smithi123 ceph-mon[193903]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:04.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:04 smithi123 ceph-mon[193903]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:04.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:04 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:04.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:04 smithi121 ceph-mon[211526]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:04.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:04 smithi121 ceph-mon[211526]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:04.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:04 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:04.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:04 smithi121 ceph-mon[219709]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:04.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:04 smithi121 ceph-mon[219709]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:05.142 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:39:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:39:04] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.43.0" 2023-09-23T20:39:05.159 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:39:05.907 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:06.807 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:06 smithi121 ceph-mon[211526]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:39:06.807 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:06 smithi121 ceph-mon[219709]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:39:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:06 smithi123 ceph-mon[193903]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:39:08.780 INFO:teuthology.orchestra.run.smithi121.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:39:08.998 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:08.999 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:08.999 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[211526]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:39:08.999 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:08.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:08.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:08.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[219709]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:39:08.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:09.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:09.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:09.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:08 smithi123 ceph-mon[193903]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-09-23T20:39:09.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:09.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:09.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:09 smithi123 ceph-mon[193903]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:09.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:09 smithi123 ceph-mon[193903]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:39:09.752 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:09.963 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:39:10.026 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:10.026 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[219709]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:10.026 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[219709]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:39:10.026 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:10.031 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:10.032 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[211526]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:39:10.032 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[211526]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:39:10.032 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:10.769 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:11.103 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:10 smithi121 ceph-mon[211526]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-09-23T20:39:11.104 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:10 smithi121 ceph-mon[219709]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-09-23T20:39:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:10 smithi123 ceph-mon[193903]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-09-23T20:39:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:12 smithi121 ceph-mon[211526]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:39:12.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:12.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:12 smithi121 ceph-mon[219709]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:39:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:12 smithi123 ceph-mon[193903]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-09-23T20:39:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:14 smithi123 ceph-mon[193903]: pgmap v114: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T20:39:14.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:14 smithi121 ceph-mon[211526]: pgmap v114: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T20:39:14.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:14 smithi121 ceph-mon[219709]: pgmap v114: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-09-23T20:39:14.782 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:39:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:39:14] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T20:39:16.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:16.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:16.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:16.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:16.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:17 smithi123 ceph-mon[193903]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:39:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:17 smithi123 ceph-mon[193903]: Reconfiguring daemon mon.a on smithi121 2023-09-23T20:39:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:17 smithi123 ceph-mon[193903]: pgmap v115: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:39:17.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:17 smithi121 ceph-mon[219709]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:39:17.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:17 smithi121 ceph-mon[219709]: Reconfiguring daemon mon.a on smithi121 2023-09-23T20:39:17.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:17 smithi121 ceph-mon[219709]: pgmap v115: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:39:17.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:17 smithi121 ceph-mon[211526]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:39:17.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:17 smithi121 ceph-mon[211526]: Reconfiguring daemon mon.a on smithi121 2023-09-23T20:39:17.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:17 smithi121 ceph-mon[211526]: pgmap v115: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:39:19.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:19.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:19.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:39:19.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:19.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: Reconfiguring daemon mgr.y on smithi121 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[219709]: pgmap v116: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:39:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:19.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:19.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:39:19.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:19.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: Reconfiguring daemon mgr.y on smithi121 2023-09-23T20:39:19.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:19 smithi121 ceph-mon[211526]: pgmap v116: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:39:19.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: Reconfiguring daemon mgr.y on smithi121 2023-09-23T20:39:19.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:19 smithi123 ceph-mon[193903]: pgmap v116: 89 pgs: 89 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:39:20.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:20 smithi121 ceph-mon[211526]: pgmap v117: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:39:20.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:20 smithi121 ceph-mon[219709]: pgmap v117: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:39:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:20 smithi123 ceph-mon[193903]: pgmap v117: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:39:21.624 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:21.624 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:21.624 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[211526]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[211526]: Reconfiguring daemon mon.c on smithi121 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[219709]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:21.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:21 smithi121 ceph-mon[219709]: Reconfiguring daemon mon.c on smithi121 2023-09-23T20:39:21.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:21.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:21.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:21 smithi123 ceph-mon[193903]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:39:21.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:21.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:21.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:21.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:21 smithi123 ceph-mon[193903]: Reconfiguring daemon mon.c on smithi121 2023-09-23T20:39:22.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:22 smithi123 ceph-mon[193903]: pgmap v118: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-09-23T20:39:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:22 smithi121 ceph-mon[219709]: pgmap v118: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-09-23T20:39:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:22 smithi121 ceph-mon[211526]: pgmap v118: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-09-23T20:39:23.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:23.926 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[211526]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.0 on smithi121 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[219709]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:23.927 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:23 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.0 on smithi121 2023-09-23T20:39:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:23 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:23 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:23 smithi123 ceph-mon[193903]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:39:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:23 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:39:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:23 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:23 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.0 on smithi121 2023-09-23T20:39:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:24 smithi121 ceph-mon[211526]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:39:25.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:39:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:39:24] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T20:39:25.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:24 smithi121 ceph-mon[219709]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:39:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:24 smithi123 ceph-mon[193903]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:39:26.409 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:26.409 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:26.409 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[211526]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.1 on smithi121 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[219709]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:26.410 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:26 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.1 on smithi121 2023-09-23T20:39:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:26 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:26 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:26 smithi123 ceph-mon[193903]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:39:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:26 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:39:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:26 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:26 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.1 on smithi121 2023-09-23T20:39:27.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:27 smithi121 ceph-mon[211526]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:39:27.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:27 smithi121 ceph-mon[219709]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:39:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:27 smithi123 ceph-mon[193903]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:39:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:39:28.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:39:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:39:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:29.155 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:29 smithi121 ceph-mon[219709]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:39:29.155 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:29 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.2 on smithi121 2023-09-23T20:39:29.155 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:29 smithi121 ceph-mon[219709]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:39:29.160 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:29 smithi121 ceph-mon[211526]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:39:29.160 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:29 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.2 on smithi121 2023-09-23T20:39:29.160 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:29 smithi121 ceph-mon[211526]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:39:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:29 smithi123 ceph-mon[193903]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:39:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:29 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.2 on smithi121 2023-09-23T20:39:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:29 smithi123 ceph-mon[193903]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-09-23T20:39:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[211526]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.3 on smithi121 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[211526]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[219709]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:39:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.3 on smithi121 2023-09-23T20:39:31.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:31 smithi121 ceph-mon[219709]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T20:39:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:31 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:31 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:31 smithi123 ceph-mon[193903]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:39:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:31 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:39:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:31 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:31 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.3 on smithi121 2023-09-23T20:39:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:31 smithi123 ceph-mon[193903]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2023-09-23T20:39:32.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:32 smithi121 ceph-mon[219709]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:39:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:32 smithi121 ceph-mon[211526]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:39:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:32 smithi123 ceph-mon[193903]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:39:33.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:33.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:33.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[219709]: Reconfiguring mds.foo.smithi121.sdlxlb (monmap changed)... 2023-09-23T20:39:33.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:33.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:33.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:33.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[219709]: Reconfiguring daemon mds.foo.smithi121.sdlxlb on smithi121 2023-09-23T20:39:33.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:33.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:33.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[211526]: Reconfiguring mds.foo.smithi121.sdlxlb (monmap changed)... 2023-09-23T20:39:33.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:33.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:33.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:33.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:33 smithi121 ceph-mon[211526]: Reconfiguring daemon mds.foo.smithi121.sdlxlb on smithi121 2023-09-23T20:39:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:33 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:33 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:33 smithi123 ceph-mon[193903]: Reconfiguring mds.foo.smithi121.sdlxlb (monmap changed)... 2023-09-23T20:39:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:33 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:33 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi121.sdlxlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:33 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:33 smithi123 ceph-mon[193903]: Reconfiguring daemon mds.foo.smithi121.sdlxlb on smithi121 2023-09-23T20:39:34.914 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:39:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:39:34] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-09-23T20:39:34.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:34 smithi121 ceph-mon[219709]: pgmap v124: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2023-09-23T20:39:34.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:34 smithi121 ceph-mon[211526]: pgmap v124: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2023-09-23T20:39:34.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:34 smithi123 ceph-mon[193903]: pgmap v124: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2023-09-23T20:39:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:35.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:35.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:35 smithi123 ceph-mon[193903]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:39:35.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:35.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:35.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:35.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:35 smithi123 ceph-mon[193903]: Reconfiguring daemon mon.b on smithi123 2023-09-23T20:39:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[211526]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:39:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[211526]: Reconfiguring daemon mon.b on smithi123 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[219709]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:35 smithi121 ceph-mon[219709]: Reconfiguring daemon mon.b on smithi123 2023-09-23T20:39:37.173 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:36 smithi121 ceph-mon[211526]: pgmap v125: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T20:39:37.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:36 smithi121 ceph-mon[219709]: pgmap v125: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T20:39:37.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:36 smithi123 ceph-mon[193903]: pgmap v125: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T20:39:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:38 smithi123 ceph-mon[193903]: pgmap v126: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T20:39:38.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:38 smithi121 ceph-mon[211526]: pgmap v126: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T20:39:38.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:38 smithi121 ceph-mon[219709]: pgmap v126: 89 pgs: 89 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:39.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:39 smithi123 ceph-mon[193903]: Reconfiguring daemon mgr.x on smithi123 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[211526]: Reconfiguring daemon mgr.x on smithi123 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:39.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:39 smithi121 ceph-mon[219709]: Reconfiguring daemon mgr.x on smithi123 2023-09-23T20:39:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:40 smithi123 ceph-mon[193903]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-09-23T20:39:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:40 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:40 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:40 smithi121 ceph-mon[211526]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-09-23T20:39:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:40 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:40 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:40 smithi121 ceph-mon[219709]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-09-23T20:39:40.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:40 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:40.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:40 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[211526]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:39:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:39:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.4 on smithi123 2023-09-23T20:39:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[219709]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:39:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:39:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:41 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.4 on smithi123 2023-09-23T20:39:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:41 smithi123 ceph-mon[193903]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:39:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:39:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:41 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.4 on smithi123 2023-09-23T20:39:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:42 smithi123 ceph-mon[193903]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:42.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:42.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:42 smithi121 ceph-mon[211526]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:42.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:42.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:42 smithi121 ceph-mon[219709]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[211526]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:39:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:39:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.5 on smithi123 2023-09-23T20:39:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[219709]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:39:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:39:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:43 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.5 on smithi123 2023-09-23T20:39:43.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:43.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:43.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:43 smithi123 ceph-mon[193903]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:39:43.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:39:43.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:43.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:43 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.5 on smithi123 2023-09-23T20:39:45.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:39:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:39:44] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T20:39:45.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:45 smithi123 ceph-mon[193903]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:45.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:45 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:45.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:45 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:45.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:45 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:39:45.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:45 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[219709]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:39:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[211526]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:45.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:45.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:45.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:39:45.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:45 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:46.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:46 smithi123 ceph-mon[193903]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:39:46.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:46 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.6 on smithi123 2023-09-23T20:39:46.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:46 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:46.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:46 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:46.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:46 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:39:46.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:46 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:46.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[211526]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:39:46.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.6 on smithi123 2023-09-23T20:39:46.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:46.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:46.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:39:46.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:46.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[219709]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:39:46.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.6 on smithi123 2023-09-23T20:39:46.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:46.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:46.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:39:46.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:46 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:47.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:47 smithi121 ceph-mon[211526]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:39:47.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:47 smithi121 ceph-mon[211526]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:39:47.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:47 smithi121 ceph-mon[211526]: Reconfiguring daemon osd.7 on smithi123 2023-09-23T20:39:47.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:47 smithi121 ceph-mon[219709]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:39:47.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:47 smithi121 ceph-mon[219709]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:39:47.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:47 smithi121 ceph-mon[219709]: Reconfiguring daemon osd.7 on smithi123 2023-09-23T20:39:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:47 smithi123 ceph-mon[193903]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:39:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:47 smithi123 ceph-mon[193903]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:39:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:47 smithi123 ceph-mon[193903]: Reconfiguring daemon osd.7 on smithi123 2023-09-23T20:39:49.198 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:49.198 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:49.198 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: Reconfiguring mds.foo.smithi123.clbyhs (monmap changed)... 2023-09-23T20:39:49.199 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:49.199 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:49.199 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:49.199 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: Reconfiguring daemon mds.foo.smithi123.clbyhs on smithi123 2023-09-23T20:39:49.199 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:49 smithi123 ceph-mon[193903]: pgmap v131: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: Reconfiguring mds.foo.smithi123.clbyhs (monmap changed)... 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: Reconfiguring daemon mds.foo.smithi123.clbyhs on smithi123 2023-09-23T20:39:49.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[211526]: pgmap v131: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: Reconfiguring mds.foo.smithi123.clbyhs (monmap changed)... 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi123.clbyhs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: Reconfiguring daemon mds.foo.smithi123.clbyhs on smithi123 2023-09-23T20:39:49.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:49 smithi121 ceph-mon[219709]: pgmap v131: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-09-23T20:39:51.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:51.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:51.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:51.338 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[211526]: pgmap v132: 89 pgs: 89 active+clean; 963 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-09-23T20:39:51.338 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:51.339 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:51.339 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:51.339 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:51 smithi121 ceph-mon[219709]: pgmap v132: 89 pgs: 89 active+clean; 963 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-09-23T20:39:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:51 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:51 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:51 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:51 smithi123 ceph-mon[193903]: pgmap v132: 89 pgs: 89 active+clean; 963 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-09-23T20:39:52.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:52 smithi123 ceph-mon[193903]: pgmap v133: 89 pgs: 89 active+clean; 963 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T20:39:52.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:52 smithi121 ceph-mon[211526]: pgmap v133: 89 pgs: 89 active+clean; 963 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T20:39:52.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:52 smithi121 ceph-mon[219709]: pgmap v133: 89 pgs: 89 active+clean; 963 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-09-23T20:39:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:54 smithi123 ceph-mon[193903]: pgmap v134: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:54.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:54 smithi121 ceph-mon[211526]: pgmap v134: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:54.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:39:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:39:54] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.43.0" 2023-09-23T20:39:54.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:54 smithi121 ceph-mon[219709]: pgmap v134: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-09-23T20:39:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[219709]: pgmap v135: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[211526]: pgmap v135: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:56.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:56.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:56.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:56 smithi123 ceph-mon[193903]: pgmap v135: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:39:58.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:59.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:39:58 smithi121 ceph-mon[211526]: pgmap v136: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:59.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:39:58 smithi121 ceph-mon[219709]: pgmap v136: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:39:59.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:39:58 smithi123 ceph-mon[193903]: pgmap v136: 89 pgs: 89 active+clean; 735 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-09-23T20:40:00.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:00 smithi121 ceph-mon[211526]: overall HEALTH_OK 2023-09-23T20:40:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:00 smithi121 ceph-mon[219709]: overall HEALTH_OK 2023-09-23T20:40:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:00 smithi123 ceph-mon[193903]: overall HEALTH_OK 2023-09-23T20:40:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:01 smithi121 ceph-mon[219709]: pgmap v137: 89 pgs: 89 active+clean; 493 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T20:40:01.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:01 smithi121 ceph-mon[211526]: pgmap v137: 89 pgs: 89 active+clean; 493 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T20:40:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:01 smithi123 ceph-mon[193903]: pgmap v137: 89 pgs: 89 active+clean; 493 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-09-23T20:40:02.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:02 smithi123 ceph-mon[193903]: pgmap v138: 89 pgs: 89 active+clean; 493 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:40:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:02 smithi121 ceph-mon[211526]: pgmap v138: 89 pgs: 89 active+clean; 493 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:40:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:02 smithi121 ceph-mon[219709]: pgmap v138: 89 pgs: 89 active+clean; 493 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-09-23T20:40:04.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:04 smithi123 ceph-mon[193903]: pgmap v139: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2023-09-23T20:40:04.780 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:40:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:40:04] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.43.0" 2023-09-23T20:40:04.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:04 smithi121 ceph-mon[211526]: pgmap v139: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2023-09-23T20:40:04.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:04 smithi121 ceph-mon[219709]: pgmap v139: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2023-09-23T20:40:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:06 smithi121 ceph-mon[211526]: pgmap v140: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T20:40:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:06 smithi121 ceph-mon[219709]: pgmap v140: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T20:40:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:06 smithi123 ceph-mon[193903]: pgmap v140: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T20:40:08.317 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 124753 objects 2023-09-23T20:40:08.317 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :159.788 2023-09-23T20:40:08.595 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T20:40:08.595 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T20:40:08.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:08 smithi123 ceph-mon[193903]: pgmap v141: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T20:40:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:08 smithi121 ceph-mon[211526]: pgmap v141: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T20:40:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:08 smithi121 ceph-mon[219709]: pgmap v141: 89 pgs: 89 active+clean; 243 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-09-23T20:40:08.933 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:10 smithi123 ceph-mon[193903]: pgmap v142: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-09-23T20:40:10.830 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:10 smithi121 ceph-mon[219709]: pgmap v142: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-09-23T20:40:10.831 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:10 smithi121 ceph-mon[211526]: pgmap v142: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-09-23T20:40:11.819 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_1' removed 2023-09-23T20:40:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/417457371' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/417457371' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/417457371' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:12.496 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:40:12.497 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:40:12.497 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T20:40:12.497 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T20:40:12.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[211526]: failed to encode map e84 with expected crc 2023-09-23T20:40:12.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[211526]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[211526]: failed to encode map e84 with expected crc 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[211526]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[219709]: failed to encode map e84 with expected crc 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[219709]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:40:12.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[219709]: failed to encode map e84 with expected crc 2023-09-23T20:40:12.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:12.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:12 smithi121 ceph-mon[219709]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:12 smithi123 ceph-mon[193903]: failed to encode map e84 with expected crc 2023-09-23T20:40:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:40:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:12 smithi123 ceph-mon[193903]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:40:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:12 smithi123 ceph-mon[193903]: failed to encode map e84 with expected crc 2023-09-23T20:40:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:12 smithi123 ceph-mon[193903]: pgmap v144: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:13.504 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:13.527 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:40:14.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:14 smithi121 ceph-mon[211526]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:14.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:14 smithi121 ceph-mon[219709]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:14 smithi123 ceph-mon[193903]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:14.784 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:15.001 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:40:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:40:14] "GET /metrics HTTP/1.1" 200 33861 "" "Prometheus/2.43.0" 2023-09-23T20:40:15.774 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_2' created 2023-09-23T20:40:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2162986307' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:40:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2162986307' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:40:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2162986307' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:40:16.725 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T20:40:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[219709]: failed to encode map e85 with expected crc 2023-09-23T20:40:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[219709]: failed to encode map e85 with expected crc 2023-09-23T20:40:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2162986307' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:40:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[219709]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T20:40:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[219709]: pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[219709]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T20:40:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[211526]: failed to encode map e85 with expected crc 2023-09-23T20:40:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[211526]: failed to encode map e85 with expected crc 2023-09-23T20:40:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2162986307' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:40:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[211526]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T20:40:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[211526]: pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:16 smithi121 ceph-mon[211526]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T20:40:17.192 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:16 smithi123 ceph-mon[193903]: failed to encode map e85 with expected crc 2023-09-23T20:40:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:16 smithi123 ceph-mon[193903]: failed to encode map e85 with expected crc 2023-09-23T20:40:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2162986307' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:40:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:16 smithi123 ceph-mon[193903]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T20:40:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:16 smithi123 ceph-mon[193903]: pgmap v147: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:16 smithi123 ceph-mon[193903]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T20:40:17.389 INFO:teuthology.orchestra.run.smithi121.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:40:18.114 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.114 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.114 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.114 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.114 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.114 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[211526]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T20:40:18.115 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:40:18.116 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:18.117 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:18.117 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:17 smithi121 ceph-mon[219709]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: failed to encode map e86 with expected crc 2023-09-23T20:40:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:40:18.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:40:18.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:18.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:18.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:17 smithi123 ceph-mon[193903]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T20:40:18.627 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T20:40:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:18 smithi121 ceph-mon[211526]: failed to encode map e87 with expected crc 2023-09-23T20:40:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:18 smithi121 ceph-mon[211526]: failed to encode map e87 with expected crc 2023-09-23T20:40:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:18 smithi121 ceph-mon[211526]: pgmap v150: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:18.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:18 smithi121 ceph-mon[219709]: failed to encode map e87 with expected crc 2023-09-23T20:40:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:18 smithi121 ceph-mon[219709]: failed to encode map e87 with expected crc 2023-09-23T20:40:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:18 smithi121 ceph-mon[219709]: pgmap v150: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:18 smithi123 ceph-mon[193903]: failed to encode map e87 with expected crc 2023-09-23T20:40:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:18 smithi123 ceph-mon[193903]: failed to encode map e87 with expected crc 2023-09-23T20:40:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:18 smithi123 ceph-mon[193903]: pgmap v150: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:19.229 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:19.824 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T20:40:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2844142703' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:40:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2844142703' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:40:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2844142703' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: failed to encode map e88 with expected crc 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2844142703' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: failed to encode map e88 with expected crc 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:20.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:20.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:20.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:21.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: failed to encode map e88 with expected crc 2023-09-23T20:40:21.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2844142703' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:40:21.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:40:21.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: failed to encode map e88 with expected crc 2023-09-23T20:40:21.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:21.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:21.126 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:21.126 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:21.126 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:21.126 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:20 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: failed to encode map e88 with expected crc 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2844142703' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: failed to encode map e88 with expected crc 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:21.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:20 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:21.256 INFO:teuthology.orchestra.run.smithi121.stdout:wait for half of osds to upgrade 2023-09-23T20:40:22.064 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T20:40:22.067 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:40:22.067 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:40:22.080 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T20:40:22.114 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T20:40:22.114 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:40:22.114 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_234442 2023-09-23T20:40:22.114 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:40:22.114 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:40:22.160 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: Upgrade: Setting container_image for all mon 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:40:22.161 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: Upgrade: Setting container_image for all crash 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[211526]: Upgrade: osd.0 is safe to restart 2023-09-23T20:40:22.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: Upgrade: Setting container_image for all mon 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:40:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:40:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:40:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: Upgrade: Setting container_image for all crash 2023-09-23T20:40:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:40:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:40:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:21 smithi121 ceph-mon[219709]: Upgrade: osd.0 is safe to restart 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: Upgrade: Setting container_image for all mon 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:40:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: Upgrade: Setting container_image for all crash 2023-09-23T20:40:22.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:22.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:40:22.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:40:22.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:21 smithi123 ceph-mon[193903]: Upgrade: osd.0 is safe to restart 2023-09-23T20:40:22.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:22 smithi121 ceph-mon[211526]: pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:22 smithi121 ceph-mon[219709]: pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:23.118 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 3259 3243 201.942 202.688 0.00420479 0.00493314 2023-09-23T20:40:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:22 smithi123 ceph-mon[193903]: pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:40:24.110 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:24.118 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 6714 6698 208.897 215.938 0.00324424 0.00477276 2023-09-23T20:40:24.781 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:40:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:40:24] "GET /metrics HTTP/1.1" 200 33861 "" "Prometheus/2.43.0" 2023-09-23T20:40:25.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[211526]: Upgrade: Updating osd.0 2023-09-23T20:40:25.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:25.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:40:25.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:25.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[211526]: Deploying daemon osd.0 on smithi121 2023-09-23T20:40:25.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[211526]: pgmap v154: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s 2023-09-23T20:40:25.119 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 9571 9555 198.711 178.562 0.00405041 0.00502307 2023-09-23T20:40:25.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[219709]: Upgrade: Updating osd.0 2023-09-23T20:40:25.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:25.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:40:25.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:25.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[219709]: Deploying daemon osd.0 on smithi121 2023-09-23T20:40:25.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:25 smithi121 ceph-mon[219709]: pgmap v154: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s 2023-09-23T20:40:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:25 smithi123 ceph-mon[193903]: Upgrade: Updating osd.0 2023-09-23T20:40:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:25 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:25 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:40:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:25 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:25 smithi123 ceph-mon[193903]: Deploying daemon osd.0 on smithi121 2023-09-23T20:40:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:25 smithi123 ceph-mon[193903]: pgmap v154: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s 2023-09-23T20:40:26.120 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 12656 12640 197.214 192.812 0.00399605 0.00506259 2023-09-23T20:40:27.120 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 15508 15492 193.42 178.25 0.0100473 0.00516379 2023-09-23T20:40:27.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:26 smithi121 ceph-mon[211526]: pgmap v155: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s wr, 536 op/s 2023-09-23T20:40:27.127 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:26 smithi121 ceph-mon[219709]: pgmap v155: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s wr, 536 op/s 2023-09-23T20:40:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:26 smithi123 ceph-mon[193903]: pgmap v155: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s wr, 536 op/s 2023-09-23T20:40:28.120 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 18005 17989 187.196 156.062 0.0043824 0.00533441 2023-09-23T20:40:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:29.121 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:28 smithi121 ceph-mon[211526]: pgmap v156: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 460 op/s 2023-09-23T20:40:29.122 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 19753 19737 176.034 109.25 0.010747 0.00567401 2023-09-23T20:40:29.122 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:28 smithi121 ceph-mon[219709]: pgmap v156: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 460 op/s 2023-09-23T20:40:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:28 smithi123 ceph-mon[193903]: pgmap v156: 89 pgs: 89 active+clean; 289 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 460 op/s 2023-09-23T20:40:29.841 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1828022650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:30.122 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 21789 21773 169.939 127.25 0.00745046 0.00587726 2023-09-23T20:40:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1828022650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1828022650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:31.123 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 23958 23942 166.087 135.562 0.00446226 0.00601519 2023-09-23T20:40:31.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:30 smithi121 ceph-mon[211526]: pgmap v157: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:40:31.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:30 smithi121 ceph-mon[219709]: pgmap v157: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:40:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:30 smithi123 ceph-mon[193903]: pgmap v157: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:40:32.131 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 26033 26017 162.432 129.688 0.00826739 0.00615153 2023-09-23T20:40:32.663 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:32 smithi121 systemd[1]: Stopping Ceph osd.0 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:40:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1836410651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:32 smithi121 ceph-mon[219709]: pgmap v158: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:40:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1836410651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:32 smithi121 ceph-mon[211526]: pgmap v158: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:40:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1836410651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:32 smithi123 ceph-mon[193903]: pgmap v158: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:40:33.128 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 27922 27906 158.35 118.062 0.00706028 0.00630993 2023-09-23T20:40:33.163 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:32 smithi121 conmon[124538]: 2023-09-23T20:40:32.904+0000 7f6429032700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:40:33.163 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:32 smithi121 conmon[124538]: 2023-09-23T20:40:32.904+0000 7f6429032700 -1 osd.0 88 *** Got signal Terminated *** 2023-09-23T20:40:33.163 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:32 smithi121 conmon[124538]: 2023-09-23T20:40:32.904+0000 7f6429032700 -1 osd.0 88 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:40:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:33 smithi123 ceph-mon[193903]: osd.0 marked itself down and dead 2023-09-23T20:40:33.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:33 smithi121 ceph-mon[219709]: osd.0 marked itself down and dead 2023-09-23T20:40:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:33 smithi121 ceph-mon[211526]: osd.0 marked itself down and dead 2023-09-23T20:40:34.129 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 28627 28611 148.836 44.0625 0.00419564 0.0063373 2023-09-23T20:40:34.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:34 smithi123 ceph-mon[193903]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:40:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:34 smithi123 ceph-mon[193903]: failed to encode map e89 with expected crc 2023-09-23T20:40:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:34 smithi123 ceph-mon[193903]: osdmap e89: 8 total, 7 up, 8 in 2023-09-23T20:40:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:34 smithi123 ceph-mon[193903]: failed to encode map e89 with expected crc 2023-09-23T20:40:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:34 smithi123 ceph-mon[193903]: pgmap v160: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-09-23T20:40:34.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[211526]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:40:34.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[211526]: failed to encode map e89 with expected crc 2023-09-23T20:40:34.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[211526]: osdmap e89: 8 total, 7 up, 8 in 2023-09-23T20:40:34.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[211526]: failed to encode map e89 with expected crc 2023-09-23T20:40:34.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[211526]: pgmap v160: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-09-23T20:40:34.778 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:40:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:40:34] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.43.0" 2023-09-23T20:40:34.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[219709]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:40:34.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[219709]: failed to encode map e89 with expected crc 2023-09-23T20:40:34.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[219709]: osdmap e89: 8 total, 7 up, 8 in 2023-09-23T20:40:34.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[219709]: failed to encode map e89 with expected crc 2023-09-23T20:40:34.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:34 smithi121 ceph-mon[219709]: pgmap v160: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-09-23T20:40:35.129 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 29785 29769 142.955 72.375 0.00178043 0.00688806 2023-09-23T20:40:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:40:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T20:40:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2215004144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:35.913 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:35 smithi121 bash[235404]: ae38a40da03256d2d18b78b27c49c36556e642847c79940ba3aa5ab09c6ee4c7 2023-09-23T20:40:35.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:40:35.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2215004144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:35.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: failed to encode map e90 with expected crc 2023-09-23T20:40:35.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2215004144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:36.129 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 32064 32048 142.914 142.438 0.00250164 0.00699096 2023-09-23T20:40:37.129 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 34314 34298 142.76 140.625 0.00706426 0.00700006 2023-09-23T20:40:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:36 smithi121 ceph-mon[219709]: pgmap v162: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 182 MiB/s wr, 2.92k op/s 2023-09-23T20:40:37.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:36 smithi121 ceph-mon[211526]: pgmap v162: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 182 MiB/s wr, 2.92k op/s 2023-09-23T20:40:37.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:36 smithi123 ceph-mon[193903]: pgmap v162: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 182 MiB/s wr, 2.92k op/s 2023-09-23T20:40:38.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 36585 36569 142.707 141.938 0.00958442 0.00700281 2023-09-23T20:40:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2093294525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:38.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2093294525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2093294525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:39.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 38834 38818 142.581 140.562 0.0106944 0.00700948 2023-09-23T20:40:39.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:38 smithi121 ceph-mon[219709]: pgmap v163: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:40:39.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:38 smithi121 ceph-mon[211526]: pgmap v163: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:40:39.165 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:39 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T20:40:39.165 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:39 smithi121 systemd[1]: Stopped Ceph osd.0 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:40:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:38 smithi123 ceph-mon[193903]: pgmap v163: 89 pgs: 23 peering, 6 stale+active+clean, 60 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T20:40:39.664 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:39 smithi121 systemd[1]: Starting Ceph osd.0 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:40:40.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 40767 40751 141.372 120.812 0.0072309 0.00706844 2023-09-23T20:40:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3793323155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3793323155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3793323155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T20:40:40.947 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:40 smithi121 bash[236058]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T20:40:41.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 42819 42803 140.681 128.25 0.00665483 0.00710411 2023-09-23T20:40:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:41 smithi121 ceph-mon[211526]: pgmap v164: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 18815/112269 objects degraded (16.759%) 2023-09-23T20:40:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:41 smithi121 ceph-mon[219709]: pgmap v164: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 18815/112269 objects degraded (16.759%) 2023-09-23T20:40:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:41 smithi123 ceph-mon[193903]: pgmap v164: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 18815/112269 objects degraded (16.759%) 2023-09-23T20:40:42.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:40:42.130637+0000 min lat: 0.00110852 max lat: 1.03039 avg lat: 0.00721052 2023-09-23T20:40:42.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:40:42.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 44405 44389 138.604 99.125 0.00646372 0.00721052 2023-09-23T20:40:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:42 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 18815/112269 objects degraded (16.759%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:42 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2023-09-23T20:40:42.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:42 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 18815/112269 objects degraded (16.759%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:42.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:42 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2023-09-23T20:40:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:42 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 18815/112269 objects degraded (16.759%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:42 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2023-09-23T20:40:43.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 46153 46137 137.207 109.25 0.0075231 0.00728413 2023-09-23T20:40:43.586 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:43.586 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:43.586 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[219709]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s; 18815/112269 objects degraded (16.759%) 2023-09-23T20:40:43.586 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3450782142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:43.587 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:43 smithi121 bash[236485]: 2acc40f5254ffa5ee96fbb41ddcdbe21488c529fbec9810a798572d4ac9edd6e 2023-09-23T20:40:43.587 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-osd[236546]: -- 172.21.15.121:0/3521021566 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563dd737a4e0 con 0x563dd734cc00 2023-09-23T20:40:43.587 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:43 smithi121 systemd[1]: Started Ceph osd.0 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:40:43.587 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:43.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:43.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[211526]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s; 18815/112269 objects degraded (16.759%) 2023-09-23T20:40:43.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3450782142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:43 smithi123 ceph-mon[193903]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s; 18815/112269 objects degraded (16.759%) 2023-09-23T20:40:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3450782142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:43.915 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:43 smithi121 conmon[236522]: 2023-09-23T20:40:43.581+0000 7f6687fa7640 -1 Falling back to public interface 2023-09-23T20:40:44.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 47982 47966 136.166 114.312 0.00955872 0.00733992 2023-09-23T20:40:44.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:44.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:44.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:44 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:44.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:44.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:44.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:44 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:44 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:45.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 49727 49711 134.988 109.062 0.00976613 0.00740291 2023-09-23T20:40:45.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:40:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:40:44] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-09-23T20:40:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:45 smithi121 ceph-mon[211526]: pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2005385073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:45 smithi121 ceph-mon[219709]: pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2005385073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:45 smithi123 ceph-mon[193903]: pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2005385073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:46.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 51475 51459 133.916 109.25 0.00599507 0.00746223 2023-09-23T20:40:47.130 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 53198 53182 132.867 107.688 0.00532057 0.00752164 2023-09-23T20:40:47.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:46 smithi121 ceph-mon[219709]: pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:47.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:46 smithi121 ceph-mon[211526]: pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:47.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:46 smithi123 ceph-mon[193903]: pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:48.137 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 54711 54695 131.361 94.5625 0.0181995 0.00760789 2023-09-23T20:40:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1884623045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:48 smithi123 ceph-mon[193903]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:48.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1884623045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:48.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[211526]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:48.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1884623045' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:48.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:48.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:48 smithi121 ceph-mon[219709]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 23570/140469 objects degraded (16.780%) 2023-09-23T20:40:49.137 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 56266 56250 130.096 97.1875 0.0102802 0.00768207 2023-09-23T20:40:50.139 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 57632 57616 128.491 85.375 0.0237738 0.00777641 2023-09-23T20:40:50.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:50 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:50.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:50 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:50.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/213257147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:50.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:50 smithi123 ceph-mon[193903]: pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s; 27815/165858 objects degraded (16.770%) 2023-09-23T20:40:50.798 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:50.798 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:50.799 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/213257147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:50.799 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[211526]: pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s; 27815/165858 objects degraded (16.770%) 2023-09-23T20:40:50.799 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:50.799 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:50.799 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/213257147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:50.799 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:50 smithi121 ceph-mon[219709]: pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s; 27815/165858 objects degraded (16.770%) 2023-09-23T20:40:51.139 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 59041 59025 127.097 88.0625 0.00604072 0.00786381 2023-09-23T20:40:51.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 27815/165858 objects degraded (16.770%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:51.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:51 smithi121 ceph-mon[211526]: from='osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:40:51.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:51 smithi121 ceph-mon[211526]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:40:51.663 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:51 smithi121 conmon[236522]: 2023-09-23T20:40:51.378+0000 7f6687fa7640 -1 osd.0 88 log_to_monitors true 2023-09-23T20:40:51.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 27815/165858 objects degraded (16.770%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:51.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:51 smithi121 ceph-mon[219709]: from='osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:40:51.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:51 smithi121 ceph-mon[219709]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:40:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 27815/165858 objects degraded (16.770%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:51 smithi123 ceph-mon[193903]: from='osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:40:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:51 smithi123 ceph-mon[193903]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:40:52.141 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 60742 60726 126.398 106.312 0.013838 0.00790734 2023-09-23T20:40:52.163 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 20:40:51 smithi121 conmon[236522]: 2023-09-23T20:40:51.851+0000 7f6678c13700 -1 osd.0 88 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:40:53.141 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 61886 61870 124.628 71.5 0.0170743 0.00794619 2023-09-23T20:40:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: failed to encode map e91 with expected crc 2023-09-23T20:40:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:40:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:40:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: failed to encode map e91 with expected crc 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: from='osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s; 27815/165858 objects degraded (16.770%) 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2771810930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: failed to encode map e91 with expected crc 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: failed to encode map e91 with expected crc 2023-09-23T20:40:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: from='osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:40:53.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:40:53.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s; 27815/165858 objects degraded (16.770%) 2023-09-23T20:40:53.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2771810930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: failed to encode map e91 with expected crc 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: failed to encode map e91 with expected crc 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: from='osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s; 27815/165858 objects degraded (16.770%) 2023-09-23T20:40:53.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2771810930' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:54.141 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 61886 61870 120.736 0 - 0.00794619 2023-09-23T20:40:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[211526]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:40:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[211526]: failed to encode map e92 with expected crc 2023-09-23T20:40:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[211526]: osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724] boot 2023-09-23T20:40:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[211526]: osdmap e92: 8 total, 8 up, 8 in 2023-09-23T20:40:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[211526]: failed to encode map e92 with expected crc 2023-09-23T20:40:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:40:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[219709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:40:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[219709]: failed to encode map e92 with expected crc 2023-09-23T20:40:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[219709]: osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724] boot 2023-09-23T20:40:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[219709]: osdmap e92: 8 total, 8 up, 8 in 2023-09-23T20:40:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[219709]: failed to encode map e92 with expected crc 2023-09-23T20:40:54.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:53 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:40:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:53 smithi123 ceph-mon[193903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:40:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:53 smithi123 ceph-mon[193903]: failed to encode map e92 with expected crc 2023-09-23T20:40:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:53 smithi123 ceph-mon[193903]: osd.0 [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724] boot 2023-09-23T20:40:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:53 smithi123 ceph-mon[193903]: osdmap e92: 8 total, 8 up, 8 in 2023-09-23T20:40:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:53 smithi123 ceph-mon[193903]: failed to encode map e92 with expected crc 2023-09-23T20:40:54.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:53 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:40:55.142 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 61886 61870 117.08 0 - 0.00794619 2023-09-23T20:40:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:40:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:40:54] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.43.0" 2023-09-23T20:40:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T20:40:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[211526]: pgmap v174: 89 pgs: 19 peering, 6 active+undersized+degraded, 9 active+undersized, 55 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 14035/180591 objects degraded (7.772%) 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:54 smithi121 ceph-mon[219709]: pgmap v174: 89 pgs: 19 peering, 6 active+undersized+degraded, 9 active+undersized, 55 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 14035/180591 objects degraded (7.772%) 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: failed to encode map e93 with expected crc 2023-09-23T20:40:55.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:54 smithi123 ceph-mon[193903]: pgmap v174: 89 pgs: 19 peering, 6 active+undersized+degraded, 9 active+undersized, 55 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 14035/180591 objects degraded (7.772%) 2023-09-23T20:40:56.141 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 62457 62441 114.688 11.8958 0.0167472 0.00871373 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1413754261' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14035/180591 objects degraded (7.772%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1413754261' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:56.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:55 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14035/180591 objects degraded (7.772%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:56.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: failed to encode map e94 with expected crc 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1413754261' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:56.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:55 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14035/180591 objects degraded (7.772%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:40:57.142 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 63513 63497 113.298 66 0.00527666 0.00882124 2023-09-23T20:40:57.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:40:56 smithi121 conmon[198376]: 2023-09-23T20:40:56.765+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:40:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: pgmap v176: 89 pgs: 19 peering, 6 active+undersized+degraded, 9 active+undersized, 55 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 14035/180591 objects degraded (7.772%) 2023-09-23T20:40:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: pgmap v176: 89 pgs: 19 peering, 6 active+undersized+degraded, 9 active+undersized, 55 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 14035/180591 objects degraded (7.772%) 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:57.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:57.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:57.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: pgmap v176: 89 pgs: 19 peering, 6 active+undersized+degraded, 9 active+undersized, 55 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 14035/180591 objects degraded (7.772%) 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:40:57.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:40:58.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 64525 64509 111.899 63.25 0.0166467 0.00893243 2023-09-23T20:40:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:40:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:40:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/471039044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:58.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:40:58.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:40:58.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:58.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:58.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/471039044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:58.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:58 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:40:58.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:58 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:40:58.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:40:58.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:58.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/471039044' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:40:59.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 65581 65565 110.66 66 0.0143894 0.00903219 2023-09-23T20:40:59.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:40:59 smithi121 ceph-mon[219709]: pgmap v177: 89 pgs: 19 peering, 4 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 14032/180591 objects degraded (7.770%) 2023-09-23T20:40:59.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:40:59 smithi121 ceph-mon[211526]: pgmap v177: 89 pgs: 19 peering, 4 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 14032/180591 objects degraded (7.770%) 2023-09-23T20:40:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:40:59 smithi123 ceph-mon[193903]: pgmap v177: 89 pgs: 19 peering, 4 active+undersized+degraded, 66 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 14032/180591 objects degraded (7.770%) 2023-09-23T20:41:00.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 66609 66593 109.439 64.25 0.00601139 0.00913264 2023-09-23T20:41:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/189792946' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:00.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/189792946' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/189792946' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:01.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 67637 67621 108.281 64.25 0.00684851 0.00922913 2023-09-23T20:41:01.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:01 smithi121 ceph-mon[211526]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 16498/194691 objects degraded (8.474%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:41:01.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14032/180591 objects degraded (7.770%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:01 smithi121 ceph-mon[219709]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 16498/194691 objects degraded (8.474%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:41:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14032/180591 objects degraded (7.770%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:01 smithi123 ceph-mon[193903]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 16498/194691 objects degraded (8.474%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:41:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14032/180591 objects degraded (7.770%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:02.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:41:02.145660+0000 min lat: 0.00110852 max lat: 2.69371 avg lat: 0.00932198 2023-09-23T20:41:02.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:41:02.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 68689 68673 107.218 65.75 0.00945481 0.00932198 2023-09-23T20:41:03.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 69659 69643 106.082 60.625 0.0118571 0.00942074 2023-09-23T20:41:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:03 smithi121 ceph-mon[219709]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 16498/194691 objects degraded (8.474%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:41:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/469515086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:03.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:03 smithi121 ceph-mon[211526]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 16498/194691 objects degraded (8.474%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:41:03.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/469515086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:03.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:03 smithi123 ceph-mon[193903]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 16498/194691 objects degraded (8.474%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:41:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/469515086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:04.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 70703 70687 105.111 65.25 0.0101164 0.00950929 2023-09-23T20:41:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:04 smithi123 ceph-mon[193903]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 16100/210117 objects degraded (7.662%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:41:04.778 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:41:04] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.43.0" 2023-09-23T20:41:04.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:04 smithi121 ceph-mon[219709]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 16100/210117 objects degraded (7.662%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:41:04.782 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:04 smithi121 ceph-mon[211526]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 16100/210117 objects degraded (7.662%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:41:05.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 71686 71670 104.096 61.4375 0.018554 0.00960093 2023-09-23T20:41:05.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3420776288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3420776288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3420776288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:06.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 72656 72640 103.108 60.625 0.0196797 0.00969383 2023-09-23T20:41:06.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16100/210117 objects degraded (7.662%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:06.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:06 smithi121 ceph-mon[211526]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 16100/210117 objects degraded (7.662%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T20:41:06.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16100/210117 objects degraded (7.662%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:06.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:06 smithi121 ceph-mon[219709]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 16100/210117 objects degraded (7.662%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T20:41:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16100/210117 objects degraded (7.662%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:06 smithi123 ceph-mon[193903]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 16100/210117 objects degraded (7.662%); 3.4 MiB/s, 55 objects/s recovering 2023-09-23T20:41:07.145 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 73660 73644 102.212 62.75 0.00956838 0.00977832 2023-09-23T20:41:08.146 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 74590 74574 101.254 58.125 0.00782401 0.00987099 2023-09-23T20:41:08.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/874311002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/874311002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:08.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/874311002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:09.146 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 75578 75562 100.414 61.75 0.0143967 0.0099531 2023-09-23T20:41:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:09 smithi121 ceph-mon[219709]: pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 16100/210117 objects degraded (7.662%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:41:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:09 smithi121 ceph-mon[211526]: pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 16100/210117 objects degraded (7.662%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:41:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:09 smithi123 ceph-mon[193903]: pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 16100/210117 objects degraded (7.662%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:41:10.146 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 76535 76519 99.5681 59.8125 0.0242841 0.0100383 2023-09-23T20:41:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:10 smithi123 ceph-mon[193903]: pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 15659/224883 objects degraded (6.963%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:41:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:10 smithi121 ceph-mon[211526]: pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 15659/224883 objects degraded (6.963%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:41:10.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:10 smithi121 ceph-mon[219709]: pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 15659/224883 objects degraded (6.963%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:41:11.146 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 77450 77434 98.7036 57.1875 0.0167944 0.010126 2023-09-23T20:41:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2895130359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15659/224883 objects degraded (6.963%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:11.783 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:11 smithi121 conmon[198376]: 2023-09-23T20:41:11.781+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:41:11.784 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2895130359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:11.784 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15659/224883 objects degraded (6.963%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:11.784 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2895130359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:11.784 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15659/224883 objects degraded (6.963%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:12.147 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 78406 78390 97.9226 59.75 0.0180396 0.0102072 2023-09-23T20:41:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:41:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[211526]: pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 15659/224883 objects degraded (6.963%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T20:41:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:41:12.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:12.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:12 smithi121 ceph-mon[219709]: pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 15659/224883 objects degraded (6.963%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T20:41:12.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:12 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:12 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:41:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:12 smithi123 ceph-mon[193903]: pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 15659/224883 objects degraded (6.963%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T20:41:13.147 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 79290 79274 97.0863 55.25 0.0325752 0.0102949 2023-09-23T20:41:13.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/601205886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:13.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/601205886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/601205886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:14.147 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 80233 80217 96.3529 58.9375 0.0197603 0.0103729 2023-09-23T20:41:14.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:14 smithi121 ceph-mon[219709]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 15157/238938 objects degraded (6.343%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:41:14.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:14 smithi121 ceph-mon[211526]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 15157/238938 objects degraded (6.343%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:41:14.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:14 smithi123 ceph-mon[193903]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 15157/238938 objects degraded (6.343%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:41:15.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 81173 81157 95.6437 58.75 0.0098486 0.0104493 2023-09-23T20:41:15.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:41:14] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-23T20:41:16.148 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 82054 82038 94.8907 55.0625 0.0166487 0.0105329 2023-09-23T20:41:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1109707788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1109707788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:16.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1109707788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:17.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 82983 82967 94.2214 58.0625 0.0078894 0.010608 2023-09-23T20:41:17.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15157/238938 objects degraded (6.343%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:17.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:16 smithi121 ceph-mon[211526]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 15157/238938 objects degraded (6.343%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:41:17.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15157/238938 objects degraded (6.343%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:17.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:16 smithi121 ceph-mon[219709]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 15157/238938 objects degraded (6.343%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:41:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15157/238938 objects degraded (6.343%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:16 smithi123 ceph-mon[193903]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 15157/238938 objects degraded (6.343%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:41:18.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 83797 83781 93.4477 50.875 0.00392158 0.0106848 2023-09-23T20:41:19.152 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 84120 84104 92.163 20.1875 0.0193752 0.0108456 2023-09-23T20:41:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[219709]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[219709]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[219709]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T20:41:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[219709]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[219709]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3094085655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[219709]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s; 15157/238938 objects degraded (6.343%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:41:19.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[211526]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[211526]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[211526]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T20:41:19.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[211526]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[211526]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3094085655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:19.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:18 smithi121 ceph-mon[211526]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s; 15157/238938 objects degraded (6.343%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:41:19.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:18 smithi123 ceph-mon[193903]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:18 smithi123 ceph-mon[193903]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:18 smithi123 ceph-mon[193903]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T20:41:19.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:18 smithi123 ceph-mon[193903]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:18 smithi123 ceph-mon[193903]: failed to encode map e95 with expected crc 2023-09-23T20:41:19.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3094085655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:19.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:18 smithi123 ceph-mon[193903]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s; 15157/238938 objects degraded (6.343%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:41:20.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 84976 84960 91.4967 53.5 0.0162174 0.0109249 2023-09-23T20:41:20.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[219709]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[219709]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[219709]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T20:41:20.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[219709]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[219709]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[219709]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[211526]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[211526]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[211526]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T20:41:20.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[211526]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[211526]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:19 smithi121 ceph-mon[211526]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:19 smithi123 ceph-mon[193903]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:19 smithi123 ceph-mon[193903]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:19 smithi123 ceph-mon[193903]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T20:41:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:19 smithi123 ceph-mon[193903]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:19 smithi123 ceph-mon[193903]: failed to encode map e96 with expected crc 2023-09-23T20:41:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:19 smithi123 ceph-mon[193903]: failed to encode map e96 with expected crc 2023-09-23T20:41:21.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 85811 85795 90.8307 52.1875 0.00764531 0.0110037 2023-09-23T20:41:21.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:20 smithi121 ceph-mon[211526]: pgmap v190: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 15064/250791 objects degraded (6.007%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:41:21.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2676895186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:21.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:20 smithi121 ceph-mon[219709]: pgmap v190: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 15064/250791 objects degraded (6.007%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:41:21.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2676895186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:21.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:20 smithi123 ceph-mon[193903]: pgmap v190: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 15064/250791 objects degraded (6.007%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:41:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2676895186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:22.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:41:22.149629+0000 min lat: 0.00110852 max lat: 2.69371 avg lat: 0.0111182 2023-09-23T20:41:22.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:41:22.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 86376 86360 89.9058 35.3125 0.012718 0.0111182 2023-09-23T20:41:22.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15064/250791 objects degraded (6.007%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:22.170 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15064/250791 objects degraded (6.007%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15064/250791 objects degraded (6.007%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:23.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 87204 87188 89.2805 51.75 0.016349 0.0111954 2023-09-23T20:41:23.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:22 smithi121 ceph-mon[211526]: pgmap v191: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 547 op/s; 14612/252078 objects degraded (5.797%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:41:23.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:22 smithi121 ceph-mon[219709]: pgmap v191: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 547 op/s; 14612/252078 objects degraded (5.797%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:41:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:22 smithi123 ceph-mon[193903]: pgmap v191: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 547 op/s; 14612/252078 objects degraded (5.797%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:41:24.149 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 88049 88033 88.6925 52.8125 0.0304066 0.0112689 2023-09-23T20:41:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/146611402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/146611402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:24.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/146611402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:25.152 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 88932 88916 88.1575 55.1875 0.030521 0.0113385 2023-09-23T20:41:25.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:24 smithi121 ceph-mon[211526]: pgmap v192: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 14276/262266 objects degraded (5.443%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:41:25.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:41:24] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-23T20:41:25.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:24 smithi121 ceph-mon[219709]: pgmap v192: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 14276/262266 objects degraded (5.443%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:41:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:24 smithi123 ceph-mon[193903]: pgmap v192: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 14276/262266 objects degraded (5.443%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:41:26.155 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 89736 89720 87.5609 50.25 0.0228831 0.0114153 2023-09-23T20:41:27.155 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 90592 90576 87.0371 53.5 0.028991 0.011484 2023-09-23T20:41:27.165 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:26 smithi121 conmon[198376]: 2023-09-23T20:41:26.812+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:27.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2721025419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:27.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:26 smithi121 ceph-mon[211526]: pgmap v193: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 14276/262266 objects degraded (5.443%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T20:41:27.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:26 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:27.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2721025419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:27.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:26 smithi121 ceph-mon[219709]: pgmap v193: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 14276/262266 objects degraded (5.443%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T20:41:27.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:26 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2721025419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:26 smithi123 ceph-mon[193903]: pgmap v193: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 14276/262266 objects degraded (5.443%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T20:41:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:26 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:28.156 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 91431 91415 86.5125 52.4375 0.0140585 0.0115534 2023-09-23T20:41:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:27 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:27 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:27 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:27 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:27 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:27 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:29.156 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 92257 92241 85.992 51.625 0.00708374 0.0116245 2023-09-23T20:41:29.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:28 smithi121 ceph-mon[211526]: pgmap v194: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 14276/263079 objects degraded (5.427%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:41:29.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/484306402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:29.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:28 smithi121 ceph-mon[219709]: pgmap v194: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 14276/263079 objects degraded (5.427%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:41:29.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/484306402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:28 smithi123 ceph-mon[193903]: pgmap v194: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 14276/263079 objects degraded (5.427%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:41:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/484306402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:30.159 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 93124 93108 85.5235 54.1875 0.0121247 0.0116875 2023-09-23T20:41:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:29 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14276/263079 objects degraded (5.427%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:29 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14276/263079 objects degraded (5.427%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:30.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:29 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14276/263079 objects degraded (5.427%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:31.158 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 93902 93886 84.9871 48.625 0.0187483 0.0117602 2023-09-23T20:41:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:30 smithi121 ceph-mon[219709]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 694 op/s; 14027/274863 objects degraded (5.103%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:41:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:30 smithi121 ceph-mon[211526]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 694 op/s; 14027/274863 objects degraded (5.103%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:41:31.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:30 smithi123 ceph-mon[193903]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 694 op/s; 14027/274863 objects degraded (5.103%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:41:32.158 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 15 94768 94753 84.5473 54.1875 0.0196051 0.0118223 2023-09-23T20:41:32.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1142942968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1142942968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1142942968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:33.159 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 15 95614 95599 84.1011 52.875 0.0144045 0.0118858 2023-09-23T20:41:33.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:32 smithi123 ceph-mon[193903]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 14027/275718 objects degraded (5.087%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:41:33.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:32 smithi121 ceph-mon[211526]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 14027/275718 objects degraded (5.087%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:41:33.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:32 smithi121 ceph-mon[219709]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 43 MiB/s wr, 691 op/s; 14027/275718 objects degraded (5.087%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:41:34.159 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 96386 96370 83.6024 48.1875 0.0310435 0.0119565 2023-09-23T20:41:34.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4163412737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4163412737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4163412737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:35.159 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 97222 97206 83.173 52.25 0.0327826 0.0120183 2023-09-23T20:41:35.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:34 smithi121 ceph-mon[211526]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 13770/287343 objects degraded (4.792%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:41:35.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:41:34] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.43.0" 2023-09-23T20:41:35.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:34 smithi121 ceph-mon[219709]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 13770/287343 objects degraded (4.792%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:41:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:34 smithi123 ceph-mon[193903]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 13770/287343 objects degraded (4.792%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:41:36.163 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 98014 97998 82.7141 49.5 0.0335061 0.0120849 2023-09-23T20:41:36.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:35 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13770/287343 objects degraded (4.792%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:35 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13770/287343 objects degraded (4.792%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:36.451 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:35 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13770/287343 objects degraded (4.792%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:37.163 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 98826 98810 82.2881 50.75 0.00943063 0.0121474 2023-09-23T20:41:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/781580121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:36 smithi121 ceph-mon[211526]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s; 13770/287343 objects degraded (4.792%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:41:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/781580121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:37.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:36 smithi121 ceph-mon[219709]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s; 13770/287343 objects degraded (4.792%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:41:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/781580121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:36 smithi123 ceph-mon[193903]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 695 op/s; 13770/287343 objects degraded (4.792%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:41:38.163 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 15 99654 99639 81.8873 51.8125 0.0229492 0.0122074 2023-09-23T20:41:38.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:38 smithi121 ceph-mon[211526]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 13770/288153 objects degraded (4.779%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:41:38.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:38 smithi121 ceph-mon[219709]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 13770/288153 objects degraded (4.779%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:41:38.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:38 smithi123 ceph-mon[193903]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 13770/288153 objects degraded (4.779%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:41:39.163 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 100419 100403 81.4441 47.75 0.00972905 0.012274 2023-09-23T20:41:39.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/608747828' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:39.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/608747828' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:39.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/608747828' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:40.166 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 101234 101218 81.0501 50.9375 0.0120051 0.0123336 2023-09-23T20:41:40.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:40 smithi121 ceph-mon[211526]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13500/299490 objects degraded (4.508%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:41:40.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:40 smithi121 ceph-mon[219709]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13500/299490 objects degraded (4.508%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:41:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:40 smithi123 ceph-mon[193903]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13500/299490 objects degraded (4.508%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:41:41.166 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 102015 101999 80.6422 48.8125 0.0275324 0.0123954 2023-09-23T20:41:41.831 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13500/299490 objects degraded (4.508%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:41.831 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1976131987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:41.832 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13500/299490 objects degraded (4.508%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:41.832 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1976131987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13500/299490 objects degraded (4.508%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1976131987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:42.167 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:41 smithi121 conmon[198376]: 2023-09-23T20:41:41.829+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:42.167 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:41:42.166800+0000 min lat: 0.00110852 max lat: 2.69371 avg lat: 0.0124572 2023-09-23T20:41:42.167 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:41:42.167 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 102791 102775 80.2406 48.5 0.016049 0.0124572 2023-09-23T20:41:42.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:42.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:42.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:42.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:42.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[219709]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 13500/300249 objects degraded (4.496%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:41:42.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:42.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:42.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:42.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:42.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:42 smithi121 ceph-mon[211526]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 13500/300249 objects degraded (4.496%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:41:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:42 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:41:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:42 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:41:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:42 smithi123 ceph-mon[193903]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 13500/300249 objects degraded (4.496%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:41:43.166 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 103597 103581 79.872 50.375 0.00861807 0.0125158 2023-09-23T20:41:44.166 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 104327 104311 79.4545 45.625 0.0285698 0.0125805 2023-09-23T20:41:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/593671902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:44.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/593671902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:44.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/593671902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:45.040 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:41:44] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-09-23T20:41:45.167 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 105134 105118 79.1044 50.4375 0.011946 0.012636 2023-09-23T20:41:45.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:45 smithi121 ceph-mon[219709]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 13212/311361 objects degraded (4.243%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:41:45.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:45 smithi121 ceph-mon[211526]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 13212/311361 objects degraded (4.243%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:41:45.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:45 smithi123 ceph-mon[193903]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 13212/311361 objects degraded (4.243%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:41:46.167 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 105912 105896 78.7417 48.625 0.00967526 0.0126947 2023-09-23T20:41:46.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13212/311361 objects degraded (4.243%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:46.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13212/311361 objects degraded (4.243%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13212/311361 objects degraded (4.243%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:47.167 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 106646 106630 78.3552 45.875 0.0129623 0.0127563 2023-09-23T20:41:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:47 smithi121 ceph-mon[219709]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 13212/311361 objects degraded (4.243%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:41:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1982700590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:47.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:47 smithi121 ceph-mon[211526]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 13212/311361 objects degraded (4.243%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:41:47.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1982700590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:47 smithi123 ceph-mon[193903]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s; 13212/311361 objects degraded (4.243%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:41:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1982700590' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:48.170 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 107429 107413 78.0107 48.9375 0.0212233 0.0128141 2023-09-23T20:41:48.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:48 smithi121 ceph-mon[219709]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s; 13212/312165 objects degraded (4.232%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:41:48.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:48 smithi121 ceph-mon[211526]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s; 13212/312165 objects degraded (4.232%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:41:48.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:48 smithi123 ceph-mon[193903]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 688 op/s; 13212/312165 objects degraded (4.232%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:41:49.175 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 108146 108130 77.6272 44.8125 0.0148536 0.0128769 2023-09-23T20:41:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1216990223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:49.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1216990223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1216990223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:50.173 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 108914 108898 77.2907 48 0.00938653 0.0129328 2023-09-23T20:41:50.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:50 smithi121 ceph-mon[211526]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 12919/322944 objects degraded (4.000%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:41:50.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:50 smithi121 ceph-mon[219709]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 12919/322944 objects degraded (4.000%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:41:50.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:50 smithi123 ceph-mon[193903]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 12919/322944 objects degraded (4.000%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:41:51.173 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 109625 109609 76.9217 44.4375 0.0129662 0.0129949 2023-09-23T20:41:51.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12919/322944 objects degraded (4.000%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:51.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12919/322944 objects degraded (4.000%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:51.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12919/322944 objects degraded (4.000%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:52.173 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.0158 2023-09-23T20:41:52.173 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 110304 2023-09-23T20:41:52.173 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T20:41:52.173 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 76.5865 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 43.4864 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 215.938 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 1225 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 695.787 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 3455 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0130529 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.0370476 2023-09-23T20:41:52.174 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 2.69371 2023-09-23T20:41:52.175 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00110852 2023-09-23T20:41:52.179 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:41:52.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2533313252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:52.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:52 smithi121 ceph-mon[219709]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 12919/323643 objects degraded (3.992%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:41:52.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2533313252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:52.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:52 smithi121 ceph-mon[211526]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 12919/323643 objects degraded (3.992%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:41:52.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2533313252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:52 smithi123 ceph-mon[193903]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 12919/323643 objects degraded (3.992%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:41:54.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:41:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:41:54] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-09-23T20:41:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[211526]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[211526]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[211526]: osdmap e97: 8 total, 8 up, 8 in 2023-09-23T20:41:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[211526]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3981059484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[211526]: pgmap v208: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 748 op/s; 12594/328041 objects degraded (3.839%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[219709]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[219709]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[219709]: osdmap e97: 8 total, 8 up, 8 in 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[219709]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3981059484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:54 smithi121 ceph-mon[219709]: pgmap v208: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 748 op/s; 12594/328041 objects degraded (3.839%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:41:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:54 smithi123 ceph-mon[193903]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:54 smithi123 ceph-mon[193903]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:54 smithi123 ceph-mon[193903]: osdmap e97: 8 total, 8 up, 8 in 2023-09-23T20:41:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:54 smithi123 ceph-mon[193903]: failed to encode map e97 with expected crc 2023-09-23T20:41:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3981059484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:54 smithi123 ceph-mon[193903]: pgmap v208: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 748 op/s; 12594/328041 objects degraded (3.839%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:41:55.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[211526]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[211526]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[211526]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T20:41:55.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[211526]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[211526]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[219709]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[219709]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[219709]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T20:41:55.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[219709]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:55 smithi121 ceph-mon[219709]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:55 smithi123 ceph-mon[193903]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:55 smithi123 ceph-mon[193903]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:55 smithi123 ceph-mon[193903]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T20:41:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:55 smithi123 ceph-mon[193903]: failed to encode map e98 with expected crc 2023-09-23T20:41:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:55 smithi123 ceph-mon[193903]: failed to encode map e98 with expected crc 2023-09-23T20:41:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12594/328041 objects degraded (3.839%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:56 smithi121 ceph-mon[211526]: pgmap v210: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 49 MiB/s wr, 902 op/s; 12594/328041 objects degraded (3.839%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:41:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12594/328041 objects degraded (3.839%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:56 smithi121 ceph-mon[219709]: pgmap v210: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 49 MiB/s wr, 902 op/s; 12594/328041 objects degraded (3.839%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:41:56.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12594/328041 objects degraded (3.839%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:41:56.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:56 smithi123 ceph-mon[193903]: pgmap v210: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 49 MiB/s wr, 902 op/s; 12594/328041 objects degraded (3.839%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:41:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4268493212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:57.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4268493212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:57.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:57.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4268493212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:58.596 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:58 smithi121 ceph-mon[211526]: pgmap v211: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 21 MiB/s wr, 476 op/s; 12594/327573 objects degraded (3.845%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:41:58.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:58 smithi121 ceph-mon[219709]: pgmap v211: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 21 MiB/s wr, 476 op/s; 12594/327573 objects degraded (3.845%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:41:58.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:58 smithi123 ceph-mon[193903]: pgmap v211: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 21 MiB/s wr, 476 op/s; 12594/327573 objects degraded (3.845%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:41:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:41:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2955771007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:59.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:41:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2955771007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:41:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:41:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2955771007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:00 smithi121 ceph-mon[211526]: pgmap v212: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 746 op/s; 12116/320418 objects degraded (3.781%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:42:00.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:00 smithi121 ceph-mon[219709]: pgmap v212: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 746 op/s; 12116/320418 objects degraded (3.781%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:42:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:00 smithi123 ceph-mon[193903]: pgmap v212: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 746 op/s; 12116/320418 objects degraded (3.781%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:42:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12116/320418 objects degraded (3.781%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:01.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12116/320418 objects degraded (3.781%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:01.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:01.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12116/320418 objects degraded (3.781%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:02.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/6357741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:02.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:02.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:02.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[211526]: pgmap v213: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.8 MiB/s wr, 494 op/s; 12116/319713 objects degraded (3.790%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:42:02.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/6357741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:02.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:02.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:02.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:02 smithi121 ceph-mon[219709]: pgmap v213: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.8 MiB/s wr, 494 op/s; 12116/319713 objects degraded (3.790%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:42:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/6357741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:02 smithi123 ceph-mon[193903]: pgmap v213: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.8 MiB/s wr, 494 op/s; 12116/319713 objects degraded (3.790%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:42:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:03 smithi121 conmon[198376]: 2023-09-23T20:42:03.450+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:42:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:04 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:04 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4037939247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:04 smithi123 ceph-mon[193903]: pgmap v214: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 KiB/s wr, 605 op/s; 11919/309948 objects degraded (3.845%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:42:05.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:42:04] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.43.0" 2023-09-23T20:42:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4037939247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:05.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[211526]: pgmap v214: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 KiB/s wr, 605 op/s; 11919/309948 objects degraded (3.845%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:42:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4037939247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:04 smithi121 ceph-mon[219709]: pgmap v214: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 KiB/s wr, 605 op/s; 11919/309948 objects degraded (3.845%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:42:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:05 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11919/309948 objects degraded (3.845%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:06.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:05 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11919/309948 objects degraded (3.845%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:06.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:05 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11919/309948 objects degraded (3.845%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:06 smithi123 ceph-mon[193903]: pgmap v215: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 KiB/s wr, 509 op/s; 11919/309948 objects degraded (3.845%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:07.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:06 smithi121 ceph-mon[211526]: pgmap v215: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 KiB/s wr, 509 op/s; 11919/309948 objects degraded (3.845%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:06 smithi121 ceph-mon[219709]: pgmap v215: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 KiB/s wr, 509 op/s; 11919/309948 objects degraded (3.845%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:08.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/362513617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/362513617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/362513617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:09.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:08 smithi123 ceph-mon[193903]: pgmap v216: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 KiB/s wr, 524 op/s; 11919/309225 objects degraded (3.854%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:08 smithi121 ceph-mon[211526]: pgmap v216: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 KiB/s wr, 524 op/s; 11919/309225 objects degraded (3.854%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:08 smithi121 ceph-mon[219709]: pgmap v216: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 KiB/s wr, 524 op/s; 11919/309225 objects degraded (3.854%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2990668869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2990668869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:10.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2990668869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:11.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:10 smithi121 ceph-mon[211526]: pgmap v217: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 11677/298776 objects degraded (3.908%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:42:11.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:10 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11919/309225 objects degraded (3.854%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:11.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:10 smithi121 ceph-mon[219709]: pgmap v217: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 11677/298776 objects degraded (3.908%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:42:11.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:10 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11919/309225 objects degraded (3.854%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:11.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:10 smithi123 ceph-mon[193903]: pgmap v217: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 11677/298776 objects degraded (3.908%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:42:11.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:10 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11919/309225 objects degraded (3.854%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3422183191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:12.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3422183191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:12.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3422183191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:13 smithi123 ceph-mon[193903]: pgmap v218: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11677/298068 objects degraded (3.918%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:42:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:13 smithi121 ceph-mon[211526]: pgmap v218: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11677/298068 objects degraded (3.918%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:42:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:13 smithi121 ceph-mon[219709]: pgmap v218: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11677/298068 objects degraded (3.918%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:42:14.779 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:14 smithi121 ceph-mon[211526]: pgmap v219: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 11442/287664 objects degraded (3.978%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:14.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:14 smithi121 ceph-mon[219709]: pgmap v219: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 11442/287664 objects degraded (3.978%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:14.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:14 smithi123 ceph-mon[193903]: pgmap v219: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 11442/287664 objects degraded (3.978%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:42:15.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:42:14] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2023-09-23T20:42:15.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/415993470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:15.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/415993470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/415993470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11442/287664 objects degraded (3.978%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:16 smithi121 ceph-mon[211526]: pgmap v220: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11442/287664 objects degraded (3.978%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:42:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11442/287664 objects degraded (3.978%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:16 smithi121 ceph-mon[219709]: pgmap v220: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11442/287664 objects degraded (3.978%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:42:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11442/287664 objects degraded (3.978%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:16 smithi123 ceph-mon[193903]: pgmap v220: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11442/287664 objects degraded (3.978%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:42:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3827822929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:18.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3827822929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3827822929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:18.892 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:18 smithi121 conmon[198376]: 2023-09-23T20:42:18.466+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:18.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:18 smithi121 ceph-mon[211526]: pgmap v221: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11442/287016 objects degraded (3.987%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:42:18.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:18 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:19.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:18 smithi121 ceph-mon[219709]: pgmap v221: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11442/287016 objects degraded (3.987%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:42:19.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:18 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:18 smithi123 ceph-mon[193903]: pgmap v221: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11442/287016 objects degraded (3.987%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:42:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:18 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:19 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:20.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:19 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:20.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/453408427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:20.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:19 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:20.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:19 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:20.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/453408427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:20.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:19 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:19 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/453408427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:21.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:20 smithi121 ceph-mon[211526]: pgmap v222: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 11194/276648 objects degraded (4.046%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:42:21.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:20 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11442/287016 objects degraded (3.987%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:21.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:20 smithi121 ceph-mon[219709]: pgmap v222: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 11194/276648 objects degraded (4.046%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:42:21.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:20 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11442/287016 objects degraded (3.987%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:21.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:20 smithi123 ceph-mon[193903]: pgmap v222: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 11194/276648 objects degraded (4.046%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:42:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:20 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11442/287016 objects degraded (3.987%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:22.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:22 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2247704506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:22.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:22 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2247704506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:22.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:22 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2247704506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:23.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:23 smithi123 ceph-mon[193903]: pgmap v223: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11194/275907 objects degraded (4.057%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:42:23.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:23 smithi121 ceph-mon[211526]: pgmap v223: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11194/275907 objects degraded (4.057%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:42:23.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:23 smithi121 ceph-mon[219709]: pgmap v223: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11194/275907 objects degraded (4.057%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:42:24.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:24 smithi121 ceph-mon[211526]: pgmap v224: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 10941/265752 objects degraded (4.117%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T20:42:24.918 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:42:24] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2023-09-23T20:42:24.921 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:24 smithi121 ceph-mon[219709]: pgmap v224: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 10941/265752 objects degraded (4.117%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T20:42:24.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:24 smithi123 ceph-mon[193903]: pgmap v224: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 10941/265752 objects degraded (4.117%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T20:42:25.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1336118749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:25.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1336118749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:25.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1336118749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:26.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10941/265752 objects degraded (4.117%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:26.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:26 smithi121 ceph-mon[211526]: pgmap v225: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 10941/265752 objects degraded (4.117%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:26.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10941/265752 objects degraded (4.117%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:26.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:26 smithi121 ceph-mon[219709]: pgmap v225: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 10941/265752 objects degraded (4.117%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10941/265752 objects degraded (4.117%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:26 smithi123 ceph-mon[193903]: pgmap v225: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 10941/265752 objects degraded (4.117%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:28.170 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:28.170 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2344986953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:28.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:28.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2344986953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2344986953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:29.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:28 smithi121 ceph-mon[219709]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 10941/265179 objects degraded (4.126%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:29.171 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:28 smithi121 ceph-mon[211526]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 10941/265179 objects degraded (4.126%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:28 smithi123 ceph-mon[193903]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 10941/265179 objects degraded (4.126%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1708794753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:30.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1708794753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1708794753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:31.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:31 smithi121 ceph-mon[219709]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10676/255072 objects degraded (4.185%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:42:31.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10941/265179 objects degraded (4.126%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:31.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:31 smithi121 ceph-mon[211526]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10676/255072 objects degraded (4.185%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:42:31.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10941/265179 objects degraded (4.126%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:31 smithi123 ceph-mon[193903]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 10676/255072 objects degraded (4.185%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:42:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10941/265179 objects degraded (4.126%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[219709]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:42:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[219709]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[219709]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[219709]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[219709]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2419423772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[219709]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10676/254373 objects degraded (4.197%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:42:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[211526]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:42:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[211526]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[211526]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[211526]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[211526]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2419423772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:33 smithi121 ceph-mon[211526]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10676/254373 objects degraded (4.197%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:42:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:33 smithi123 ceph-mon[193903]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:42:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:33 smithi123 ceph-mon[193903]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:33 smithi123 ceph-mon[193903]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:33 smithi123 ceph-mon[193903]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:33 smithi123 ceph-mon[193903]: failed to encode map e99 with expected crc 2023-09-23T20:42:33.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2419423772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:33.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:33 smithi123 ceph-mon[193903]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10676/254373 objects degraded (4.197%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T20:42:33.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:33 smithi121 conmon[198376]: 2023-09-23T20:42:33.487+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[211526]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[211526]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[211526]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T20:42:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[211526]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[219709]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[219709]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[219709]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T20:42:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[219709]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:34 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:34 smithi123 ceph-mon[193903]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:34 smithi123 ceph-mon[193903]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:34 smithi123 ceph-mon[193903]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T20:42:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:34 smithi123 ceph-mon[193903]: failed to encode map e100 with expected crc 2023-09-23T20:42:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:34 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:35.126 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:42:34] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2023-09-23T20:42:35.130 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:35.130 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:35.130 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[219709]: pgmap v231: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 10407/247068 objects degraded (4.212%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:42:35.130 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1537239954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:35.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:35.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:35.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[211526]: pgmap v231: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 10407/247068 objects degraded (4.212%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:42:35.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1537239954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:35 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:35 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:35 smithi123 ceph-mon[193903]: pgmap v231: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 10407/247068 objects degraded (4.212%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:42:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1537239954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:36.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10407/247068 objects degraded (4.212%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:36.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10407/247068 objects degraded (4.212%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10407/247068 objects degraded (4.212%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:37 smithi121 ceph-mon[211526]: pgmap v232: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10407/247068 objects degraded (4.212%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:42:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:37 smithi121 ceph-mon[219709]: pgmap v232: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10407/247068 objects degraded (4.212%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:42:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:37 smithi123 ceph-mon[193903]: pgmap v232: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10407/247068 objects degraded (4.212%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:42:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3256430312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3256430312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3256430312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:39 smithi123 ceph-mon[193903]: pgmap v233: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 10242/245505 objects degraded (4.172%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:39 smithi121 ceph-mon[211526]: pgmap v233: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 10242/245505 objects degraded (4.172%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:39.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:39 smithi121 ceph-mon[219709]: pgmap v233: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 10242/245505 objects degraded (4.172%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3846578486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:40.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3846578486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:40.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3846578486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:41 smithi123 ceph-mon[193903]: pgmap v234: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 9999/236073 objects degraded (4.236%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:42:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10242/245505 objects degraded (4.172%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:41 smithi121 ceph-mon[211526]: pgmap v234: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 9999/236073 objects degraded (4.236%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:42:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10242/245505 objects degraded (4.172%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:41 smithi121 ceph-mon[219709]: pgmap v234: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 9999/236073 objects degraded (4.236%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:42:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10242/245505 objects degraded (4.172%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:42.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:42.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:43 smithi123 ceph-mon[193903]: pgmap v235: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 9999/234753 objects degraded (4.259%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:42:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2770724156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:43 smithi121 ceph-mon[211526]: pgmap v235: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 9999/234753 objects degraded (4.259%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:42:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2770724156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:43 smithi121 ceph-mon[219709]: pgmap v235: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 9999/234753 objects degraded (4.259%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:42:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2770724156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:44 smithi121 ceph-mon[211526]: pgmap v236: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 9754/225315 objects degraded (4.329%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:42:44.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:44 smithi121 ceph-mon[219709]: pgmap v236: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 9754/225315 objects degraded (4.329%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:42:44.914 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:42:44] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T20:42:44.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:44 smithi123 ceph-mon[193903]: pgmap v236: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 9754/225315 objects degraded (4.329%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:42:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3393532601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3393532601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3393532601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:46.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 9754/225315 objects degraded (4.329%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:46.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:46 smithi121 ceph-mon[211526]: pgmap v237: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9754/225315 objects degraded (4.329%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:46.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 9754/225315 objects degraded (4.329%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:46.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:46 smithi121 ceph-mon[219709]: pgmap v237: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9754/225315 objects degraded (4.329%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:46.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 9754/225315 objects degraded (4.329%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:46.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:46 smithi123 ceph-mon[193903]: pgmap v237: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9754/225315 objects degraded (4.329%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:48.170 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1809656616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:48.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1809656616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:48.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1809656616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:48.910 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:48 smithi121 conmon[198376]: 2023-09-23T20:42:48.516+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:48.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:48 smithi121 ceph-mon[211526]: pgmap v238: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 9754/223944 objects degraded (4.356%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:49.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:48 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:49.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:48 smithi121 ceph-mon[219709]: pgmap v238: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 9754/223944 objects degraded (4.356%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:49.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:48 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:49.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:48 smithi123 ceph-mon[193903]: pgmap v238: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 9754/223944 objects degraded (4.356%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:42:49.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:48 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:49 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:49 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:49 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:49 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:49 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:42:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:49 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:42:51.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2431846169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:51.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:50 smithi123 ceph-mon[193903]: pgmap v239: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 9501/214809 objects degraded (4.423%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:42:51.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:50 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 9754/223944 objects degraded (4.356%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2431846169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:50 smithi121 ceph-mon[211526]: pgmap v239: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 9501/214809 objects degraded (4.423%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:42:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:50 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 9754/223944 objects degraded (4.356%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2431846169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:50 smithi121 ceph-mon[219709]: pgmap v239: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 9501/214809 objects degraded (4.423%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:42:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:50 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 9754/223944 objects degraded (4.356%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:53.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:53 smithi121 ceph-mon[211526]: pgmap v240: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 9501/213423 objects degraded (4.452%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:53.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/529556792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:53.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:53 smithi121 ceph-mon[219709]: pgmap v240: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 9501/213423 objects degraded (4.452%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:53.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/529556792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:53 smithi123 ceph-mon[193903]: pgmap v240: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 9501/213423 objects degraded (4.452%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:42:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/529556792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:54.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:42:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:42:54] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T20:42:54.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:54 smithi121 ceph-mon[211526]: pgmap v241: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 9223/204147 objects degraded (4.518%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:42:54.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:54 smithi121 ceph-mon[219709]: pgmap v241: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 9223/204147 objects degraded (4.518%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:42:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:54 smithi123 ceph-mon[193903]: pgmap v241: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 9223/204147 objects degraded (4.518%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:42:55.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1067076823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:55.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1067076823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1067076823' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 9223/204147 objects degraded (4.518%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:56 smithi121 ceph-mon[211526]: pgmap v242: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9223/204147 objects degraded (4.518%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:42:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 9223/204147 objects degraded (4.518%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:56 smithi121 ceph-mon[219709]: pgmap v242: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9223/204147 objects degraded (4.518%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:42:56.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 9223/204147 objects degraded (4.518%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:42:56.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:56 smithi123 ceph-mon[193903]: pgmap v242: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9223/204147 objects degraded (4.518%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:42:58.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:59.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3148457857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:59.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:42:58 smithi123 ceph-mon[193903]: pgmap v243: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9223/202890 objects degraded (4.546%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:42:59.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3148457857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:59.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:42:58 smithi121 ceph-mon[211526]: pgmap v243: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9223/202890 objects degraded (4.546%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:42:59.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3148457857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:42:59.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:42:58 smithi121 ceph-mon[219709]: pgmap v243: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 9223/202890 objects degraded (4.546%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:43:00.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:00 smithi121 ceph-mon[211526]: pgmap v244: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 8955/193917 objects degraded (4.618%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:43:00.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:00 smithi121 ceph-mon[219709]: pgmap v244: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 8955/193917 objects degraded (4.618%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:43:00.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:00 smithi123 ceph-mon[193903]: pgmap v244: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 8955/193917 objects degraded (4.618%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:43:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/322738794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 8955/193917 objects degraded (4.618%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/322738794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 8955/193917 objects degraded (4.618%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/322738794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 8955/193917 objects degraded (4.618%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:02 smithi121 ceph-mon[211526]: pgmap v245: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8955/192609 objects degraded (4.649%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:43:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:02 smithi121 ceph-mon[219709]: pgmap v245: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8955/192609 objects degraded (4.649%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:43:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:02 smithi123 ceph-mon[193903]: pgmap v245: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8955/192609 objects degraded (4.649%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:43:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/608761094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/608761094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/608761094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:04.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:04 smithi121 ceph-mon[211526]: pgmap v246: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 8656/183531 objects degraded (4.716%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:43:04.918 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:43:04] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-09-23T20:43:04.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:04 smithi121 ceph-mon[219709]: pgmap v246: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 8656/183531 objects degraded (4.716%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:43:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:04 smithi123 ceph-mon[193903]: pgmap v246: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 8656/183531 objects degraded (4.716%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:43:06.171 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4026983497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:06.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4026983497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4026983497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 8656/183531 objects degraded (4.716%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:06 smithi121 ceph-mon[211526]: pgmap v247: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 8656/183531 objects degraded (4.716%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:43:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 8656/183531 objects degraded (4.716%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:06 smithi121 ceph-mon[219709]: pgmap v247: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 8656/183531 objects degraded (4.716%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:43:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 8656/183531 objects degraded (4.716%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:06 smithi123 ceph-mon[193903]: pgmap v247: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 8656/183531 objects degraded (4.716%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:43:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3168559852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[211526]: pgmap v248: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 8656/182496 objects degraded (4.743%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:43:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3168559852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:08 smithi121 ceph-mon[219709]: pgmap v248: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 8656/182496 objects degraded (4.743%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:43:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3168559852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:08 smithi123 ceph-mon[193903]: pgmap v248: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 8656/182496 objects degraded (4.743%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:43:09.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T20:43:09.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:09.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:09.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:09.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:09.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:09 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:43:09.920 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:09 smithi121 conmon[198376]: 2023-09-23T20:43:09.471+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:43:09.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: failed to encode map e101 with expected crc 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:43:09.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:09.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:09.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:09 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-09-23T20:43:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[211526]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[211526]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[211526]: osdmap e102: 8 total, 8 up, 8 in 2023-09-23T20:43:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[211526]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[211526]: pgmap v251: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 8368/174129 objects degraded (4.806%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:43:10.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[219709]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[219709]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[219709]: osdmap e102: 8 total, 8 up, 8 in 2023-09-23T20:43:10.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[219709]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:10 smithi121 ceph-mon[219709]: pgmap v251: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 8368/174129 objects degraded (4.806%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:43:10.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:10 smithi123 ceph-mon[193903]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:10 smithi123 ceph-mon[193903]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:10 smithi123 ceph-mon[193903]: osdmap e102: 8 total, 8 up, 8 in 2023-09-23T20:43:10.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:10 smithi123 ceph-mon[193903]: failed to encode map e102 with expected crc 2023-09-23T20:43:10.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:10 smithi123 ceph-mon[193903]: pgmap v251: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 8368/174129 objects degraded (4.806%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:43:11.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3388346381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:11.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 8368/174129 objects degraded (4.806%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:11.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3388346381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:11.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 8368/174129 objects degraded (4.806%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3388346381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 8368/174129 objects degraded (4.806%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:12 smithi121 ceph-mon[211526]: pgmap v252: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 8368/173094 objects degraded (4.834%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:43:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:12 smithi121 ceph-mon[219709]: pgmap v252: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 8368/173094 objects degraded (4.834%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:43:12.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:12 smithi123 ceph-mon[193903]: pgmap v252: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 8368/173094 objects degraded (4.834%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:43:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3451255188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3451255188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3451255188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:14.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:43:14] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.43.0" 2023-09-23T20:43:14.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:14 smithi121 ceph-mon[211526]: pgmap v253: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 8022/165411 objects degraded (4.850%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:43:14.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:14 smithi121 ceph-mon[219709]: pgmap v253: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 8022/165411 objects degraded (4.850%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:43:14.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:14 smithi123 ceph-mon[193903]: pgmap v253: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 8022/165411 objects degraded (4.850%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:43:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:15 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 8022/165411 objects degraded (4.850%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:15 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 8022/165411 objects degraded (4.850%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:16.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:15 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 8022/165411 objects degraded (4.850%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/267416481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:16 smithi123 ceph-mon[193903]: pgmap v254: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8022/165411 objects degraded (4.850%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:43:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/267416481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:16 smithi121 ceph-mon[211526]: pgmap v254: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8022/165411 objects degraded (4.850%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:43:17.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/267416481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:17.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:16 smithi121 ceph-mon[219709]: pgmap v254: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 8022/165411 objects degraded (4.850%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T20:43:19.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:19 smithi121 ceph-mon[219709]: pgmap v255: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 8022/164070 objects degraded (4.889%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:43:19.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2055795303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:19.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:19 smithi121 ceph-mon[211526]: pgmap v255: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 8022/164070 objects degraded (4.889%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:43:19.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2055795303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:19.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:19 smithi123 ceph-mon[193903]: pgmap v255: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 8022/164070 objects degraded (4.889%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:43:19.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2055795303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:20 smithi121 ceph-mon[211526]: pgmap v256: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 7750/154968 objects degraded (5.001%); 3.0 MiB/s, 56 objects/s recovering 2023-09-23T20:43:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:20 smithi121 ceph-mon[219709]: pgmap v256: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 7750/154968 objects degraded (5.001%); 3.0 MiB/s, 56 objects/s recovering 2023-09-23T20:43:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:20 smithi123 ceph-mon[193903]: pgmap v256: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 7750/154968 objects degraded (5.001%); 3.0 MiB/s, 56 objects/s recovering 2023-09-23T20:43:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7750/154968 objects degraded (5.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/376689977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7750/154968 objects degraded (5.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/376689977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:21.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7750/154968 objects degraded (5.001%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:21.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/376689977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:22.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:22 smithi121 ceph-mon[211526]: pgmap v257: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 7750/153705 objects degraded (5.042%); 2.8 MiB/s, 51 objects/s recovering 2023-09-23T20:43:22.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:22 smithi121 ceph-mon[219709]: pgmap v257: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 7750/153705 objects degraded (5.042%); 2.8 MiB/s, 51 objects/s recovering 2023-09-23T20:43:22.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:22 smithi123 ceph-mon[193903]: pgmap v257: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 7750/153705 objects degraded (5.042%); 2.8 MiB/s, 51 objects/s recovering 2023-09-23T20:43:23.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/560882514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/560882514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:24.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/560882514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:24.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[211526]: pgmap v258: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7487/144567 objects degraded (5.179%); 2.8 MiB/s, 73 objects/s recovering 2023-09-23T20:43:24.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:24.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:24.750 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:24.761 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:24 smithi121 conmon[198376]: 2023-09-23T20:43:24.493+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:24.761 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[219709]: pgmap v258: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7487/144567 objects degraded (5.179%); 2.8 MiB/s, 73 objects/s recovering 2023-09-23T20:43:24.761 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:24.761 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:24.762 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:24 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:25.175 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:43:24] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.43.0" 2023-09-23T20:43:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:24 smithi123 ceph-mon[193903]: pgmap v258: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 7487/144567 objects degraded (5.179%); 2.8 MiB/s, 73 objects/s recovering 2023-09-23T20:43:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:24 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:24 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:24 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:25 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7487/144567 objects degraded (5.179%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:26.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:25 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7487/144567 objects degraded (5.179%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:26.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:25 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7487/144567 objects degraded (5.179%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1993994810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:26 smithi123 ceph-mon[193903]: pgmap v259: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 7487/144567 objects degraded (5.179%); 981 KiB/s, 44 objects/s recovering 2023-09-23T20:43:27.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1993994810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:27.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:26 smithi121 ceph-mon[211526]: pgmap v259: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 7487/144567 objects degraded (5.179%); 981 KiB/s, 44 objects/s recovering 2023-09-23T20:43:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1993994810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:27.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:26 smithi121 ceph-mon[219709]: pgmap v259: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 7487/144567 objects degraded (5.179%); 981 KiB/s, 44 objects/s recovering 2023-09-23T20:43:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:28.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:29 smithi121 ceph-mon[211526]: pgmap v260: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 7487/143244 objects degraded (5.227%); 981 KiB/s, 44 objects/s recovering 2023-09-23T20:43:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/51444853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:29.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:29 smithi121 ceph-mon[219709]: pgmap v260: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 7487/143244 objects degraded (5.227%); 981 KiB/s, 44 objects/s recovering 2023-09-23T20:43:29.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/51444853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:29 smithi123 ceph-mon[193903]: pgmap v260: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 7487/143244 objects degraded (5.227%); 981 KiB/s, 44 objects/s recovering 2023-09-23T20:43:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/51444853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:30.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:30 smithi121 ceph-mon[211526]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 7223/134265 objects degraded (5.380%); 981 KiB/s, 66 objects/s recovering 2023-09-23T20:43:30.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:30 smithi121 ceph-mon[219709]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 7223/134265 objects degraded (5.380%); 981 KiB/s, 66 objects/s recovering 2023-09-23T20:43:30.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:30 smithi123 ceph-mon[193903]: pgmap v261: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 7223/134265 objects degraded (5.380%); 981 KiB/s, 66 objects/s recovering 2023-09-23T20:43:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7223/134265 objects degraded (5.380%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3177390159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7223/134265 objects degraded (5.380%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3177390159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7223/134265 objects degraded (5.380%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3177390159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:32.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:32 smithi121 ceph-mon[211526]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 7223/132915 objects degraded (5.434%); 0 B/s, 43 objects/s recovering 2023-09-23T20:43:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:32 smithi121 ceph-mon[219709]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 7223/132915 objects degraded (5.434%); 0 B/s, 43 objects/s recovering 2023-09-23T20:43:32.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:32 smithi123 ceph-mon[193903]: pgmap v262: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 7223/132915 objects degraded (5.434%); 0 B/s, 43 objects/s recovering 2023-09-23T20:43:34.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4162222617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4162222617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4162222617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:34 smithi121 ceph-mon[219709]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 6932/123672 objects degraded (5.605%); 0 B/s, 68 objects/s recovering 2023-09-23T20:43:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:34 smithi121 ceph-mon[211526]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 6932/123672 objects degraded (5.605%); 0 B/s, 68 objects/s recovering 2023-09-23T20:43:35.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:43:34] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T20:43:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:34 smithi123 ceph-mon[193903]: pgmap v263: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 6932/123672 objects degraded (5.605%); 0 B/s, 68 objects/s recovering 2023-09-23T20:43:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:35 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6932/123672 objects degraded (5.605%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:36.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:35 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6932/123672 objects degraded (5.605%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:35 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6932/123672 objects degraded (5.605%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2011297763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:36 smithi121 ceph-mon[211526]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 6932/123672 objects degraded (5.605%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2011297763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:37.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:36 smithi121 ceph-mon[219709]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 6932/123672 objects degraded (5.605%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2011297763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:36 smithi123 ceph-mon[193903]: pgmap v264: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 6932/123672 objects degraded (5.605%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:38.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:38 smithi121 ceph-mon[211526]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 6932/120462 objects degraded (5.755%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:38.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:38 smithi121 ceph-mon[219709]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 6932/120462 objects degraded (5.755%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:38.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:38 smithi123 ceph-mon[193903]: pgmap v265: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 6932/120462 objects degraded (5.755%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:39.916 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:39 smithi121 conmon[198376]: 2023-09-23T20:43:39.529+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4204746554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:39 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:39.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4204746554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:39.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:39 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:39.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4204746554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:39 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:40 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:40 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:40 smithi121 ceph-mon[211526]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6642/113265 objects degraded (5.864%); 0 B/s, 70 objects/s recovering 2023-09-23T20:43:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:40 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:40 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:40 smithi121 ceph-mon[219709]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6642/113265 objects degraded (5.864%); 0 B/s, 70 objects/s recovering 2023-09-23T20:43:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:40 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:40 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:40 smithi123 ceph-mon[193903]: pgmap v266: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6642/113265 objects degraded (5.864%); 0 B/s, 70 objects/s recovering 2023-09-23T20:43:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6642/113265 objects degraded (5.864%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2724092970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:41.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6642/113265 objects degraded (5.864%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:41.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2724092970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:41.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6642/113265 objects degraded (5.864%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:41.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2724092970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:42.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:42 smithi121 ceph-mon[219709]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 6642/111909 objects degraded (5.935%); 0 B/s, 48 objects/s recovering 2023-09-23T20:43:42.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:42.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:42 smithi121 ceph-mon[211526]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 6642/111909 objects degraded (5.935%); 0 B/s, 48 objects/s recovering 2023-09-23T20:43:42.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:42 smithi123 ceph-mon[193903]: pgmap v267: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 6642/111909 objects degraded (5.935%); 0 B/s, 48 objects/s recovering 2023-09-23T20:43:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T20:43:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2157638776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:44.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[219709]: pgmap v269: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 6337/102891 objects degraded (6.159%); 0 B/s, 59 objects/s recovering 2023-09-23T20:43:44.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:43:44] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-09-23T20:43:44.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T20:43:44.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2157638776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:44.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:44 smithi121 ceph-mon[211526]: pgmap v269: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 6337/102891 objects degraded (6.159%); 0 B/s, 59 objects/s recovering 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: failed to encode map e103 with expected crc 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2157638776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:44 smithi123 ceph-mon[193903]: pgmap v269: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 6337/102891 objects degraded (6.159%); 0 B/s, 59 objects/s recovering 2023-09-23T20:43:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[211526]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[211526]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[211526]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T20:43:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[211526]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[219709]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[219709]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[219709]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T20:43:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:45 smithi121 ceph-mon[219709]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:45 smithi123 ceph-mon[193903]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:45 smithi123 ceph-mon[193903]: failed to encode map e104 with expected crc 2023-09-23T20:43:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:45 smithi123 ceph-mon[193903]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T20:43:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:45 smithi123 ceph-mon[193903]: failed to encode map e104 with expected crc 2023-09-23T20:43:46.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6337/102891 objects degraded (6.159%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:46.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:46 smithi123 ceph-mon[193903]: pgmap v271: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6337/102891 objects degraded (6.159%); 0 B/s, 74 objects/s recovering 2023-09-23T20:43:46.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2133101422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:47.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6337/102891 objects degraded (6.159%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:47.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:46 smithi121 ceph-mon[211526]: pgmap v271: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6337/102891 objects degraded (6.159%); 0 B/s, 74 objects/s recovering 2023-09-23T20:43:47.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2133101422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:47.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6337/102891 objects degraded (6.159%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:47.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:46 smithi121 ceph-mon[219709]: pgmap v271: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6337/102891 objects degraded (6.159%); 0 B/s, 74 objects/s recovering 2023-09-23T20:43:47.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2133101422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:49 smithi121 ceph-mon[211526]: pgmap v272: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6337/99606 objects degraded (6.362%); 0 B/s, 38 objects/s recovering 2023-09-23T20:43:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/493358399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:49.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:49 smithi121 ceph-mon[219709]: pgmap v272: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6337/99606 objects degraded (6.362%); 0 B/s, 38 objects/s recovering 2023-09-23T20:43:49.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/493358399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:49.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:49 smithi123 ceph-mon[193903]: pgmap v272: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6337/99606 objects degraded (6.362%); 0 B/s, 38 objects/s recovering 2023-09-23T20:43:49.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/493358399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:50.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:50 smithi121 ceph-mon[211526]: pgmap v273: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6014/93912 objects degraded (6.404%); 0 B/s, 78 objects/s recovering 2023-09-23T20:43:50.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:50 smithi121 ceph-mon[219709]: pgmap v273: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6014/93912 objects degraded (6.404%); 0 B/s, 78 objects/s recovering 2023-09-23T20:43:50.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:50 smithi123 ceph-mon[193903]: pgmap v273: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6014/93912 objects degraded (6.404%); 0 B/s, 78 objects/s recovering 2023-09-23T20:43:51.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6014/93912 objects degraded (6.404%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:51.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6014/93912 objects degraded (6.404%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:51.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6014/93912 objects degraded (6.404%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:52.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1427492273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:52 smithi121 ceph-mon[211526]: pgmap v274: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 6014/92562 objects degraded (6.497%); 0 B/s, 36 objects/s recovering 2023-09-23T20:43:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1427492273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:52 smithi121 ceph-mon[219709]: pgmap v274: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 6014/92562 objects degraded (6.497%); 0 B/s, 36 objects/s recovering 2023-09-23T20:43:52.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1427492273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:52 smithi123 ceph-mon[193903]: pgmap v274: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 6014/92562 objects degraded (6.497%); 0 B/s, 36 objects/s recovering 2023-09-23T20:43:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3503093285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:54.779 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:54 smithi121 conmon[198376]: 2023-09-23T20:43:54.554+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:54.780 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:43:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:43:54] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-09-23T20:43:54.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3503093285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:54.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3503093285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:55 smithi123 ceph-mon[193903]: pgmap v275: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 5776/83682 objects degraded (6.902%); 0 B/s, 56 objects/s recovering 2023-09-23T20:43:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:55 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:55 smithi121 ceph-mon[219709]: pgmap v275: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 5776/83682 objects degraded (6.902%); 0 B/s, 56 objects/s recovering 2023-09-23T20:43:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:55 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:55.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:55 smithi121 ceph-mon[211526]: pgmap v275: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 5776/83682 objects degraded (6.902%); 0 B/s, 56 objects/s recovering 2023-09-23T20:43:55.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:55 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:56 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:56 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:56.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5776/83682 objects degraded (6.902%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:56.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:56 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:56.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:56 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:56.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5776/83682 objects degraded (6.902%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:56 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:43:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:56 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:43:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5776/83682 objects degraded (6.902%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:43:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:57 smithi121 ceph-mon[219709]: pgmap v276: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5776/83682 objects degraded (6.902%); 0 B/s, 47 objects/s recovering 2023-09-23T20:43:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1804195578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:57.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:57 smithi121 ceph-mon[211526]: pgmap v276: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5776/83682 objects degraded (6.902%); 0 B/s, 47 objects/s recovering 2023-09-23T20:43:57.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1804195578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:57.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:57 smithi123 ceph-mon[193903]: pgmap v276: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 5776/83682 objects degraded (6.902%); 0 B/s, 47 objects/s recovering 2023-09-23T20:43:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1804195578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:43:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:59.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:43:59 smithi121 ceph-mon[219709]: pgmap v277: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 5776/80574 objects degraded (7.169%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:59.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:43:59 smithi121 ceph-mon[211526]: pgmap v277: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 5776/80574 objects degraded (7.169%); 0 B/s, 46 objects/s recovering 2023-09-23T20:43:59.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:43:59 smithi123 ceph-mon[193903]: pgmap v277: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 5776/80574 objects degraded (7.169%); 0 B/s, 46 objects/s recovering 2023-09-23T20:44:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1670331165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1670331165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1670331165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:01 smithi121 ceph-mon[219709]: pgmap v278: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 5540/73545 objects degraded (7.533%); 0 B/s, 66 objects/s recovering 2023-09-23T20:44:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5540/73545 objects degraded (7.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:01 smithi121 ceph-mon[211526]: pgmap v278: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 5540/73545 objects degraded (7.533%); 0 B/s, 66 objects/s recovering 2023-09-23T20:44:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5540/73545 objects degraded (7.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:01 smithi123 ceph-mon[193903]: pgmap v278: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 5540/73545 objects degraded (7.533%); 0 B/s, 66 objects/s recovering 2023-09-23T20:44:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5540/73545 objects degraded (7.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4251340168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4251340168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4251340168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:03.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:03 smithi121 ceph-mon[211526]: pgmap v279: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5540/72258 objects degraded (7.667%); 0 B/s, 39 objects/s recovering 2023-09-23T20:44:03.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:03 smithi121 ceph-mon[219709]: pgmap v279: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5540/72258 objects degraded (7.667%); 0 B/s, 39 objects/s recovering 2023-09-23T20:44:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:03 smithi123 ceph-mon[193903]: pgmap v279: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 5540/72258 objects degraded (7.667%); 0 B/s, 39 objects/s recovering 2023-09-23T20:44:04.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:04 smithi121 ceph-mon[219709]: pgmap v280: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 5279/63375 objects degraded (8.330%); 0 B/s, 61 objects/s recovering 2023-09-23T20:44:04.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:04 smithi121 ceph-mon[211526]: pgmap v280: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 5279/63375 objects degraded (8.330%); 0 B/s, 61 objects/s recovering 2023-09-23T20:44:04.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:44:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:44:04] "GET /metrics HTTP/1.1" 200 36347 "" "Prometheus/2.43.0" 2023-09-23T20:44:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:04 smithi123 ceph-mon[193903]: pgmap v280: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 5279/63375 objects degraded (8.330%); 0 B/s, 61 objects/s recovering 2023-09-23T20:44:05.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1520126133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:05.923 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1520126133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1520126133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5279/63375 objects degraded (8.330%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:06 smithi121 ceph-mon[211526]: pgmap v281: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5279/63375 objects degraded (8.330%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5279/63375 objects degraded (8.330%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:06 smithi121 ceph-mon[219709]: pgmap v281: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5279/63375 objects degraded (8.330%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5279/63375 objects degraded (8.330%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:06 smithi123 ceph-mon[193903]: pgmap v281: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5279/63375 objects degraded (8.330%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:08.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3252291668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:08.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3252291668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:08.420 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3252291668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:09.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:08 smithi123 ceph-mon[193903]: pgmap v282: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5279/59517 objects degraded (8.870%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:08 smithi121 ceph-mon[211526]: pgmap v282: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5279/59517 objects degraded (8.870%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:08 smithi121 ceph-mon[219709]: pgmap v282: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5279/59517 objects degraded (8.870%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/138257268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:10.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:10.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/138257268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:10.421 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:10.421 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/138257268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:10.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:44:10 smithi123 conmon[172845]: logger=cleanup t=2023-09-23T20:44:10.390392611Z level=info msg="Completed cleanup jobs" duration=141.816527ms 2023-09-23T20:44:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:10 smithi121 ceph-mon[211526]: pgmap v283: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5032/53127 objects degraded (9.472%); 0 B/s, 61 objects/s recovering 2023-09-23T20:44:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:10 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5279/59517 objects degraded (8.870%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:11.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:10 smithi121 ceph-mon[219709]: pgmap v283: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5032/53127 objects degraded (9.472%); 0 B/s, 61 objects/s recovering 2023-09-23T20:44:11.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:10 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5279/59517 objects degraded (8.870%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:11.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:10 smithi123 ceph-mon[193903]: pgmap v283: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5032/53127 objects degraded (9.472%); 0 B/s, 61 objects/s recovering 2023-09-23T20:44:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:10 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5279/59517 objects degraded (8.870%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:13.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/350022195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:13.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:13 smithi123 ceph-mon[193903]: pgmap v284: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5032/51786 objects degraded (9.717%); 0 B/s, 42 objects/s recovering 2023-09-23T20:44:13.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:13.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:13.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/350022195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:13.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[211526]: pgmap v284: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5032/51786 objects degraded (9.717%); 0 B/s, 42 objects/s recovering 2023-09-23T20:44:13.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:13.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:13.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/350022195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:13.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[219709]: pgmap v284: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5032/51786 objects degraded (9.717%); 0 B/s, 42 objects/s recovering 2023-09-23T20:44:13.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:13.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:14.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:14 smithi121 ceph-mon[211526]: pgmap v285: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 908 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4778/43623 objects degraded (10.953%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:14.922 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:44:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:44:14] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-09-23T20:44:14.922 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:14 smithi121 ceph-mon[219709]: pgmap v285: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 908 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4778/43623 objects degraded (10.953%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:14 smithi123 ceph-mon[193903]: pgmap v285: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 908 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4778/43623 objects degraded (10.953%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1936966109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:15 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:15 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1936966109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:15 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:15 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:15.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1936966109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:15 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:15 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:16.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:44:16 smithi121 conmon[198376]: 2023-09-23T20:44:16.413+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:44:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4778/43623 objects degraded (10.953%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[219709]: pgmap v286: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 908 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 4778/43623 objects degraded (10.953%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4778/43623 objects degraded (10.953%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:44:16.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:16 smithi121 ceph-mon[211526]: pgmap v286: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 908 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 4778/43623 objects degraded (10.953%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:16.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4778/43623 objects degraded (10.953%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:16.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:16.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:16.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:16.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:16.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:44:16.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:16.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:16.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:44:16.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:16 smithi123 ceph-mon[193903]: pgmap v286: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 908 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 4778/43623 objects degraded (10.953%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:18.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/962675086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/962675086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/962675086' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:19.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:19 smithi123 ceph-mon[193903]: pgmap v287: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 832 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 4778/39960 objects degraded (11.957%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:19.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:19 smithi121 ceph-mon[211526]: pgmap v287: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 832 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 4778/39960 objects degraded (11.957%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:19 smithi121 ceph-mon[219709]: pgmap v287: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 832 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 4778/39960 objects degraded (11.957%); 0 B/s, 41 objects/s recovering 2023-09-23T20:44:20.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/511150203' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:20.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/511150203' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/511150203' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:21 smithi121 ceph-mon[211526]: pgmap v288: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 702 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 4518/33747 objects degraded (13.388%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4778/39960 objects degraded (11.957%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:21.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:21 smithi121 ceph-mon[219709]: pgmap v288: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 702 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 4518/33747 objects degraded (13.388%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:21.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4778/39960 objects degraded (11.957%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:21.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:21 smithi123 ceph-mon[193903]: pgmap v288: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 702 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 4518/33747 objects degraded (13.388%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:21.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4778/39960 objects degraded (11.957%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:22 smithi121 ceph-mon[211526]: pgmap v289: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 675 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 4518/32463 objects degraded (13.917%); 0 B/s, 42 objects/s recovering 2023-09-23T20:44:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:22 smithi121 ceph-mon[219709]: pgmap v289: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 675 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 4518/32463 objects degraded (13.917%); 0 B/s, 42 objects/s recovering 2023-09-23T20:44:22.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:22 smithi123 ceph-mon[193903]: pgmap v289: 89 pgs: 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 675 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 4518/32463 objects degraded (13.917%); 0 B/s, 42 objects/s recovering 2023-09-23T20:44:23.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/234898715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:23.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/234898715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/234898715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:24.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:44:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:44:24] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-09-23T20:44:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T20:44:24.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[211526]: pgmap v291: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4483/23973 objects degraded (18.700%); 0 B/s, 29 objects/s recovering 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:24 smithi121 ceph-mon[219709]: pgmap v291: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4483/23973 objects degraded (18.700%); 0 B/s, 29 objects/s recovering 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: failed to encode map e105 with expected crc 2023-09-23T20:44:24.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:24 smithi123 ceph-mon[193903]: pgmap v291: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4483/23973 objects degraded (18.700%); 0 B/s, 29 objects/s recovering 2023-09-23T20:44:25.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T20:44:25.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3650182663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3650182663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: failed to encode map e106 with expected crc 2023-09-23T20:44:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3650182663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:26.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4483/23973 objects degraded (18.700%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:26.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:26 smithi121 ceph-mon[211526]: pgmap v293: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 4483/23973 objects degraded (18.700%); 0 B/s, 36 objects/s recovering 2023-09-23T20:44:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4483/23973 objects degraded (18.700%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:26.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:26 smithi121 ceph-mon[219709]: pgmap v293: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 4483/23973 objects degraded (18.700%); 0 B/s, 36 objects/s recovering 2023-09-23T20:44:26.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4483/23973 objects degraded (18.700%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:26.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:26 smithi123 ceph-mon[193903]: pgmap v293: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 498 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 4483/23973 objects degraded (18.700%); 0 B/s, 36 objects/s recovering 2023-09-23T20:44:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/844175631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:28.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:28.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/844175631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:28.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:28.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/844175631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:28 smithi123 ceph-mon[193903]: pgmap v294: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 410 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 4217/19722 objects degraded (21.382%); 0 B/s, 37 objects/s recovering 2023-09-23T20:44:29.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:28 smithi121 ceph-mon[211526]: pgmap v294: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 410 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 4217/19722 objects degraded (21.382%); 0 B/s, 37 objects/s recovering 2023-09-23T20:44:29.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:28 smithi121 ceph-mon[219709]: pgmap v294: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 410 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 4217/19722 objects degraded (21.382%); 0 B/s, 37 objects/s recovering 2023-09-23T20:44:30.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3961839366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:30.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3961839366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3961839366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:31 smithi121 ceph-mon[211526]: pgmap v295: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 313 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 3911/15087 objects degraded (25.923%); 0 B/s, 75 objects/s recovering 2023-09-23T20:44:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4217/19722 objects degraded (21.382%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:31 smithi121 ceph-mon[219709]: pgmap v295: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 313 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 3911/15087 objects degraded (25.923%); 0 B/s, 75 objects/s recovering 2023-09-23T20:44:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4217/19722 objects degraded (21.382%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:31 smithi123 ceph-mon[193903]: pgmap v295: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 313 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 3911/15087 objects degraded (25.923%); 0 B/s, 75 objects/s recovering 2023-09-23T20:44:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4217/19722 objects degraded (21.382%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:32.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:32 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:32.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:32 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:32.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:32 smithi121 ceph-mon[211526]: Upgrade: osd.1 is safe to restart 2023-09-23T20:44:32.645 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:32 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:32.645 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:32 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:32.645 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:32 smithi121 ceph-mon[219709]: Upgrade: osd.1 is safe to restart 2023-09-23T20:44:32.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:32 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:32.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:32 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:44:32.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:32 smithi123 ceph-mon[193903]: Upgrade: osd.1 is safe to restart 2023-09-23T20:44:33.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:33 smithi121 ceph-mon[219709]: pgmap v296: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 3911/13215 objects degraded (29.595%); 0 B/s, 68 objects/s recovering 2023-09-23T20:44:33.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/332295214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:33.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:33 smithi121 ceph-mon[211526]: pgmap v296: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 3911/13215 objects degraded (29.595%); 0 B/s, 68 objects/s recovering 2023-09-23T20:44:33.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/332295214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:33 smithi123 ceph-mon[193903]: pgmap v296: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 274 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 3911/13215 objects degraded (29.595%); 0 B/s, 68 objects/s recovering 2023-09-23T20:44:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/332295214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:34.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:34 smithi121 ceph-mon[219709]: pgmap v297: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3731/5283 objects degraded (70.623%); 0 B/s, 75 objects/s recovering 2023-09-23T20:44:34.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:34 smithi121 ceph-mon[211526]: pgmap v297: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3731/5283 objects degraded (70.623%); 0 B/s, 75 objects/s recovering 2023-09-23T20:44:34.922 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:44:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:44:34] "GET /metrics HTTP/1.1" 200 36274 "" "Prometheus/2.43.0" 2023-09-23T20:44:34.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:34 smithi123 ceph-mon[193903]: pgmap v297: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3731/5283 objects degraded (70.623%); 0 B/s, 75 objects/s recovering 2023-09-23T20:44:35.969 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[211526]: Upgrade: Updating osd.1 2023-09-23T20:44:35.969 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[211526]: Deploying daemon osd.1 on smithi121 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3521856036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[219709]: Upgrade: Updating osd.1 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[219709]: Deploying daemon osd.1 on smithi121 2023-09-23T20:44:35.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3521856036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:35 smithi123 ceph-mon[193903]: Upgrade: Updating osd.1 2023-09-23T20:44:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:44:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:35 smithi123 ceph-mon[193903]: Deploying daemon osd.1 on smithi121 2023-09-23T20:44:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3521856036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:36.350 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 110304 objects 2023-09-23T20:44:36.350 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :164.17 2023-09-23T20:44:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3731/5283 objects degraded (70.623%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:36.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:36 smithi123 ceph-mon[193903]: pgmap v298: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 3731/5283 objects degraded (70.623%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3731/5283 objects degraded (70.623%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:36 smithi121 ceph-mon[211526]: pgmap v298: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 3731/5283 objects degraded (70.623%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3731/5283 objects degraded (70.623%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:36 smithi121 ceph-mon[219709]: pgmap v298: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 109 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 3731/5283 objects degraded (70.623%); 0 B/s, 63 objects/s recovering 2023-09-23T20:44:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/754175798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:38.319 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T20:44:38.320 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T20:44:38.361 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/754175798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:38.361 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/754175798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:39.206 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:38 smithi121 ceph-mon[219709]: pgmap v299: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 52 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3731/2550 objects degraded (146.314%); 0 B/s, 62 objects/s recovering 2023-09-23T20:44:39.206 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:38 smithi121 ceph-mon[211526]: pgmap v299: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 52 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3731/2550 objects degraded (146.314%); 0 B/s, 62 objects/s recovering 2023-09-23T20:44:39.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:38 smithi123 ceph-mon[193903]: pgmap v299: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 52 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3731/2550 objects degraded (146.314%); 0 B/s, 62 objects/s recovering 2023-09-23T20:44:39.288 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:39.825 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:39 smithi121 systemd[1]: Stopping Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:40.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1300075451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:40.664 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:40 smithi121 conmon[133456]: 2023-09-23T20:44:40.314+0000 7f190227e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:44:40.664 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:40 smithi121 conmon[133456]: 2023-09-23T20:44:40.314+0000 7f190227e700 -1 osd.1 106 *** Got signal Terminated *** 2023-09-23T20:44:40.664 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:40 smithi121 conmon[133456]: 2023-09-23T20:44:40.314+0000 7f190227e700 -1 osd.1 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:44:40.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1300075451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1300075451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:41.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:41 smithi121 ceph-mon[211526]: osd.1 marked itself down and dead 2023-09-23T20:44:41.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:41 smithi121 ceph-mon[211526]: pgmap v300: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 3126/72 objects degraded (4341.667%); 0 B/s, 90 objects/s recovering 2023-09-23T20:44:41.396 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3731/2550 objects degraded (146.314%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:41.397 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:41 smithi121 ceph-mon[219709]: osd.1 marked itself down and dead 2023-09-23T20:44:41.397 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:41 smithi121 ceph-mon[219709]: pgmap v300: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 3126/72 objects degraded (4341.667%); 0 B/s, 90 objects/s recovering 2023-09-23T20:44:41.397 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3731/2550 objects degraded (146.314%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:41 smithi123 ceph-mon[193903]: osd.1 marked itself down and dead 2023-09-23T20:44:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:41 smithi123 ceph-mon[193903]: pgmap v300: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 3126/72 objects degraded (4341.667%); 0 B/s, 90 objects/s recovering 2023-09-23T20:44:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3731/2550 objects degraded (146.314%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:44:42.289 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_2' removed 2023-09-23T20:44:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[211526]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:44:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[211526]: failed to encode map e107 with expected crc 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[211526]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[211526]: failed to encode map e107 with expected crc 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4090120892' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[219709]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[219709]: failed to encode map e107 with expected crc 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[219709]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[219709]: failed to encode map e107 with expected crc 2023-09-23T20:44:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4090120892' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:44:42.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:42 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:44:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:42 smithi123 ceph-mon[193903]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:44:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:42 smithi123 ceph-mon[193903]: failed to encode map e107 with expected crc 2023-09-23T20:44:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:42 smithi123 ceph-mon[193903]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T20:44:42.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:42 smithi123 ceph-mon[193903]: failed to encode map e107 with expected crc 2023-09-23T20:44:42.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4090120892' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:44:42.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:42 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:44:43.413 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:43 smithi121 bash[249228]: b5379433baefa9da0f78124a1587613fb89772a463325eb94cafa707c9fc03f6 2023-09-23T20:44:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: pgmap v303: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:44:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/145199422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:43.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: pgmap v303: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:44:43.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/145199422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:43.438 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:44:43.438 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:44:43.439 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T20:44:43.439 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: failed to encode map e108 with expected crc 2023-09-23T20:44:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: pgmap v303: 73 pgs: 4 peering, 7 stale+active+clean, 62 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:44:43.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/145199422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:44.105 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:44 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:44:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:44 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3126/72 objects degraded (4341.667%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T20:44:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:44 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:44:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:44 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3126/72 objects degraded (4341.667%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T20:44:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:44 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:44:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:44 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3126/72 objects degraded (4341.667%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T20:44:45.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:44:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:44:44] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.43.0" 2023-09-23T20:44:45.663 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:45 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T20:44:45.664 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:45 smithi121 systemd[1]: Stopped Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:45.664 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:45 smithi121 systemd[1]: Starting Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:44:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:45 smithi121 ceph-mon[211526]: pgmap v304: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:44:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3963945113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:45.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:45 smithi121 ceph-mon[219709]: pgmap v304: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:44:45.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3963945113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:45 smithi123 ceph-mon[193903]: pgmap v304: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:44:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3963945113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:46 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:46.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:46 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:46 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:44:47.164 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:46 smithi121 bash[250463]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T20:44:47.300 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_3' created 2023-09-23T20:44:47.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:47 smithi121 ceph-mon[219709]: pgmap v305: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:44:47.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/296962747' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:44:47.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1967037061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:47.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:47 smithi121 ceph-mon[211526]: pgmap v305: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:44:47.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/296962747' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:44:47.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1967037061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:47 smithi123 ceph-mon[193903]: pgmap v305: 73 pgs: 2 active+undersized+degraded, 4 peering, 26 active+undersized, 41 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-09-23T20:44:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/296962747' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:44:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1967037061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:48.370 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T20:44:48.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[211526]: failed to encode map e109 with expected crc 2023-09-23T20:44:48.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[211526]: failed to encode map e109 with expected crc 2023-09-23T20:44:48.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/296962747' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:44:48.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[211526]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T20:44:48.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[219709]: failed to encode map e109 with expected crc 2023-09-23T20:44:48.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[219709]: failed to encode map e109 with expected crc 2023-09-23T20:44:48.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/296962747' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:44:48.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-mon[219709]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T20:44:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:48 smithi123 ceph-mon[193903]: failed to encode map e109 with expected crc 2023-09-23T20:44:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:48 smithi123 ceph-mon[193903]: failed to encode map e109 with expected crc 2023-09-23T20:44:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/296962747' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:44:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:48 smithi123 ceph-mon[193903]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T20:44:48.898 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:49.123 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:48 smithi121 ceph-osd[251159]: -- 172.21.15.121:0/1165630246 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b709cc04e0 con 0x55b709c92c00 2023-09-23T20:44:49.123 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:48 smithi121 bash[251043]: 015c027d535f9bed98928477a1e628a4be5b5b93dd854c9916a2a5cc9fa5d398 2023-09-23T20:44:49.123 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:48 smithi121 systemd[1]: Started Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:44:49.413 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:49 smithi121 conmon[251098]: 2023-09-23T20:44:49.302+0000 7fe007905640 -1 Falling back to public interface 2023-09-23T20:44:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: osdmap e110: 8 total, 7 up, 8 in 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: pgmap v308: 89 pgs: 10 unknown, 3 active+undersized+degraded, 30 active+undersized, 46 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:49.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: osdmap e110: 8 total, 7 up, 8 in 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: pgmap v308: 89 pgs: 10 unknown, 3 active+undersized+degraded, 30 active+undersized, 46 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:49.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:49 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:49.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: osdmap e110: 8 total, 7 up, 8 in 2023-09-23T20:44:49.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: failed to encode map e110 with expected crc 2023-09-23T20:44:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: pgmap v308: 89 pgs: 10 unknown, 3 active+undersized+degraded, 30 active+undersized, 46 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:49 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:44:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:44:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[211526]: failed to encode map e111 with expected crc 2023-09-23T20:44:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[211526]: failed to encode map e111 with expected crc 2023-09-23T20:44:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[211526]: osdmap e111: 8 total, 7 up, 8 in 2023-09-23T20:44:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1731193018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:50.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:44:50.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[219709]: failed to encode map e111 with expected crc 2023-09-23T20:44:50.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[219709]: failed to encode map e111 with expected crc 2023-09-23T20:44:50.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[219709]: osdmap e111: 8 total, 7 up, 8 in 2023-09-23T20:44:50.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1731193018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:50.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:50 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:44:50.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:50 smithi123 ceph-mon[193903]: failed to encode map e111 with expected crc 2023-09-23T20:44:50.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:50 smithi123 ceph-mon[193903]: failed to encode map e111 with expected crc 2023-09-23T20:44:50.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:50 smithi123 ceph-mon[193903]: osdmap e111: 8 total, 7 up, 8 in 2023-09-23T20:44:50.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1731193018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:51.319 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T20:44:51.605 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[211526]: failed to encode map e111 with expected crc 2023-09-23T20:44:51.654 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[211526]: pgmap v310: 89 pgs: 4 unknown, 3 active+undersized+degraded, 35 active+undersized, 47 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:51.654 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/707835500' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:44:51.654 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:44:51.654 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:51.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[219709]: failed to encode map e111 with expected crc 2023-09-23T20:44:51.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[219709]: pgmap v310: 89 pgs: 4 unknown, 3 active+undersized+degraded, 35 active+undersized, 47 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:51.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/707835500' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:44:51.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:44:51.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:51 smithi123 ceph-mon[193903]: failed to encode map e111 with expected crc 2023-09-23T20:44:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:51 smithi123 ceph-mon[193903]: pgmap v310: 89 pgs: 4 unknown, 3 active+undersized+degraded, 35 active+undersized, 47 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/707835500' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:44:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:44:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[211526]: failed to encode map e112 with expected crc 2023-09-23T20:44:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[211526]: failed to encode map e112 with expected crc 2023-09-23T20:44:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:44:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[211526]: osdmap e112: 8 total, 7 up, 8 in 2023-09-23T20:44:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/512863993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:52.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[219709]: failed to encode map e112 with expected crc 2023-09-23T20:44:52.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[219709]: failed to encode map e112 with expected crc 2023-09-23T20:44:52.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:44:52.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[219709]: osdmap e112: 8 total, 7 up, 8 in 2023-09-23T20:44:52.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/512863993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:52 smithi123 ceph-mon[193903]: failed to encode map e112 with expected crc 2023-09-23T20:44:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:52 smithi123 ceph-mon[193903]: failed to encode map e112 with expected crc 2023-09-23T20:44:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:52 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:44:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:52 smithi123 ceph-mon[193903]: osdmap e112: 8 total, 7 up, 8 in 2023-09-23T20:44:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/512863993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:53.271 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T20:44:53.273 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:44:53.274 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:44:53.327 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T20:44:53.327 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:44:53.327 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_252781 2023-09-23T20:44:53.327 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:44:53.328 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:44:53.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:53 smithi121 ceph-mon[211526]: pgmap v312: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:53.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:53 smithi121 ceph-mon[219709]: pgmap v312: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:53.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:53 smithi123 ceph-mon[193903]: pgmap v312: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-09-23T20:44:54.328 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 3666 3650 228.111 228.125 0.00486451 0.00437309 2023-09-23T20:44:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:54.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:54.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/192850425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:54.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[211526]: pgmap v313: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 60 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 686/2922 objects degraded (23.477%) 2023-09-23T20:44:54.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:44:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:44:54] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.43.0" 2023-09-23T20:44:54.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:54.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:54.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/192850425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:54.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:54 smithi121 ceph-mon[219709]: pgmap v313: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 60 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 686/2922 objects degraded (23.477%) 2023-09-23T20:44:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:54 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:54 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/192850425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:54 smithi123 ceph-mon[193903]: pgmap v313: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 60 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 686/2922 objects degraded (23.477%) 2023-09-23T20:44:55.328 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 7113 7097 221.718 215.438 0.00384765 0.00449614 2023-09-23T20:44:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:56.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:56.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:44:56.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 10367 10351 215.557 203.375 0.00284387 0.00463104 2023-09-23T20:44:57.163 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:56 smithi121 conmon[251098]: 2023-09-23T20:44:56.961+0000 7fe007905640 -1 osd.1 106 log_to_monitors true 2023-09-23T20:44:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 686/2922 objects degraded (23.477%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:56 smithi121 ceph-mon[211526]: pgmap v314: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 60 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s wr, 118 op/s; 686/2922 objects degraded (23.477%) 2023-09-23T20:44:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 686/2922 objects degraded (23.477%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:56 smithi121 ceph-mon[219709]: pgmap v314: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 60 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s wr, 118 op/s; 686/2922 objects degraded (23.477%) 2023-09-23T20:44:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 686/2922 objects degraded (23.477%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:44:57.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:56 smithi123 ceph-mon[193903]: pgmap v314: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 60 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 MiB/s wr, 118 op/s; 686/2922 objects degraded (23.477%) 2023-09-23T20:44:57.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 13666 13650 213.21 206.188 0.0020168 0.00468497 2023-09-23T20:44:58.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:57 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:44:58.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/305632438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:58.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:58.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 17120 17104 213.739 215.875 0.00166766 0.00467229 2023-09-23T20:44:58.413 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:44:58 smithi121 conmon[251098]: 2023-09-23T20:44:58.018+0000 7fdff8571700 -1 osd.1 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:44:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:57 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:44:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/305632438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:57 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:44:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/305632438' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:44:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:59.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 19300 19284 200.825 136.25 0.00190387 0.00470905 2023-09-23T20:44:59.358 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[219709]: failed to encode map e113 with expected crc 2023-09-23T20:44:59.358 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:44:59.358 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[219709]: osdmap e113: 8 total, 7 up, 8 in 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[219709]: failed to encode map e113 with expected crc 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[219709]: pgmap v316: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 658 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 2526/31644 objects degraded (7.983%) 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[211526]: failed to encode map e113 with expected crc 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[211526]: osdmap e113: 8 total, 7 up, 8 in 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[211526]: failed to encode map e113 with expected crc 2023-09-23T20:44:59.359 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:58 smithi121 ceph-mon[211526]: pgmap v316: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 658 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 2526/31644 objects degraded (7.983%) 2023-09-23T20:44:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:58 smithi123 ceph-mon[193903]: failed to encode map e113 with expected crc 2023-09-23T20:44:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:58 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:44:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:58 smithi123 ceph-mon[193903]: osdmap e113: 8 total, 7 up, 8 in 2023-09-23T20:44:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:58 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:44:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:58 smithi123 ceph-mon[193903]: failed to encode map e113 with expected crc 2023-09-23T20:44:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:58 smithi123 ceph-mon[193903]: pgmap v316: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 658 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 2526/31644 objects degraded (7.983%) 2023-09-23T20:45:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:59 smithi123 ceph-mon[193903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:45:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:59 smithi123 ceph-mon[193903]: failed to encode map e114 with expected crc 2023-09-23T20:45:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:59 smithi123 ceph-mon[193903]: osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709] boot 2023-09-23T20:45:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:59 smithi123 ceph-mon[193903]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T20:45:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:59 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:45:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:44:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1596209485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:00.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 19300 19284 172.139 0 - 0.00470905 2023-09-23T20:45:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[219709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:45:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[219709]: failed to encode map e114 with expected crc 2023-09-23T20:45:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[219709]: osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709] boot 2023-09-23T20:45:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[219709]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T20:45:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:45:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1596209485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[211526]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:45:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[211526]: failed to encode map e114 with expected crc 2023-09-23T20:45:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[211526]: osd.1 [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709] boot 2023-09-23T20:45:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[211526]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T20:45:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:45:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:44:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1596209485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:01.330 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 19341 19325 150.944 1.28125 0.00335034 0.00657777 2023-09-23T20:45:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: pgmap v319: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.0 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 7806/50712 objects degraded (15.393%) 2023-09-23T20:45:01.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2526/31644 objects degraded (7.983%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T20:45:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: pgmap v319: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.0 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 7806/50712 objects degraded (15.393%) 2023-09-23T20:45:01.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:00 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2526/31644 objects degraded (7.983%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.545 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.545 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.545 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T20:45:01.545 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: failed to encode map e115 with expected crc 2023-09-23T20:45:01.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: pgmap v319: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.0 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 7806/50712 objects degraded (15.393%) 2023-09-23T20:45:01.547 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:00 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2526/31644 objects degraded (7.983%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:02.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[211526]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[211526]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[211526]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T20:45:02.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[211526]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[211526]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[211526]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2282249257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:02.264 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:02 smithi121 conmon[198376]: 2023-09-23T20:45:02.259+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:45:02.264 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[219709]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.264 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[219709]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.264 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[219709]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T20:45:02.264 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[219709]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.264 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[219709]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[219709]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2282249257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:02.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 20686 20670 143.513 84.0625 0.0044304 0.00695897 2023-09-23T20:45:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:02 smithi123 ceph-mon[193903]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:02 smithi123 ceph-mon[193903]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:02 smithi123 ceph-mon[193903]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T20:45:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:02 smithi123 ceph-mon[193903]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:02 smithi123 ceph-mon[193903]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:02 smithi123 ceph-mon[193903]: failed to encode map e116 with expected crc 2023-09-23T20:45:02.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2282249257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:03.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 22511 22495 140.567 114.062 0.00544297 0.00710847 2023-09-23T20:45:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[211526]: pgmap v321: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 27 active+undersized, 53 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 MiB/s wr, 3.17k op/s; 8117/56223 objects degraded (14.437%); 912 KiB/s, 14 objects/s recovering 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:45:03.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:03 smithi121 ceph-mon[219709]: pgmap v321: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 27 active+undersized, 53 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 MiB/s wr, 3.17k op/s; 8117/56223 objects degraded (14.437%); 912 KiB/s, 14 objects/s recovering 2023-09-23T20:45:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T20:45:03.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:03 smithi123 ceph-mon[193903]: pgmap v321: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 27 active+undersized, 53 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 MiB/s wr, 3.17k op/s; 8117/56223 objects degraded (14.437%); 912 KiB/s, 14 objects/s recovering 2023-09-23T20:45:04.329 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 24219 24203 137.493 106.75 0.00996433 0.00726694 2023-09-23T20:45:04.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2573119748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:04.776 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:45:04] "GET /metrics HTTP/1.1" 200 36332 "" "Prometheus/2.43.0" 2023-09-23T20:45:04.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2573119748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2573119748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:05.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 25922 25906 134.887 106.438 0.00496431 0.00740782 2023-09-23T20:45:05.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:05 smithi121 ceph-mon[211526]: pgmap v322: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 8158/67299 objects degraded (12.122%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:45:05.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:05 smithi121 ceph-mon[219709]: pgmap v322: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 8158/67299 objects degraded (12.122%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:45:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:05 smithi123 ceph-mon[193903]: pgmap v322: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 8158/67299 objects degraded (12.122%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:45:06.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 27681 27665 132.959 109.938 0.00684574 0.00751468 2023-09-23T20:45:06.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 8158/67299 objects degraded (12.122%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 8158/67299 objects degraded (12.122%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 8158/67299 objects degraded (12.122%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:07.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 29134 29118 129.949 90.8125 0.0062265 0.00768736 2023-09-23T20:45:07.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:07 smithi121 ceph-mon[211526]: pgmap v323: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 8158/67299 objects degraded (12.122%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:45:07.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3918627763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:07.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:07 smithi121 ceph-mon[219709]: pgmap v323: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 8158/67299 objects degraded (12.122%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:45:07.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3918627763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:07 smithi123 ceph-mon[193903]: pgmap v323: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 8158/67299 objects degraded (12.122%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:45:07.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3918627763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:08.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 30816 30800 128.294 105.125 0.0101631 0.00778887 2023-09-23T20:45:09.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 32399 32383 126.459 98.9375 0.0168518 0.00790095 2023-09-23T20:45:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:09 smithi121 ceph-mon[211526]: pgmap v324: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7634/83097 objects degraded (9.187%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:45:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:09 smithi121 ceph-mon[219709]: pgmap v324: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7634/83097 objects degraded (9.187%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:45:09.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:09 smithi123 ceph-mon[193903]: pgmap v324: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 7634/83097 objects degraded (9.187%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:45:10.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 33994 33978 124.884 99.6875 0.00494049 0.0080017 2023-09-23T20:45:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2265602004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:10.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:10 smithi121 ceph-mon[211526]: pgmap v325: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 7130/92298 objects degraded (7.725%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:45:10.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2265602004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:10.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:10 smithi121 ceph-mon[219709]: pgmap v325: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 7130/92298 objects degraded (7.725%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:45:10.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2265602004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:10.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:10 smithi123 ceph-mon[193903]: pgmap v325: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 7130/92298 objects degraded (7.725%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:45:11.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 35651 35635 123.699 103.562 0.00417537 0.00807786 2023-09-23T20:45:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7130/92298 objects degraded (7.725%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:11.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7130/92298 objects degraded (7.725%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7130/92298 objects degraded (7.725%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:12.333 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 37172 37156 122.192 95.0625 0.00956094 0.00817882 2023-09-23T20:45:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4017204562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[219709]: pgmap v326: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 6558/98277 objects degraded (6.673%); 10 MiB/s, 161 objects/s recovering 2023-09-23T20:45:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4017204562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:12 smithi121 ceph-mon[211526]: pgmap v326: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 6558/98277 objects degraded (6.673%); 10 MiB/s, 161 objects/s recovering 2023-09-23T20:45:12.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4017204562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:12 smithi123 ceph-mon[193903]: pgmap v326: 89 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 6558/98277 objects degraded (6.673%); 10 MiB/s, 161 objects/s recovering 2023-09-23T20:45:13.332 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:45:13.333122+0000 min lat: 0.00107183 max lat: 2.2766 avg lat: 0.00817492 2023-09-23T20:45:13.333 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:45:13.333 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 37928 37912 118.445 47.25 0.00199447 0.00817492 2023-09-23T20:45:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[211526]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T20:45:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[211526]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[211526]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[211526]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[211526]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[211526]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[219709]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T20:45:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[219709]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[219709]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[219709]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[219709]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:13 smithi121 ceph-mon[219709]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:13 smithi123 ceph-mon[193903]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T20:45:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:13 smithi123 ceph-mon[193903]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:13 smithi123 ceph-mon[193903]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:13 smithi123 ceph-mon[193903]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:13 smithi123 ceph-mon[193903]: failed to encode map e117 with expected crc 2023-09-23T20:45:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:13 smithi123 ceph-mon[193903]: failed to encode map e117 with expected crc 2023-09-23T20:45:14.333 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 38308 38292 113.936 23.75 0.00263608 0.00852965 2023-09-23T20:45:14.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:45:14] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-09-23T20:45:14.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[211526]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[211526]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[211526]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T20:45:14.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[211526]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[211526]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[211526]: pgmap v329: 89 pgs: 1 creating+peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 6322/112281 objects degraded (5.631%); 14 MiB/s, 229 objects/s recovering 2023-09-23T20:45:14.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[219709]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[219709]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[219709]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T20:45:14.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[219709]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[219709]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:14 smithi121 ceph-mon[219709]: pgmap v329: 89 pgs: 1 creating+peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 6322/112281 objects degraded (5.631%); 14 MiB/s, 229 objects/s recovering 2023-09-23T20:45:14.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:14 smithi123 ceph-mon[193903]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:14 smithi123 ceph-mon[193903]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:14 smithi123 ceph-mon[193903]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T20:45:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:14 smithi123 ceph-mon[193903]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:14 smithi123 ceph-mon[193903]: failed to encode map e118 with expected crc 2023-09-23T20:45:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:14 smithi123 ceph-mon[193903]: pgmap v329: 89 pgs: 1 creating+peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 6322/112281 objects degraded (5.631%); 14 MiB/s, 229 objects/s recovering 2023-09-23T20:45:15.334 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 39460 39444 112.025 72 0.00516146 0.00892126 2023-09-23T20:45:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2107171405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[211526]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[211526]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[211526]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[211526]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[211526]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[211526]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2107171405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[219709]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[219709]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[219709]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T20:45:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[219709]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[219709]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:15 smithi121 ceph-mon[219709]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2107171405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:15 smithi123 ceph-mon[193903]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:15 smithi123 ceph-mon[193903]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:15 smithi123 ceph-mon[193903]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T20:45:15.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:15 smithi123 ceph-mon[193903]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:15 smithi123 ceph-mon[193903]: failed to encode map e119 with expected crc 2023-09-23T20:45:15.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:15 smithi123 ceph-mon[193903]: failed to encode map e119 with expected crc 2023-09-23T20:45:16.335 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 41097 41081 111.598 102.312 0.00727369 0.00895499 2023-09-23T20:45:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6322/112281 objects degraded (5.631%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:16 smithi121 ceph-mon[219709]: pgmap v331: 89 pgs: 1 creating+peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 6322/112281 objects degraded (5.631%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T20:45:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6322/112281 objects degraded (5.631%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:16 smithi121 ceph-mon[211526]: pgmap v331: 89 pgs: 1 creating+peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 6322/112281 objects degraded (5.631%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T20:45:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6322/112281 objects degraded (5.631%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:16 smithi123 ceph-mon[193903]: pgmap v331: 89 pgs: 1 creating+peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 6322/112281 objects degraded (5.631%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T20:45:17.335 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 42640 42624 110.966 96.4375 0.0047541 0.00900629 2023-09-23T20:45:17.664 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:17 smithi121 conmon[198376]: 2023-09-23T20:45:17.276+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:45:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1218247155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:17 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:17 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:45:18.335 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 44246 44230 110.542 100.375 0.00628086 0.00903982 2023-09-23T20:45:18.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1218247155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:18.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:18.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:18.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:45:18.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1218247155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:18.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:18.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:18.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:17 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:45:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:18 smithi123 ceph-mon[193903]: pgmap v332: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 5731/126267 objects degraded (4.539%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T20:45:19.335 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 45799 45783 110.02 97.0625 0.0138523 0.00908504 2023-09-23T20:45:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:18 smithi121 ceph-mon[211526]: pgmap v332: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 5731/126267 objects degraded (4.539%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T20:45:19.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:18 smithi121 ceph-mon[219709]: pgmap v332: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 5731/126267 objects degraded (4.539%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T20:45:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2550537697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:20.336 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 47307 47291 109.436 94.25 0.0238269 0.00913332 2023-09-23T20:45:20.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2550537697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2550537697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:21.336 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 48849 48833 108.969 96.375 0.0136433 0.0091712 2023-09-23T20:45:21.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:20 smithi121 ceph-mon[219709]: pgmap v333: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 5095/132675 objects degraded (3.840%); 12 MiB/s, 184 objects/s recovering 2023-09-23T20:45:21.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:20 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5731/126267 objects degraded (4.539%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:21.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:20 smithi121 ceph-mon[211526]: pgmap v333: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 5095/132675 objects degraded (3.840%); 12 MiB/s, 184 objects/s recovering 2023-09-23T20:45:21.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:20 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5731/126267 objects degraded (4.539%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:20 smithi123 ceph-mon[193903]: pgmap v333: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 5095/132675 objects degraded (3.840%); 12 MiB/s, 184 objects/s recovering 2023-09-23T20:45:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:20 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5731/126267 objects degraded (4.539%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:22.336 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 50026 50010 107.749 73.5625 0.00275887 0.00920673 2023-09-23T20:45:23.336 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 50470 50454 105.082 27.75 0.0116496 0.00951152 2023-09-23T20:45:23.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T20:45:23.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2047826185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[219709]: pgmap v335: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5095/136905 objects degraded (3.722%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2047826185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:23.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:23 smithi121 ceph-mon[211526]: pgmap v335: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5095/136905 objects degraded (3.722%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:45:23.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T20:45:23.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: failed to encode map e120 with expected crc 2023-09-23T20:45:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2047826185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:23 smithi123 ceph-mon[193903]: pgmap v335: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5095/136905 objects degraded (3.722%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:45:24.336 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 51954 51938 104.684 92.75 0.0122006 0.00954705 2023-09-23T20:45:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[211526]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[211526]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[211526]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T20:45:24.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[211526]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[211526]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[211526]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[219709]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[219709]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[219709]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T20:45:24.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[219709]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[219709]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:24 smithi121 ceph-mon[219709]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:24 smithi123 ceph-mon[193903]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:24 smithi123 ceph-mon[193903]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:24 smithi123 ceph-mon[193903]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T20:45:24.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:24 smithi123 ceph-mon[193903]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:24 smithi123 ceph-mon[193903]: failed to encode map e121 with expected crc 2023-09-23T20:45:24.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:24 smithi123 ceph-mon[193903]: failed to encode map e121 with expected crc 2023-09-23T20:45:25.041 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:45:24] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-09-23T20:45:25.041 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:25 smithi121 ceph-mon[211526]: pgmap v337: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 4467/151938 objects degraded (2.940%); 14 MiB/s, 231 objects/s recovering 2023-09-23T20:45:25.041 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1345723126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:25.336 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 53333 53317 104.106 86.1875 0.00933038 0.00960031 2023-09-23T20:45:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:25 smithi121 ceph-mon[219709]: pgmap v337: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 4467/151938 objects degraded (2.940%); 14 MiB/s, 231 objects/s recovering 2023-09-23T20:45:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1345723126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:25 smithi123 ceph-mon[193903]: pgmap v337: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 4467/151938 objects degraded (2.940%); 14 MiB/s, 231 objects/s recovering 2023-09-23T20:45:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1345723126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:26.337 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 54775 54759 103.681 90.125 0.00863321 0.0096407 2023-09-23T20:45:26.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4467/151938 objects degraded (2.940%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:26.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4467/151938 objects degraded (2.940%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:26.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4467/151938 objects degraded (2.940%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:27.337 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 56101 56085 103.069 82.875 0.0153237 0.00969764 2023-09-23T20:45:27.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:27 smithi121 ceph-mon[211526]: pgmap v338: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 4467/151938 objects degraded (2.940%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:45:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:27 smithi121 ceph-mon[219709]: pgmap v338: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 4467/151938 objects degraded (2.940%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:45:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:27 smithi123 ceph-mon[193903]: pgmap v338: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 4467/151938 objects degraded (2.940%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:45:28.337 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 57492 57476 102.608 86.9375 0.0165582 0.00974153 2023-09-23T20:45:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/583888589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:28.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/583888589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:28.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/583888589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:29.337 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 58808 58792 102.043 82.25 0.0181803 0.00979425 2023-09-23T20:45:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:29 smithi121 ceph-mon[211526]: pgmap v339: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 3969/165846 objects degraded (2.393%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:45:29.420 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:29 smithi121 ceph-mon[219709]: pgmap v339: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 3969/165846 objects degraded (2.393%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:45:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:29 smithi123 ceph-mon[193903]: pgmap v339: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 3969/165846 objects degraded (2.393%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:45:30.339 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 60101 60085 101.463 80.8125 0.0101427 0.00985126 2023-09-23T20:45:30.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2903043459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2903043459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2903043459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:31.339 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 61442 61426 100.998 83.8125 0.0059151 0.00989612 2023-09-23T20:45:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:31 smithi121 ceph-mon[211526]: pgmap v340: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 3969/172308 objects degraded (2.303%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:45:31.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3969/165846 objects degraded (2.393%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:31 smithi121 ceph-mon[219709]: pgmap v340: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 3969/172308 objects degraded (2.303%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:45:31.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3969/165846 objects degraded (2.393%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:31 smithi123 ceph-mon[193903]: pgmap v340: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 3969/172308 objects degraded (2.303%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:45:31.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3969/165846 objects degraded (2.393%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:32.339 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 62634 62618 100.319 74.5 0.00397662 0.00993404 2023-09-23T20:45:33.339 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:45:33.340095+0000 min lat: 0.00107183 max lat: 2.2766 avg lat: 0.0101733 2023-09-23T20:45:33.340 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:45:33.340 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 15 62916 62901 98.2532 17.6875 0.00429842 0.0101733 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: Upgrade: osd.2 is safe to restart 2023-09-23T20:45:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/596203322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[211526]: pgmap v342: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 3969/176055 objects degraded (2.254%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: Upgrade: osd.2 is safe to restart 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/596203322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:33 smithi121 ceph-mon[219709]: pgmap v342: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 3969/176055 objects degraded (2.254%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: failed to encode map e122 with expected crc 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: Upgrade: osd.2 is safe to restart 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/596203322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:33 smithi123 ceph-mon[193903]: pgmap v342: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 3969/176055 objects degraded (2.254%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:45:34.341 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 64016 64000 97.5321 68.6875 0.0128492 0.0102478 2023-09-23T20:45:34.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[211526]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[211526]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[211526]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T20:45:34.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[211526]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[211526]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[219709]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[219709]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[219709]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T20:45:34.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[219709]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:34 smithi121 ceph-mon[219709]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:34 smithi123 ceph-mon[193903]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:34 smithi123 ceph-mon[193903]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:34 smithi123 ceph-mon[193903]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T20:45:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:34 smithi123 ceph-mon[193903]: failed to encode map e123 with expected crc 2023-09-23T20:45:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:34 smithi123 ceph-mon[193903]: failed to encode map e123 with expected crc 2023-09-23T20:45:35.117 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:45:34] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T20:45:35.341 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 64832 64816 96.4216 51 0.00966516 0.0103651 2023-09-23T20:45:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:35 smithi121 ceph-mon[211526]: pgmap v344: 89 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 3334/188952 objects degraded (1.764%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T20:45:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2651826461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:35.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:35 smithi121 ceph-mon[219709]: pgmap v344: 89 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 3334/188952 objects degraded (1.764%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T20:45:35.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2651826461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:35 smithi123 ceph-mon[193903]: pgmap v344: 89 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 3334/188952 objects degraded (1.764%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T20:45:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2651826461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:36.343 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 66021 66005 95.9035 74.3125 0.0156176 0.0104223 2023-09-23T20:45:37.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[211526]: Upgrade: Updating osd.2 2023-09-23T20:45:37.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:37.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:45:37.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:37.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[211526]: Deploying daemon osd.2 on smithi121 2023-09-23T20:45:37.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3334/188952 objects degraded (1.764%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:37.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[211526]: pgmap v345: 89 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 3334/188952 objects degraded (1.764%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:45:37.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[219709]: Upgrade: Updating osd.2 2023-09-23T20:45:37.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:37.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:45:37.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:37.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[219709]: Deploying daemon osd.2 on smithi121 2023-09-23T20:45:37.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3334/188952 objects degraded (1.764%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:37.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:36 smithi121 ceph-mon[219709]: pgmap v345: 89 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 3334/188952 objects degraded (1.764%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:45:37.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:36 smithi123 ceph-mon[193903]: Upgrade: Updating osd.2 2023-09-23T20:45:37.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:36 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:37.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:36 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:45:37.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:36 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:37.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:36 smithi123 ceph-mon[193903]: Deploying daemon osd.2 on smithi121 2023-09-23T20:45:37.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3334/188952 objects degraded (1.764%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:37.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:36 smithi123 ceph-mon[193903]: pgmap v345: 89 pgs: 1 creating+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 3334/188952 objects degraded (1.764%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:45:37.343 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 67178 67162 95.3673 72.3125 0.0111044 0.0104805 2023-09-23T20:45:38.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3110222815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:38.343 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 68361 68345 94.8911 73.9375 0.0101628 0.0105326 2023-09-23T20:45:38.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3110222815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:38.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3110222815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:39.343 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 69512 69496 94.3919 71.9375 0.0156657 0.0105886 2023-09-23T20:45:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:38 smithi121 ceph-mon[211526]: pgmap v346: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 3334/200823 objects degraded (1.660%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:45:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:38 smithi121 ceph-mon[219709]: pgmap v346: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 3334/200823 objects degraded (1.660%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:45:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:38 smithi123 ceph-mon[193903]: pgmap v346: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 3334/200823 objects degraded (1.660%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:45:40.139 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3919829815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:40.345 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 70583 70567 93.8041 66.9375 0.0281925 0.0106549 2023-09-23T20:45:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3919829815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:40.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3919829815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:41.346 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 71772 71756 93.3957 74.3125 0.0100764 0.0107016 2023-09-23T20:45:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:41 smithi121 ceph-mon[211526]: pgmap v347: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 2990/204852 objects degraded (1.460%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:45:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3334/200823 objects degraded (1.660%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:41 smithi121 ceph-mon[219709]: pgmap v347: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 2990/204852 objects degraded (1.460%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:45:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3334/200823 objects degraded (1.660%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:41 smithi123 ceph-mon[193903]: pgmap v347: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 2990/204852 objects degraded (1.460%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:45:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3334/200823 objects degraded (1.660%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:41.913 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:41 smithi121 systemd[1]: Stopping Ceph osd.2 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:45:42.346 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 72834 72818 92.8443 66.375 0.00604858 0.0107623 2023-09-23T20:45:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:42 smithi121 ceph-mon[211526]: osd.2 marked itself down and dead 2023-09-23T20:45:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:42 smithi121 ceph-mon[219709]: osd.2 marked itself down and dead 2023-09-23T20:45:42.663 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:42 smithi121 conmon[142308]: 2023-09-23T20:45:42.240+0000 7fe7065f3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:45:42.664 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:42 smithi121 conmon[142308]: 2023-09-23T20:45:42.240+0000 7fe7065f3700 -1 osd.2 123 *** Got signal Terminated *** 2023-09-23T20:45:42.664 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:42 smithi121 conmon[142308]: 2023-09-23T20:45:42.240+0000 7fe7065f3700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:45:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:42 smithi123 ceph-mon[193903]: osd.2 marked itself down and dead 2023-09-23T20:45:43.346 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 72864 72848 91.0254 1.875 0.00615468 0.0108558 2023-09-23T20:45:43.413 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:43 smithi121 bash[258944]: 2c9b742b8c06f0e48e462dba084b90730697befe3d29ae44dc969bc73413b12d 2023-09-23T20:45:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:45:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: osdmap e124: 8 total, 7 up, 8 in 2023-09-23T20:45:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: failed to encode map e124 with expected crc 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: failed to encode map e124 with expected crc 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3467154244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[219709]: pgmap v349: 89 pgs: 3 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2990/207984 objects degraded (1.438%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: osdmap e124: 8 total, 7 up, 8 in 2023-09-23T20:45:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: failed to encode map e124 with expected crc 2023-09-23T20:45:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: failed to encode map e124 with expected crc 2023-09-23T20:45:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3467154244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:43 smithi121 ceph-mon[211526]: pgmap v349: 89 pgs: 3 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2990/207984 objects degraded (1.438%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: osdmap e124: 8 total, 7 up, 8 in 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: failed to encode map e124 with expected crc 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: failed to encode map e124 with expected crc 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3467154244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:43 smithi123 ceph-mon[193903]: pgmap v349: 89 pgs: 3 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2990/207984 objects degraded (1.438%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:45:44.346 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 74174 74158 90.8459 81.875 0.00610606 0.0110031 2023-09-23T20:45:44.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: osdmap e125: 8 total, 7 up, 8 in 2023-09-23T20:45:44.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[211526]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: osdmap e125: 8 total, 7 up, 8 in 2023-09-23T20:45:44.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:44 smithi121 ceph-mon[219709]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: osdmap e125: 8 total, 7 up, 8 in 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: failed to encode map e125 with expected crc 2023-09-23T20:45:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:44 smithi123 ceph-mon[193903]: failed to encode map e125 with expected crc 2023-09-23T20:45:45.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:45:44] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2023-09-23T20:45:45.347 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 75385 75369 90.5544 75.6875 0.0151623 0.0110383 2023-09-23T20:45:45.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:45 smithi121 ceph-mon[211526]: pgmap v351: 89 pgs: 64 active+clean, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 peering; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 16038/219363 objects degraded (7.311%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:45:45.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/987381389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:45.670 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:45 smithi121 ceph-mon[219709]: pgmap v351: 89 pgs: 64 active+clean, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 peering; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 16038/219363 objects degraded (7.311%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:45:45.671 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/987381389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:45 smithi123 ceph-mon[193903]: pgmap v351: 89 pgs: 64 active+clean, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 peering; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 16038/219363 objects degraded (7.311%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:45:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/987381389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:46.347 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 76653 76637 90.3411 79.25 0.00427923 0.0110641 2023-09-23T20:45:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16038/219363 objects degraded (7.311%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16038/219363 objects degraded (7.311%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16038/219363 objects degraded (7.311%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:47.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 77969 77953 90.1896 82.25 0.0128714 0.0110834 2023-09-23T20:45:47.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:47 smithi121 ceph-mon[211526]: pgmap v352: 89 pgs: 64 active+clean, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 peering; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 16038/219363 objects degraded (7.311%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:45:47.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:47 smithi121 ceph-mon[219709]: pgmap v352: 89 pgs: 64 active+clean, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 peering; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 16038/219363 objects degraded (7.311%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:45:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:47 smithi123 ceph-mon[193903]: pgmap v352: 89 pgs: 64 active+clean, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 peering; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 16038/219363 objects degraded (7.311%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:45:48.165 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:47 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T20:45:48.165 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:47 smithi121 systemd[1]: Stopped Ceph osd.2 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:45:48.166 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:48 smithi121 systemd[1]: Starting Ceph osd.2 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:45:48.348 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 79189 79173 89.936 76.25 0.00794202 0.0111149 2023-09-23T20:45:48.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/78300144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:48.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/78300144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/78300144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:49.348 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 80407 80391 89.6894 76.125 0.00774438 0.0111451 2023-09-23T20:45:49.627 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:49 smithi121 ceph-mon[211526]: pgmap v353: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 31641/232512 objects degraded (13.608%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:45:49.627 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:45:49.627 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:45:49.627 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T20:45:49.627 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:45:49.628 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:45:49.628 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T20:45:49.628 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:45:49.628 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:49 smithi121 bash[259630]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T20:45:49.628 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:49 smithi121 ceph-mon[219709]: pgmap v353: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 31641/232512 objects degraded (13.608%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:45:49.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:49 smithi123 ceph-mon[193903]: pgmap v353: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 31641/232512 objects degraded (13.608%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:45:50.348 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 15 81597 81582 89.4217 74.4375 0.013071 0.0111784 2023-09-23T20:45:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/620038938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/620038938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:50.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/620038938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:51.348 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 82526 82510 88.88 58 0.0159522 0.0112462 2023-09-23T20:45:51.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:51 smithi121 ceph-mon[211526]: pgmap v354: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 31667/237327 objects degraded (13.343%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:45:51.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31641/232512 objects degraded (13.608%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:51.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:51 smithi121 ceph-mon[219709]: pgmap v354: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 31667/237327 objects degraded (13.343%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:45:51.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31641/232512 objects degraded (13.608%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:51 smithi123 ceph-mon[193903]: pgmap v354: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 31667/237327 objects degraded (13.343%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:45:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31641/232512 objects degraded (13.608%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:52.353 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 83713 83697 88.6274 74.1875 0.0198931 0.0112781 2023-09-23T20:45:52.663 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:52 smithi121 bash[260066]: 97892133c587b4d3f535ea944318951689195933f0feafb99af69bf2d90370db 2023-09-23T20:45:52.663 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:52 smithi121 systemd[1]: Started Ceph osd.2 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:45:53.164 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:45:52 smithi121 conmon[260103]: 2023-09-23T20:45:52.669+0000 7ffbd7353640 -1 Falling back to public interface 2023-09-23T20:45:53.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:45:53.351225+0000 min lat: 0.00107183 max lat: 2.2766 avg lat: 0.0113279 2023-09-23T20:45:53.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:45:53.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 84758 84742 88.2388 65.3125 0.0182852 0.0113279 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: pgmap v355: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 32016/240474 objects degraded (13.314%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1592238371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: pgmap v355: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 32016/240474 objects degraded (13.314%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1592238371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:53.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: pgmap v355: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 32016/240474 objects degraded (13.314%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:45:53.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1592238371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:53.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:53.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:54.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 15 85881 85866 87.9439 70.25 0.0219618 0.0113658 2023-09-23T20:45:54.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:54 smithi121 ceph-mon[211526]: pgmap v356: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 33476/254133 objects degraded (13.173%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:45:54.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:54 smithi121 ceph-mon[219709]: pgmap v356: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 33476/254133 objects degraded (13.173%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:45:54.918 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:45:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:45:54] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2023-09-23T20:45:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:54 smithi123 ceph-mon[193903]: pgmap v356: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 33476/254133 objects degraded (13.173%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T20:45:55.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 86858 86842 87.5094 61 0.0115592 0.0114221 2023-09-23T20:45:55.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2444956294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2444956294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2444956294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:56.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 87967 87951 87.2205 69.3125 0.00804058 0.0114608 2023-09-23T20:45:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 33476/254133 objects degraded (13.173%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:56 smithi121 ceph-mon[219709]: pgmap v357: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 33476/254133 objects degraded (13.173%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:45:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 33476/254133 objects degraded (13.173%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:56 smithi121 ceph-mon[211526]: pgmap v357: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 33476/254133 objects degraded (13.173%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:45:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 33476/254133 objects degraded (13.173%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:45:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:56 smithi123 ceph-mon[193903]: pgmap v357: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 33476/254133 objects degraded (13.173%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:45:57.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 89016 89000 86.8821 65.5625 0.0090491 0.0115048 2023-09-23T20:45:58.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 89981 89965 86.4733 60.3125 0.0117049 0.0115593 2023-09-23T20:45:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1614291420' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1614291420' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:45:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1614291420' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:45:59.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 91071 91055 86.1953 68.125 0.00667292 0.0115964 2023-09-23T20:45:59.474 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:45:59 smithi121 ceph-mon[211526]: pgmap v358: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 35137/265959 objects degraded (13.211%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:45:59.475 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:45:59 smithi121 ceph-mon[219709]: pgmap v358: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 35137/265959 objects degraded (13.211%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:45:59.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:45:59 smithi123 ceph-mon[193903]: pgmap v358: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 35137/265959 objects degraded (13.211%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:46:00.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 92023 92007 85.7969 59.5 0.00338724 0.0116155 2023-09-23T20:46:01.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 92358 92342 84.8433 20.9375 0.0115725 0.0117818 2023-09-23T20:46:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T20:46:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1629536822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: pgmap v360: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 35041/269856 objects degraded (12.985%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 35137/265959 objects degraded (13.211%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1629536822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: pgmap v360: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 35041/269856 objects degraded (12.985%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:46:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 35137/265959 objects degraded (13.211%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: failed to encode map e126 with expected crc 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1629536822' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: pgmap v360: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 35041/269856 objects degraded (12.985%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:46:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 35137/265959 objects degraded (13.211%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:02.351 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 93268 93252 84.438 56.875 0.015648 0.0118375 2023-09-23T20:46:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[211526]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[211526]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T20:46:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[211526]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[211526]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[211526]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[211526]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[211526]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[219709]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[219709]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[219709]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[219709]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[219709]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[219709]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:02 smithi121 ceph-mon[219709]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:02 smithi123 ceph-mon[193903]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:02 smithi123 ceph-mon[193903]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T20:46:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:02 smithi123 ceph-mon[193903]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:02 smithi123 ceph-mon[193903]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:02 smithi123 ceph-mon[193903]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:02 smithi123 ceph-mon[193903]: failed to encode map e127 with expected crc 2023-09-23T20:46:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:02 smithi123 ceph-mon[193903]: failed to encode map e127 with expected crc 2023-09-23T20:46:03.210 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:46:02 smithi121 conmon[260103]: 2023-09-23T20:46:02.928+0000 7ffbd7353640 -1 osd.2 123 log_to_monitors true 2023-09-23T20:46:03.211 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 20:46:03 smithi121 conmon[260103]: 2023-09-23T20:46:03.078+0000 7ffbc87c0700 -1 osd.2 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:46:03.213 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:03 smithi121 ceph-mon[211526]: pgmap v362: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s; 35307/272241 objects degraded (12.969%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:46:03.213 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:03 smithi121 ceph-mon[211526]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:46:03.213 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2728048279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:03.213 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:03 smithi121 ceph-mon[219709]: pgmap v362: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s; 35307/272241 objects degraded (12.969%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:46:03.214 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:03 smithi121 ceph-mon[219709]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:46:03.214 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2728048279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:03.352 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 94235 94219 84.0951 60.4375 0.0148119 0.0118864 2023-09-23T20:46:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:03 smithi123 ceph-mon[193903]: pgmap v362: 89 pgs: 61 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 754 op/s; 35307/272241 objects degraded (12.969%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:46:03.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:03 smithi123 ceph-mon[193903]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:46:03.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2728048279' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:04.353 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 95065 95049 83.6413 51.875 0.00335799 0.0119037 2023-09-23T20:46:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[219709]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[219709]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:46:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[219709]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T20:46:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[219709]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:46:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[219709]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[219709]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[211526]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[211526]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:46:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[211526]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T20:46:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[211526]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:46:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[211526]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:04 smithi121 ceph-mon[211526]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:04 smithi123 ceph-mon[193903]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:04 smithi123 ceph-mon[193903]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:46:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:04 smithi123 ceph-mon[193903]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T20:46:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:04 smithi123 ceph-mon[193903]: from='osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:46:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:04 smithi123 ceph-mon[193903]: failed to encode map e128 with expected crc 2023-09-23T20:46:04.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:04 smithi123 ceph-mon[193903]: failed to encode map e128 with expected crc 2023-09-23T20:46:05.061 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:46:04] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.43.0" 2023-09-23T20:46:05.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[211526]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:46:05.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[211526]: osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913] boot 2023-09-23T20:46:05.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[211526]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T20:46:05.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[211526]: failed to encode map e129 with expected crc 2023-09-23T20:46:05.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:46:05.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[211526]: pgmap v365: 89 pgs: 62 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 36237/282516 objects degraded (12.827%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:46:05.352 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 95065 95049 82.4799 0 - 0.0119037 2023-09-23T20:46:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[219709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:46:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[219709]: osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913] boot 2023-09-23T20:46:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[219709]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T20:46:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[219709]: failed to encode map e129 with expected crc 2023-09-23T20:46:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:46:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:05 smithi121 ceph-mon[219709]: pgmap v365: 89 pgs: 62 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 36237/282516 objects degraded (12.827%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:46:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:05 smithi123 ceph-mon[193903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:46:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:05 smithi123 ceph-mon[193903]: osd.2 [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913] boot 2023-09-23T20:46:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:05 smithi123 ceph-mon[193903]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T20:46:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:05 smithi123 ceph-mon[193903]: failed to encode map e129 with expected crc 2023-09-23T20:46:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:05 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:46:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:05 smithi123 ceph-mon[193903]: pgmap v365: 89 pgs: 62 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 36237/282516 objects degraded (12.827%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:46:06.081 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.082 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1594470667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:06.083 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 36237/282516 objects degraded (12.827%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:46:06.352 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 95070 95054 81.354 0.15625 2.28511 0.0120234 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1594470667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:06.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 36237/282516 objects degraded (12.827%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:46:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: failed to encode map e130 with expected crc 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1594470667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 36237/282516 objects degraded (12.827%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:46:07.353 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 95999 95983 81.0393 58.0625 0.0262184 0.0123347 2023-09-23T20:46:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[219709]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[219709]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T20:46:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[219709]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[219709]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[219709]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[219709]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[219709]: pgmap v368: 89 pgs: 62 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 36237/282516 objects degraded (12.827%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T20:46:07.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[211526]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[211526]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T20:46:07.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[211526]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[211526]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[211526]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[211526]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:07 smithi121 ceph-mon[211526]: pgmap v368: 89 pgs: 62 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 36237/282516 objects degraded (12.827%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T20:46:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:07 smithi123 ceph-mon[193903]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:07 smithi123 ceph-mon[193903]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T20:46:07.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:07 smithi123 ceph-mon[193903]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:07 smithi123 ceph-mon[193903]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:07 smithi123 ceph-mon[193903]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:07 smithi123 ceph-mon[193903]: failed to encode map e131 with expected crc 2023-09-23T20:46:07.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:07 smithi123 ceph-mon[193903]: pgmap v368: 89 pgs: 62 active+clean, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 36237/282516 objects degraded (12.827%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T20:46:08.355 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 96800 96784 80.6241 50.0625 0.0177356 0.0123978 2023-09-23T20:46:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/220946753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:08.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/220946753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:08.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/220946753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:09.355 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 97731 97715 80.3284 58.1875 0.0142637 0.0124437 2023-09-23T20:46:09.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:09 smithi121 ceph-mon[219709]: pgmap v369: 89 pgs: 75 active+clean, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 13680/288432 objects degraded (4.743%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:46:09.419 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:09 smithi121 ceph-mon[211526]: pgmap v369: 89 pgs: 75 active+clean, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 13680/288432 objects degraded (4.743%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:46:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:09 smithi123 ceph-mon[193903]: pgmap v369: 89 pgs: 75 active+clean, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 13680/288432 objects degraded (4.743%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:46:10.355 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 98562 98546 79.9597 51.9375 0.0046474 0.0124848 2023-09-23T20:46:10.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2398296398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:10.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2398296398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:10.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2398296398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:11.355 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 99275 99259 79.506 44.5625 0.0183538 0.0125722 2023-09-23T20:46:11.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[211526]: pgmap v370: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 25 MiB/s wr, 396 op/s; 8609/290256 objects degraded (2.966%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13680/288432 objects degraded (4.743%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[219709]: pgmap v370: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 25 MiB/s wr, 396 op/s; 8609/290256 objects degraded (2.966%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:46:11.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13680/288432 objects degraded (4.743%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:46:11.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:11.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:11.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:11 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:11 smithi121 conmon[198376]: 2023-09-23T20:46:11.589+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:46:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:11 smithi123 ceph-mon[193903]: pgmap v370: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 25 MiB/s wr, 396 op/s; 8609/290256 objects degraded (2.966%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:46:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13680/288432 objects degraded (4.743%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:46:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:11 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:11 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:11 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:11 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:11 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:12.356 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 100212 100196 79.2407 58.5625 0.0104703 0.0126152 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:46:12.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:46:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:13.356 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:46:13.356367+0000 min lat: 0.00107183 max lat: 2.29721 avg lat: 0.0126232 2023-09-23T20:46:13.356 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:46:13.356 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 100464 100448 78.4472 15.75 0.00328545 0.0126232 2023-09-23T20:46:13.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[211526]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[211526]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T20:46:13.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[211526]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[211526]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[211526]: pgmap v372: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 28 MiB/s wr, 448 op/s; 8609/292596 objects degraded (2.942%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:46:13.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3676477132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:13.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[219709]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[219709]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T20:46:13.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[219709]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[219709]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[219709]: pgmap v372: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 28 MiB/s wr, 448 op/s; 8609/292596 objects degraded (2.942%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:46:13.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3676477132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:13 smithi123 ceph-mon[193903]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:13 smithi123 ceph-mon[193903]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T20:46:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:13 smithi123 ceph-mon[193903]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:13 smithi123 ceph-mon[193903]: failed to encode map e132 with expected crc 2023-09-23T20:46:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:13 smithi123 ceph-mon[193903]: pgmap v372: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 28 MiB/s wr, 448 op/s; 8609/292596 objects degraded (2.942%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T20:46:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3676477132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:14.359 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 101242 101226 78.0755 48.625 0.0137213 0.012803 2023-09-23T20:46:14.778 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:46:14] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-09-23T20:46:14.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[211526]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[211526]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T20:46:14.779 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[211526]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.779 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[211526]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[219709]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[219709]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T20:46:14.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[219709]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:14 smithi121 ceph-mon[219709]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:14 smithi123 ceph-mon[193903]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:14 smithi123 ceph-mon[193903]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T20:46:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:14 smithi123 ceph-mon[193903]: failed to encode map e133 with expected crc 2023-09-23T20:46:14.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:14 smithi123 ceph-mon[193903]: failed to encode map e133 with expected crc 2023-09-23T20:46:15.360 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 102115 102099 77.7888 54.5625 0.00359421 0.0128477 2023-09-23T20:46:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:15 smithi121 ceph-mon[211526]: pgmap v374: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 8148/301488 objects degraded (2.703%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T20:46:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2024023193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:15.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:15 smithi121 ceph-mon[219709]: pgmap v374: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 8148/301488 objects degraded (2.703%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T20:46:15.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2024023193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:15 smithi123 ceph-mon[193903]: pgmap v374: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 8148/301488 objects degraded (2.703%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T20:46:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2024023193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:16.360 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 102927 102911 77.4632 50.75 0.0131481 0.0129049 2023-09-23T20:46:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 8148/301488 objects degraded (2.703%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 8148/301488 objects degraded (2.703%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 8148/301488 objects degraded (2.703%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:17.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 103874 103858 77.2421 59.1875 0.0100752 0.0129417 2023-09-23T20:46:17.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:17 smithi121 ceph-mon[211526]: pgmap v375: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s; 8148/301488 objects degraded (2.703%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T20:46:17.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:17 smithi121 ceph-mon[219709]: pgmap v375: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s; 8148/301488 objects degraded (2.703%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T20:46:17.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:17 smithi123 ceph-mon[193903]: pgmap v375: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s; 8148/301488 objects degraded (2.703%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T20:46:18.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 104733 104717 76.965 53.6875 0.0277238 0.0129877 2023-09-23T20:46:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1503792517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:18.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1503792517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:18.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1503792517' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:19.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 105665 105649 76.7474 58.25 0.0120967 0.0130251 2023-09-23T20:46:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:19 smithi121 ceph-mon[211526]: pgmap v376: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s; 7737/311439 objects degraded (2.484%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:46:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:19 smithi121 ceph-mon[219709]: pgmap v376: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s; 7737/311439 objects degraded (2.484%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:46:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:19 smithi123 ceph-mon[193903]: pgmap v376: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s; 7737/311439 objects degraded (2.484%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:46:20.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 106540 106524 76.4939 54.6875 0.0212597 0.0130674 2023-09-23T20:46:20.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/43239022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:20.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/43239022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/43239022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:21.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 107436 107420 76.261 56 0.00800235 0.0131084 2023-09-23T20:46:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:21 smithi121 ceph-mon[211526]: pgmap v377: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 7737/314001 objects degraded (2.464%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:46:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7737/314001 objects degraded (2.464%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:21 smithi121 ceph-mon[219709]: pgmap v377: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 7737/314001 objects degraded (2.464%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:46:21.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7737/314001 objects degraded (2.464%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:21 smithi123 ceph-mon[193903]: pgmap v377: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 7737/314001 objects degraded (2.464%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T20:46:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7737/314001 objects degraded (2.464%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:22.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 108339 108323 76.0383 56.4375 0.0185958 0.0131462 2023-09-23T20:46:23.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.007 2023-09-23T20:46:23.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 109172 2023-09-23T20:46:23.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T20:46:23.364 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 75.808 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 43.4136 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 228.125 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 1212 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 694.637 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 3650 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0131874 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.0498102 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 2.29721 2023-09-23T20:46:23.365 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00107183 2023-09-23T20:46:23.371 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:46:23.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:23 smithi121 ceph-mon[211526]: pgmap v378: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 7737/316629 objects degraded (2.444%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T20:46:23.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1340002830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:23.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:23 smithi121 ceph-mon[219709]: pgmap v378: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 7737/316629 objects degraded (2.444%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T20:46:23.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1340002830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:23 smithi123 ceph-mon[193903]: pgmap v378: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 7737/316629 objects degraded (2.444%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T20:46:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1340002830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:25.169 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:46:24] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-09-23T20:46:25.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:25 smithi121 ceph-mon[211526]: pgmap v379: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 92 B/s rd, 59 MiB/s wr, 958 op/s; 7303/326412 objects degraded (2.237%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:46:25.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:25 smithi121 ceph-mon[219709]: pgmap v379: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 92 B/s rd, 59 MiB/s wr, 958 op/s; 7303/326412 objects degraded (2.237%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:46:25.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:25 smithi123 ceph-mon[193903]: pgmap v379: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 92 B/s rd, 59 MiB/s wr, 958 op/s; 7303/326412 objects degraded (2.237%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:46:26.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1805806532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:26.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7303/326412 objects degraded (2.237%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:26.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1805806532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:26.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7303/326412 objects degraded (2.237%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:26.916 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:26 smithi121 conmon[198376]: 2023-09-23T20:46:26.618+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:26.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1805806532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:26.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7303/326412 objects degraded (2.237%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[211526]: pgmap v380: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 716 op/s; 7303/326412 objects degraded (2.237%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:46:27.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[211526]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[219709]: pgmap v380: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 716 op/s; 7303/326412 objects degraded (2.237%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:27.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:27.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[219709]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T20:46:27.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:27 smithi123 ceph-mon[193903]: pgmap v380: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 716 op/s; 7303/326412 objects degraded (2.237%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:46:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:27 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:27 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:27 smithi123 ceph-mon[193903]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T20:46:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[219709]: failed to encode map e134 with expected crc 2023-09-23T20:46:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[219709]: failed to encode map e134 with expected crc 2023-09-23T20:46:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2443134167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[219709]: failed to encode map e135 with expected crc 2023-09-23T20:46:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[219709]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T20:46:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[219709]: failed to encode map e135 with expected crc 2023-09-23T20:46:28.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[219709]: pgmap v383: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 965 op/s; 6980/318015 objects degraded (2.195%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:46:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[211526]: failed to encode map e134 with expected crc 2023-09-23T20:46:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[211526]: failed to encode map e134 with expected crc 2023-09-23T20:46:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2443134167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[211526]: failed to encode map e135 with expected crc 2023-09-23T20:46:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[211526]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T20:46:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[211526]: failed to encode map e135 with expected crc 2023-09-23T20:46:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:28 smithi121 ceph-mon[211526]: pgmap v383: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 965 op/s; 6980/318015 objects degraded (2.195%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:46:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:28 smithi123 ceph-mon[193903]: failed to encode map e134 with expected crc 2023-09-23T20:46:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:28 smithi123 ceph-mon[193903]: failed to encode map e134 with expected crc 2023-09-23T20:46:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2443134167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:28 smithi123 ceph-mon[193903]: failed to encode map e135 with expected crc 2023-09-23T20:46:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:28 smithi123 ceph-mon[193903]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T20:46:28.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:28 smithi123 ceph-mon[193903]: failed to encode map e135 with expected crc 2023-09-23T20:46:28.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:28 smithi123 ceph-mon[193903]: pgmap v383: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 965 op/s; 6980/318015 objects degraded (2.195%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:46:29.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:29 smithi121 ceph-mon[211526]: failed to encode map e135 with expected crc 2023-09-23T20:46:29.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:29 smithi121 ceph-mon[211526]: failed to encode map e135 with expected crc 2023-09-23T20:46:29.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:29 smithi121 ceph-mon[219709]: failed to encode map e135 with expected crc 2023-09-23T20:46:29.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:29 smithi121 ceph-mon[219709]: failed to encode map e135 with expected crc 2023-09-23T20:46:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:29 smithi123 ceph-mon[193903]: failed to encode map e135 with expected crc 2023-09-23T20:46:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:29 smithi123 ceph-mon[193903]: failed to encode map e135 with expected crc 2023-09-23T20:46:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3709645314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:30 smithi121 ceph-mon[211526]: pgmap v384: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 29 MiB/s wr, 948 op/s; 6980/315774 objects degraded (2.210%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:46:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3709645314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:30 smithi121 ceph-mon[219709]: pgmap v384: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 29 MiB/s wr, 948 op/s; 6980/315774 objects degraded (2.210%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:46:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3709645314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:30 smithi123 ceph-mon[193903]: pgmap v384: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 29 MiB/s wr, 948 op/s; 6980/315774 objects degraded (2.210%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:46:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6980/315774 objects degraded (2.210%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:31.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6980/315774 objects degraded (2.210%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6980/315774 objects degraded (2.210%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:32.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:32 smithi123 ceph-mon[193903]: pgmap v385: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 1.9 MiB/s wr, 597 op/s; 6595/313542 objects degraded (2.103%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:46:33.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:32 smithi121 ceph-mon[219709]: pgmap v385: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 1.9 MiB/s wr, 597 op/s; 6595/313542 objects degraded (2.103%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:46:33.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:32 smithi121 ceph-mon[211526]: pgmap v385: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 1.9 MiB/s wr, 597 op/s; 6595/313542 objects degraded (2.103%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:46:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/729150208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:34.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/729150208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:34.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/729150208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:34.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:34 smithi123 ceph-mon[193903]: pgmap v386: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.9 MiB/s wr, 1.04k op/s; 6595/302865 objects degraded (2.178%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:46:35.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:34 smithi121 ceph-mon[219709]: pgmap v386: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.9 MiB/s wr, 1.04k op/s; 6595/302865 objects degraded (2.178%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:46:35.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:34 smithi121 ceph-mon[211526]: pgmap v386: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.9 MiB/s wr, 1.04k op/s; 6595/302865 objects degraded (2.178%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:46:35.170 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:46:34] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.43.0" 2023-09-23T20:46:35.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3324910500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:36.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3324910500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:36.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3324910500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:36.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6595/302865 objects degraded (2.178%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:36.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:36 smithi123 ceph-mon[193903]: pgmap v387: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.6 MiB/s wr, 873 op/s; 6595/302865 objects degraded (2.178%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:46:37.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6595/302865 objects degraded (2.178%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:37.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:36 smithi121 ceph-mon[211526]: pgmap v387: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.6 MiB/s wr, 873 op/s; 6595/302865 objects degraded (2.178%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:46:37.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6595/302865 objects degraded (2.178%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:37.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:36 smithi121 ceph-mon[219709]: pgmap v387: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.6 MiB/s wr, 873 op/s; 6595/302865 objects degraded (2.178%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:46:38.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2423263170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:38.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2423263170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2423263170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:39.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:39 smithi121 ceph-mon[211526]: pgmap v388: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 424 KiB/s wr, 1.04k op/s; 6288/292473 objects degraded (2.150%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:46:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:39 smithi121 ceph-mon[219709]: pgmap v388: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 424 KiB/s wr, 1.04k op/s; 6288/292473 objects degraded (2.150%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:46:39.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:39 smithi123 ceph-mon[193903]: pgmap v388: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 424 KiB/s wr, 1.04k op/s; 6288/292473 objects degraded (2.150%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:46:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:40 smithi123 ceph-mon[193903]: pgmap v389: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 6288/290076 objects degraded (2.168%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:46:41.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:40 smithi121 ceph-mon[211526]: pgmap v389: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 6288/290076 objects degraded (2.168%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:46:41.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:40 smithi121 ceph-mon[219709]: pgmap v389: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 6288/290076 objects degraded (2.168%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:46:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3460242133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:41.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6288/290076 objects degraded (2.168%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:41.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3460242133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6288/290076 objects degraded (2.168%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:41.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:41 smithi121 conmon[198376]: 2023-09-23T20:46:41.645+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:41.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3460242133' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:41.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6288/290076 objects degraded (2.168%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:41.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:42.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:42 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:42 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:42 smithi123 ceph-mon[193903]: pgmap v390: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 5959/287736 objects degraded (2.071%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:46:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:43.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[211526]: pgmap v390: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 5959/287736 objects degraded (2.071%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:46:43.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:43.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:43.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:43.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:42 smithi121 ceph-mon[219709]: pgmap v390: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 5959/287736 objects degraded (2.071%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:46:43.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1850934599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:44.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1850934599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:44.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1850934599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:45.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:46:44] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-23T20:46:45.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:44 smithi121 ceph-mon[211526]: pgmap v391: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 999 op/s; 5959/277530 objects degraded (2.147%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:46:45.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:44 smithi121 ceph-mon[219709]: pgmap v391: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 999 op/s; 5959/277530 objects degraded (2.147%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:46:45.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:44 smithi123 ceph-mon[193903]: pgmap v391: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 999 op/s; 5959/277530 objects degraded (2.147%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:46:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/313758217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:46.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/313758217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:46.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/313758217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:47.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5959/277530 objects degraded (2.147%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:47.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:46 smithi121 ceph-mon[219709]: pgmap v392: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 5959/277530 objects degraded (2.147%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:46:47.172 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5959/277530 objects degraded (2.147%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:47.172 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:46 smithi121 ceph-mon[211526]: pgmap v392: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 5959/277530 objects degraded (2.147%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:46:47.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5959/277530 objects degraded (2.147%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:47.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:46 smithi123 ceph-mon[193903]: pgmap v392: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 5959/277530 objects degraded (2.147%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:46:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:46:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4018629745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[211526]: pgmap v394: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 5593/267537 objects degraded (2.091%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4018629745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:49 smithi121 ceph-mon[219709]: pgmap v394: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 5593/267537 objects degraded (2.091%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: failed to encode map e136 with expected crc 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4018629745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:49 smithi123 ceph-mon[193903]: pgmap v394: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 5593/267537 objects degraded (2.091%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:46:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[211526]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[211526]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[211526]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:46:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[211526]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[211526]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[219709]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[219709]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[219709]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:46:50.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[219709]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:50 smithi121 ceph-mon[219709]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:50 smithi123 ceph-mon[193903]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:50 smithi123 ceph-mon[193903]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:50 smithi123 ceph-mon[193903]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:46:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:50 smithi123 ceph-mon[193903]: failed to encode map e137 with expected crc 2023-09-23T20:46:50.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:50 smithi123 ceph-mon[193903]: failed to encode map e137 with expected crc 2023-09-23T20:46:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:51 smithi121 ceph-mon[219709]: pgmap v396: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 5593/265584 objects degraded (2.106%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:46:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/544388643' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5593/267537 objects degraded (2.091%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:51 smithi121 ceph-mon[211526]: pgmap v396: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 5593/265584 objects degraded (2.106%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:46:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/544388643' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5593/267537 objects degraded (2.091%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:51 smithi123 ceph-mon[193903]: pgmap v396: 89 pgs: 84 active+clean, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 5593/265584 objects degraded (2.106%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:46:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/544388643' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5593/267537 objects degraded (2.091%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:52.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:52 smithi123 ceph-mon[193903]: pgmap v397: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 5575/263580 objects degraded (2.115%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:46:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:52 smithi121 ceph-mon[211526]: pgmap v397: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 5575/263580 objects degraded (2.115%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:46:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:52 smithi121 ceph-mon[219709]: pgmap v397: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 5575/263580 objects degraded (2.115%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:46:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/364703711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:54.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/364703711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:54.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/364703711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:54 smithi123 ceph-mon[193903]: pgmap v398: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 5141/254871 objects degraded (2.017%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:46:55.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:46:54] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2023-09-23T20:46:55.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:54 smithi121 ceph-mon[211526]: pgmap v398: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 5141/254871 objects degraded (2.017%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:46:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:54 smithi121 ceph-mon[219709]: pgmap v398: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 5141/254871 objects degraded (2.017%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:46:56.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 5141/254871 objects degraded (2.017%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:56.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:56 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/927423569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:56.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 5141/254871 objects degraded (2.017%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:56.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:56 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/927423569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 5141/254871 objects degraded (2.017%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:46:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:56 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/927423569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:57.079 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:46:56 smithi121 conmon[198376]: 2023-09-23T20:46:56.670+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:57.079 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:57 smithi121 ceph-mon[211526]: pgmap v399: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5141/254871 objects degraded (2.017%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:46:57.079 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:57.079 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:57 smithi121 ceph-mon[219709]: pgmap v399: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5141/254871 objects degraded (2.017%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:46:57.080 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:57.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:57 smithi123 ceph-mon[193903]: pgmap v399: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5141/254871 objects degraded (2.017%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:46:57.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:58 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:58 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:58 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:58 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:58 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:46:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:58 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:46:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:59 smithi121 ceph-mon[211526]: pgmap v400: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 4816/245679 objects degraded (1.960%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:46:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:46:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3863580073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:59 smithi121 ceph-mon[219709]: pgmap v400: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 4816/245679 objects degraded (1.960%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:46:59.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:46:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3863580073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:46:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:59 smithi123 ceph-mon[193903]: pgmap v400: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 4816/245679 objects degraded (1.960%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:46:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:46:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3863580073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:00 smithi123 ceph-mon[193903]: pgmap v401: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 4816/242703 objects degraded (1.984%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:47:01.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:00 smithi121 ceph-mon[211526]: pgmap v401: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 4816/242703 objects degraded (1.984%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:47:01.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:00 smithi121 ceph-mon[219709]: pgmap v401: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 4816/242703 objects degraded (1.984%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:47:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4816/242703 objects degraded (1.984%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1893828634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:02.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4816/242703 objects degraded (1.984%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:02.169 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1893828634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:02.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4816/242703 objects degraded (1.984%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:02.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1893828634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:02.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:02 smithi123 ceph-mon[193903]: pgmap v402: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4816/240483 objects degraded (2.003%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:47:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:02 smithi121 ceph-mon[211526]: pgmap v402: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4816/240483 objects degraded (2.003%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:47:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:02 smithi121 ceph-mon[219709]: pgmap v402: 89 pgs: 84 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4816/240483 objects degraded (2.003%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:47:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2818302492' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:04.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2818302492' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2818302492' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:05.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:47:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:47:04] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2023-09-23T20:47:05.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:04 smithi121 ceph-mon[211526]: pgmap v403: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 4453/230709 objects degraded (1.930%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:47:05.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:04 smithi121 ceph-mon[219709]: pgmap v403: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 4453/230709 objects degraded (1.930%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:47:05.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:04 smithi123 ceph-mon[193903]: pgmap v403: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 4453/230709 objects degraded (1.930%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T20:47:06.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4453/230709 objects degraded (1.930%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:06.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4453/230709 objects degraded (1.930%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:06.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4453/230709 objects degraded (1.930%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1238756629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:07 smithi121 ceph-mon[211526]: pgmap v404: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 4453/230709 objects degraded (1.930%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:47:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1238756629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:07 smithi121 ceph-mon[219709]: pgmap v404: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 4453/230709 objects degraded (1.930%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:47:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1238756629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:07 smithi123 ceph-mon[193903]: pgmap v404: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 4453/230709 objects degraded (1.930%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:47:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[211526]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[211526]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[211526]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:47:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[211526]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[219709]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[219709]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[219709]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:47:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:08 smithi121 ceph-mon[219709]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:08 smithi123 ceph-mon[193903]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:08 smithi123 ceph-mon[193903]: failed to encode map e138 with expected crc 2023-09-23T20:47:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:08 smithi123 ceph-mon[193903]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:47:08.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:08 smithi123 ceph-mon[193903]: failed to encode map e138 with expected crc 2023-09-23T20:47:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[211526]: failed to encode map e139 with expected crc 2023-09-23T20:47:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[211526]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:47:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[211526]: failed to encode map e139 with expected crc 2023-09-23T20:47:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[211526]: pgmap v407: 89 pgs: 85 active+clean, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 4220/222864 objects degraded (1.894%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:47:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3870644547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[219709]: failed to encode map e139 with expected crc 2023-09-23T20:47:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[219709]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:47:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[219709]: failed to encode map e139 with expected crc 2023-09-23T20:47:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[219709]: pgmap v407: 89 pgs: 85 active+clean, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 4220/222864 objects degraded (1.894%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:47:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3870644547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:09 smithi123 ceph-mon[193903]: failed to encode map e139 with expected crc 2023-09-23T20:47:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:09 smithi123 ceph-mon[193903]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:47:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:09 smithi123 ceph-mon[193903]: failed to encode map e139 with expected crc 2023-09-23T20:47:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:09 smithi123 ceph-mon[193903]: pgmap v407: 89 pgs: 85 active+clean, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 4220/222864 objects degraded (1.894%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T20:47:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3870644547' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:11 smithi121 ceph-mon[211526]: pgmap v408: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 3953/220416 objects degraded (1.793%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:47:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3953/220416 objects degraded (1.793%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:11.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1328029109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:11.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:11 smithi121 ceph-mon[219709]: pgmap v408: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 3953/220416 objects degraded (1.793%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:47:11.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3953/220416 objects degraded (1.793%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:11.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1328029109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:11 smithi123 ceph-mon[193903]: pgmap v408: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 3953/220416 objects degraded (1.793%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:47:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3953/220416 objects degraded (1.793%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1328029109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:12.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:12 smithi123 ceph-mon[193903]: pgmap v409: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 3953/218583 objects degraded (1.808%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:47:13.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:13.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:13.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:12 smithi121 ceph-mon[211526]: pgmap v409: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 3953/218583 objects degraded (1.808%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:47:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:12 smithi121 ceph-mon[219709]: pgmap v409: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 3953/218583 objects degraded (1.808%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:47:14.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3372941971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:14.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3372941971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3372941971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:15.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:47:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:47:14] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2023-09-23T20:47:15.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:14 smithi121 ceph-mon[211526]: pgmap v410: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 3658/209208 objects degraded (1.748%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T20:47:15.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:14 smithi121 ceph-mon[219709]: pgmap v410: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 3658/209208 objects degraded (1.748%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T20:47:15.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:14 smithi123 ceph-mon[193903]: pgmap v410: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 3658/209208 objects degraded (1.748%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T20:47:16.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3658/209208 objects degraded (1.748%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:16.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3658/209208 objects degraded (1.748%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:16.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3658/209208 objects degraded (1.748%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:17.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:17.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2141399406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:17.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:17 smithi123 ceph-mon[193903]: pgmap v411: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3658/209208 objects degraded (1.748%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:47:17.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:17.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:17.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2141399406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:17.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[219709]: pgmap v411: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3658/209208 objects degraded (1.748%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:47:17.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:17.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:17.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2141399406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:17.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:17 smithi121 ceph-mon[211526]: pgmap v411: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3658/209208 objects degraded (1.748%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:47:19.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:19 smithi121 ceph-mon[219709]: pgmap v412: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 3658/201240 objects degraded (1.818%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:19.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/12608585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:19.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:19 smithi121 ceph-mon[211526]: pgmap v412: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 3658/201240 objects degraded (1.818%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:19.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/12608585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:19 smithi123 ceph-mon[193903]: pgmap v412: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 3658/201240 objects degraded (1.818%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/12608585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:20 smithi121 ceph-mon[211526]: pgmap v413: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 3404/199425 objects degraded (1.707%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:47:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:20 smithi121 ceph-mon[219709]: pgmap v413: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 3404/199425 objects degraded (1.707%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:47:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:20 smithi123 ceph-mon[193903]: pgmap v413: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 3404/199425 objects degraded (1.707%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:47:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3404/199425 objects degraded (1.707%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3629645767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3404/199425 objects degraded (1.707%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3629645767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:22.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3404/199425 objects degraded (1.707%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:22.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:22.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:22.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3629645767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:22.663 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:47:22 smithi121 conmon[198376]: 2023-09-23T20:47:22.260+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:47:22.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:22.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:22.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:22.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:22.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:22.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:22.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:22.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:22.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:22.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:47:22.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:22 smithi123 ceph-mon[193903]: pgmap v414: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 3404/197367 objects degraded (1.725%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:47:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:23.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:23.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:23.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:23.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:23.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:23.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:23.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:47:23.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[211526]: pgmap v414: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 3404/197367 objects degraded (1.725%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:47:23.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:23.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:23.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:23.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:23.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:23.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:23.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:47:23.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:23.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:23.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:47:23.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:22 smithi121 ceph-mon[219709]: pgmap v414: 89 pgs: 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 3404/197367 objects degraded (1.725%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:47:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3410290539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3410290539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:24.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3410290539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:25.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:47:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:47:24] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2023-09-23T20:47:25.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:25 smithi121 ceph-mon[211526]: pgmap v415: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 3061/188322 objects degraded (1.625%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:47:25.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:25 smithi121 ceph-mon[219709]: pgmap v415: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 3061/188322 objects degraded (1.625%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:47:25.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:25 smithi123 ceph-mon[193903]: pgmap v415: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 3061/188322 objects degraded (1.625%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:47:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3061/188322 objects degraded (1.625%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3061/188322 objects degraded (1.625%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:26.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3061/188322 objects degraded (1.625%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:27 smithi121 ceph-mon[211526]: pgmap v416: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 3061/188322 objects degraded (1.625%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:47:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3067121200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:27 smithi121 ceph-mon[219709]: pgmap v416: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 3061/188322 objects degraded (1.625%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:47:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3067121200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:27.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:27 smithi123 ceph-mon[193903]: pgmap v416: 89 pgs: 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 3061/188322 objects degraded (1.625%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:47:27.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3067121200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:29.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[211526]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:47:29.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[211526]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[211526]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[211526]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[211526]: pgmap v418: 89 pgs: 85 active+clean, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 3005/178836 objects degraded (1.680%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:47:29.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1209727158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:29.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[219709]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:47:29.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[219709]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[219709]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.670 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[219709]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.670 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[219709]: pgmap v418: 89 pgs: 85 active+clean, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 3005/178836 objects degraded (1.680%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:47:29.670 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1209727158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:29 smithi123 ceph-mon[193903]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:47:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:29 smithi123 ceph-mon[193903]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:29 smithi123 ceph-mon[193903]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:29 smithi123 ceph-mon[193903]: failed to encode map e140 with expected crc 2023-09-23T20:47:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:29 smithi123 ceph-mon[193903]: pgmap v418: 89 pgs: 85 active+clean, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 3005/178836 objects degraded (1.680%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:47:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1209727158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:30.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[211526]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[211526]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[211526]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:47:30.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[211526]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[211526]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[219709]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[219709]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[219709]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:47:30.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[219709]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:30 smithi121 ceph-mon[219709]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:30 smithi123 ceph-mon[193903]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:30 smithi123 ceph-mon[193903]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:30 smithi123 ceph-mon[193903]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:47:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:30 smithi123 ceph-mon[193903]: failed to encode map e141 with expected crc 2023-09-23T20:47:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:30 smithi123 ceph-mon[193903]: failed to encode map e141 with expected crc 2023-09-23T20:47:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:31 smithi121 ceph-mon[211526]: pgmap v420: 89 pgs: 1 peering, 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 2729/176892 objects degraded (1.543%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:47:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3005/178836 objects degraded (1.680%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:31 smithi121 ceph-mon[219709]: pgmap v420: 89 pgs: 1 peering, 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 2729/176892 objects degraded (1.543%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:47:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3005/178836 objects degraded (1.680%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:31.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:31 smithi123 ceph-mon[193903]: pgmap v420: 89 pgs: 1 peering, 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 2729/176892 objects degraded (1.543%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:47:31.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3005/178836 objects degraded (1.680%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:32.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/875931980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/875931980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:32.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/875931980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:33.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:33 smithi121 ceph-mon[211526]: pgmap v421: 89 pgs: 1 peering, 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 2729/174960 objects degraded (1.560%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:47:33.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:33 smithi121 ceph-mon[219709]: pgmap v421: 89 pgs: 1 peering, 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 2729/174960 objects degraded (1.560%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:47:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:33 smithi123 ceph-mon[193903]: pgmap v421: 89 pgs: 1 peering, 85 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 2729/174960 objects degraded (1.560%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:47:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:34 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3608021631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:34 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3608021631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:34.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:34 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3608021631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:35.169 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:47:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:47:34] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2023-09-23T20:47:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:35 smithi121 ceph-mon[211526]: pgmap v422: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 2338/166602 objects degraded (1.403%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:47:35.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:35 smithi121 ceph-mon[219709]: pgmap v422: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 2338/166602 objects degraded (1.403%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:47:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:35 smithi123 ceph-mon[193903]: pgmap v422: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 2338/166602 objects degraded (1.403%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T20:47:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2338/166602 objects degraded (1.403%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2338/166602 objects degraded (1.403%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:36.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2338/166602 objects degraded (1.403%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:37.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:37 smithi121 ceph-mon[211526]: pgmap v423: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 2338/166602 objects degraded (1.403%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:47:37.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2296211962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:37.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:37 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:37.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:37 smithi121 ceph-mon[219709]: pgmap v423: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 2338/166602 objects degraded (1.403%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:47:37.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2296211962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:37.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:37 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:37.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:37 smithi123 ceph-mon[193903]: pgmap v423: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 2338/166602 objects degraded (1.403%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:47:37.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2296211962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:37.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:37 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:38 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:38 smithi121 ceph-mon[211526]: Upgrade: osd.3 is safe to restart 2023-09-23T20:47:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:38 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:38 smithi121 ceph-mon[219709]: Upgrade: osd.3 is safe to restart 2023-09-23T20:47:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:38 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:47:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:38 smithi123 ceph-mon[193903]: Upgrade: osd.3 is safe to restart 2023-09-23T20:47:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:39 smithi121 ceph-mon[211526]: pgmap v424: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 2016/157278 objects degraded (1.282%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:47:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1245539289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:39.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:39 smithi121 ceph-mon[219709]: pgmap v424: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 2016/157278 objects degraded (1.282%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:47:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1245539289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:39 smithi123 ceph-mon[193903]: pgmap v424: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 2016/157278 objects degraded (1.282%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:47:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1245539289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:41.661 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[211526]: pgmap v425: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 2016/155064 objects degraded (1.300%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:47:41.661 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2016/155064 objects degraded (1.300%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:41.661 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:41.661 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:47:41.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:41.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[219709]: pgmap v425: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 2016/155064 objects degraded (1.300%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:47:41.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2016/155064 objects degraded (1.300%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:41.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:41.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:47:41.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:41 smithi123 ceph-mon[193903]: pgmap v425: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 2016/155064 objects degraded (1.300%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:47:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2016/155064 objects degraded (1.300%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:47:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:42.703 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[219709]: Upgrade: Updating osd.3 2023-09-23T20:47:42.703 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[219709]: Deploying daemon osd.3 on smithi121 2023-09-23T20:47:42.703 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/127482742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:42.703 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:42.704 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[211526]: Upgrade: Updating osd.3 2023-09-23T20:47:42.704 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[211526]: Deploying daemon osd.3 on smithi121 2023-09-23T20:47:42.704 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/127482742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:42.704 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:42.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:42 smithi123 ceph-mon[193903]: Upgrade: Updating osd.3 2023-09-23T20:47:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:42 smithi123 ceph-mon[193903]: Deploying daemon osd.3 on smithi121 2023-09-23T20:47:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/127482742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:43 smithi121 ceph-mon[219709]: pgmap v426: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 2016/153171 objects degraded (1.316%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:47:43.920 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:43 smithi121 ceph-mon[211526]: pgmap v426: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 2016/153171 objects degraded (1.316%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:47:43.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:43 smithi123 ceph-mon[193903]: pgmap v426: 89 pgs: 86 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 2016/153171 objects degraded (1.316%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:47:44.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2640552023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:44.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:44 smithi123 ceph-mon[193903]: pgmap v427: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 1674/145389 objects degraded (1.151%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:47:45.021 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:47:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:47:44] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T20:47:45.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2640552023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:45.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:44 smithi121 ceph-mon[211526]: pgmap v427: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 1674/145389 objects degraded (1.151%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:47:45.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2640552023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:45.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:44 smithi121 ceph-mon[219709]: pgmap v427: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 1674/145389 objects degraded (1.151%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:47:46.270 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1674/145389 objects degraded (1.151%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:46.271 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1674/145389 objects degraded (1.151%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1674/145389 objects degraded (1.151%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:46.663 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:46 smithi121 systemd[1]: Stopping Ceph osd.3 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:47:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:47 smithi121 ceph-mon[219709]: pgmap v428: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1674/145389 objects degraded (1.151%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:47:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1800545674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:47 smithi121 ceph-mon[219709]: osd.3 marked itself down and dead 2023-09-23T20:47:47.413 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:47 smithi121 conmon[151425]: 2023-09-23T20:47:47.000+0000 7fef35033700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:47:47.413 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:47 smithi121 conmon[151425]: 2023-09-23T20:47:47.000+0000 7fef35033700 -1 osd.3 141 *** Got signal Terminated *** 2023-09-23T20:47:47.413 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:47 smithi121 conmon[151425]: 2023-09-23T20:47:47.000+0000 7fef35033700 -1 osd.3 141 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:47:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:47 smithi121 ceph-mon[211526]: pgmap v428: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1674/145389 objects degraded (1.151%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:47:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1800545674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:47 smithi121 ceph-mon[211526]: osd.3 marked itself down and dead 2023-09-23T20:47:47.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:47 smithi123 ceph-mon[193903]: pgmap v428: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1674/145389 objects degraded (1.151%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:47:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1800545674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:47 smithi123 ceph-mon[193903]: osd.3 marked itself down and dead 2023-09-23T20:47:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[211526]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:47:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[211526]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[211526]: osdmap e142: 8 total, 7 up, 8 in 2023-09-23T20:47:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[211526]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[211526]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[211526]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[211526]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[219709]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[219709]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[219709]: osdmap e142: 8 total, 7 up, 8 in 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[219709]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[219709]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[219709]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:48 smithi121 ceph-mon[219709]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.415 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:47 smithi121 bash[268708]: dcc588b3144bcecf101ef38c2017de0f78a2c7bbb860f9433d450a6ae0dff2ca 2023-09-23T20:47:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:48 smithi123 ceph-mon[193903]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:47:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:48 smithi123 ceph-mon[193903]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:48 smithi123 ceph-mon[193903]: osdmap e142: 8 total, 7 up, 8 in 2023-09-23T20:47:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:48 smithi123 ceph-mon[193903]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:48 smithi123 ceph-mon[193903]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:48 smithi123 ceph-mon[193903]: failed to encode map e142 with expected crc 2023-09-23T20:47:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:48 smithi123 ceph-mon[193903]: failed to encode map e142 with expected crc 2023-09-23T20:47:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: osdmap e143: 8 total, 7 up, 8 in 2023-09-23T20:47:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[211526]: pgmap v431: 89 pgs: 14 active+undersized, 8 peering, 6 stale+active+clean, 56 active+clean, 3 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10155/139281 objects degraded (7.291%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:47:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: osdmap e143: 8 total, 7 up, 8 in 2023-09-23T20:47:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:49 smithi121 ceph-mon[219709]: pgmap v431: 89 pgs: 14 active+undersized, 8 peering, 6 stale+active+clean, 56 active+clean, 3 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10155/139281 objects degraded (7.291%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:47:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: osdmap e143: 8 total, 7 up, 8 in 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: failed to encode map e143 with expected crc 2023-09-23T20:47:49.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:49 smithi123 ceph-mon[193903]: pgmap v431: 89 pgs: 14 active+undersized, 8 peering, 6 stale+active+clean, 56 active+clean, 3 active+undersized+degraded, 2 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10155/139281 objects degraded (7.291%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T20:47:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:47:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[211526]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[211526]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T20:47:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[211526]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[211526]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3610047957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:50.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:47:50.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[219709]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[219709]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T20:47:50.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[219709]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[219709]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3610047957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:50.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:50 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:47:50.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:50 smithi123 ceph-mon[193903]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:50 smithi123 ceph-mon[193903]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T20:47:50.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:50 smithi123 ceph-mon[193903]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:50 smithi123 ceph-mon[193903]: failed to encode map e144 with expected crc 2023-09-23T20:47:50.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3610047957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:51 smithi121 ceph-mon[211526]: pgmap v433: 89 pgs: 22 active+undersized, 9 peering, 50 active+clean, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 15603/137373 objects degraded (11.358%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:51.427 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10155/139281 objects degraded (7.291%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:51.427 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:51 smithi121 ceph-mon[219709]: pgmap v433: 89 pgs: 22 active+undersized, 9 peering, 50 active+clean, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 15603/137373 objects degraded (11.358%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:51.428 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10155/139281 objects degraded (7.291%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:51 smithi123 ceph-mon[193903]: pgmap v433: 89 pgs: 22 active+undersized, 9 peering, 50 active+clean, 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 15603/137373 objects degraded (11.358%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10155/139281 objects degraded (7.291%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:47:52.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1813690686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:52.459 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1813690686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:52.459 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:51 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T20:47:52.459 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:51 smithi121 systemd[1]: Stopped Ceph osd.3 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:47:52.459 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:52 smithi121 systemd[1]: Starting Ceph osd.3 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:47:52.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1813690686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:53 smithi121 ceph-mon[211526]: pgmap v434: 89 pgs: 28 active+undersized, 1 peering, 50 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 18355/135735 objects degraded (13.523%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:53.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:53 smithi121 ceph-mon[219709]: pgmap v434: 89 pgs: 28 active+undersized, 1 peering, 50 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 18355/135735 objects degraded (13.523%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:53 smithi123 ceph-mon[193903]: pgmap v434: 89 pgs: 28 active+undersized, 1 peering, 50 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 18355/135735 objects degraded (13.523%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:47:53.899 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:47:53.899 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T20:47:53.900 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T20:47:53.900 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T20:47:53.900 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T20:47:53.900 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T20:47:53.900 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T20:47:53.900 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:53 smithi121 bash[269368]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T20:47:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:54 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T20:47:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:54 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T20:47:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:54 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T20:47:55.159 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:47:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:47:54] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T20:47:55.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2863667400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:55.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:55 smithi121 ceph-mon[219709]: pgmap v435: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 16768/126486 objects degraded (13.257%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:47:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2863667400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:55 smithi121 ceph-mon[211526]: pgmap v435: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 16768/126486 objects degraded (13.257%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:47:55.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2863667400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:55.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:55 smithi123 ceph-mon[193903]: pgmap v435: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 16768/126486 objects degraded (13.257%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:47:56.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16768/126486 objects degraded (13.257%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:47:56.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:56.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:56.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:56.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16768/126486 objects degraded (13.257%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:47:56.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:56.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:56.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:56.254 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:55 smithi121 bash[269783]: 5bca5bb9cc3f5ea5e93c93fe5cf62ed4a7e1fa08a11e1130e83da0f0a2e4732d 2023-09-23T20:47:56.254 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:55 smithi121 systemd[1]: Started Ceph osd.3 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:47:56.254 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:47:56 smithi121 conmon[269821]: 2023-09-23T20:47:56.249+0000 7fb801dba640 -1 Falling back to public interface 2023-09-23T20:47:56.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16768/126486 objects degraded (13.257%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:47:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:56.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:57.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:57 smithi123 ceph-mon[193903]: pgmap v436: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 16768/126486 objects degraded (13.257%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:47:57.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3481667678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:57.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:57 smithi121 ceph-mon[211526]: pgmap v436: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 16768/126486 objects degraded (13.257%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:47:57.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3481667678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:57.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:57 smithi121 ceph-mon[219709]: pgmap v436: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 16768/126486 objects degraded (13.257%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:47:57.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3481667678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:47:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:58.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:58.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:47:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:59.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:47:59 smithi121 ceph-mon[211526]: pgmap v437: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 15580/118857 objects degraded (13.108%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:47:59.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:47:59 smithi121 ceph-mon[219709]: pgmap v437: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 15580/118857 objects degraded (13.108%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:47:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:47:59 smithi123 ceph-mon[193903]: pgmap v437: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 15580/118857 objects degraded (13.108%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:48:00.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3602192553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:00.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3602192553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:00.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3602192553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:01 smithi123 ceph-mon[193903]: pgmap v438: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 14831/114417 objects degraded (12.962%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:48:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15580/118857 objects degraded (13.108%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:48:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[211526]: pgmap v438: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 14831/114417 objects degraded (12.962%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:48:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15580/118857 objects degraded (13.108%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:48:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[219709]: pgmap v438: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 14831/114417 objects degraded (12.962%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:48:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:01.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15580/118857 objects degraded (13.108%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:48:01.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1050062962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:02.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1050062962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:02.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:02.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:02.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1050062962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:02.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:02.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:03.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:03 smithi123 ceph-mon[193903]: pgmap v439: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 14560/112137 objects degraded (12.984%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:48:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:03 smithi123 ceph-mon[193903]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T20:48:03.813 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:03 smithi121 ceph-mon[211526]: pgmap v439: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 14560/112137 objects degraded (12.984%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:48:03.813 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:03 smithi121 ceph-mon[211526]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T20:48:03.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:03 smithi121 ceph-mon[219709]: pgmap v439: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 14560/112137 objects degraded (12.984%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:48:03.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:03 smithi121 ceph-mon[219709]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T20:48:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:04 smithi123 ceph-mon[193903]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:04 smithi123 ceph-mon[193903]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:04 smithi123 ceph-mon[193903]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:04 smithi123 ceph-mon[193903]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:04 smithi123 ceph-mon[193903]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:04 smithi123 ceph-mon[193903]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:04 smithi123 ceph-mon[193903]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[211526]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[211526]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[211526]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[211526]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[211526]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[211526]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[211526]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T20:48:04.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[219709]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[219709]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[219709]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[219709]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[219709]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[219709]: failed to encode map e145 with expected crc 2023-09-23T20:48:04.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:04 smithi121 ceph-mon[219709]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T20:48:05.109 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:48:05 smithi121 conmon[269821]: 2023-09-23T20:48:05.102+0000 7fb801dba640 -1 osd.3 141 log_to_monitors true 2023-09-23T20:48:05.110 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:48:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:48:04] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-09-23T20:48:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: pgmap v442: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+undersized+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 12954/102228 objects degraded (12.672%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:48:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/171601764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: from='osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:48:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:05 smithi123 ceph-mon[193903]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:48:05.913 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 20:48:05 smithi121 conmon[269821]: 2023-09-23T20:48:05.510+0000 7fb7ef21f700 -1 osd.3 141 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:48:05.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: pgmap v442: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+undersized+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 12954/102228 objects degraded (12.672%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:48:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/171601764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: from='osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:48:05.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[211526]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: failed to encode map e146 with expected crc 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: pgmap v442: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+undersized+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 12954/102228 objects degraded (12.672%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/171601764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: from='osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:48:05.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:05 smithi121 ceph-mon[219709]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:48:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[211526]: failed to encode map e147 with expected crc 2023-09-23T20:48:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[211526]: failed to encode map e147 with expected crc 2023-09-23T20:48:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[211526]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:48:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[211526]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T20:48:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[211526]: from='osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:48:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[211526]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:48:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12954/102228 objects degraded (12.672%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:48:06.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[219709]: failed to encode map e147 with expected crc 2023-09-23T20:48:06.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[219709]: failed to encode map e147 with expected crc 2023-09-23T20:48:06.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[219709]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:48:06.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[219709]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T20:48:06.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[219709]: from='osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:48:06.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[219709]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:48:06.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12954/102228 objects degraded (12.672%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:48:06.965 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T20:48:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:06 smithi123 ceph-mon[193903]: failed to encode map e147 with expected crc 2023-09-23T20:48:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:06 smithi123 ceph-mon[193903]: failed to encode map e147 with expected crc 2023-09-23T20:48:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:06 smithi123 ceph-mon[193903]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:48:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:06 smithi123 ceph-mon[193903]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T20:48:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:06 smithi123 ceph-mon[193903]: from='osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:48:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:06 smithi123 ceph-mon[193903]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:48:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12954/102228 objects degraded (12.672%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:48:07.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[219709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:48:07.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[219709]: osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631] boot 2023-09-23T20:48:07.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[219709]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:48:07.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[219709]: failed to encode map e148 with expected crc 2023-09-23T20:48:07.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:48:07.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[219709]: pgmap v445: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+undersized+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12954/102228 objects degraded (12.672%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:48:07.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1068927089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:07.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[211526]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:48:07.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[211526]: osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631] boot 2023-09-23T20:48:07.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[211526]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:48:07.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[211526]: failed to encode map e148 with expected crc 2023-09-23T20:48:07.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:48:07.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[211526]: pgmap v445: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+undersized+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12954/102228 objects degraded (12.672%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:48:07.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1068927089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:07.954 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:48:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:07 smithi123 ceph-mon[193903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:48:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:07 smithi123 ceph-mon[193903]: osd.3 [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631] boot 2023-09-23T20:48:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:07 smithi123 ceph-mon[193903]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:48:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:07 smithi123 ceph-mon[193903]: failed to encode map e148 with expected crc 2023-09-23T20:48:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:07 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:48:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:07 smithi123 ceph-mon[193903]: pgmap v445: 89 pgs: 28 active+undersized, 51 active+clean, 1 active+undersized+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12954/102228 objects degraded (12.672%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:48:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1068927089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:48:08.515 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.515 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.515 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.515 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.515 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.515 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:48:08.516 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.516 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[211526]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:08 smithi121 ceph-mon[219709]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.929 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:48:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: failed to encode map e149 with expected crc 2023-09-23T20:48:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:08 smithi123 ceph-mon[193903]: failed to encode map e149 with expected crc 2023-09-23T20:48:09.413 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:48:09 smithi121 conmon[198376]: 2023-09-23T20:48:09.153+0000 7f479dcb1700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:48:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: pgmap v448: 89 pgs: 3 remapped+peering, 6 peering, 15 active+undersized, 60 active+clean, 5 active+undersized+degraded; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 4268/97764 objects degraded (4.366%) 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:09.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: pgmap v448: 89 pgs: 3 remapped+peering, 6 peering, 15 active+undersized, 60 active+clean, 5 active+undersized+degraded; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 4268/97764 objects degraded (4.366%) 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: failed to encode map e150 with expected crc 2023-09-23T20:48:09.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: pgmap v448: 89 pgs: 3 remapped+peering, 6 peering, 15 active+undersized, 60 active+clean, 5 active+undersized+degraded; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 4268/97764 objects degraded (4.366%) 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:09.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:48:10.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:10 smithi121 ceph-mon[219709]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:48:10.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:10 smithi121 ceph-mon[219709]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:48:10.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:10 smithi121 ceph-mon[211526]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:48:10.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:10 smithi121 ceph-mon[211526]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:48:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:10 smithi123 ceph-mon[193903]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:48:10.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:10 smithi123 ceph-mon[193903]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T20:48:11.571 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:11 smithi121 ceph-mon[211526]: pgmap v449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 remapped+peering, 6 peering, 78 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 1748/96423 objects degraded (1.813%) 2023-09-23T20:48:11.571 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4268/97764 objects degraded (4.366%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:11.571 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:11 smithi121 ceph-mon[219709]: pgmap v449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 remapped+peering, 6 peering, 78 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 1748/96423 objects degraded (1.813%) 2023-09-23T20:48:11.572 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4268/97764 objects degraded (4.366%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:11 smithi123 ceph-mon[193903]: pgmap v449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 remapped+peering, 6 peering, 78 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 1748/96423 objects degraded (1.813%) 2023-09-23T20:48:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4268/97764 objects degraded (4.366%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:12.013 INFO:teuthology.orchestra.run.smithi121.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[211526]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[211526]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:13.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[211526]: pgmap v450: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 2630/94860 objects degraded (2.773%); 0 B/s, 0 objects/s recovering 2023-09-23T20:48:13.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[219709]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:13.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[219709]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:13.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:13.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:13.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:13.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:13 smithi121 ceph-mon[219709]: pgmap v450: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 2630/94860 objects degraded (2.773%); 0 B/s, 0 objects/s recovering 2023-09-23T20:48:13.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:13 smithi123 ceph-mon[193903]: from='client.44891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:13.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:13 smithi123 ceph-mon[193903]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:48:13.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:13.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:13.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:13.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:13 smithi123 ceph-mon[193903]: pgmap v450: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 3 remapped+peering, 83 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 2630/94860 objects degraded (2.773%); 0 B/s, 0 objects/s recovering 2023-09-23T20:48:13.645 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:48:14.401 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:15.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:48:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:48:14] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2023-09-23T20:48:15.961 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:15 smithi121 ceph-mon[211526]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 4340/86199 objects degraded (5.035%); 0 B/s, 38 objects/s recovering 2023-09-23T20:48:15.961 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:15 smithi121 ceph-mon[219709]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 4340/86199 objects degraded (5.035%); 0 B/s, 38 objects/s recovering 2023-09-23T20:48:15.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:15 smithi123 ceph-mon[193903]: pgmap v451: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 4340/86199 objects degraded (5.035%); 0 B/s, 38 objects/s recovering 2023-09-23T20:48:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4340/86199 objects degraded (5.035%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4340/86199 objects degraded (5.035%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:16.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4340/86199 objects degraded (5.035%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:17.768 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:48:17.768 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.a smithi121 *:9093,9094 running (14m) - 18m 29.6M - 0.25.0 c8568f914cd2 a7c6818f0f7f 2023-09-23T20:48:17.768 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.a smithi123 *:3000 running (14m) - 18m 90.8M - 9.4.7 2c41d148cca3 c2e1f7abecff 2023-09-23T20:48:17.768 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi121.sdlxlb smithi121 running (17m) - 17m 20.7M - 17.2.6-1123-g61431f6b 0db14aa01094 c7389c893171 2023-09-23T20:48:17.768 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi123.clbyhs smithi123 running (17m) - 17m 16.9M - 17.2.6-1123-g61431f6b 0db14aa01094 1704af1c5a0c 2023-09-23T20:48:17.768 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.x smithi123 *:8443,9283,8765 running (13m) - 22m 452M - 18.2.0-460-gbb0daeff e32c2a64c126 df813ef15dd6 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.y smithi121 *:8443,9283,8765 running (13m) - 24m 525M - 18.2.0-460-gbb0daeff e32c2a64c126 adf60a30334d 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a smithi121 running (12m) - 24m 116M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 41cbfbe73b6e 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b smithi123 running (9m) - 22m 71.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 5eccbfe5a4b7 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c smithi121 running (11m) - 22m 84.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 828ec0e17fe9 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.a smithi121 *:9100 running (14m) - 18m 21.5M - 1.5.0 0da6a335fe13 ab86679c1d7d 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.b smithi123 *:9100 running (14m) - 17m 18.6M - 1.5.0 0da6a335fe13 5f079d4704ad 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (7m) - 22m 3646M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 2acc40f5254f 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (3m) - 21m 2703M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 015c027d535f 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (2m) - 21m 1166M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 97892133c587 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (22s) - 21m 12.9M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 5bca5bb9cc3f 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi123 running (20m) - 20m 6164M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 470c2a4d010b 2023-09-23T20:48:17.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi123 running (20m) - 20m 7244M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ab39a1466c34 2023-09-23T20:48:17.770 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi123 running (19m) - 19m 5387M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 3f6aba9e3a6b 2023-09-23T20:48:17.770 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi123 running (19m) - 19m 5992M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 514f59904aaa 2023-09-23T20:48:17.770 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.a smithi123 *:9095 running (13m) - 19m 67.7M - 2.43.0 a07b618ecd1d 9b8ac663186b 2023-09-23T20:48:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:17 smithi123 ceph-mon[193903]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 4340/86199 objects degraded (5.035%); 0 B/s, 34 objects/s recovering 2023-09-23T20:48:18.011 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:17 smithi121 ceph-mon[211526]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 4340/86199 objects degraded (5.035%); 0 B/s, 34 objects/s recovering 2023-09-23T20:48:18.012 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:17 smithi121 ceph-mon[219709]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 4340/86199 objects degraded (5.035%); 0 B/s, 34 objects/s recovering 2023-09-23T20:48:18.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:18 smithi123 ceph-mon[193903]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:18 smithi123 ceph-mon[193903]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4340/77544 objects degraded (5.597%); 0 B/s, 66 objects/s recovering 2023-09-23T20:48:18.984 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:48:18.987 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:18 smithi121 ceph-mon[211526]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:18.987 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:18 smithi121 ceph-mon[211526]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4340/77544 objects degraded (5.597%); 0 B/s, 66 objects/s recovering 2023-09-23T20:48:18.987 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:18 smithi121 ceph-mon[219709]: from='client.34962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:18.987 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:18 smithi121 ceph-mon[219709]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4340/77544 objects degraded (5.597%); 0 B/s, 66 objects/s recovering 2023-09-23T20:48:19.792 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[211526]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:48:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[211526]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[211526]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[211526]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[211526]: pgmap v455: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 4340/74553 objects degraded (5.821%); 0 B/s, 83 objects/s recovering 2023-09-23T20:48:21.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4340/77544 objects degraded (5.597%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:21.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[219709]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:48:21.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[219709]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[219709]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[219709]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[219709]: pgmap v455: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 4340/74553 objects degraded (5.821%); 0 B/s, 83 objects/s recovering 2023-09-23T20:48:21.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4340/77544 objects degraded (5.597%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:21 smithi123 ceph-mon[193903]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:48:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:21 smithi123 ceph-mon[193903]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:21 smithi123 ceph-mon[193903]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:21 smithi123 ceph-mon[193903]: failed to encode map e151 with expected crc 2023-09-23T20:48:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:21 smithi123 ceph-mon[193903]: pgmap v455: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 4340/74553 objects degraded (5.821%); 0 B/s, 83 objects/s recovering 2023-09-23T20:48:21.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4340/77544 objects degraded (5.597%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[211526]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[211526]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:48:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[211526]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[211526]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[211526]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[219709]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[219709]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:48:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[219709]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[219709]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:22 smithi121 ceph-mon[219709]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:22 smithi123 ceph-mon[193903]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:22 smithi123 ceph-mon[193903]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:48:22.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:22 smithi123 ceph-mon[193903]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:22 smithi123 ceph-mon[193903]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:22 smithi123 ceph-mon[193903]: failed to encode map e152 with expected crc 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.a smithi121 *:9093,9094 running (14m) - 18m 29.6M - 0.25.0 c8568f914cd2 a7c6818f0f7f 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.a smithi123 *:3000 running (14m) - 18m 90.8M - 9.4.7 2c41d148cca3 c2e1f7abecff 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi121.sdlxlb smithi121 running (17m) - 17m 20.7M - 17.2.6-1123-g61431f6b 0db14aa01094 c7389c893171 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foo.smithi123.clbyhs smithi123 running (17m) - 17m 16.9M - 17.2.6-1123-g61431f6b 0db14aa01094 1704af1c5a0c 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.x smithi123 *:8443,9283,8765 running (13m) - 22m 452M - 18.2.0-460-gbb0daeff e32c2a64c126 df813ef15dd6 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.y smithi121 *:8443,9283,8765 running (14m) - 24m 525M - 18.2.0-460-gbb0daeff e32c2a64c126 adf60a30334d 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a smithi121 running (12m) - 24m 116M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 41cbfbe73b6e 2023-09-23T20:48:22.809 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b smithi123 running (9m) - 23m 71.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 5eccbfe5a4b7 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c smithi121 running (11m) - 22m 84.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 828ec0e17fe9 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.a smithi121 *:9100 running (14m) - 18m 21.5M - 1.5.0 0da6a335fe13 ab86679c1d7d 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.b smithi123 *:9100 running (14m) - 18m 18.6M - 1.5.0 0da6a335fe13 5f079d4704ad 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (7m) - 22m 3646M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 2acc40f5254f 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (3m) - 21m 2703M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 015c027d535f 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (2m) - 21m 1166M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 97892133c587 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (27s) - 21m 12.9M 2500M 18.2.0-460-gbb0daeff e32c2a64c126 5bca5bb9cc3f 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi123 running (20m) - 20m 6164M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 470c2a4d010b 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi123 running (20m) - 20m 7244M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ab39a1466c34 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi123 running (20m) - 20m 5387M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 3f6aba9e3a6b 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi123 running (19m) - 19m 5992M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 514f59904aaa 2023-09-23T20:48:22.810 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.a smithi123 *:9095 running (13m) - 19m 67.7M - 2.43.0 a07b618ecd1d 9b8ac663186b 2023-09-23T20:48:23.487 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:23 smithi123 ceph-mon[193903]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 3458/72762 objects degraded (4.752%); 0 B/s, 101 objects/s recovering 2023-09-23T20:48:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:23 smithi121 ceph-mon[211526]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 3458/72762 objects degraded (4.752%); 0 B/s, 101 objects/s recovering 2023-09-23T20:48:23.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:23 smithi121 ceph-mon[219709]: pgmap v457: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 3458/72762 objects degraded (4.752%); 0 B/s, 101 objects/s recovering 2023-09-23T20:48:24.121 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T20:48:24.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:24 smithi121 ceph-mon[219709]: from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:24.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:24 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:24.426 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:24 smithi121 ceph-mon[211526]: from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:24.427 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:24 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:24.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:24 smithi123 ceph-mon[193903]: from='client.34965 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:48:24.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:24 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:24.860 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:25.167 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:48:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:48:24] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2023-09-23T20:48:25.494 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:25 smithi121 ceph-mon[211526]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 3458/66009 objects degraded (5.239%); 0 B/s, 101 objects/s recovering 2023-09-23T20:48:25.499 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:25 smithi121 ceph-mon[219709]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 3458/66009 objects degraded (5.239%); 0 B/s, 101 objects/s recovering 2023-09-23T20:48:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:25 smithi123 ceph-mon[193903]: pgmap v458: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 3458/66009 objects degraded (5.239%); 0 B/s, 101 objects/s recovering 2023-09-23T20:48:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3458/66009 objects degraded (5.239%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3458/66009 objects degraded (5.239%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:26.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3458/66009 objects degraded (5.239%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:27 smithi121 ceph-mon[211526]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 3458/65409 objects degraded (5.287%); 0 B/s, 56 objects/s recovering 2023-09-23T20:48:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:27 smithi121 ceph-mon[219709]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 3458/65409 objects degraded (5.287%); 0 B/s, 56 objects/s recovering 2023-09-23T20:48:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:27 smithi123 ceph-mon[193903]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 3458/65409 objects degraded (5.287%); 0 B/s, 56 objects/s recovering 2023-09-23T20:48:28.044 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 4, 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:48:28.045 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2023-09-23T20:48:28.046 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 2 2023-09-23T20:48:28.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:48:28.046 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2023-09-23T20:48:28.046 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 6, 2023-09-23T20:48:28.046 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T20:48:28.046 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:48:28.046 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:48:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3407668830' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:28.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3407668830' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3407668830' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:48:29.285 DEBUG:teuthology.parallel:result is None 2023-09-23T20:48:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:29 smithi121 ceph-mon[211526]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 3458/58191 objects degraded (5.942%); 0 B/s, 102 objects/s recovering 2023-09-23T20:48:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:29 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:29.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:29 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:29 smithi121 ceph-mon[219709]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 3458/58191 objects degraded (5.942%); 0 B/s, 102 objects/s recovering 2023-09-23T20:48:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:29 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:29 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:29.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:29 smithi123 ceph-mon[193903]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 3458/58191 objects degraded (5.942%); 0 B/s, 102 objects/s recovering 2023-09-23T20:48:29.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:29 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:29.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:29 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:30.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:30 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:30 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:30 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:30 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:30 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:30 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:30 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:30 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:48:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:48:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 3458/54918 objects degraded (6.297%); 0 B/s, 74 objects/s recovering 2023-09-23T20:48:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3458/58191 objects degraded (5.942%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:31.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:48:31.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 3458/54918 objects degraded (6.297%); 0 B/s, 74 objects/s recovering 2023-09-23T20:48:31.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3458/58191 objects degraded (5.942%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: failed to encode map e153 with expected crc 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: pgmap v462: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 3458/54918 objects degraded (6.297%); 0 B/s, 74 objects/s recovering 2023-09-23T20:48:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3458/58191 objects degraded (5.942%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[211526]: failed to encode map e153 with expected crc 2023-09-23T20:48:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[211526]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[211526]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[211526]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[211526]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[211526]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:48:32.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[219709]: failed to encode map e153 with expected crc 2023-09-23T20:48:32.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[219709]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[219709]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[219709]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[219709]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:32 smithi121 ceph-mon[219709]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:48:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:32 smithi123 ceph-mon[193903]: failed to encode map e153 with expected crc 2023-09-23T20:48:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:32 smithi123 ceph-mon[193903]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:32 smithi123 ceph-mon[193903]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:32 smithi123 ceph-mon[193903]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:32 smithi123 ceph-mon[193903]: failed to encode map e154 with expected crc 2023-09-23T20:48:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:32 smithi123 ceph-mon[193903]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:48:33.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:33 smithi121 ceph-mon[211526]: failed to encode map e154 with expected crc 2023-09-23T20:48:33.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:33 smithi121 ceph-mon[211526]: pgmap v464: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 3458/53232 objects degraded (6.496%); 0 B/s, 74 objects/s recovering 2023-09-23T20:48:33.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:33 smithi121 ceph-mon[219709]: failed to encode map e154 with expected crc 2023-09-23T20:48:33.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:33 smithi121 ceph-mon[219709]: pgmap v464: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 3458/53232 objects degraded (6.496%); 0 B/s, 74 objects/s recovering 2023-09-23T20:48:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:33 smithi123 ceph-mon[193903]: failed to encode map e154 with expected crc 2023-09-23T20:48:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:33 smithi123 ceph-mon[193903]: pgmap v464: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 3458/53232 objects degraded (6.496%); 0 B/s, 74 objects/s recovering 2023-09-23T20:48:35.166 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:48:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:48:34] "GET /metrics HTTP/1.1" 200 36339 "" "Prometheus/2.43.0" 2023-09-23T20:48:35.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:35 smithi123 ceph-mon[193903]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 957 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2614/45966 objects degraded (5.687%); 0 B/s, 128 objects/s recovering 2023-09-23T20:48:36.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:35 smithi121 ceph-mon[219709]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 957 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2614/45966 objects degraded (5.687%); 0 B/s, 128 objects/s recovering 2023-09-23T20:48:36.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:35 smithi121 ceph-mon[211526]: pgmap v465: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 957 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2614/45966 objects degraded (5.687%); 0 B/s, 128 objects/s recovering 2023-09-23T20:48:36.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2614/45966 objects degraded (5.687%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2614/45966 objects degraded (5.687%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2614/45966 objects degraded (5.687%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:37.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:37 smithi123 ceph-mon[193903]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 945 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 2614/45393 objects degraded (5.759%); 0 B/s, 75 objects/s recovering 2023-09-23T20:48:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:37 smithi121 ceph-mon[211526]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 945 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 2614/45393 objects degraded (5.759%); 0 B/s, 75 objects/s recovering 2023-09-23T20:48:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:37 smithi121 ceph-mon[219709]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 945 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 2614/45393 objects degraded (5.759%); 0 B/s, 75 objects/s recovering 2023-09-23T20:48:39.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:39 smithi123 ceph-mon[193903]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 766 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 2614/36816 objects degraded (7.100%); 0 B/s, 73 objects/s recovering 2023-09-23T20:48:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:39 smithi121 ceph-mon[219709]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 766 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 2614/36816 objects degraded (7.100%); 0 B/s, 73 objects/s recovering 2023-09-23T20:48:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:39 smithi121 ceph-mon[211526]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 766 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 2614/36816 objects degraded (7.100%); 0 B/s, 73 objects/s recovering 2023-09-23T20:48:41.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:41 smithi123 ceph-mon[193903]: pgmap v468: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 688 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2614/33090 objects degraded (7.900%); 0 B/s, 93 objects/s recovering 2023-09-23T20:48:41.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:41 smithi123 ceph-mon[193903]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T20:48:41.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:41 smithi123 ceph-mon[193903]: failed to encode map e155 with expected crc 2023-09-23T20:48:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:41 smithi123 ceph-mon[193903]: failed to encode map e155 with expected crc 2023-09-23T20:48:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:41 smithi123 ceph-mon[193903]: failed to encode map e155 with expected crc 2023-09-23T20:48:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:41 smithi123 ceph-mon[193903]: failed to encode map e155 with expected crc 2023-09-23T20:48:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2614/33090 objects degraded (7.900%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:42.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[211526]: pgmap v468: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 688 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2614/33090 objects degraded (7.900%); 0 B/s, 93 objects/s recovering 2023-09-23T20:48:42.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[211526]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T20:48:42.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[211526]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[211526]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[211526]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[211526]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2614/33090 objects degraded (7.900%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[219709]: pgmap v468: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 688 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2614/33090 objects degraded (7.900%); 0 B/s, 93 objects/s recovering 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[219709]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[219709]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[219709]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[219709]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[219709]: failed to encode map e155 with expected crc 2023-09-23T20:48:42.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2614/33090 objects degraded (7.900%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[211526]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[211526]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[211526]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[211526]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[211526]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 651 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2614/31299 objects degraded (8.352%); 0 B/s, 41 objects/s recovering 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[219709]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[219709]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[219709]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T20:48:43.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[219709]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:43.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:42 smithi121 ceph-mon[219709]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 651 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2614/31299 objects degraded (8.352%); 0 B/s, 41 objects/s recovering 2023-09-23T20:48:43.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:42 smithi123 ceph-mon[193903]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:42 smithi123 ceph-mon[193903]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:42 smithi123 ceph-mon[193903]: osdmap e156: 8 total, 8 up, 8 in 2023-09-23T20:48:43.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:42 smithi123 ceph-mon[193903]: failed to encode map e156 with expected crc 2023-09-23T20:48:43.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:43.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:42 smithi123 ceph-mon[193903]: pgmap v471: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 651 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2614/31299 objects degraded (8.352%); 0 B/s, 41 objects/s recovering 2023-09-23T20:48:44.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:43 smithi121 ceph-mon[211526]: failed to encode map e156 with expected crc 2023-09-23T20:48:44.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:43 smithi121 ceph-mon[219709]: failed to encode map e156 with expected crc 2023-09-23T20:48:44.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:43 smithi123 ceph-mon[193903]: failed to encode map e156 with expected crc 2023-09-23T20:48:45.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:44 smithi121 ceph-mon[211526]: pgmap v472: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 493 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 1713/23706 objects degraded (7.226%); 0 B/s, 103 objects/s recovering 2023-09-23T20:48:45.165 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:48:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:48:44] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.43.0" 2023-09-23T20:48:45.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:44 smithi121 ceph-mon[219709]: pgmap v472: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 493 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 1713/23706 objects degraded (7.226%); 0 B/s, 103 objects/s recovering 2023-09-23T20:48:45.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:44 smithi123 ceph-mon[193903]: pgmap v472: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 493 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 1713/23706 objects degraded (7.226%); 0 B/s, 103 objects/s recovering 2023-09-23T20:48:46.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1713/23706 objects degraded (7.226%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:46.420 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1713/23706 objects degraded (7.226%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1713/23706 objects degraded (7.226%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:47 smithi121 ceph-mon[219709]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 478 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 1713/23001 objects degraded (7.448%); 0 B/s, 103 objects/s recovering 2023-09-23T20:48:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:47 smithi121 ceph-mon[211526]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 478 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 1713/23001 objects degraded (7.448%); 0 B/s, 103 objects/s recovering 2023-09-23T20:48:47.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:47 smithi123 ceph-mon[193903]: pgmap v473: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 478 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 1713/23001 objects degraded (7.448%); 0 B/s, 103 objects/s recovering 2023-09-23T20:48:49.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:49 smithi123 ceph-mon[193903]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 306 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 1713/14730 objects degraded (11.629%); 0 B/s, 62 objects/s recovering 2023-09-23T20:48:50.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:49 smithi121 ceph-mon[219709]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 306 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 1713/14730 objects degraded (11.629%); 0 B/s, 62 objects/s recovering 2023-09-23T20:48:50.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:49 smithi121 ceph-mon[211526]: pgmap v474: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 306 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 1713/14730 objects degraded (11.629%); 0 B/s, 62 objects/s recovering 2023-09-23T20:48:51.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:51 smithi123 ceph-mon[193903]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 226 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 1713/10911 objects degraded (15.700%); 0 B/s, 84 objects/s recovering 2023-09-23T20:48:51.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:51 smithi123 ceph-mon[193903]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T20:48:51.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:51 smithi123 ceph-mon[193903]: failed to encode map e157 with expected crc 2023-09-23T20:48:51.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:51 smithi123 ceph-mon[193903]: failed to encode map e157 with expected crc 2023-09-23T20:48:51.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:51 smithi123 ceph-mon[193903]: failed to encode map e157 with expected crc 2023-09-23T20:48:51.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1713/10911 objects degraded (15.700%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[211526]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 226 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 1713/10911 objects degraded (15.700%); 0 B/s, 84 objects/s recovering 2023-09-23T20:48:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[211526]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T20:48:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[211526]: failed to encode map e157 with expected crc 2023-09-23T20:48:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[211526]: failed to encode map e157 with expected crc 2023-09-23T20:48:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[211526]: failed to encode map e157 with expected crc 2023-09-23T20:48:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1713/10911 objects degraded (15.700%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[219709]: pgmap v475: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 226 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 1713/10911 objects degraded (15.700%); 0 B/s, 84 objects/s recovering 2023-09-23T20:48:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[219709]: osdmap e157: 8 total, 8 up, 8 in 2023-09-23T20:48:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[219709]: failed to encode map e157 with expected crc 2023-09-23T20:48:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[219709]: failed to encode map e157 with expected crc 2023-09-23T20:48:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[219709]: failed to encode map e157 with expected crc 2023-09-23T20:48:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1713/10911 objects degraded (15.700%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:48:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[219709]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[219709]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[219709]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T20:48:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[219709]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[211526]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[211526]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[211526]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T20:48:53.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:52 smithi121 ceph-mon[211526]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:52 smithi123 ceph-mon[193903]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:52 smithi123 ceph-mon[193903]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:52 smithi123 ceph-mon[193903]: osdmap e158: 8 total, 8 up, 8 in 2023-09-23T20:48:53.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:52 smithi123 ceph-mon[193903]: failed to encode map e158 with expected crc 2023-09-23T20:48:53.709 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 109172 objects 2023-09-23T20:48:53.709 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :150.336 2023-09-23T20:48:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:53 smithi121 ceph-mon[219709]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 184 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1713/8895 objects degraded (19.258%); 0 B/s, 42 objects/s recovering 2023-09-23T20:48:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:53 smithi121 ceph-mon[211526]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 184 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1713/8895 objects degraded (19.258%); 0 B/s, 42 objects/s recovering 2023-09-23T20:48:54.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:53 smithi123 ceph-mon[193903]: pgmap v478: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 184 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1713/8895 objects degraded (19.258%); 0 B/s, 42 objects/s recovering 2023-09-23T20:48:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:48:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:48:54] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.43.0" 2023-09-23T20:48:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:54 smithi121 ceph-mon[211526]: pgmap v479: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 863/2070 objects degraded (41.691%); 0 B/s, 99 objects/s recovering 2023-09-23T20:48:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:54 smithi121 ceph-mon[219709]: pgmap v479: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 863/2070 objects degraded (41.691%); 0 B/s, 99 objects/s recovering 2023-09-23T20:48:55.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:54 smithi123 ceph-mon[193903]: pgmap v479: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 863/2070 objects degraded (41.691%); 0 B/s, 99 objects/s recovering 2023-09-23T20:48:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[219709]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[219709]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[219709]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T20:48:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[219709]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[219709]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[219709]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[211526]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[211526]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[211526]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T20:48:56.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[211526]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[211526]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:55 smithi121 ceph-mon[211526]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:55 smithi123 ceph-mon[193903]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:55 smithi123 ceph-mon[193903]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:55 smithi123 ceph-mon[193903]: osdmap e159: 8 total, 8 up, 8 in 2023-09-23T20:48:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:55 smithi123 ceph-mon[193903]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:55 smithi123 ceph-mon[193903]: failed to encode map e159 with expected crc 2023-09-23T20:48:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:55 smithi123 ceph-mon[193903]: failed to encode map e159 with expected crc 2023-09-23T20:48:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[211526]: failed to encode map e159 with expected crc 2023-09-23T20:48:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[211526]: failed to encode map e160 with expected crc 2023-09-23T20:48:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[211526]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T20:48:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 863/2070 objects degraded (41.691%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:48:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[211526]: pgmap v482: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 20 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 863/999 objects degraded (86.386%); 0 B/s, 106 objects/s recovering 2023-09-23T20:48:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[219709]: failed to encode map e159 with expected crc 2023-09-23T20:48:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[219709]: failed to encode map e160 with expected crc 2023-09-23T20:48:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[219709]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T20:48:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 863/2070 objects degraded (41.691%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:48:57.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:56 smithi121 ceph-mon[219709]: pgmap v482: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 20 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 863/999 objects degraded (86.386%); 0 B/s, 106 objects/s recovering 2023-09-23T20:48:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:56 smithi123 ceph-mon[193903]: failed to encode map e159 with expected crc 2023-09-23T20:48:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:56 smithi123 ceph-mon[193903]: failed to encode map e160 with expected crc 2023-09-23T20:48:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:56 smithi123 ceph-mon[193903]: osdmap e160: 8 total, 8 up, 8 in 2023-09-23T20:48:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 863/2070 objects degraded (41.691%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:48:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:56 smithi123 ceph-mon[193903]: pgmap v482: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 20 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 863/999 objects degraded (86.386%); 0 B/s, 106 objects/s recovering 2023-09-23T20:48:57.328 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T20:48:57.328 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T20:48:58.161 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T20:48:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: failed to encode map e160 with expected crc 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: osdmap e161: 8 total, 8 up, 8 in 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: failed to encode map e161 with expected crc 2023-09-23T20:48:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[219709]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[219709]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[219709]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T20:48:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[219709]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[219709]: pgmap v485: 89 pgs: 1 peering, 88 active+clean; 646 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 0 B/s, 259 objects/s recovering 2023-09-23T20:48:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[211526]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[211526]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[211526]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T20:48:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[211526]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:48:59 smithi121 ceph-mon[211526]: pgmap v485: 89 pgs: 1 peering, 88 active+clean; 646 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 0 B/s, 259 objects/s recovering 2023-09-23T20:48:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:59 smithi123 ceph-mon[193903]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:59 smithi123 ceph-mon[193903]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:59 smithi123 ceph-mon[193903]: osdmap e162: 8 total, 8 up, 8 in 2023-09-23T20:48:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:59 smithi123 ceph-mon[193903]: failed to encode map e162 with expected crc 2023-09-23T20:48:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:48:59 smithi123 ceph-mon[193903]: pgmap v485: 89 pgs: 1 peering, 88 active+clean; 646 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 0 B/s, 259 objects/s recovering 2023-09-23T20:49:00.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:00 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 863/999 objects degraded (86.386%), 1 pg degraded) 2023-09-23T20:49:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:00 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T20:49:00.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:00 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 863/999 objects degraded (86.386%), 1 pg degraded) 2023-09-23T20:49:00.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:00 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T20:49:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:00 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 863/999 objects degraded (86.386%), 1 pg degraded) 2023-09-23T20:49:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:00 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T20:49:01.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:01 smithi121 ceph-mon[219709]: pgmap v486: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 0 B/s, 238 objects/s recovering 2023-09-23T20:49:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:01 smithi121 ceph-mon[211526]: pgmap v486: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 0 B/s, 238 objects/s recovering 2023-09-23T20:49:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:01 smithi123 ceph-mon[193903]: pgmap v486: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 0 B/s, 238 objects/s recovering 2023-09-23T20:49:02.103 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_3' removed 2023-09-23T20:49:02.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3221518742' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:02.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:02 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:02.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3221518742' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:02.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:02 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3221518742' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:02 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:02.956 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:49:02.956 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T20:49:02.957 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:02.957 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:49:02.969 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T20:49:02.969 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T20:49:02.978 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:49:02.978 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:49:02.978 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:49:03.381 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[211526]: failed to encode map e163 with expected crc 2023-09-23T20:49:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:49:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[211526]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T20:49:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[211526]: failed to encode map e163 with expected crc 2023-09-23T20:49:03.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[211526]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[219709]: failed to encode map e163 with expected crc 2023-09-23T20:49:03.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:49:03.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[219709]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T20:49:03.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[219709]: failed to encode map e163 with expected crc 2023-09-23T20:49:03.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:03 smithi121 ceph-mon[219709]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:03.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:03 smithi123 ceph-mon[193903]: failed to encode map e163 with expected crc 2023-09-23T20:49:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:49:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:03 smithi123 ceph-mon[193903]: osdmap e163: 8 total, 8 up, 8 in 2023-09-23T20:49:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:03 smithi123 ceph-mon[193903]: failed to encode map e163 with expected crc 2023-09-23T20:49:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:03 smithi123 ceph-mon[193903]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:04 smithi121 ceph-mon[211526]: failed to encode map e163 with expected crc 2023-09-23T20:49:04.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:04 smithi121 ceph-mon[211526]: failed to encode map e163 with expected crc 2023-09-23T20:49:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:04 smithi121 ceph-mon[219709]: failed to encode map e163 with expected crc 2023-09-23T20:49:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:04 smithi121 ceph-mon[219709]: failed to encode map e163 with expected crc 2023-09-23T20:49:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:04 smithi123 ceph-mon[193903]: failed to encode map e163 with expected crc 2023-09-23T20:49:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:04 smithi123 ceph-mon[193903]: failed to encode map e163 with expected crc 2023-09-23T20:49:04.825 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:49:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:49:04] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.43.0" 2023-09-23T20:49:05.301 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:05 smithi121 ceph-mon[211526]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:05 smithi121 ceph-mon[219709]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:05 smithi123 ceph-mon[193903]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:06.075 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":35,"active_gid":25066,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":467299182},{"type":"v1","addr":"172.21.15.121:6801","nonce":467299182}]},"active_addr":"172.21.15.121:6801/467299182","active_change":"2023-09-23T20:35:42.237144+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25063,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":78,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":3826251942}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":952501386}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":4155218119}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":2188733328}]}]} 2023-09-23T20:49:06.080 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:49:06.080 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:49:06.080 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:49:06.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1852899423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:49:06.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1852899423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:49:06.431 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1852899423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:49:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:07 smithi121 ceph-mon[219709]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:07 smithi121 ceph-mon[211526]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:07 smithi123 ceph-mon[193903]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:08.250 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:08.250 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":163,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:49:02.066000+0000","last_up_change":"2023-09-23T20:48:06.470732+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:30:20.536872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:30:21.339996+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":92,"up_thru":153,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6803","nonce":3792083724}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6805","nonce":3792083724}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6809","nonce":3792083724}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6807","nonce":3792083724}]},"public_addr":"172.21.15.121:6803/3792083724","cluster_addr":"172.21.15.121:6805/3792083724","heartbeat_back_addr":"172.21.15.121:6809/3792083724","heartbeat_front_addr":"172.21.15.121:6807/3792083724","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":114,"up_thru":161,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6811","nonce":1737064709}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6813","nonce":1737064709}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6817","nonce":1737064709}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6815","nonce":1737064709}]},"public_addr":"172.21.15.121:6811/1737064709","cluster_addr":"172.21.15.121:6813/1737064709","heartbeat_back_addr":"172.21.15.121:6817/1737064709","heartbeat_front_addr":"172.21.15.121:6815/1737064709","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":123,"up_from":129,"up_thru":148,"down_at":124,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6819","nonce":2050616913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6821","nonce":2050616913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6825","nonce":2050616913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6823","nonce":2050616913}]},"public_addr":"172.21.15.121:6819/2050616913","cluster_addr":"172.21.15.121:6821/2050616913","heartbeat_back_addr":"172.21.15.121:6825/2050616913","heartbeat_front_addr":"172.21.15.121:6823/2050616913","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":141,"up_from":148,"up_thru":159,"down_at":142,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6827","nonce":3257078631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6829","nonce":3257078631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6833","nonce":3257078631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6831","nonce":3257078631}]},"public_addr":"172.21.15.121:6827/3257078631","cluster_addr":"172.21.15.121:6829/3257078631","heartbeat_back_addr":"172.21.15.121:6833/3257078631","heartbeat_front_addr":"172.21.15.121:6831/3257078631","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":156,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":123},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":141},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/961670864":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/2952381553":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/2000763757":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/4233508401":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.123:0/2412578238":"2023-09-24T20:34:36.437609+0000","172.21.15.121:6800/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/3564595757":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/167205692":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/623964166":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/3933818352":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/1025991692":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6834/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6835/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/2713351150":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/156999919":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/985594636":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/719574644":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/1669790000":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/85909652":"2023-09-24T20:35:42.236940+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:49:08.478 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:49:08 smithi123 conmon[172845]: logger=sqlstore.transactions t=2023-09-23T20:49:08.007704926Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:49:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1685566628' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:49:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1685566628' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:49:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1685566628' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:49:09.056 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:49:09.056 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:49:09.404 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:09 smithi121 ceph-mon[211526]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:09 smithi121 ceph-mon[219709]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:09 smithi123 ceph-mon[193903]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:11.288 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:11.289 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":163,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T20:49:02.066000+0000","last_up_change":"2023-09-23T20:48:06.470732+0000","last_in_change":"2023-09-23T20:28:34.189703+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:30:20.536872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:30:21.339996+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":92,"up_thru":153,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6803","nonce":3792083724}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6805","nonce":3792083724}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6809","nonce":3792083724}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6807","nonce":3792083724}]},"public_addr":"172.21.15.121:6803/3792083724","cluster_addr":"172.21.15.121:6805/3792083724","heartbeat_back_addr":"172.21.15.121:6809/3792083724","heartbeat_front_addr":"172.21.15.121:6807/3792083724","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":114,"up_thru":161,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6811","nonce":1737064709}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6813","nonce":1737064709}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6817","nonce":1737064709}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":1737064709},{"type":"v1","addr":"172.21.15.121:6815","nonce":1737064709}]},"public_addr":"172.21.15.121:6811/1737064709","cluster_addr":"172.21.15.121:6813/1737064709","heartbeat_back_addr":"172.21.15.121:6817/1737064709","heartbeat_front_addr":"172.21.15.121:6815/1737064709","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":123,"up_from":129,"up_thru":148,"down_at":124,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6819","nonce":2050616913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6821","nonce":2050616913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6825","nonce":2050616913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6823","nonce":2050616913}]},"public_addr":"172.21.15.121:6819/2050616913","cluster_addr":"172.21.15.121:6821/2050616913","heartbeat_back_addr":"172.21.15.121:6825/2050616913","heartbeat_front_addr":"172.21.15.121:6823/2050616913","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":141,"up_from":148,"up_thru":159,"down_at":142,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6827","nonce":3257078631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6829","nonce":3257078631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6833","nonce":3257078631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6831","nonce":3257078631}]},"public_addr":"172.21.15.121:6827/3257078631","cluster_addr":"172.21.15.121:6829/3257078631","heartbeat_back_addr":"172.21.15.121:6833/3257078631","heartbeat_front_addr":"172.21.15.121:6831/3257078631","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":156,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6809","nonce":3299108602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6811","nonce":3299108602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6815","nonce":3299108602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3299108602},{"type":"v1","addr":"172.21.15.123:6813","nonce":3299108602}]},"public_addr":"172.21.15.123:6809/3299108602","cluster_addr":"172.21.15.123:6811/3299108602","heartbeat_back_addr":"172.21.15.123:6815/3299108602","heartbeat_front_addr":"172.21.15.123:6813/3299108602","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":148,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":123},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":141},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/961670864":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/2952381553":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/2000763757":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/4233508401":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.123:0/2412578238":"2023-09-24T20:34:36.437609+0000","172.21.15.121:6800/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/3564595757":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/167205692":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/623964166":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/3933818352":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/1025991692":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6834/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6835/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/2713351150":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/156999919":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/985594636":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/719574644":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/1669790000":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/85909652":"2023-09-24T20:35:42.236940+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:49:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:11 smithi123 ceph-mon[193903]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/341204727' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:49:12.038 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:11 smithi121 ceph-mon[211526]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:12.038 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/341204727' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:49:12.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:11 smithi121 ceph-mon[219709]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:12.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/341204727' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:49:12.055 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:49:12.056 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:49:12.056 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:49:12.056 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:49:12.057 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:49:12.057 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:49:12.057 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:49:12.058 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:49:13.223 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.224 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.383 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.383 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.588 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.592 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:13 smithi121 ceph-mon[211526]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:13 smithi121 ceph-mon[219709]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:13.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:13 smithi123 ceph-mon[193903]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:13.958 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.960 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:15.127 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:49:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:49:14] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2023-09-23T20:49:15.754 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:15 smithi121 ceph-mon[211526]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:15.755 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:15 smithi121 ceph-mon[219709]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:15.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:15 smithi123 ceph-mon[193903]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:17 smithi121 ceph-mon[211526]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:17 smithi121 ceph-mon[219709]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:17.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:17 smithi123 ceph-mon[193903]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:19 smithi123 ceph-mon[193903]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:19 smithi121 ceph-mon[211526]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:19 smithi121 ceph-mon[219709]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:21.027 INFO:teuthology.orchestra.run.smithi121.stdout:167503724801 2023-09-23T20:49:21.028 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:49:21.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:21 smithi121 ceph-mon[211526]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:21.818 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:21 smithi121 ceph-mon[219709]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:21 smithi123 ceph-mon[193903]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:23.298 INFO:teuthology.orchestra.run.smithi121.stdout:489626271798 2023-09-23T20:49:23.299 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:49:23.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:23 smithi123 ceph-mon[193903]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:24.138 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:23 smithi121 ceph-mon[219709]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:24.138 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:23 smithi121 ceph-mon[211526]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:24.424 INFO:teuthology.orchestra.run.smithi121.stdout:236223201527 2023-09-23T20:49:24.424 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:49:24.519 INFO:teuthology.orchestra.run.smithi121.stdout:635655159825 2023-09-23T20:49:24.519 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:49:25.019 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:49:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:49:24] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2023-09-23T20:49:25.050 INFO:teuthology.orchestra.run.smithi121.stdout:133143986437 2023-09-23T20:49:25.051 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:49:25.052 INFO:teuthology.orchestra.run.smithi121.stdout:554050781225 2023-09-23T20:49:25.052 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:49:25.637 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:25.659 INFO:teuthology.orchestra.run.smithi121.stdout:201863463164 2023-09-23T20:49:25.660 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:49:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:25 smithi121 ceph-mon[211526]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:25.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:25 smithi121 ceph-mon[219709]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:25 smithi123 ceph-mon[193903]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:26.588 INFO:teuthology.orchestra.run.smithi121.stdout:395136991336 2023-09-23T20:49:26.588 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:49:26.616 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:27.007 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:27.007 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:27 smithi121 ceph-mon[211526]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:27 smithi121 ceph-mon[219709]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:27 smithi123 ceph-mon[193903]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:28.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:28.342 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:28.387 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:28.571 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:29.000 INFO:teuthology.orchestra.run.smithi121.stdout:167503724802 2023-09-23T20:49:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:29 smithi121 ceph-mon[211526]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2119183417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:49:29.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:29 smithi121 ceph-mon[219709]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:29.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2119183417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:49:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:29 smithi123 ceph-mon[193903]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2119183417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:49:30.221 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:30 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:30.221 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:30 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:30 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:30.634 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724801 got 167503724802 for osd.5 2023-09-23T20:49:30.634 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:31 smithi123 ceph-mon[193903]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:31 smithi121 ceph-mon[211526]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:31 smithi121 ceph-mon[219709]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:32.196 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:32.247 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:32.588 INFO:teuthology.orchestra.run.smithi121.stdout:489626271800 2023-09-23T20:49:32.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1400133666' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:49:32.845 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1400133666' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:49:32.941 INFO:teuthology.orchestra.run.smithi121.stdout:236223201529 2023-09-23T20:49:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1400133666' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:49:33.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[211526]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:33.943 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2466957275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:49:33.943 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:33.943 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:33.943 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[219709]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:33.943 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2466957275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:49:33.943 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:33.943 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:33 smithi123 ceph-mon[193903]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2466957275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:49:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:33 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:33 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:34.091 INFO:tasks.cephadm.ceph_manager.ceph:need seq 489626271798 got 489626271800 for osd.1 2023-09-23T20:49:34.091 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:34.905 INFO:teuthology.orchestra.run.smithi121.stdout:133143986440 2023-09-23T20:49:35.066 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:49:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:49:34] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2023-09-23T20:49:35.111 INFO:teuthology.orchestra.run.smithi121.stdout:635655159827 2023-09-23T20:49:35.616 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201527 got 236223201529 for osd.7 2023-09-23T20:49:35.616 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:35.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:35 smithi121 ceph-mon[211526]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:35.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1940921996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:49:35.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3131810204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:49:35.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:35 smithi121 ceph-mon[219709]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:35.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1940921996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:49:35.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3131810204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:49:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:35 smithi123 ceph-mon[193903]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1940921996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:49:35.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3131810204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:49:37.208 INFO:tasks.cephadm.ceph_manager.ceph:need seq 635655159825 got 635655159827 for osd.3 2023-09-23T20:49:37.208 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:37 smithi121 ceph-mon[211526]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:37 smithi121 ceph-mon[219709]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:37 smithi123 ceph-mon[193903]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:37.686 INFO:teuthology.orchestra.run.smithi121.stdout:201863463167 2023-09-23T20:49:38.113 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986437 got 133143986440 for osd.4 2023-09-23T20:49:38.113 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1009948990' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:49:38.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1009948990' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:49:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1009948990' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:49:38.716 INFO:teuthology.orchestra.run.smithi121.stdout:395136991338 2023-09-23T20:49:38.823 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463164 got 201863463167 for osd.6 2023-09-23T20:49:38.823 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:39.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:39 smithi121 ceph-mon[219709]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:39.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2861444137' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:49:39.287 INFO:teuthology.orchestra.run.smithi121.stdout:554050781229 2023-09-23T20:49:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:39 smithi121 ceph-mon[211526]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2861444137' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:49:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:39 smithi123 ceph-mon[193903]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2861444137' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:49:40.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1314387610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:49:40.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1314387610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:49:40.401 INFO:tasks.cephadm.ceph_manager.ceph:need seq 395136991336 got 395136991338 for osd.0 2023-09-23T20:49:40.401 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:40.466 INFO:tasks.cephadm.ceph_manager.ceph:need seq 554050781225 got 554050781229 for osd.2 2023-09-23T20:49:40.466 DEBUG:teuthology.parallel:result is None 2023-09-23T20:49:40.466 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:49:40.466 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:49:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1314387610' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:49:40.863 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:41.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:41 smithi121 ceph-mon[211526]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:41.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:41 smithi121 ceph-mon[219709]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:41 smithi123 ceph-mon[193903]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:42.859 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:42.859 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:43.398 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:43.400 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:49:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:43 smithi123 ceph-mon[193903]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:43.741 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:43 smithi121 ceph-mon[211526]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:43.741 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:43 smithi121 ceph-mon[219709]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:44.355 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-09-23T20:49:42.637070+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2272776,"kb_used_data":14984,"kb_used_omap":20,"kb_used_meta":2257707,"kb_avail":747688440,"statfs":{"total":767960285184,"available":765632962560,"internally_reserved":0,"allocated":15343616,"data_stored":10367957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2311892622},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002753"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058647+0000","last_change":"2023-09-23T20:46:05.100324+0000","last_active":"2023-09-23T20:48:58.058647+0000","last_peered":"2023-09-23T20:48:58.058647+0000","last_clean":"2023-09-23T20:48:58.058647+0000","last_became_active":"2023-09-23T20:46:05.099981+0000","last_became_peered":"2023-09-23T20:46:05.099981+0000","last_unstale":"2023-09-23T20:48:58.058647+0000","last_undegraded":"2023-09-23T20:48:58.058647+0000","last_fullsized":"2023-09-23T20:48:58.058647+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:19.876971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055345+0000","last_change":"2023-09-23T20:48:07.517973+0000","last_active":"2023-09-23T20:48:58.055345+0000","last_peered":"2023-09-23T20:48:58.055345+0000","last_clean":"2023-09-23T20:48:58.055345+0000","last_became_active":"2023-09-23T20:48:07.517681+0000","last_became_peered":"2023-09-23T20:48:07.517681+0000","last_unstale":"2023-09-23T20:48:58.055345+0000","last_undegraded":"2023-09-23T20:48:58.055345+0000","last_fullsized":"2023-09-23T20:48:58.055345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.789594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.056881+0000","last_change":"2023-09-23T20:48:07.541333+0000","last_active":"2023-09-23T20:48:58.056881+0000","last_peered":"2023-09-23T20:48:58.056881+0000","last_clean":"2023-09-23T20:48:58.056881+0000","last_became_active":"2023-09-23T20:48:07.541014+0000","last_became_peered":"2023-09-23T20:48:07.541014+0000","last_unstale":"2023-09-23T20:48:58.056881+0000","last_undegraded":"2023-09-23T20:48:58.056881+0000","last_fullsized":"2023-09-23T20:48:58.056881+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:48.678651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140248+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:48:58.140248+0000","last_peered":"2023-09-23T20:48:58.140248+0000","last_clean":"2023-09-23T20:48:58.140248+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:48:58.140248+0000","last_undegraded":"2023-09-23T20:48:58.140248+0000","last_fullsized":"2023-09-23T20:48:58.140248+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054913+0000","last_change":"2023-09-23T20:48:07.540435+0000","last_active":"2023-09-23T20:48:58.054913+0000","last_peered":"2023-09-23T20:48:58.054913+0000","last_clean":"2023-09-23T20:48:58.054913+0000","last_became_active":"2023-09-23T20:48:07.540062+0000","last_became_peered":"2023-09-23T20:48:07.540062+0000","last_unstale":"2023-09-23T20:48:58.054913+0000","last_undegraded":"2023-09-23T20:48:58.054913+0000","last_fullsized":"2023-09-23T20:48:58.054913+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:19.449218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060678+0000","last_change":"2023-09-23T20:45:00.009034+0000","last_active":"2023-09-23T20:48:58.060678+0000","last_peered":"2023-09-23T20:48:58.060678+0000","last_clean":"2023-09-23T20:48:58.060678+0000","last_became_active":"2023-09-23T20:45:00.008706+0000","last_became_peered":"2023-09-23T20:45:00.008706+0000","last_unstale":"2023-09-23T20:48:58.060678+0000","last_undegraded":"2023-09-23T20:48:58.060678+0000","last_fullsized":"2023-09-23T20:48:58.060678+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:29.204761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140415+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:48:58.140415+0000","last_peered":"2023-09-23T20:48:58.140415+0000","last_clean":"2023-09-23T20:48:58.140415+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:48:58.140415+0000","last_undegraded":"2023-09-23T20:48:58.140415+0000","last_fullsized":"2023-09-23T20:48:58.140415+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054946+0000","last_change":"2023-09-23T20:48:07.533499+0000","last_active":"2023-09-23T20:48:58.054946+0000","last_peered":"2023-09-23T20:48:58.054946+0000","last_clean":"2023-09-23T20:48:58.054946+0000","last_became_active":"2023-09-23T20:48:07.520669+0000","last_became_peered":"2023-09-23T20:48:07.520669+0000","last_unstale":"2023-09-23T20:48:58.054946+0000","last_undegraded":"2023-09-23T20:48:58.054946+0000","last_fullsized":"2023-09-23T20:48:58.054946+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:55.109803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058059+0000","last_change":"2023-09-23T20:45:00.007125+0000","last_active":"2023-09-23T20:48:58.058059+0000","last_peered":"2023-09-23T20:48:58.058059+0000","last_clean":"2023-09-23T20:48:58.058059+0000","last_became_active":"2023-09-23T20:45:00.006403+0000","last_became_peered":"2023-09-23T20:45:00.006403+0000","last_unstale":"2023-09-23T20:48:58.058059+0000","last_undegraded":"2023-09-23T20:48:58.058059+0000","last_fullsized":"2023-09-23T20:48:58.058059+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:27.906490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055087+0000","last_change":"2023-09-23T20:48:07.525099+0000","last_active":"2023-09-23T20:48:58.055087+0000","last_peered":"2023-09-23T20:48:58.055087+0000","last_clean":"2023-09-23T20:48:58.055087+0000","last_became_active":"2023-09-23T20:48:07.524868+0000","last_became_peered":"2023-09-23T20:48:07.524868+0000","last_unstale":"2023-09-23T20:48:58.055087+0000","last_undegraded":"2023-09-23T20:48:58.055087+0000","last_fullsized":"2023-09-23T20:48:58.055087+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:19.340586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140203+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:48:58.140203+0000","last_peered":"2023-09-23T20:48:58.140203+0000","last_clean":"2023-09-23T20:48:58.140203+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:48:58.140203+0000","last_undegraded":"2023-09-23T20:48:58.140203+0000","last_fullsized":"2023-09-23T20:48:58.140203+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595558+0000","last_change":"2023-09-23T20:46:05.099313+0000","last_active":"2023-09-23T20:49:02.595558+0000","last_peered":"2023-09-23T20:49:02.595558+0000","last_clean":"2023-09-23T20:49:02.595558+0000","last_became_active":"2023-09-23T20:46:05.099055+0000","last_became_peered":"2023-09-23T20:46:05.099055+0000","last_unstale":"2023-09-23T20:49:02.595558+0000","last_undegraded":"2023-09-23T20:49:02.595558+0000","last_fullsized":"2023-09-23T20:49:02.595558+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:51.820608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058022+0000","last_change":"2023-09-23T20:30:22.107118+0000","last_active":"2023-09-23T20:48:58.058022+0000","last_peered":"2023-09-23T20:48:58.058022+0000","last_clean":"2023-09-23T20:48:58.058022+0000","last_became_active":"2023-09-23T20:30:22.106839+0000","last_became_peered":"2023-09-23T20:30:22.106839+0000","last_unstale":"2023-09-23T20:48:58.058022+0000","last_undegraded":"2023-09-23T20:48:58.058022+0000","last_fullsized":"2023-09-23T20:48:58.058022+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:02.981333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058390+0000","last_change":"2023-09-23T20:45:00.007712+0000","last_active":"2023-09-23T20:48:58.058390+0000","last_peered":"2023-09-23T20:48:58.058390+0000","last_clean":"2023-09-23T20:48:58.058390+0000","last_became_active":"2023-09-23T20:45:00.007399+0000","last_became_peered":"2023-09-23T20:45:00.007399+0000","last_unstale":"2023-09-23T20:48:58.058390+0000","last_undegraded":"2023-09-23T20:48:58.058390+0000","last_fullsized":"2023-09-23T20:48:58.058390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:48.842612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139939+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:48:58.139939+0000","last_peered":"2023-09-23T20:48:58.139939+0000","last_clean":"2023-09-23T20:48:58.139939+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:48:58.139939+0000","last_undegraded":"2023-09-23T20:48:58.139939+0000","last_fullsized":"2023-09-23T20:48:58.139939+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055261+0000","last_change":"2023-09-23T20:48:07.540014+0000","last_active":"2023-09-23T20:48:58.055261+0000","last_peered":"2023-09-23T20:48:58.055261+0000","last_clean":"2023-09-23T20:48:58.055261+0000","last_became_active":"2023-09-23T20:48:07.539722+0000","last_became_peered":"2023-09-23T20:48:07.539722+0000","last_unstale":"2023-09-23T20:48:58.055261+0000","last_undegraded":"2023-09-23T20:48:58.055261+0000","last_fullsized":"2023-09-23T20:48:58.055261+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.937232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140140+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:48:58.140140+0000","last_peered":"2023-09-23T20:48:58.140140+0000","last_clean":"2023-09-23T20:48:58.140140+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:48:58.140140+0000","last_undegraded":"2023-09-23T20:48:58.140140+0000","last_fullsized":"2023-09-23T20:48:58.140140+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058777+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:48:58.058777+0000","last_peered":"2023-09-23T20:48:58.058777+0000","last_clean":"2023-09-23T20:48:58.058777+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:48:58.058777+0000","last_undegraded":"2023-09-23T20:48:58.058777+0000","last_fullsized":"2023-09-23T20:48:58.058777+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139702+0000","last_change":"2023-09-23T20:30:22.110586+0000","last_active":"2023-09-23T20:48:58.139702+0000","last_peered":"2023-09-23T20:48:58.139702+0000","last_clean":"2023-09-23T20:48:58.139702+0000","last_became_active":"2023-09-23T20:30:22.110248+0000","last_became_peered":"2023-09-23T20:30:22.110248+0000","last_unstale":"2023-09-23T20:48:58.139702+0000","last_undegraded":"2023-09-23T20:48:58.139702+0000","last_fullsized":"2023-09-23T20:48:58.139702+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:29.867944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140341+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:48:58.140341+0000","last_peered":"2023-09-23T20:48:58.140341+0000","last_clean":"2023-09-23T20:48:58.140341+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:48:58.140341+0000","last_undegraded":"2023-09-23T20:48:58.140341+0000","last_fullsized":"2023-09-23T20:48:58.140341+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060100+0000","last_change":"2023-09-23T20:40:53.862653+0000","last_active":"2023-09-23T20:48:58.060100+0000","last_peered":"2023-09-23T20:48:58.060100+0000","last_clean":"2023-09-23T20:48:58.060100+0000","last_became_active":"2023-09-23T20:40:53.862259+0000","last_became_peered":"2023-09-23T20:40:53.862259+0000","last_unstale":"2023-09-23T20:48:58.060100+0000","last_undegraded":"2023-09-23T20:48:58.060100+0000","last_fullsized":"2023-09-23T20:48:58.060100+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:28.626871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055685+0000","last_change":"2023-09-23T20:48:07.557905+0000","last_active":"2023-09-23T20:48:58.055685+0000","last_peered":"2023-09-23T20:48:58.055685+0000","last_clean":"2023-09-23T20:48:58.055685+0000","last_became_active":"2023-09-23T20:48:07.557568+0000","last_became_peered":"2023-09-23T20:48:07.557568+0000","last_unstale":"2023-09-23T20:48:58.055685+0000","last_undegraded":"2023-09-23T20:48:58.055685+0000","last_fullsized":"2023-09-23T20:48:58.055685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:14.229704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140303+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:48:58.140303+0000","last_peered":"2023-09-23T20:48:58.140303+0000","last_clean":"2023-09-23T20:48:58.140303+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:48:58.140303+0000","last_undegraded":"2023-09-23T20:48:58.140303+0000","last_fullsized":"2023-09-23T20:48:58.140303+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058595+0000","last_change":"2023-09-23T20:30:22.772475+0000","last_active":"2023-09-23T20:48:58.058595+0000","last_peered":"2023-09-23T20:48:58.058595+0000","last_clean":"2023-09-23T20:48:58.058595+0000","last_became_active":"2023-09-23T20:30:22.772159+0000","last_became_peered":"2023-09-23T20:30:22.772159+0000","last_unstale":"2023-09-23T20:48:58.058595+0000","last_undegraded":"2023-09-23T20:48:58.058595+0000","last_fullsized":"2023-09-23T20:48:58.058595+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:08.373446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140167+0000","last_change":"2023-09-23T20:45:00.003948+0000","last_active":"2023-09-23T20:48:58.140167+0000","last_peered":"2023-09-23T20:48:58.140167+0000","last_clean":"2023-09-23T20:48:58.140167+0000","last_became_active":"2023-09-23T20:45:00.003639+0000","last_became_peered":"2023-09-23T20:45:00.003639+0000","last_unstale":"2023-09-23T20:48:58.140167+0000","last_undegraded":"2023-09-23T20:48:58.140167+0000","last_fullsized":"2023-09-23T20:48:58.140167+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:59.619696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057894+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.057894+0000","last_peered":"2023-09-23T20:48:58.057894+0000","last_clean":"2023-09-23T20:48:58.057894+0000","last_became_active":"2023-09-23T20:48:07.544029+0000","last_became_peered":"2023-09-23T20:48:07.544029+0000","last_unstale":"2023-09-23T20:48:58.057894+0000","last_undegraded":"2023-09-23T20:48:58.057894+0000","last_fullsized":"2023-09-23T20:48:58.057894+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:52.671448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140012+0000","last_change":"2023-09-23T20:30:22.113582+0000","last_active":"2023-09-23T20:48:58.140012+0000","last_peered":"2023-09-23T20:48:58.140012+0000","last_clean":"2023-09-23T20:48:58.140012+0000","last_became_active":"2023-09-23T20:30:22.113291+0000","last_became_peered":"2023-09-23T20:30:22.113291+0000","last_unstale":"2023-09-23T20:48:58.140012+0000","last_undegraded":"2023-09-23T20:48:58.140012+0000","last_fullsized":"2023-09-23T20:48:58.140012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:19.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060226+0000","last_change":"2023-09-23T20:46:05.097625+0000","last_active":"2023-09-23T20:48:58.060226+0000","last_peered":"2023-09-23T20:48:58.060226+0000","last_clean":"2023-09-23T20:48:58.060226+0000","last_became_active":"2023-09-23T20:46:05.097296+0000","last_became_peered":"2023-09-23T20:46:05.097296+0000","last_unstale":"2023-09-23T20:48:58.060226+0000","last_undegraded":"2023-09-23T20:48:58.060226+0000","last_fullsized":"2023-09-23T20:48:58.060226+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:34.564149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058440+0000","last_change":"2023-09-23T20:48:07.522410+0000","last_active":"2023-09-23T20:48:58.058440+0000","last_peered":"2023-09-23T20:48:58.058440+0000","last_clean":"2023-09-23T20:48:58.058440+0000","last_became_active":"2023-09-23T20:48:07.522058+0000","last_became_peered":"2023-09-23T20:48:07.522058+0000","last_unstale":"2023-09-23T20:48:58.058440+0000","last_undegraded":"2023-09-23T20:48:58.058440+0000","last_fullsized":"2023-09-23T20:48:58.058440+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:24.844663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140085+0000","last_change":"2023-09-23T20:45:00.002738+0000","last_active":"2023-09-23T20:48:58.140085+0000","last_peered":"2023-09-23T20:48:58.140085+0000","last_clean":"2023-09-23T20:48:58.140085+0000","last_became_active":"2023-09-23T20:45:00.002403+0000","last_became_peered":"2023-09-23T20:45:00.002403+0000","last_unstale":"2023-09-23T20:48:58.140085+0000","last_undegraded":"2023-09-23T20:48:58.140085+0000","last_fullsized":"2023-09-23T20:48:58.140085+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:54.551205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054983+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:48:58.054983+0000","last_peered":"2023-09-23T20:48:58.054983+0000","last_clean":"2023-09-23T20:48:58.054983+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:48:58.054983+0000","last_undegraded":"2023-09-23T20:48:58.054983+0000","last_fullsized":"2023-09-23T20:48:58.054983+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057922+0000","last_change":"2023-09-23T20:48:07.520873+0000","last_active":"2023-09-23T20:48:58.057922+0000","last_peered":"2023-09-23T20:48:58.057922+0000","last_clean":"2023-09-23T20:48:58.057922+0000","last_became_active":"2023-09-23T20:48:07.520517+0000","last_became_peered":"2023-09-23T20:48:07.520517+0000","last_unstale":"2023-09-23T20:48:58.057922+0000","last_undegraded":"2023-09-23T20:48:58.057922+0000","last_fullsized":"2023-09-23T20:48:58.057922+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:43.626732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060090+0000","last_change":"2023-09-23T20:45:00.009049+0000","last_active":"2023-09-23T20:48:58.060090+0000","last_peered":"2023-09-23T20:48:58.060090+0000","last_clean":"2023-09-23T20:48:58.060090+0000","last_became_active":"2023-09-23T20:45:00.008700+0000","last_became_peered":"2023-09-23T20:45:00.008700+0000","last_unstale":"2023-09-23T20:48:58.060090+0000","last_undegraded":"2023-09-23T20:48:58.060090+0000","last_fullsized":"2023-09-23T20:48:58.060090+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:52.333134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057799+0000","last_change":"2023-09-23T20:46:05.106502+0000","last_active":"2023-09-23T20:48:58.057799+0000","last_peered":"2023-09-23T20:48:58.057799+0000","last_clean":"2023-09-23T20:48:58.057799+0000","last_became_active":"2023-09-23T20:46:05.106134+0000","last_became_peered":"2023-09-23T20:46:05.106134+0000","last_unstale":"2023-09-23T20:48:58.057799+0000","last_undegraded":"2023-09-23T20:48:58.057799+0000","last_fullsized":"2023-09-23T20:48:58.057799+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:39.257351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057632+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:48:58.057632+0000","last_peered":"2023-09-23T20:48:58.057632+0000","last_clean":"2023-09-23T20:48:58.057632+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:48:58.057632+0000","last_undegraded":"2023-09-23T20:48:58.057632+0000","last_fullsized":"2023-09-23T20:48:58.057632+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058202+0000","last_change":"2023-09-23T20:45:00.008536+0000","last_active":"2023-09-23T20:48:58.058202+0000","last_peered":"2023-09-23T20:48:58.058202+0000","last_clean":"2023-09-23T20:48:58.058202+0000","last_became_active":"2023-09-23T20:45:00.008220+0000","last_became_peered":"2023-09-23T20:45:00.008220+0000","last_unstale":"2023-09-23T20:48:58.058202+0000","last_undegraded":"2023-09-23T20:48:58.058202+0000","last_fullsized":"2023-09-23T20:48:58.058202+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:04.017031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054990+0000","last_change":"2023-09-23T20:48:07.540484+0000","last_active":"2023-09-23T20:48:58.054990+0000","last_peered":"2023-09-23T20:48:58.054990+0000","last_clean":"2023-09-23T20:48:58.054990+0000","last_became_active":"2023-09-23T20:48:07.540219+0000","last_became_peered":"2023-09-23T20:48:07.540219+0000","last_unstale":"2023-09-23T20:48:58.054990+0000","last_undegraded":"2023-09-23T20:48:58.054990+0000","last_fullsized":"2023-09-23T20:48:58.054990+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:54.792744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139793+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:48:58.139793+0000","last_peered":"2023-09-23T20:48:58.139793+0000","last_clean":"2023-09-23T20:48:58.139793+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:48:58.139793+0000","last_undegraded":"2023-09-23T20:48:58.139793+0000","last_fullsized":"2023-09-23T20:48:58.139793+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595372+0000","last_change":"2023-09-23T20:45:00.001830+0000","last_active":"2023-09-23T20:49:02.595372+0000","last_peered":"2023-09-23T20:49:02.595372+0000","last_clean":"2023-09-23T20:49:02.595372+0000","last_became_active":"2023-09-23T20:45:00.001532+0000","last_became_peered":"2023-09-23T20:45:00.001532+0000","last_unstale":"2023-09-23T20:49:02.595372+0000","last_undegraded":"2023-09-23T20:49:02.595372+0000","last_fullsized":"2023-09-23T20:49:02.595372+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:17.499694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058858+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:48:58.058858+0000","last_peered":"2023-09-23T20:48:58.058858+0000","last_clean":"2023-09-23T20:48:58.058858+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:48:58.058858+0000","last_undegraded":"2023-09-23T20:48:58.058858+0000","last_fullsized":"2023-09-23T20:48:58.058858+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595237+0000","last_change":"2023-09-23T20:45:00.004588+0000","last_active":"2023-09-23T20:49:02.595237+0000","last_peered":"2023-09-23T20:49:02.595237+0000","last_clean":"2023-09-23T20:49:02.595237+0000","last_became_active":"2023-09-23T20:45:00.004185+0000","last_became_peered":"2023-09-23T20:45:00.004185+0000","last_unstale":"2023-09-23T20:49:02.595237+0000","last_undegraded":"2023-09-23T20:49:02.595237+0000","last_fullsized":"2023-09-23T20:49:02.595237+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:47.406920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.059029+0000","last_change":"2023-09-23T20:45:00.007439+0000","last_active":"2023-09-23T20:48:58.059029+0000","last_peered":"2023-09-23T20:48:58.059029+0000","last_clean":"2023-09-23T20:48:58.059029+0000","last_became_active":"2023-09-23T20:45:00.007046+0000","last_became_peered":"2023-09-23T20:45:00.007046+0000","last_unstale":"2023-09-23T20:48:58.059029+0000","last_undegraded":"2023-09-23T20:48:58.059029+0000","last_fullsized":"2023-09-23T20:48:58.059029+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:42.363434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058490+0000","last_change":"2023-09-23T20:40:53.853109+0000","last_active":"2023-09-23T20:48:58.058490+0000","last_peered":"2023-09-23T20:48:58.058490+0000","last_clean":"2023-09-23T20:48:58.058490+0000","last_became_active":"2023-09-23T20:40:53.852788+0000","last_became_peered":"2023-09-23T20:40:53.852788+0000","last_unstale":"2023-09-23T20:48:58.058490+0000","last_undegraded":"2023-09-23T20:48:58.058490+0000","last_fullsized":"2023-09-23T20:48:58.058490+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:55.251424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058273+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:48:58.058273+0000","last_peered":"2023-09-23T20:48:58.058273+0000","last_clean":"2023-09-23T20:48:58.058273+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:48:58.058273+0000","last_undegraded":"2023-09-23T20:48:58.058273+0000","last_fullsized":"2023-09-23T20:48:58.058273+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139764+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:48:58.139764+0000","last_peered":"2023-09-23T20:48:58.139764+0000","last_clean":"2023-09-23T20:48:58.139764+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:48:58.139764+0000","last_undegraded":"2023-09-23T20:48:58.139764+0000","last_fullsized":"2023-09-23T20:48:58.139764+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.268923+0000","last_change":"2023-09-23T20:46:05.075939+0000","last_active":"2023-09-23T20:48:58.268923+0000","last_peered":"2023-09-23T20:48:58.268923+0000","last_clean":"2023-09-23T20:48:58.268923+0000","last_became_active":"2023-09-23T20:46:05.075634+0000","last_became_peered":"2023-09-23T20:46:05.075634+0000","last_unstale":"2023-09-23T20:48:58.268923+0000","last_undegraded":"2023-09-23T20:48:58.268923+0000","last_fullsized":"2023-09-23T20:48:58.268923+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:46.867950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.596359+0000","last_change":"2023-09-23T20:45:00.002611+0000","last_active":"2023-09-23T20:49:02.596359+0000","last_peered":"2023-09-23T20:49:02.596359+0000","last_clean":"2023-09-23T20:49:02.596359+0000","last_became_active":"2023-09-23T20:45:00.002331+0000","last_became_peered":"2023-09-23T20:45:00.002331+0000","last_unstale":"2023-09-23T20:49:02.596359+0000","last_undegraded":"2023-09-23T20:49:02.596359+0000","last_fullsized":"2023-09-23T20:49:02.596359+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:19.968130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058681+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:48:58.058681+0000","last_peered":"2023-09-23T20:48:58.058681+0000","last_clean":"2023-09-23T20:48:58.058681+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:48:58.058681+0000","last_undegraded":"2023-09-23T20:48:58.058681+0000","last_fullsized":"2023-09-23T20:48:58.058681+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":503,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139669+0000","last_change":"2023-09-23T20:40:53.853903+0000","last_active":"2023-09-23T20:48:58.139669+0000","last_peered":"2023-09-23T20:48:58.139669+0000","last_clean":"2023-09-23T20:48:58.139669+0000","last_became_active":"2023-09-23T20:40:53.852537+0000","last_became_peered":"2023-09-23T20:40:53.852537+0000","last_unstale":"2023-09-23T20:48:58.139669+0000","last_undegraded":"2023-09-23T20:48:58.139669+0000","last_fullsized":"2023-09-23T20:48:58.139669+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:49.454779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055032+0000","last_change":"2023-09-23T20:48:07.521001+0000","last_active":"2023-09-23T20:48:58.055032+0000","last_peered":"2023-09-23T20:48:58.055032+0000","last_clean":"2023-09-23T20:48:58.055032+0000","last_became_active":"2023-09-23T20:48:07.520697+0000","last_became_peered":"2023-09-23T20:48:07.520697+0000","last_unstale":"2023-09-23T20:48:58.055032+0000","last_undegraded":"2023-09-23T20:48:58.055032+0000","last_fullsized":"2023-09-23T20:48:58.055032+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:21.120614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060569+0000","last_change":"2023-09-23T20:30:22.772777+0000","last_active":"2023-09-23T20:48:58.060569+0000","last_peered":"2023-09-23T20:48:58.060569+0000","last_clean":"2023-09-23T20:48:58.060569+0000","last_became_active":"2023-09-23T20:30:22.772481+0000","last_became_peered":"2023-09-23T20:30:22.772481+0000","last_unstale":"2023-09-23T20:48:58.060569+0000","last_undegraded":"2023-09-23T20:48:58.060569+0000","last_fullsized":"2023-09-23T20:48:58.060569+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:53.724607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139632+0000","last_change":"2023-09-23T20:45:00.001911+0000","last_active":"2023-09-23T20:48:58.139632+0000","last_peered":"2023-09-23T20:48:58.139632+0000","last_clean":"2023-09-23T20:48:58.139632+0000","last_became_active":"2023-09-23T20:45:00.001531+0000","last_became_peered":"2023-09-23T20:45:00.001531+0000","last_unstale":"2023-09-23T20:48:58.139632+0000","last_undegraded":"2023-09-23T20:48:58.139632+0000","last_fullsized":"2023-09-23T20:48:58.139632+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:17.927950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.074369+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:49:02.074369+0000","last_peered":"2023-09-23T20:49:02.074369+0000","last_clean":"2023-09-23T20:49:02.074369+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:49:02.074369+0000","last_undegraded":"2023-09-23T20:49:02.074369+0000","last_fullsized":"2023-09-23T20:49:02.074369+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076247+0000","last_change":"2023-09-23T20:46:05.077316+0000","last_active":"2023-09-23T20:49:02.076247+0000","last_peered":"2023-09-23T20:49:02.076247+0000","last_clean":"2023-09-23T20:49:02.076247+0000","last_became_active":"2023-09-23T20:46:05.077044+0000","last_became_peered":"2023-09-23T20:46:05.077044+0000","last_unstale":"2023-09-23T20:49:02.076247+0000","last_undegraded":"2023-09-23T20:49:02.076247+0000","last_fullsized":"2023-09-23T20:49:02.076247+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:56.517854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140070+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:48:58.140070+0000","last_peered":"2023-09-23T20:48:58.140070+0000","last_clean":"2023-09-23T20:48:58.140070+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:48:58.140070+0000","last_undegraded":"2023-09-23T20:48:58.140070+0000","last_fullsized":"2023-09-23T20:48:58.140070+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":87,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595415+0000","last_change":"2023-09-23T20:46:05.079434+0000","last_active":"2023-09-23T20:49:02.595415+0000","last_peered":"2023-09-23T20:49:02.595415+0000","last_clean":"2023-09-23T20:49:02.595415+0000","last_became_active":"2023-09-23T20:46:05.079111+0000","last_became_peered":"2023-09-23T20:46:05.079111+0000","last_unstale":"2023-09-23T20:49:02.595415+0000","last_undegraded":"2023-09-23T20:49:02.595415+0000","last_fullsized":"2023-09-23T20:49:02.595415+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.869457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072216+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:49:02.072216+0000","last_peered":"2023-09-23T20:49:02.072216+0000","last_clean":"2023-09-23T20:49:02.072216+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:49:02.072216+0000","last_undegraded":"2023-09-23T20:49:02.072216+0000","last_fullsized":"2023-09-23T20:49:02.072216+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055650+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:48:58.055650+0000","last_peered":"2023-09-23T20:48:58.055650+0000","last_clean":"2023-09-23T20:48:58.055650+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:48:58.055650+0000","last_undegraded":"2023-09-23T20:48:58.055650+0000","last_fullsized":"2023-09-23T20:48:58.055650+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595600+0000","last_change":"2023-09-23T20:45:00.005180+0000","last_active":"2023-09-23T20:49:02.595600+0000","last_peered":"2023-09-23T20:49:02.595600+0000","last_clean":"2023-09-23T20:49:02.595600+0000","last_became_active":"2023-09-23T20:45:00.004791+0000","last_became_peered":"2023-09-23T20:45:00.004791+0000","last_unstale":"2023-09-23T20:49:02.595600+0000","last_undegraded":"2023-09-23T20:49:02.595600+0000","last_fullsized":"2023-09-23T20:49:02.595600+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:37.690560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055025+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:48:58.055025+0000","last_peered":"2023-09-23T20:48:58.055025+0000","last_clean":"2023-09-23T20:48:58.055025+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:48:58.055025+0000","last_undegraded":"2023-09-23T20:48:58.055025+0000","last_fullsized":"2023-09-23T20:48:58.055025+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060682+0000","last_change":"2023-09-23T20:46:05.073849+0000","last_active":"2023-09-23T20:48:58.060682+0000","last_peered":"2023-09-23T20:48:58.060682+0000","last_clean":"2023-09-23T20:48:58.060682+0000","last_became_active":"2023-09-23T20:46:05.073536+0000","last_became_peered":"2023-09-23T20:46:05.073536+0000","last_unstale":"2023-09-23T20:48:58.060682+0000","last_undegraded":"2023-09-23T20:48:58.060682+0000","last_fullsized":"2023-09-23T20:48:58.060682+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.086216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076148+0000","last_change":"2023-09-23T20:45:00.005077+0000","last_active":"2023-09-23T20:49:02.076148+0000","last_peered":"2023-09-23T20:49:02.076148+0000","last_clean":"2023-09-23T20:49:02.076148+0000","last_became_active":"2023-09-23T20:45:00.004383+0000","last_became_peered":"2023-09-23T20:45:00.004383+0000","last_unstale":"2023-09-23T20:49:02.076148+0000","last_undegraded":"2023-09-23T20:49:02.076148+0000","last_fullsized":"2023-09-23T20:49:02.076148+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:45.178382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057732+0000","last_change":"2023-09-23T20:30:22.131210+0000","last_active":"2023-09-23T20:48:58.057732+0000","last_peered":"2023-09-23T20:48:58.057732+0000","last_clean":"2023-09-23T20:48:58.057732+0000","last_became_active":"2023-09-23T20:30:22.130879+0000","last_became_peered":"2023-09-23T20:30:22.130879+0000","last_unstale":"2023-09-23T20:48:58.057732+0000","last_undegraded":"2023-09-23T20:48:58.057732+0000","last_fullsized":"2023-09-23T20:48:58.057732+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:10.733014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":99,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140129+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:48:58.140129+0000","last_peered":"2023-09-23T20:48:58.140129+0000","last_clean":"2023-09-23T20:48:58.140129+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:48:58.140129+0000","last_undegraded":"2023-09-23T20:48:58.140129+0000","last_fullsized":"2023-09-23T20:48:58.140129+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139585+0000","last_change":"2023-09-23T20:48:07.539824+0000","last_active":"2023-09-23T20:48:58.139585+0000","last_peered":"2023-09-23T20:48:58.139585+0000","last_clean":"2023-09-23T20:48:58.139585+0000","last_became_active":"2023-09-23T20:48:07.539530+0000","last_became_peered":"2023-09-23T20:48:07.539530+0000","last_unstale":"2023-09-23T20:48:58.139585+0000","last_undegraded":"2023-09-23T20:48:58.139585+0000","last_fullsized":"2023-09-23T20:48:58.139585+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:21.997036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058551+0000","last_change":"2023-09-23T20:45:00.008268+0000","last_active":"2023-09-23T20:48:58.058551+0000","last_peered":"2023-09-23T20:48:58.058551+0000","last_clean":"2023-09-23T20:48:58.058551+0000","last_became_active":"2023-09-23T20:45:00.006393+0000","last_became_peered":"2023-09-23T20:45:00.006393+0000","last_unstale":"2023-09-23T20:48:58.058551+0000","last_undegraded":"2023-09-23T20:48:58.058551+0000","last_fullsized":"2023-09-23T20:48:58.058551+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:41.255405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058217+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.058217+0000","last_peered":"2023-09-23T20:48:58.058217+0000","last_clean":"2023-09-23T20:48:58.058217+0000","last_became_active":"2023-09-23T20:48:07.544066+0000","last_became_peered":"2023-09-23T20:48:07.544066+0000","last_unstale":"2023-09-23T20:48:58.058217+0000","last_undegraded":"2023-09-23T20:48:58.058217+0000","last_fullsized":"2023-09-23T20:48:58.058217+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:28.905486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058955+0000","last_change":"2023-09-23T20:30:22.104198+0000","last_active":"2023-09-23T20:48:58.058955+0000","last_peered":"2023-09-23T20:48:58.058955+0000","last_clean":"2023-09-23T20:48:58.058955+0000","last_became_active":"2023-09-23T20:30:22.103970+0000","last_became_peered":"2023-09-23T20:30:22.103970+0000","last_unstale":"2023-09-23T20:48:58.058955+0000","last_undegraded":"2023-09-23T20:48:58.058955+0000","last_fullsized":"2023-09-23T20:48:58.058955+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:32.820350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058740+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:48:58.058740+0000","last_peered":"2023-09-23T20:48:58.058740+0000","last_clean":"2023-09-23T20:48:58.058740+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:48:58.058740+0000","last_undegraded":"2023-09-23T20:48:58.058740+0000","last_fullsized":"2023-09-23T20:48:58.058740+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055468+0000","last_change":"2023-09-23T20:48:07.521311+0000","last_active":"2023-09-23T20:48:58.055468+0000","last_peered":"2023-09-23T20:48:58.055468+0000","last_clean":"2023-09-23T20:48:58.055468+0000","last_became_active":"2023-09-23T20:48:07.521030+0000","last_became_peered":"2023-09-23T20:48:07.521030+0000","last_unstale":"2023-09-23T20:48:58.055468+0000","last_undegraded":"2023-09-23T20:48:58.055468+0000","last_fullsized":"2023-09-23T20:48:58.055468+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.849053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055021+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:48:58.055021+0000","last_peered":"2023-09-23T20:48:58.055021+0000","last_clean":"2023-09-23T20:48:58.055021+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:48:58.055021+0000","last_undegraded":"2023-09-23T20:48:58.055021+0000","last_fullsized":"2023-09-23T20:48:58.055021+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072229+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:49:02.072229+0000","last_peered":"2023-09-23T20:49:02.072229+0000","last_clean":"2023-09-23T20:49:02.072229+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:49:02.072229+0000","last_undegraded":"2023-09-23T20:49:02.072229+0000","last_fullsized":"2023-09-23T20:49:02.072229+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057625+0000","last_change":"2023-09-23T20:45:00.004734+0000","last_active":"2023-09-23T20:48:58.057625+0000","last_peered":"2023-09-23T20:48:58.057625+0000","last_clean":"2023-09-23T20:48:58.057625+0000","last_became_active":"2023-09-23T20:45:00.004413+0000","last_became_peered":"2023-09-23T20:45:00.004413+0000","last_unstale":"2023-09-23T20:48:58.057625+0000","last_undegraded":"2023-09-23T20:48:58.057625+0000","last_fullsized":"2023-09-23T20:48:58.057625+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:57.454649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201531,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":2180,"kb_used_omap":0,"kb_used_meta":296384,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":2232320,"data_stored":1659199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":303497216},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":4.4119999999999999,"15min":2.488},"min":{"1min":0.371,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.7990000000000004,"5min":225.56999999999999,"15min":225.56999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.4730000000000001,"5min":4.6440000000000001,"15min":2.8570000000000002},"min":{"1min":0.36599999999999999,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":8.1549999999999994,"5min":225.61099999999999,"15min":225.61099999999999},"last":0.47799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.132,"5min":3.2690000000000001,"15min":1.867},"min":{"1min":0.377,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":128.97999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.443},{"interface":"front","average":{"1min":10.537000000000001,"5min":6.306,"15min":2.8900000000000001},"min":{"1min":0.432,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":128.81200000000001,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":4.1689999999999996,"15min":3.4820000000000002},"min":{"1min":0.40000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.2249999999999996,"5min":142.321,"15min":142.321},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.3140000000000001,"15min":2.6619999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.8149999999999999,"5min":142.29300000000001,"15min":142.29300000000001},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.209,"5min":10.209,"15min":10.209},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":138.345,"5min":138.345,"15min":138.345},"last":0.372},{"interface":"front","average":{"1min":9.423,"5min":9.423,"15min":9.423},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":133.76400000000001,"5min":133.76400000000001,"15min":133.76400000000001},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4220000000000002,"5min":1.4770000000000001,"15min":1.7270000000000001},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.216},"max":{"1min":9.1400000000000006,"5min":9.1400000000000006,"15min":103.541},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.3839999999999999,"5min":1.478,"15min":1.893},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.4649999999999999,"5min":8.4649999999999999,"15min":107.514},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":1.5509999999999999,"15min":1.927},"min":{"1min":0.32300000000000001,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":9.4440000000000008,"5min":9.4440000000000008,"15min":67.180999999999997},"last":0.316},{"interface":"front","average":{"1min":2.9969999999999999,"5min":1.762,"15min":1.8879999999999999},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.2110000000000003,"5min":13.156000000000001,"15min":67.135000000000005},"last":0.45300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:48:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5550000000000002,"5min":1.647,"15min":1.653},"min":{"1min":0.38700000000000001,"5min":0.30399999999999999,"15min":0.28899999999999998},"max":{"1min":10.509,"5min":10.509,"15min":16.032},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.0619999999999998,"5min":1.2929999999999999,"15min":1.381},"min":{"1min":0.35999999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.4350000000000005,"5min":8.4350000000000005,"15min":11.672000000000001},"last":0.48899999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463168,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":255164,"kb_used_data":2164,"kb_used_omap":0,"kb_used_meta":252992,"kb_avail":93489988,"statfs":{"total":95995035648,"available":95733747712,"internally_reserved":0,"allocated":2215936,"data_stored":1658846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":259063808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":7.2859999999999996,"5min":3.4649999999999999,"15min":2.4359999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.20399999999999999},"max":{"1min":92.164000000000001,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":7.2859999999999996,"5min":3.04,"15min":2.431},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":92.138000000000005,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.48499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.5110000000000001,"5min":2.907,"15min":2.073},"min":{"1min":0.30399999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":96.623999999999995,"5min":96.623999999999995,"15min":96.623999999999995},"last":0.46500000000000002},{"interface":"front","average":{"1min":6.5579999999999998,"5min":2.8090000000000002,"15min":1.839},"min":{"1min":0.30399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":96.254000000000005,"5min":96.254000000000005,"15min":96.254000000000005},"last":0.58399999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.3319999999999999,"15min":2.3029999999999999},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.7009999999999996,"5min":8.9849999999999994,"15min":8.9849999999999994},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.5209999999999999,"5min":2.6269999999999998,"15min":2.6070000000000002},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":10.59,"5min":10.59,"15min":10.59},"last":0.621}]},{"osd":3,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.125,"5min":6.125,"15min":6.125},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":79.668000000000006,"5min":79.668000000000006,"15min":79.668000000000006},"last":0.502},{"interface":"front","average":{"1min":5.9980000000000002,"5min":5.9980000000000002,"15min":5.9980000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":79.700000000000003,"5min":79.700000000000003,"15min":79.700000000000003},"last":0.63200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.2709999999999999,"15min":1.341},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":9.0709999999999997,"5min":9.0709999999999997,"15min":16.247},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.756,"15min":1.6479999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.27000000000000002},"max":{"1min":9.8650000000000002,"5min":9.8650000000000002,"15min":17.163},"last":0.376}]},{"osd":5,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.556,"15min":1.518},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.26000000000000001},"max":{"1min":9.0340000000000007,"5min":9.1969999999999992,"15min":11.484999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.214,"5min":1.7490000000000001,"15min":1.631},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":9.8949999999999996,"5min":9.8949999999999996,"15min":17.321000000000002},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:49:29 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":1.5589999999999999,"15min":1.4259999999999999},"min":{"1min":0.40200000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2590000000000003,"5min":9.3249999999999993,"15min":19.451000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.605,"15min":1.6299999999999999},"min":{"1min":0.40000000000000002,"5min":0.36099999999999999,"15min":0.32100000000000001},"max":{"1min":9.7959999999999994,"5min":9.7959999999999994,"15min":16.295999999999999},"last":0.435}]}]},{"osd":1,"up_from":114,"seq":489626271802,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236760,"kb_used_data":1744,"kb_used_omap":4,"kb_used_meta":235003,"kb_avail":93508392,"statfs":{"total":95995035648,"available":95752593408,"internally_reserved":0,"allocated":1785856,"data_stored":1086600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":240643982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":3.206,"5min":2.4340000000000002,"15min":2.1160000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.191},"max":{"1min":48.350999999999999,"5min":48.350999999999999,"15min":48.350999999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":4.7350000000000003,"5min":2.698,"15min":2.048},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":52.018000000000001,"5min":52.018000000000001,"15min":52.018000000000001},"last":0.25700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":2.0350000000000001,"15min":2.1789999999999998},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.7479999999999993,"5min":11.593999999999999,"15min":11.593999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.613,"15min":2.855},"min":{"1min":0.29499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.465999999999999,"5min":14.153,"15min":14.153},"last":0.32200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.73,"15min":2.73},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.4969999999999999,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.04,"5min":2.04,"15min":2.04},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.5630000000000006,"5min":8.5630000000000006,"15min":8.5630000000000006},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.663,"15min":1.349},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.439,"5min":10.439,"15min":10.439},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.496,"5min":1.431,"15min":1.1870000000000001},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":10.34,"5min":10.34,"15min":10.34},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":2.4950000000000001,"15min":1.853},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.5410000000000004,"5min":10.177,"15min":10.177},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.415,"5min":1.635,"15min":1.268},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.3890000000000002,"5min":6.8680000000000003,"15min":6.8680000000000003},"last":0.74099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.9990000000000001,"15min":1.7010000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.25,"5min":10.25,"15min":10.25},"last":0.752},{"interface":"front","average":{"1min":1.008,"5min":1.581,"15min":1.3149999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.4489999999999998,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":0.57899999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.871,"15min":1.4790000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.3499999999999996,"5min":7.4480000000000004,"15min":7.4480000000000004},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.948,"5min":2.0569999999999999,"15min":1.911},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.0259999999999998,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.72699999999999998}]}]},{"osd":0,"up_from":92,"seq":395136991340,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":402740,"kb_used_data":2348,"kb_used_omap":5,"kb_used_meta":400378,"kb_avail":93342412,"statfs":{"total":95995035648,"available":95582629888,"internally_reserved":0,"allocated":2404352,"data_stored":1676026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":409987706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5780000000000003,"5min":3.2480000000000002,"15min":2.4420000000000002},"min":{"1min":0.20000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.140000000000001,"5min":64.140000000000001,"15min":64.140000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":4.5860000000000003,"5min":3.3679999999999999,"15min":2.5270000000000001},"min":{"1min":0.20999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":59.570999999999998,"5min":59.570999999999998,"15min":59.570999999999998},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":2.319,"15min":2.0569999999999999},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.5839999999999996,"5min":49.652000000000001,"15min":49.652000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.8,"5min":4.0250000000000004,"15min":4.3360000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4720000000000004,"5min":49.595999999999997,"15min":49.595999999999997},"last":0.377}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":41.810000000000002,"5min":41.810000000000002,"15min":41.810000000000002},"last":0.39700000000000002},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":41.880000000000003,"5min":41.880000000000003,"15min":41.880000000000003},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.9750000000000001,"15min":1.7589999999999999},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.3559999999999999,"5min":12.973000000000001,"15min":12.973000000000001},"last":0.627},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.6850000000000001,"15min":2.1419999999999999},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":8.234,"5min":12.148,"15min":12.148},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":2.6040000000000001,"15min":2.1749999999999998},"min":{"1min":0.40600000000000003,"5min":0.30199999999999999,"15min":0.28799999999999998},"max":{"1min":5.6550000000000002,"5min":14.051,"15min":14.051},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.9410000000000001,"15min":2.2989999999999999},"min":{"1min":0.371,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.3119999999999994,"5min":12.332000000000001,"15min":12.332000000000001},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.1219999999999999,"15min":2.3580000000000001},"min":{"1min":0.39600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":7.8959999999999999,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.1659999999999999,"5min":2.2519999999999998,"15min":1.708},"min":{"1min":0.36299999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.1189999999999998,"5min":20.399999999999999,"15min":20.399999999999999},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":1.829,"15min":2.4550000000000001},"min":{"1min":0.46700000000000003,"5min":0.41299999999999998,"15min":0.35099999999999998},"max":{"1min":7.3780000000000001,"5min":12.23,"15min":12.23},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.9350000000000001,"5min":2.2149999999999999,"15min":1.7849999999999999},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.7430000000000003,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.65000000000000002}]}]},{"osd":2,"up_from":129,"seq":554050781229,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270952,"kb_used_data":1696,"kb_used_omap":7,"kb_used_meta":269240,"kb_avail":93474200,"statfs":{"total":95995035648,"available":95717580800,"internally_reserved":0,"allocated":1736704,"data_stored":1078264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":275701841},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.5070000000000001,"15min":2.9980000000000002},"min":{"1min":0.21299999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.5490000000000004,"5min":14.327,"15min":14.327},"last":0.31},{"interface":"front","average":{"1min":1.024,"5min":2.323,"15min":2.7589999999999999},"min":{"1min":0.217,"5min":0.186,"15min":0.186},"max":{"1min":6.2359999999999998,"5min":14.757999999999999,"15min":14.757999999999999},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":2.5880000000000001,"15min":2.8900000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.4039999999999999,"5min":13.023,"15min":13.023},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.3759999999999999,"15min":3.0019999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.0419999999999998,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":0.32900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:49:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.959,"5min":11.959,"15min":11.959},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.6539999999999999,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.819,"15min":1.9370000000000001},"min":{"1min":0.34100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.8440000000000003,"5min":7.8440000000000003,"15min":7.8440000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.411,"5min":2.3490000000000002,"15min":2.7109999999999999},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.317},"max":{"1min":6.2789999999999999,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.61199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.5630000000000002,"15min":3.0539999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.3760000000000003,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.7509999999999999,"5min":2.5030000000000001,"15min":2.8610000000000002},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3630000000000004,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.67500000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.7210000000000001,"15min":3.29},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.3449999999999998,"5min":7.476,"15min":7.476},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.1379999999999999,"5min":2.806,"15min":3.0790000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":9.5220000000000002,"5min":12.595000000000001,"15min":12.595000000000001},"last":0.71099999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.2320000000000002,"15min":2.5350000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.4909999999999997,"5min":6.4989999999999997,"15min":6.4989999999999997},"last":0.68799999999999994},{"interface":"front","average":{"1min":1.669,"5min":2.5259999999999998,"15min":2.8839999999999999},"min":{"1min":0.41399999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.5979999999999999,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":0.53400000000000003}]}]},{"osd":3,"up_from":148,"seq":635655159829,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":135308,"kb_used_data":1668,"kb_used_omap":2,"kb_used_meta":133629,"kb_avail":93609844,"statfs":{"total":95995035648,"available":95856480256,"internally_reserved":0,"allocated":1708032,"data_stored":1075110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":136836149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":14.332000000000001,"5min":14.332000000000001,"15min":14.332000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":253.68100000000001,"5min":253.68100000000001,"15min":253.68100000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":13.208,"5min":13.208,"15min":13.208},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":253.46199999999999,"5min":253.46199999999999,"15min":253.46199999999999},"last":0.27900000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":10.417999999999999,"5min":10.417999999999999,"15min":10.417999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":172.77099999999999,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":9.4079999999999995,"5min":9.4079999999999995,"15min":9.4079999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":172.36600000000001,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.4619999999999997,"5min":5.4619999999999997,"15min":5.4619999999999997},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.307}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.4749999999999996,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.2170000000000001,"15min":2.2170000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":16.408999999999999,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.5750000000000002,"5min":3.5750000000000002,"15min":3.5750000000000002},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.958,"5min":1.958,"15min":1.958},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":9.7780000000000005,"5min":9.7780000000000005,"15min":9.7780000000000005},"last":0.67000000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":1.972,"15min":1.972},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.7509999999999994,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":0.69199999999999995},{"interface":"front","average":{"1min":2.6419999999999999,"5min":2.6419999999999999,"15min":2.6419999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":16.451000000000001,"5min":16.451000000000001,"15min":16.451000000000001},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.5640000000000001,"15min":1.5640000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.9450000000000003,"5min":4.9450000000000003,"15min":4.9450000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.351,"5min":2.351,"15min":2.351},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.64100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986441,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307912,"kb_used_data":1600,"kb_used_omap":0,"kb_used_meta":306304,"kb_avail":93437240,"statfs":{"total":95995035648,"available":95679733760,"internally_reserved":0,"allocated":1638400,"data_stored":1069084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":313655296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.52,"15min":1.839},"min":{"1min":0.216,"5min":0.216,"15min":0.214},"max":{"1min":8.0519999999999996,"5min":12.823,"15min":12.823},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.508,"5min":1.998,"15min":1.321},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.22600000000000001},"max":{"1min":4.3090000000000002,"5min":10.413,"15min":13.211},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":8.4019999999999992,"5min":6.6639999999999997,"15min":3.23},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":125.35599999999999,"5min":286.31099999999998,"15min":286.31099999999998},"last":0.67500000000000004},{"interface":"front","average":{"1min":8.0459999999999994,"5min":6.7679999999999998,"15min":3.996},"min":{"1min":0.312,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":125.21899999999999,"5min":285.97000000000003,"15min":285.97000000000003},"last":0.66300000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.48,"15min":1.375},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.988,"5min":2.347,"15min":2.3620000000000001},"min":{"1min":0.39300000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.9450000000000003,"5min":12.773999999999999,"15min":12.773999999999999},"last":0.61799999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":7.5990000000000002,"5min":7.5990000000000002,"15min":7.5990000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":107.285,"5min":107.285,"15min":107.285},"last":0.53500000000000003},{"interface":"front","average":{"1min":7.1920000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":107.303,"5min":107.303,"15min":107.303},"last":0.75600000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.2090000000000001,"15min":1.387},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.1740000000000004,"5min":7.1740000000000004,"15min":17.713999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.512,"5min":1.2,"15min":1.5},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.9939999999999998,"5min":9.0660000000000007,"15min":16.149000000000001},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.4630000000000001,"15min":1.5369999999999999},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.4109999999999996,"5min":16.271000000000001,"15min":21.920999999999999},"last":0.5},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.234,"15min":1.345},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.28599999999999998},"max":{"1min":3.431,"5min":11.718,"15min":11.718},"last":0.63900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":1.4630000000000001,"15min":1.8600000000000001},"min":{"1min":0.33300000000000002,"5min":0.315,"15min":0.31},"max":{"1min":7.1559999999999997,"5min":9.8390000000000004,"15min":32.878},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.4430000000000001,"5min":1.2769999999999999,"15min":1.637},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":6.4199999999999999,"5min":7.8399999999999999,"15min":23.457000000000001},"last":0.60699999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724805,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":365368,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":363776,"kb_avail":93379784,"statfs":{"total":95995035648,"available":95620898816,"internally_reserved":0,"allocated":1622016,"data_stored":1064828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":372506624},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.4330000000000001,"15min":1.7669999999999999},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.7559999999999998,"5min":10.94,"15min":12.880000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.274,"5min":1.754,"15min":1.3859999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":10.028,"5min":10.955,"15min":10.955},"last":0.53600000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":4.5990000000000002,"15min":2.2130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":12.529999999999999,"5min":323.541,"15min":323.541},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.262,"5min":4.7270000000000003,"15min":2.5219999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.6900000000000004,"5min":324.072,"15min":324.072},"last":0.40100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.137,"5min":2.415,"15min":2.423},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.585000000000001,"5min":13.394,"15min":13.394},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.827,"15min":1.663},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7430000000000003,"5min":14.263,"15min":14.263},"last":0.48899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.863,"15min":1.863},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.295,"5min":10.295,"15min":10.295},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.1080000000000005,"5min":8.1080000000000005,"15min":8.1080000000000005},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:48:47 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":0.95099999999999996,"15min":1.2050000000000001},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.216},"max":{"1min":5.8529999999999998,"5min":5.8529999999999998,"15min":9.8800000000000008},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.3,"5min":1.3360000000000001,"15min":1.6000000000000001},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.23999999999999999},"max":{"1min":4.7309999999999999,"5min":8.3170000000000002,"15min":16.869},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":1.325,"15min":1.544},"min":{"1min":0.316,"5min":0.28000000000000003,"15min":0.27000000000000002},"max":{"1min":6.1689999999999996,"5min":8.3010000000000002,"15min":17},"last":0.624},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.2050000000000001,"15min":1.427},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.9870000000000001,"5min":9.6370000000000005,"15min":14.08},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:48:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":1.649,"15min":1.895},"min":{"1min":0.33100000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.5579999999999998,"5min":14.217000000000001,"15min":17.071999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.97999999999999998,"15min":1.222},"min":{"1min":0.29899999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.3620000000000001,"5min":5.601,"15min":8.6370000000000005},"last":0.73899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:49:44.360 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:49:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[211526]: from='client.35091 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:44.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:44.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[219709]: from='client.35091 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:44.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:44.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:44.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.710 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:44 smithi123 ceph-mon[193903]: from='client.35091 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:44 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:44 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:49:45.119 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:49:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:49:44] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2023-09-23T20:49:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:45 smithi123 ceph-mon[193903]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:45 smithi121 ceph-mon[211526]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:45 smithi121 ceph-mon[219709]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:46.663 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:46.665 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:49:47.380 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:47 smithi121 ceph-mon[219709]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:47.380 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:47 smithi121 ceph-mon[219709]: from='client.35097 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:47.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:47 smithi121 ceph-mon[211526]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:47.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:47 smithi121 ceph-mon[211526]: from='client.35097 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:47.398 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":510,"stamp":"2023-09-23T20:49:46.637924+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2272776,"kb_used_data":14984,"kb_used_omap":20,"kb_used_meta":2257707,"kb_avail":747688440,"statfs":{"total":767960285184,"available":765632962560,"internally_reserved":0,"allocated":15343616,"data_stored":10367957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2311892622},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002625"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058647+0000","last_change":"2023-09-23T20:46:05.100324+0000","last_active":"2023-09-23T20:48:58.058647+0000","last_peered":"2023-09-23T20:48:58.058647+0000","last_clean":"2023-09-23T20:48:58.058647+0000","last_became_active":"2023-09-23T20:46:05.099981+0000","last_became_peered":"2023-09-23T20:46:05.099981+0000","last_unstale":"2023-09-23T20:48:58.058647+0000","last_undegraded":"2023-09-23T20:48:58.058647+0000","last_fullsized":"2023-09-23T20:48:58.058647+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:19.876971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055345+0000","last_change":"2023-09-23T20:48:07.517973+0000","last_active":"2023-09-23T20:48:58.055345+0000","last_peered":"2023-09-23T20:48:58.055345+0000","last_clean":"2023-09-23T20:48:58.055345+0000","last_became_active":"2023-09-23T20:48:07.517681+0000","last_became_peered":"2023-09-23T20:48:07.517681+0000","last_unstale":"2023-09-23T20:48:58.055345+0000","last_undegraded":"2023-09-23T20:48:58.055345+0000","last_fullsized":"2023-09-23T20:48:58.055345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.789594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.056881+0000","last_change":"2023-09-23T20:48:07.541333+0000","last_active":"2023-09-23T20:48:58.056881+0000","last_peered":"2023-09-23T20:48:58.056881+0000","last_clean":"2023-09-23T20:48:58.056881+0000","last_became_active":"2023-09-23T20:48:07.541014+0000","last_became_peered":"2023-09-23T20:48:07.541014+0000","last_unstale":"2023-09-23T20:48:58.056881+0000","last_undegraded":"2023-09-23T20:48:58.056881+0000","last_fullsized":"2023-09-23T20:48:58.056881+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:48.678651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140248+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:48:58.140248+0000","last_peered":"2023-09-23T20:48:58.140248+0000","last_clean":"2023-09-23T20:48:58.140248+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:48:58.140248+0000","last_undegraded":"2023-09-23T20:48:58.140248+0000","last_fullsized":"2023-09-23T20:48:58.140248+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054913+0000","last_change":"2023-09-23T20:48:07.540435+0000","last_active":"2023-09-23T20:48:58.054913+0000","last_peered":"2023-09-23T20:48:58.054913+0000","last_clean":"2023-09-23T20:48:58.054913+0000","last_became_active":"2023-09-23T20:48:07.540062+0000","last_became_peered":"2023-09-23T20:48:07.540062+0000","last_unstale":"2023-09-23T20:48:58.054913+0000","last_undegraded":"2023-09-23T20:48:58.054913+0000","last_fullsized":"2023-09-23T20:48:58.054913+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:19.449218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060678+0000","last_change":"2023-09-23T20:45:00.009034+0000","last_active":"2023-09-23T20:48:58.060678+0000","last_peered":"2023-09-23T20:48:58.060678+0000","last_clean":"2023-09-23T20:48:58.060678+0000","last_became_active":"2023-09-23T20:45:00.008706+0000","last_became_peered":"2023-09-23T20:45:00.008706+0000","last_unstale":"2023-09-23T20:48:58.060678+0000","last_undegraded":"2023-09-23T20:48:58.060678+0000","last_fullsized":"2023-09-23T20:48:58.060678+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:29.204761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140415+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:48:58.140415+0000","last_peered":"2023-09-23T20:48:58.140415+0000","last_clean":"2023-09-23T20:48:58.140415+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:48:58.140415+0000","last_undegraded":"2023-09-23T20:48:58.140415+0000","last_fullsized":"2023-09-23T20:48:58.140415+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054946+0000","last_change":"2023-09-23T20:48:07.533499+0000","last_active":"2023-09-23T20:48:58.054946+0000","last_peered":"2023-09-23T20:48:58.054946+0000","last_clean":"2023-09-23T20:48:58.054946+0000","last_became_active":"2023-09-23T20:48:07.520669+0000","last_became_peered":"2023-09-23T20:48:07.520669+0000","last_unstale":"2023-09-23T20:48:58.054946+0000","last_undegraded":"2023-09-23T20:48:58.054946+0000","last_fullsized":"2023-09-23T20:48:58.054946+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:55.109803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058059+0000","last_change":"2023-09-23T20:45:00.007125+0000","last_active":"2023-09-23T20:48:58.058059+0000","last_peered":"2023-09-23T20:48:58.058059+0000","last_clean":"2023-09-23T20:48:58.058059+0000","last_became_active":"2023-09-23T20:45:00.006403+0000","last_became_peered":"2023-09-23T20:45:00.006403+0000","last_unstale":"2023-09-23T20:48:58.058059+0000","last_undegraded":"2023-09-23T20:48:58.058059+0000","last_fullsized":"2023-09-23T20:48:58.058059+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:27.906490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055087+0000","last_change":"2023-09-23T20:48:07.525099+0000","last_active":"2023-09-23T20:48:58.055087+0000","last_peered":"2023-09-23T20:48:58.055087+0000","last_clean":"2023-09-23T20:48:58.055087+0000","last_became_active":"2023-09-23T20:48:07.524868+0000","last_became_peered":"2023-09-23T20:48:07.524868+0000","last_unstale":"2023-09-23T20:48:58.055087+0000","last_undegraded":"2023-09-23T20:48:58.055087+0000","last_fullsized":"2023-09-23T20:48:58.055087+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:19.340586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140203+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:48:58.140203+0000","last_peered":"2023-09-23T20:48:58.140203+0000","last_clean":"2023-09-23T20:48:58.140203+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:48:58.140203+0000","last_undegraded":"2023-09-23T20:48:58.140203+0000","last_fullsized":"2023-09-23T20:48:58.140203+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595558+0000","last_change":"2023-09-23T20:46:05.099313+0000","last_active":"2023-09-23T20:49:02.595558+0000","last_peered":"2023-09-23T20:49:02.595558+0000","last_clean":"2023-09-23T20:49:02.595558+0000","last_became_active":"2023-09-23T20:46:05.099055+0000","last_became_peered":"2023-09-23T20:46:05.099055+0000","last_unstale":"2023-09-23T20:49:02.595558+0000","last_undegraded":"2023-09-23T20:49:02.595558+0000","last_fullsized":"2023-09-23T20:49:02.595558+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:51.820608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058022+0000","last_change":"2023-09-23T20:30:22.107118+0000","last_active":"2023-09-23T20:48:58.058022+0000","last_peered":"2023-09-23T20:48:58.058022+0000","last_clean":"2023-09-23T20:48:58.058022+0000","last_became_active":"2023-09-23T20:30:22.106839+0000","last_became_peered":"2023-09-23T20:30:22.106839+0000","last_unstale":"2023-09-23T20:48:58.058022+0000","last_undegraded":"2023-09-23T20:48:58.058022+0000","last_fullsized":"2023-09-23T20:48:58.058022+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:02.981333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058390+0000","last_change":"2023-09-23T20:45:00.007712+0000","last_active":"2023-09-23T20:48:58.058390+0000","last_peered":"2023-09-23T20:48:58.058390+0000","last_clean":"2023-09-23T20:48:58.058390+0000","last_became_active":"2023-09-23T20:45:00.007399+0000","last_became_peered":"2023-09-23T20:45:00.007399+0000","last_unstale":"2023-09-23T20:48:58.058390+0000","last_undegraded":"2023-09-23T20:48:58.058390+0000","last_fullsized":"2023-09-23T20:48:58.058390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:48.842612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139939+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:48:58.139939+0000","last_peered":"2023-09-23T20:48:58.139939+0000","last_clean":"2023-09-23T20:48:58.139939+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:48:58.139939+0000","last_undegraded":"2023-09-23T20:48:58.139939+0000","last_fullsized":"2023-09-23T20:48:58.139939+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055261+0000","last_change":"2023-09-23T20:48:07.540014+0000","last_active":"2023-09-23T20:48:58.055261+0000","last_peered":"2023-09-23T20:48:58.055261+0000","last_clean":"2023-09-23T20:48:58.055261+0000","last_became_active":"2023-09-23T20:48:07.539722+0000","last_became_peered":"2023-09-23T20:48:07.539722+0000","last_unstale":"2023-09-23T20:48:58.055261+0000","last_undegraded":"2023-09-23T20:48:58.055261+0000","last_fullsized":"2023-09-23T20:48:58.055261+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.937232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140140+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:48:58.140140+0000","last_peered":"2023-09-23T20:48:58.140140+0000","last_clean":"2023-09-23T20:48:58.140140+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:48:58.140140+0000","last_undegraded":"2023-09-23T20:48:58.140140+0000","last_fullsized":"2023-09-23T20:48:58.140140+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058777+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:48:58.058777+0000","last_peered":"2023-09-23T20:48:58.058777+0000","last_clean":"2023-09-23T20:48:58.058777+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:48:58.058777+0000","last_undegraded":"2023-09-23T20:48:58.058777+0000","last_fullsized":"2023-09-23T20:48:58.058777+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139702+0000","last_change":"2023-09-23T20:30:22.110586+0000","last_active":"2023-09-23T20:48:58.139702+0000","last_peered":"2023-09-23T20:48:58.139702+0000","last_clean":"2023-09-23T20:48:58.139702+0000","last_became_active":"2023-09-23T20:30:22.110248+0000","last_became_peered":"2023-09-23T20:30:22.110248+0000","last_unstale":"2023-09-23T20:48:58.139702+0000","last_undegraded":"2023-09-23T20:48:58.139702+0000","last_fullsized":"2023-09-23T20:48:58.139702+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:29.867944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140341+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:48:58.140341+0000","last_peered":"2023-09-23T20:48:58.140341+0000","last_clean":"2023-09-23T20:48:58.140341+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:48:58.140341+0000","last_undegraded":"2023-09-23T20:48:58.140341+0000","last_fullsized":"2023-09-23T20:48:58.140341+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060100+0000","last_change":"2023-09-23T20:40:53.862653+0000","last_active":"2023-09-23T20:48:58.060100+0000","last_peered":"2023-09-23T20:48:58.060100+0000","last_clean":"2023-09-23T20:48:58.060100+0000","last_became_active":"2023-09-23T20:40:53.862259+0000","last_became_peered":"2023-09-23T20:40:53.862259+0000","last_unstale":"2023-09-23T20:48:58.060100+0000","last_undegraded":"2023-09-23T20:48:58.060100+0000","last_fullsized":"2023-09-23T20:48:58.060100+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:28.626871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055685+0000","last_change":"2023-09-23T20:48:07.557905+0000","last_active":"2023-09-23T20:48:58.055685+0000","last_peered":"2023-09-23T20:48:58.055685+0000","last_clean":"2023-09-23T20:48:58.055685+0000","last_became_active":"2023-09-23T20:48:07.557568+0000","last_became_peered":"2023-09-23T20:48:07.557568+0000","last_unstale":"2023-09-23T20:48:58.055685+0000","last_undegraded":"2023-09-23T20:48:58.055685+0000","last_fullsized":"2023-09-23T20:48:58.055685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:14.229704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140303+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:48:58.140303+0000","last_peered":"2023-09-23T20:48:58.140303+0000","last_clean":"2023-09-23T20:48:58.140303+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:48:58.140303+0000","last_undegraded":"2023-09-23T20:48:58.140303+0000","last_fullsized":"2023-09-23T20:48:58.140303+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058595+0000","last_change":"2023-09-23T20:30:22.772475+0000","last_active":"2023-09-23T20:48:58.058595+0000","last_peered":"2023-09-23T20:48:58.058595+0000","last_clean":"2023-09-23T20:48:58.058595+0000","last_became_active":"2023-09-23T20:30:22.772159+0000","last_became_peered":"2023-09-23T20:30:22.772159+0000","last_unstale":"2023-09-23T20:48:58.058595+0000","last_undegraded":"2023-09-23T20:48:58.058595+0000","last_fullsized":"2023-09-23T20:48:58.058595+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:08.373446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140167+0000","last_change":"2023-09-23T20:45:00.003948+0000","last_active":"2023-09-23T20:48:58.140167+0000","last_peered":"2023-09-23T20:48:58.140167+0000","last_clean":"2023-09-23T20:48:58.140167+0000","last_became_active":"2023-09-23T20:45:00.003639+0000","last_became_peered":"2023-09-23T20:45:00.003639+0000","last_unstale":"2023-09-23T20:48:58.140167+0000","last_undegraded":"2023-09-23T20:48:58.140167+0000","last_fullsized":"2023-09-23T20:48:58.140167+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:59.619696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057894+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.057894+0000","last_peered":"2023-09-23T20:48:58.057894+0000","last_clean":"2023-09-23T20:48:58.057894+0000","last_became_active":"2023-09-23T20:48:07.544029+0000","last_became_peered":"2023-09-23T20:48:07.544029+0000","last_unstale":"2023-09-23T20:48:58.057894+0000","last_undegraded":"2023-09-23T20:48:58.057894+0000","last_fullsized":"2023-09-23T20:48:58.057894+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:52.671448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140012+0000","last_change":"2023-09-23T20:30:22.113582+0000","last_active":"2023-09-23T20:48:58.140012+0000","last_peered":"2023-09-23T20:48:58.140012+0000","last_clean":"2023-09-23T20:48:58.140012+0000","last_became_active":"2023-09-23T20:30:22.113291+0000","last_became_peered":"2023-09-23T20:30:22.113291+0000","last_unstale":"2023-09-23T20:48:58.140012+0000","last_undegraded":"2023-09-23T20:48:58.140012+0000","last_fullsized":"2023-09-23T20:48:58.140012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:19.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060226+0000","last_change":"2023-09-23T20:46:05.097625+0000","last_active":"2023-09-23T20:48:58.060226+0000","last_peered":"2023-09-23T20:48:58.060226+0000","last_clean":"2023-09-23T20:48:58.060226+0000","last_became_active":"2023-09-23T20:46:05.097296+0000","last_became_peered":"2023-09-23T20:46:05.097296+0000","last_unstale":"2023-09-23T20:48:58.060226+0000","last_undegraded":"2023-09-23T20:48:58.060226+0000","last_fullsized":"2023-09-23T20:48:58.060226+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:34.564149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058440+0000","last_change":"2023-09-23T20:48:07.522410+0000","last_active":"2023-09-23T20:48:58.058440+0000","last_peered":"2023-09-23T20:48:58.058440+0000","last_clean":"2023-09-23T20:48:58.058440+0000","last_became_active":"2023-09-23T20:48:07.522058+0000","last_became_peered":"2023-09-23T20:48:07.522058+0000","last_unstale":"2023-09-23T20:48:58.058440+0000","last_undegraded":"2023-09-23T20:48:58.058440+0000","last_fullsized":"2023-09-23T20:48:58.058440+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:24.844663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140085+0000","last_change":"2023-09-23T20:45:00.002738+0000","last_active":"2023-09-23T20:48:58.140085+0000","last_peered":"2023-09-23T20:48:58.140085+0000","last_clean":"2023-09-23T20:48:58.140085+0000","last_became_active":"2023-09-23T20:45:00.002403+0000","last_became_peered":"2023-09-23T20:45:00.002403+0000","last_unstale":"2023-09-23T20:48:58.140085+0000","last_undegraded":"2023-09-23T20:48:58.140085+0000","last_fullsized":"2023-09-23T20:48:58.140085+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:54.551205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054983+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:48:58.054983+0000","last_peered":"2023-09-23T20:48:58.054983+0000","last_clean":"2023-09-23T20:48:58.054983+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:48:58.054983+0000","last_undegraded":"2023-09-23T20:48:58.054983+0000","last_fullsized":"2023-09-23T20:48:58.054983+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057922+0000","last_change":"2023-09-23T20:48:07.520873+0000","last_active":"2023-09-23T20:48:58.057922+0000","last_peered":"2023-09-23T20:48:58.057922+0000","last_clean":"2023-09-23T20:48:58.057922+0000","last_became_active":"2023-09-23T20:48:07.520517+0000","last_became_peered":"2023-09-23T20:48:07.520517+0000","last_unstale":"2023-09-23T20:48:58.057922+0000","last_undegraded":"2023-09-23T20:48:58.057922+0000","last_fullsized":"2023-09-23T20:48:58.057922+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:43.626732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060090+0000","last_change":"2023-09-23T20:45:00.009049+0000","last_active":"2023-09-23T20:48:58.060090+0000","last_peered":"2023-09-23T20:48:58.060090+0000","last_clean":"2023-09-23T20:48:58.060090+0000","last_became_active":"2023-09-23T20:45:00.008700+0000","last_became_peered":"2023-09-23T20:45:00.008700+0000","last_unstale":"2023-09-23T20:48:58.060090+0000","last_undegraded":"2023-09-23T20:48:58.060090+0000","last_fullsized":"2023-09-23T20:48:58.060090+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:52.333134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057799+0000","last_change":"2023-09-23T20:46:05.106502+0000","last_active":"2023-09-23T20:48:58.057799+0000","last_peered":"2023-09-23T20:48:58.057799+0000","last_clean":"2023-09-23T20:48:58.057799+0000","last_became_active":"2023-09-23T20:46:05.106134+0000","last_became_peered":"2023-09-23T20:46:05.106134+0000","last_unstale":"2023-09-23T20:48:58.057799+0000","last_undegraded":"2023-09-23T20:48:58.057799+0000","last_fullsized":"2023-09-23T20:48:58.057799+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:39.257351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057632+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:48:58.057632+0000","last_peered":"2023-09-23T20:48:58.057632+0000","last_clean":"2023-09-23T20:48:58.057632+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:48:58.057632+0000","last_undegraded":"2023-09-23T20:48:58.057632+0000","last_fullsized":"2023-09-23T20:48:58.057632+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058202+0000","last_change":"2023-09-23T20:45:00.008536+0000","last_active":"2023-09-23T20:48:58.058202+0000","last_peered":"2023-09-23T20:48:58.058202+0000","last_clean":"2023-09-23T20:48:58.058202+0000","last_became_active":"2023-09-23T20:45:00.008220+0000","last_became_peered":"2023-09-23T20:45:00.008220+0000","last_unstale":"2023-09-23T20:48:58.058202+0000","last_undegraded":"2023-09-23T20:48:58.058202+0000","last_fullsized":"2023-09-23T20:48:58.058202+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:04.017031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054990+0000","last_change":"2023-09-23T20:48:07.540484+0000","last_active":"2023-09-23T20:48:58.054990+0000","last_peered":"2023-09-23T20:48:58.054990+0000","last_clean":"2023-09-23T20:48:58.054990+0000","last_became_active":"2023-09-23T20:48:07.540219+0000","last_became_peered":"2023-09-23T20:48:07.540219+0000","last_unstale":"2023-09-23T20:48:58.054990+0000","last_undegraded":"2023-09-23T20:48:58.054990+0000","last_fullsized":"2023-09-23T20:48:58.054990+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:54.792744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139793+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:48:58.139793+0000","last_peered":"2023-09-23T20:48:58.139793+0000","last_clean":"2023-09-23T20:48:58.139793+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:48:58.139793+0000","last_undegraded":"2023-09-23T20:48:58.139793+0000","last_fullsized":"2023-09-23T20:48:58.139793+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595372+0000","last_change":"2023-09-23T20:45:00.001830+0000","last_active":"2023-09-23T20:49:02.595372+0000","last_peered":"2023-09-23T20:49:02.595372+0000","last_clean":"2023-09-23T20:49:02.595372+0000","last_became_active":"2023-09-23T20:45:00.001532+0000","last_became_peered":"2023-09-23T20:45:00.001532+0000","last_unstale":"2023-09-23T20:49:02.595372+0000","last_undegraded":"2023-09-23T20:49:02.595372+0000","last_fullsized":"2023-09-23T20:49:02.595372+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:17.499694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058858+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:48:58.058858+0000","last_peered":"2023-09-23T20:48:58.058858+0000","last_clean":"2023-09-23T20:48:58.058858+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:48:58.058858+0000","last_undegraded":"2023-09-23T20:48:58.058858+0000","last_fullsized":"2023-09-23T20:48:58.058858+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595237+0000","last_change":"2023-09-23T20:45:00.004588+0000","last_active":"2023-09-23T20:49:02.595237+0000","last_peered":"2023-09-23T20:49:02.595237+0000","last_clean":"2023-09-23T20:49:02.595237+0000","last_became_active":"2023-09-23T20:45:00.004185+0000","last_became_peered":"2023-09-23T20:45:00.004185+0000","last_unstale":"2023-09-23T20:49:02.595237+0000","last_undegraded":"2023-09-23T20:49:02.595237+0000","last_fullsized":"2023-09-23T20:49:02.595237+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:47.406920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.059029+0000","last_change":"2023-09-23T20:45:00.007439+0000","last_active":"2023-09-23T20:48:58.059029+0000","last_peered":"2023-09-23T20:48:58.059029+0000","last_clean":"2023-09-23T20:48:58.059029+0000","last_became_active":"2023-09-23T20:45:00.007046+0000","last_became_peered":"2023-09-23T20:45:00.007046+0000","last_unstale":"2023-09-23T20:48:58.059029+0000","last_undegraded":"2023-09-23T20:48:58.059029+0000","last_fullsized":"2023-09-23T20:48:58.059029+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:42.363434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058490+0000","last_change":"2023-09-23T20:40:53.853109+0000","last_active":"2023-09-23T20:48:58.058490+0000","last_peered":"2023-09-23T20:48:58.058490+0000","last_clean":"2023-09-23T20:48:58.058490+0000","last_became_active":"2023-09-23T20:40:53.852788+0000","last_became_peered":"2023-09-23T20:40:53.852788+0000","last_unstale":"2023-09-23T20:48:58.058490+0000","last_undegraded":"2023-09-23T20:48:58.058490+0000","last_fullsized":"2023-09-23T20:48:58.058490+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:55.251424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058273+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:48:58.058273+0000","last_peered":"2023-09-23T20:48:58.058273+0000","last_clean":"2023-09-23T20:48:58.058273+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:48:58.058273+0000","last_undegraded":"2023-09-23T20:48:58.058273+0000","last_fullsized":"2023-09-23T20:48:58.058273+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139764+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:48:58.139764+0000","last_peered":"2023-09-23T20:48:58.139764+0000","last_clean":"2023-09-23T20:48:58.139764+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:48:58.139764+0000","last_undegraded":"2023-09-23T20:48:58.139764+0000","last_fullsized":"2023-09-23T20:48:58.139764+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.268923+0000","last_change":"2023-09-23T20:46:05.075939+0000","last_active":"2023-09-23T20:48:58.268923+0000","last_peered":"2023-09-23T20:48:58.268923+0000","last_clean":"2023-09-23T20:48:58.268923+0000","last_became_active":"2023-09-23T20:46:05.075634+0000","last_became_peered":"2023-09-23T20:46:05.075634+0000","last_unstale":"2023-09-23T20:48:58.268923+0000","last_undegraded":"2023-09-23T20:48:58.268923+0000","last_fullsized":"2023-09-23T20:48:58.268923+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:46.867950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.596359+0000","last_change":"2023-09-23T20:45:00.002611+0000","last_active":"2023-09-23T20:49:02.596359+0000","last_peered":"2023-09-23T20:49:02.596359+0000","last_clean":"2023-09-23T20:49:02.596359+0000","last_became_active":"2023-09-23T20:45:00.002331+0000","last_became_peered":"2023-09-23T20:45:00.002331+0000","last_unstale":"2023-09-23T20:49:02.596359+0000","last_undegraded":"2023-09-23T20:49:02.596359+0000","last_fullsized":"2023-09-23T20:49:02.596359+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:19.968130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058681+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:48:58.058681+0000","last_peered":"2023-09-23T20:48:58.058681+0000","last_clean":"2023-09-23T20:48:58.058681+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:48:58.058681+0000","last_undegraded":"2023-09-23T20:48:58.058681+0000","last_fullsized":"2023-09-23T20:48:58.058681+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":503,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139669+0000","last_change":"2023-09-23T20:40:53.853903+0000","last_active":"2023-09-23T20:48:58.139669+0000","last_peered":"2023-09-23T20:48:58.139669+0000","last_clean":"2023-09-23T20:48:58.139669+0000","last_became_active":"2023-09-23T20:40:53.852537+0000","last_became_peered":"2023-09-23T20:40:53.852537+0000","last_unstale":"2023-09-23T20:48:58.139669+0000","last_undegraded":"2023-09-23T20:48:58.139669+0000","last_fullsized":"2023-09-23T20:48:58.139669+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:49.454779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055032+0000","last_change":"2023-09-23T20:48:07.521001+0000","last_active":"2023-09-23T20:48:58.055032+0000","last_peered":"2023-09-23T20:48:58.055032+0000","last_clean":"2023-09-23T20:48:58.055032+0000","last_became_active":"2023-09-23T20:48:07.520697+0000","last_became_peered":"2023-09-23T20:48:07.520697+0000","last_unstale":"2023-09-23T20:48:58.055032+0000","last_undegraded":"2023-09-23T20:48:58.055032+0000","last_fullsized":"2023-09-23T20:48:58.055032+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:21.120614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060569+0000","last_change":"2023-09-23T20:30:22.772777+0000","last_active":"2023-09-23T20:48:58.060569+0000","last_peered":"2023-09-23T20:48:58.060569+0000","last_clean":"2023-09-23T20:48:58.060569+0000","last_became_active":"2023-09-23T20:30:22.772481+0000","last_became_peered":"2023-09-23T20:30:22.772481+0000","last_unstale":"2023-09-23T20:48:58.060569+0000","last_undegraded":"2023-09-23T20:48:58.060569+0000","last_fullsized":"2023-09-23T20:48:58.060569+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:53.724607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139632+0000","last_change":"2023-09-23T20:45:00.001911+0000","last_active":"2023-09-23T20:48:58.139632+0000","last_peered":"2023-09-23T20:48:58.139632+0000","last_clean":"2023-09-23T20:48:58.139632+0000","last_became_active":"2023-09-23T20:45:00.001531+0000","last_became_peered":"2023-09-23T20:45:00.001531+0000","last_unstale":"2023-09-23T20:48:58.139632+0000","last_undegraded":"2023-09-23T20:48:58.139632+0000","last_fullsized":"2023-09-23T20:48:58.139632+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:17.927950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.074369+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:49:02.074369+0000","last_peered":"2023-09-23T20:49:02.074369+0000","last_clean":"2023-09-23T20:49:02.074369+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:49:02.074369+0000","last_undegraded":"2023-09-23T20:49:02.074369+0000","last_fullsized":"2023-09-23T20:49:02.074369+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076247+0000","last_change":"2023-09-23T20:46:05.077316+0000","last_active":"2023-09-23T20:49:02.076247+0000","last_peered":"2023-09-23T20:49:02.076247+0000","last_clean":"2023-09-23T20:49:02.076247+0000","last_became_active":"2023-09-23T20:46:05.077044+0000","last_became_peered":"2023-09-23T20:46:05.077044+0000","last_unstale":"2023-09-23T20:49:02.076247+0000","last_undegraded":"2023-09-23T20:49:02.076247+0000","last_fullsized":"2023-09-23T20:49:02.076247+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:56.517854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140070+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:48:58.140070+0000","last_peered":"2023-09-23T20:48:58.140070+0000","last_clean":"2023-09-23T20:48:58.140070+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:48:58.140070+0000","last_undegraded":"2023-09-23T20:48:58.140070+0000","last_fullsized":"2023-09-23T20:48:58.140070+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":87,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595415+0000","last_change":"2023-09-23T20:46:05.079434+0000","last_active":"2023-09-23T20:49:02.595415+0000","last_peered":"2023-09-23T20:49:02.595415+0000","last_clean":"2023-09-23T20:49:02.595415+0000","last_became_active":"2023-09-23T20:46:05.079111+0000","last_became_peered":"2023-09-23T20:46:05.079111+0000","last_unstale":"2023-09-23T20:49:02.595415+0000","last_undegraded":"2023-09-23T20:49:02.595415+0000","last_fullsized":"2023-09-23T20:49:02.595415+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.869457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072216+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:49:02.072216+0000","last_peered":"2023-09-23T20:49:02.072216+0000","last_clean":"2023-09-23T20:49:02.072216+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:49:02.072216+0000","last_undegraded":"2023-09-23T20:49:02.072216+0000","last_fullsized":"2023-09-23T20:49:02.072216+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055650+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:48:58.055650+0000","last_peered":"2023-09-23T20:48:58.055650+0000","last_clean":"2023-09-23T20:48:58.055650+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:48:58.055650+0000","last_undegraded":"2023-09-23T20:48:58.055650+0000","last_fullsized":"2023-09-23T20:48:58.055650+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595600+0000","last_change":"2023-09-23T20:45:00.005180+0000","last_active":"2023-09-23T20:49:02.595600+0000","last_peered":"2023-09-23T20:49:02.595600+0000","last_clean":"2023-09-23T20:49:02.595600+0000","last_became_active":"2023-09-23T20:45:00.004791+0000","last_became_peered":"2023-09-23T20:45:00.004791+0000","last_unstale":"2023-09-23T20:49:02.595600+0000","last_undegraded":"2023-09-23T20:49:02.595600+0000","last_fullsized":"2023-09-23T20:49:02.595600+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:37.690560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055025+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:48:58.055025+0000","last_peered":"2023-09-23T20:48:58.055025+0000","last_clean":"2023-09-23T20:48:58.055025+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:48:58.055025+0000","last_undegraded":"2023-09-23T20:48:58.055025+0000","last_fullsized":"2023-09-23T20:48:58.055025+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060682+0000","last_change":"2023-09-23T20:46:05.073849+0000","last_active":"2023-09-23T20:48:58.060682+0000","last_peered":"2023-09-23T20:48:58.060682+0000","last_clean":"2023-09-23T20:48:58.060682+0000","last_became_active":"2023-09-23T20:46:05.073536+0000","last_became_peered":"2023-09-23T20:46:05.073536+0000","last_unstale":"2023-09-23T20:48:58.060682+0000","last_undegraded":"2023-09-23T20:48:58.060682+0000","last_fullsized":"2023-09-23T20:48:58.060682+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.086216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076148+0000","last_change":"2023-09-23T20:45:00.005077+0000","last_active":"2023-09-23T20:49:02.076148+0000","last_peered":"2023-09-23T20:49:02.076148+0000","last_clean":"2023-09-23T20:49:02.076148+0000","last_became_active":"2023-09-23T20:45:00.004383+0000","last_became_peered":"2023-09-23T20:45:00.004383+0000","last_unstale":"2023-09-23T20:49:02.076148+0000","last_undegraded":"2023-09-23T20:49:02.076148+0000","last_fullsized":"2023-09-23T20:49:02.076148+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:45.178382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057732+0000","last_change":"2023-09-23T20:30:22.131210+0000","last_active":"2023-09-23T20:48:58.057732+0000","last_peered":"2023-09-23T20:48:58.057732+0000","last_clean":"2023-09-23T20:48:58.057732+0000","last_became_active":"2023-09-23T20:30:22.130879+0000","last_became_peered":"2023-09-23T20:30:22.130879+0000","last_unstale":"2023-09-23T20:48:58.057732+0000","last_undegraded":"2023-09-23T20:48:58.057732+0000","last_fullsized":"2023-09-23T20:48:58.057732+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:10.733014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":99,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140129+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:48:58.140129+0000","last_peered":"2023-09-23T20:48:58.140129+0000","last_clean":"2023-09-23T20:48:58.140129+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:48:58.140129+0000","last_undegraded":"2023-09-23T20:48:58.140129+0000","last_fullsized":"2023-09-23T20:48:58.140129+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139585+0000","last_change":"2023-09-23T20:48:07.539824+0000","last_active":"2023-09-23T20:48:58.139585+0000","last_peered":"2023-09-23T20:48:58.139585+0000","last_clean":"2023-09-23T20:48:58.139585+0000","last_became_active":"2023-09-23T20:48:07.539530+0000","last_became_peered":"2023-09-23T20:48:07.539530+0000","last_unstale":"2023-09-23T20:48:58.139585+0000","last_undegraded":"2023-09-23T20:48:58.139585+0000","last_fullsized":"2023-09-23T20:48:58.139585+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:21.997036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058551+0000","last_change":"2023-09-23T20:45:00.008268+0000","last_active":"2023-09-23T20:48:58.058551+0000","last_peered":"2023-09-23T20:48:58.058551+0000","last_clean":"2023-09-23T20:48:58.058551+0000","last_became_active":"2023-09-23T20:45:00.006393+0000","last_became_peered":"2023-09-23T20:45:00.006393+0000","last_unstale":"2023-09-23T20:48:58.058551+0000","last_undegraded":"2023-09-23T20:48:58.058551+0000","last_fullsized":"2023-09-23T20:48:58.058551+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:41.255405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058217+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.058217+0000","last_peered":"2023-09-23T20:48:58.058217+0000","last_clean":"2023-09-23T20:48:58.058217+0000","last_became_active":"2023-09-23T20:48:07.544066+0000","last_became_peered":"2023-09-23T20:48:07.544066+0000","last_unstale":"2023-09-23T20:48:58.058217+0000","last_undegraded":"2023-09-23T20:48:58.058217+0000","last_fullsized":"2023-09-23T20:48:58.058217+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:28.905486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058955+0000","last_change":"2023-09-23T20:30:22.104198+0000","last_active":"2023-09-23T20:48:58.058955+0000","last_peered":"2023-09-23T20:48:58.058955+0000","last_clean":"2023-09-23T20:48:58.058955+0000","last_became_active":"2023-09-23T20:30:22.103970+0000","last_became_peered":"2023-09-23T20:30:22.103970+0000","last_unstale":"2023-09-23T20:48:58.058955+0000","last_undegraded":"2023-09-23T20:48:58.058955+0000","last_fullsized":"2023-09-23T20:48:58.058955+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:32.820350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058740+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:48:58.058740+0000","last_peered":"2023-09-23T20:48:58.058740+0000","last_clean":"2023-09-23T20:48:58.058740+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:48:58.058740+0000","last_undegraded":"2023-09-23T20:48:58.058740+0000","last_fullsized":"2023-09-23T20:48:58.058740+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055468+0000","last_change":"2023-09-23T20:48:07.521311+0000","last_active":"2023-09-23T20:48:58.055468+0000","last_peered":"2023-09-23T20:48:58.055468+0000","last_clean":"2023-09-23T20:48:58.055468+0000","last_became_active":"2023-09-23T20:48:07.521030+0000","last_became_peered":"2023-09-23T20:48:07.521030+0000","last_unstale":"2023-09-23T20:48:58.055468+0000","last_undegraded":"2023-09-23T20:48:58.055468+0000","last_fullsized":"2023-09-23T20:48:58.055468+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.849053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055021+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:48:58.055021+0000","last_peered":"2023-09-23T20:48:58.055021+0000","last_clean":"2023-09-23T20:48:58.055021+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:48:58.055021+0000","last_undegraded":"2023-09-23T20:48:58.055021+0000","last_fullsized":"2023-09-23T20:48:58.055021+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072229+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:49:02.072229+0000","last_peered":"2023-09-23T20:49:02.072229+0000","last_clean":"2023-09-23T20:49:02.072229+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:49:02.072229+0000","last_undegraded":"2023-09-23T20:49:02.072229+0000","last_fullsized":"2023-09-23T20:49:02.072229+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057625+0000","last_change":"2023-09-23T20:45:00.004734+0000","last_active":"2023-09-23T20:48:58.057625+0000","last_peered":"2023-09-23T20:48:58.057625+0000","last_clean":"2023-09-23T20:48:58.057625+0000","last_became_active":"2023-09-23T20:45:00.004413+0000","last_became_peered":"2023-09-23T20:45:00.004413+0000","last_unstale":"2023-09-23T20:48:58.057625+0000","last_undegraded":"2023-09-23T20:48:58.057625+0000","last_fullsized":"2023-09-23T20:48:58.057625+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:57.454649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201532,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":2180,"kb_used_omap":0,"kb_used_meta":296384,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":2232320,"data_stored":1659199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":303497216},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":4.4119999999999999,"15min":2.488},"min":{"1min":0.371,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.7990000000000004,"5min":225.56999999999999,"15min":225.56999999999999},"last":0.311},{"interface":"front","average":{"1min":1.4730000000000001,"5min":4.6440000000000001,"15min":2.8570000000000002},"min":{"1min":0.36599999999999999,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":8.1549999999999994,"5min":225.61099999999999,"15min":225.61099999999999},"last":0.48199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.132,"5min":3.2690000000000001,"15min":1.867},"min":{"1min":0.377,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":128.97999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":10.537000000000001,"5min":6.306,"15min":2.8900000000000001},"min":{"1min":0.432,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":128.81200000000001,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.58399999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":4.1689999999999996,"15min":3.4820000000000002},"min":{"1min":0.40000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.2249999999999996,"5min":142.321,"15min":142.321},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.3140000000000001,"15min":2.6619999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.8149999999999999,"5min":142.29300000000001,"15min":142.29300000000001},"last":0.54500000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.209,"5min":10.209,"15min":10.209},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":138.345,"5min":138.345,"15min":138.345},"last":0.51900000000000002},{"interface":"front","average":{"1min":9.423,"5min":9.423,"15min":9.423},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":133.76400000000001,"5min":133.76400000000001,"15min":133.76400000000001},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.361,"15min":1.6379999999999999},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.216},"max":{"1min":3.4319999999999999,"5min":9.1400000000000006,"15min":103.541},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.2949999999999999,"15min":1.75},"min":{"1min":0.30199999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.843,"5min":8.4649999999999999,"15min":107.514},"last":0.65700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.498,"15min":1.8580000000000001},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":6.6260000000000003,"5min":9.4440000000000008,"15min":67.180999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.181,"5min":1.7250000000000001,"15min":1.77},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":7.9109999999999996,"5min":13.156000000000001,"15min":67.135000000000005},"last":0.67000000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.679,"15min":1.5780000000000001},"min":{"1min":0.41999999999999998,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":5.0220000000000002,"5min":10.509,"15min":16.032},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.1830000000000001,"15min":1.3280000000000001},"min":{"1min":0.41399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9269999999999996,"5min":8.4350000000000005,"15min":11.672000000000001},"last":0.68799999999999994}]}]},{"osd":6,"up_from":47,"seq":201863463169,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":255164,"kb_used_data":2164,"kb_used_omap":0,"kb_used_meta":252992,"kb_avail":93489988,"statfs":{"total":95995035648,"available":95733747712,"internally_reserved":0,"allocated":2215936,"data_stored":1658846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":259063808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":7.2859999999999996,"5min":3.4649999999999999,"15min":2.4359999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.20399999999999999},"max":{"1min":92.164000000000001,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.435},{"interface":"front","average":{"1min":7.2859999999999996,"5min":3.04,"15min":2.431},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":92.138000000000005,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.56699999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.5110000000000001,"5min":2.907,"15min":2.073},"min":{"1min":0.30399999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":96.623999999999995,"5min":96.623999999999995,"15min":96.623999999999995},"last":0.499},{"interface":"front","average":{"1min":6.5579999999999998,"5min":2.8090000000000002,"15min":1.839},"min":{"1min":0.30399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":96.254000000000005,"5min":96.254000000000005,"15min":96.254000000000005},"last":0.373}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.3319999999999999,"15min":2.3029999999999999},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.7009999999999996,"5min":8.9849999999999994,"15min":8.9849999999999994},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.5209999999999999,"5min":2.6269999999999998,"15min":2.6070000000000002},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":10.59,"5min":10.59,"15min":10.59},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.125,"5min":6.125,"15min":6.125},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":79.668000000000006,"5min":79.668000000000006,"15min":79.668000000000006},"last":0.41099999999999998},{"interface":"front","average":{"1min":5.9980000000000002,"5min":5.9980000000000002,"15min":5.9980000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":79.700000000000003,"5min":79.700000000000003,"15min":79.700000000000003},"last":0.59799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.2709999999999999,"15min":1.341},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":9.0709999999999997,"5min":9.0709999999999997,"15min":16.247},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.756,"15min":1.6479999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.27000000000000002},"max":{"1min":9.8650000000000002,"5min":9.8650000000000002,"15min":17.163},"last":0.61399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.556,"15min":1.518},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.26000000000000001},"max":{"1min":9.0340000000000007,"5min":9.1969999999999992,"15min":11.484999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.214,"5min":1.7490000000000001,"15min":1.631},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":9.8949999999999996,"5min":9.8949999999999996,"15min":17.321000000000002},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:49:29 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":1.5589999999999999,"15min":1.4259999999999999},"min":{"1min":0.40200000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2590000000000003,"5min":9.3249999999999993,"15min":19.451000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.605,"15min":1.6299999999999999},"min":{"1min":0.40000000000000002,"5min":0.36099999999999999,"15min":0.32100000000000001},"max":{"1min":9.7959999999999994,"5min":9.7959999999999994,"15min":16.295999999999999},"last":0.58399999999999996}]}]},{"osd":1,"up_from":114,"seq":489626271803,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236760,"kb_used_data":1744,"kb_used_omap":4,"kb_used_meta":235003,"kb_avail":93508392,"statfs":{"total":95995035648,"available":95752593408,"internally_reserved":0,"allocated":1785856,"data_stored":1086600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":240643982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":3.206,"5min":2.4340000000000002,"15min":2.1160000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.191},"max":{"1min":48.350999999999999,"5min":48.350999999999999,"15min":48.350999999999999},"last":0.316},{"interface":"front","average":{"1min":4.7350000000000003,"5min":2.698,"15min":2.048},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":52.018000000000001,"5min":52.018000000000001,"15min":52.018000000000001},"last":0.222}]},{"osd":2,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":2.0350000000000001,"15min":2.1789999999999998},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.7479999999999993,"5min":11.593999999999999,"15min":11.593999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.613,"15min":2.855},"min":{"1min":0.29499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.465999999999999,"5min":14.153,"15min":14.153},"last":0.29199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.73,"15min":2.73},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.4969999999999999,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":2.04,"5min":2.04,"15min":2.04},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.5630000000000006,"5min":8.5630000000000006,"15min":8.5630000000000006},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.663,"15min":1.349},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.439,"5min":10.439,"15min":10.439},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.496,"5min":1.431,"15min":1.1870000000000001},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":10.34,"5min":10.34,"15min":10.34},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":2.4950000000000001,"15min":1.853},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.5410000000000004,"5min":10.177,"15min":10.177},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.415,"5min":1.635,"15min":1.268},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.3890000000000002,"5min":6.8680000000000003,"15min":6.8680000000000003},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.9990000000000001,"15min":1.7010000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.25,"5min":10.25,"15min":10.25},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.008,"5min":1.581,"15min":1.3149999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.4489999999999998,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.871,"15min":1.4790000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.3499999999999996,"5min":7.4480000000000004,"15min":7.4480000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.948,"5min":2.0569999999999999,"15min":1.911},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.0259999999999998,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.499}]}]},{"osd":0,"up_from":92,"seq":395136991340,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":402740,"kb_used_data":2348,"kb_used_omap":5,"kb_used_meta":400378,"kb_avail":93342412,"statfs":{"total":95995035648,"available":95582629888,"internally_reserved":0,"allocated":2404352,"data_stored":1676026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":409987706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5780000000000003,"5min":3.2480000000000002,"15min":2.4420000000000002},"min":{"1min":0.20000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.140000000000001,"5min":64.140000000000001,"15min":64.140000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":4.5860000000000003,"5min":3.3679999999999999,"15min":2.5270000000000001},"min":{"1min":0.20999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":59.570999999999998,"5min":59.570999999999998,"15min":59.570999999999998},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":2.319,"15min":2.0569999999999999},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.5839999999999996,"5min":49.652000000000001,"15min":49.652000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.8,"5min":4.0250000000000004,"15min":4.3360000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4720000000000004,"5min":49.595999999999997,"15min":49.595999999999997},"last":0.377}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":41.810000000000002,"5min":41.810000000000002,"15min":41.810000000000002},"last":0.39700000000000002},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":41.880000000000003,"5min":41.880000000000003,"15min":41.880000000000003},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.9750000000000001,"15min":1.7589999999999999},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.3559999999999999,"5min":12.973000000000001,"15min":12.973000000000001},"last":0.627},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.6850000000000001,"15min":2.1419999999999999},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":8.234,"5min":12.148,"15min":12.148},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":2.6040000000000001,"15min":2.1749999999999998},"min":{"1min":0.40600000000000003,"5min":0.30199999999999999,"15min":0.28799999999999998},"max":{"1min":5.6550000000000002,"5min":14.051,"15min":14.051},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.9410000000000001,"15min":2.2989999999999999},"min":{"1min":0.371,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.3119999999999994,"5min":12.332000000000001,"15min":12.332000000000001},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.1219999999999999,"15min":2.3580000000000001},"min":{"1min":0.39600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":7.8959999999999999,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.1659999999999999,"5min":2.2519999999999998,"15min":1.708},"min":{"1min":0.36299999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.1189999999999998,"5min":20.399999999999999,"15min":20.399999999999999},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":1.829,"15min":2.4550000000000001},"min":{"1min":0.46700000000000003,"5min":0.41299999999999998,"15min":0.35099999999999998},"max":{"1min":7.3780000000000001,"5min":12.23,"15min":12.23},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.9350000000000001,"5min":2.2149999999999999,"15min":1.7849999999999999},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.7430000000000003,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.65000000000000002}]}]},{"osd":2,"up_from":129,"seq":554050781230,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270952,"kb_used_data":1696,"kb_used_omap":7,"kb_used_meta":269240,"kb_avail":93474200,"statfs":{"total":95995035648,"available":95717580800,"internally_reserved":0,"allocated":1736704,"data_stored":1078264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":275701841},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.5070000000000001,"15min":2.9980000000000002},"min":{"1min":0.21299999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.5490000000000004,"5min":14.327,"15min":14.327},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.024,"5min":2.323,"15min":2.7589999999999999},"min":{"1min":0.217,"5min":0.186,"15min":0.186},"max":{"1min":6.2359999999999998,"5min":14.757999999999999,"15min":14.757999999999999},"last":0.311}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":2.5880000000000001,"15min":2.8900000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.4039999999999999,"5min":13.023,"15min":13.023},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.3759999999999999,"15min":3.0019999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.0419999999999998,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":0.29399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.959,"5min":11.959,"15min":11.959},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.6539999999999999,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.52800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.819,"15min":1.9370000000000001},"min":{"1min":0.34100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.8440000000000003,"5min":7.8440000000000003,"15min":7.8440000000000003},"last":0.751},{"interface":"front","average":{"1min":1.411,"5min":2.3490000000000002,"15min":2.7109999999999999},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.317},"max":{"1min":6.2789999999999999,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.61199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.5630000000000002,"15min":3.0539999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.3760000000000003,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.7509999999999999,"5min":2.5030000000000001,"15min":2.8610000000000002},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3630000000000004,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.45300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.7210000000000001,"15min":3.29},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.3449999999999998,"5min":7.476,"15min":7.476},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.1379999999999999,"5min":2.806,"15min":3.0790000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":9.5220000000000002,"5min":12.595000000000001,"15min":12.595000000000001},"last":0.72199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.2320000000000002,"15min":2.5350000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.4909999999999997,"5min":6.4989999999999997,"15min":6.4989999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.669,"5min":2.5259999999999998,"15min":2.8839999999999999},"min":{"1min":0.41399999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.5979999999999999,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":0.67100000000000004}]}]},{"osd":3,"up_from":148,"seq":635655159830,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":135308,"kb_used_data":1668,"kb_used_omap":2,"kb_used_meta":133629,"kb_avail":93609844,"statfs":{"total":95995035648,"available":95856480256,"internally_reserved":0,"allocated":1708032,"data_stored":1075110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":136836149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":14.332000000000001,"5min":14.332000000000001,"15min":14.332000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":253.68100000000001,"5min":253.68100000000001,"15min":253.68100000000001},"last":0.188},{"interface":"front","average":{"1min":13.208,"5min":13.208,"15min":13.208},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":253.46199999999999,"5min":253.46199999999999,"15min":253.46199999999999},"last":0.26500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":10.417999999999999,"5min":10.417999999999999,"15min":10.417999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":172.77099999999999,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":9.4079999999999995,"5min":9.4079999999999995,"15min":9.4079999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":172.36600000000001,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.4619999999999997,"5min":5.4619999999999997,"15min":5.4619999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.4749999999999996,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.2170000000000001,"15min":2.2170000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":16.408999999999999,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.65500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.5750000000000002,"5min":3.5750000000000002,"15min":3.5750000000000002},"last":0.629},{"interface":"front","average":{"1min":1.958,"5min":1.958,"15min":1.958},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":9.7780000000000005,"5min":9.7780000000000005,"15min":9.7780000000000005},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":1.972,"15min":1.972},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.7509999999999994,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":0.79500000000000004},{"interface":"front","average":{"1min":2.6419999999999999,"5min":2.6419999999999999,"15min":2.6419999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":16.451000000000001,"5min":16.451000000000001,"15min":16.451000000000001},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.5640000000000001,"15min":1.5640000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.9450000000000003,"5min":4.9450000000000003,"15min":4.9450000000000003},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.351,"5min":2.351,"15min":2.351},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.60699999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986442,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307912,"kb_used_data":1600,"kb_used_omap":0,"kb_used_meta":306304,"kb_avail":93437240,"statfs":{"total":95995035648,"available":95679733760,"internally_reserved":0,"allocated":1638400,"data_stored":1069084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":313655296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.52,"15min":1.839},"min":{"1min":0.216,"5min":0.216,"15min":0.214},"max":{"1min":8.0519999999999996,"5min":12.823,"15min":12.823},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.508,"5min":1.998,"15min":1.321},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.22600000000000001},"max":{"1min":4.3090000000000002,"5min":10.413,"15min":13.211},"last":0.58799999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":8.4019999999999992,"5min":6.6639999999999997,"15min":3.23},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":125.35599999999999,"5min":286.31099999999998,"15min":286.31099999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":8.0459999999999994,"5min":6.7679999999999998,"15min":3.996},"min":{"1min":0.312,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":125.21899999999999,"5min":285.97000000000003,"15min":285.97000000000003},"last":0.504}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.48,"15min":1.375},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.988,"5min":2.347,"15min":2.3620000000000001},"min":{"1min":0.39300000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.9450000000000003,"5min":12.773999999999999,"15min":12.773999999999999},"last":0.65000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":7.5990000000000002,"5min":7.5990000000000002,"15min":7.5990000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":107.285,"5min":107.285,"15min":107.285},"last":0.53200000000000003},{"interface":"front","average":{"1min":7.1920000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":107.303,"5min":107.303,"15min":107.303},"last":0.48399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.2390000000000001,"15min":1.258},"min":{"1min":0.32500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.2000000000000002,"5min":7.1740000000000004,"15min":17.713999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.1879999999999999,"15min":1.331},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.891,"5min":9.0660000000000007,"15min":14.455},"last":0.68000000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.4630000000000001,"15min":1.5369999999999999},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.4109999999999996,"5min":16.271000000000001,"15min":21.920999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.234,"15min":1.345},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.28599999999999998},"max":{"1min":3.431,"5min":11.718,"15min":11.718},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.4750000000000001,"15min":1.6679999999999999},"min":{"1min":0.42499999999999999,"5min":0.315,"15min":0.31},"max":{"1min":6.0039999999999996,"5min":9.8390000000000004,"15min":21.792999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.321,"15min":1.47},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.317},"max":{"1min":3.306,"5min":7.8399999999999999,"15min":23.457000000000001},"last":0.60799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724806,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":365368,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":363776,"kb_avail":93379784,"statfs":{"total":95995035648,"available":95620898816,"internally_reserved":0,"allocated":1622016,"data_stored":1064828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":372506624},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.4330000000000001,"15min":1.7669999999999999},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.7559999999999998,"5min":10.94,"15min":12.880000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.274,"5min":1.754,"15min":1.3859999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":10.028,"5min":10.955,"15min":10.955},"last":0.58999999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":4.5990000000000002,"15min":2.2130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":12.529999999999999,"5min":323.541,"15min":323.541},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.262,"5min":4.7270000000000003,"15min":2.5219999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.6900000000000004,"5min":324.072,"15min":324.072},"last":0.48399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.137,"5min":2.415,"15min":2.423},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.585000000000001,"5min":13.394,"15min":13.394},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.827,"15min":1.663},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7430000000000003,"5min":14.263,"15min":14.263},"last":0.55100000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.863,"15min":1.863},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.295,"5min":10.295,"15min":10.295},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.1080000000000005,"5min":8.1080000000000005,"15min":8.1080000000000005},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:48:47 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":0.95099999999999996,"15min":1.2050000000000001},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.216},"max":{"1min":5.8529999999999998,"5min":5.8529999999999998,"15min":9.8800000000000008},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.3,"5min":1.3360000000000001,"15min":1.6000000000000001},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.23999999999999999},"max":{"1min":4.7309999999999999,"5min":8.3170000000000002,"15min":16.869},"last":0.61199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":1.325,"15min":1.544},"min":{"1min":0.316,"5min":0.28000000000000003,"15min":0.27000000000000002},"max":{"1min":6.1689999999999996,"5min":8.3010000000000002,"15min":17},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.2050000000000001,"15min":1.427},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.9870000000000001,"5min":9.6370000000000005,"15min":14.08},"last":0.624}]},{"osd":7,"last update":"Sat Sep 23 20:49:40 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.5960000000000001,"15min":1.6910000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.28199999999999997},"max":{"1min":2.9300000000000002,"5min":14.217000000000001,"15min":17.071999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.96699999999999997,"15min":1.1259999999999999},"min":{"1min":0.40300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0499999999999998,"5min":5.601,"15min":8.6370000000000005},"last":0.70599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:49:47.404 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:49:47.404 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:49:47.404 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:49:47.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:49:47.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:47 smithi123 ceph-mon[193903]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:47 smithi123 ceph-mon[193903]: from='client.35097 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:47.756 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:49.419 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:49.420 INFO:teuthology.orchestra.run.smithi121.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:49:49.696 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:49 smithi121 ceph-mon[211526]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:49.697 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2851749462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:49:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:49 smithi123 ceph-mon[193903]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2851749462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:49:50.080 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:49 smithi121 ceph-mon[219709]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:50.080 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2851749462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:49:50.096 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:49:50.096 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:49:50.106 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T20:49:50.107 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T20:49:50.115 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T20:49:50.116 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:49:50.116 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T20:49:50.117 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T20:49:50.118 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T20:49:50.118 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T20:49:50.118 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T20:49:50.118 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:49:50.118 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:49:50.462 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:51.765 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:51 smithi121 ceph-mon[219709]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:51.766 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:51 smithi121 ceph-mon[211526]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:51.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:51 smithi123 ceph-mon[193903]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:52.187 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:49:52.821 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":512,"stamp":"2023-09-23T20:49:50.638879+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2272776,"kb_used_data":14984,"kb_used_omap":20,"kb_used_meta":2257707,"kb_avail":747688440,"statfs":{"total":767960285184,"available":765632962560,"internally_reserved":0,"allocated":15343616,"data_stored":10367957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2311892622},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002651"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058647+0000","last_change":"2023-09-23T20:46:05.100324+0000","last_active":"2023-09-23T20:48:58.058647+0000","last_peered":"2023-09-23T20:48:58.058647+0000","last_clean":"2023-09-23T20:48:58.058647+0000","last_became_active":"2023-09-23T20:46:05.099981+0000","last_became_peered":"2023-09-23T20:46:05.099981+0000","last_unstale":"2023-09-23T20:48:58.058647+0000","last_undegraded":"2023-09-23T20:48:58.058647+0000","last_fullsized":"2023-09-23T20:48:58.058647+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:19.876971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055345+0000","last_change":"2023-09-23T20:48:07.517973+0000","last_active":"2023-09-23T20:48:58.055345+0000","last_peered":"2023-09-23T20:48:58.055345+0000","last_clean":"2023-09-23T20:48:58.055345+0000","last_became_active":"2023-09-23T20:48:07.517681+0000","last_became_peered":"2023-09-23T20:48:07.517681+0000","last_unstale":"2023-09-23T20:48:58.055345+0000","last_undegraded":"2023-09-23T20:48:58.055345+0000","last_fullsized":"2023-09-23T20:48:58.055345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.789594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":87,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595140+0000","last_change":"2023-09-23T20:48:07.541333+0000","last_active":"2023-09-23T20:49:02.595140+0000","last_peered":"2023-09-23T20:49:02.595140+0000","last_clean":"2023-09-23T20:49:02.595140+0000","last_became_active":"2023-09-23T20:48:07.541014+0000","last_became_peered":"2023-09-23T20:48:07.541014+0000","last_unstale":"2023-09-23T20:49:02.595140+0000","last_undegraded":"2023-09-23T20:49:02.595140+0000","last_fullsized":"2023-09-23T20:49:02.595140+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:48.678651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140248+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:48:58.140248+0000","last_peered":"2023-09-23T20:48:58.140248+0000","last_clean":"2023-09-23T20:48:58.140248+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:48:58.140248+0000","last_undegraded":"2023-09-23T20:48:58.140248+0000","last_fullsized":"2023-09-23T20:48:58.140248+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054913+0000","last_change":"2023-09-23T20:48:07.540435+0000","last_active":"2023-09-23T20:48:58.054913+0000","last_peered":"2023-09-23T20:48:58.054913+0000","last_clean":"2023-09-23T20:48:58.054913+0000","last_became_active":"2023-09-23T20:48:07.540062+0000","last_became_peered":"2023-09-23T20:48:07.540062+0000","last_unstale":"2023-09-23T20:48:58.054913+0000","last_undegraded":"2023-09-23T20:48:58.054913+0000","last_fullsized":"2023-09-23T20:48:58.054913+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:19.449218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060678+0000","last_change":"2023-09-23T20:45:00.009034+0000","last_active":"2023-09-23T20:48:58.060678+0000","last_peered":"2023-09-23T20:48:58.060678+0000","last_clean":"2023-09-23T20:48:58.060678+0000","last_became_active":"2023-09-23T20:45:00.008706+0000","last_became_peered":"2023-09-23T20:45:00.008706+0000","last_unstale":"2023-09-23T20:48:58.060678+0000","last_undegraded":"2023-09-23T20:48:58.060678+0000","last_fullsized":"2023-09-23T20:48:58.060678+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:29.204761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140415+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:48:58.140415+0000","last_peered":"2023-09-23T20:48:58.140415+0000","last_clean":"2023-09-23T20:48:58.140415+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:48:58.140415+0000","last_undegraded":"2023-09-23T20:48:58.140415+0000","last_fullsized":"2023-09-23T20:48:58.140415+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054946+0000","last_change":"2023-09-23T20:48:07.533499+0000","last_active":"2023-09-23T20:48:58.054946+0000","last_peered":"2023-09-23T20:48:58.054946+0000","last_clean":"2023-09-23T20:48:58.054946+0000","last_became_active":"2023-09-23T20:48:07.520669+0000","last_became_peered":"2023-09-23T20:48:07.520669+0000","last_unstale":"2023-09-23T20:48:58.054946+0000","last_undegraded":"2023-09-23T20:48:58.054946+0000","last_fullsized":"2023-09-23T20:48:58.054946+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:55.109803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058059+0000","last_change":"2023-09-23T20:45:00.007125+0000","last_active":"2023-09-23T20:48:58.058059+0000","last_peered":"2023-09-23T20:48:58.058059+0000","last_clean":"2023-09-23T20:48:58.058059+0000","last_became_active":"2023-09-23T20:45:00.006403+0000","last_became_peered":"2023-09-23T20:45:00.006403+0000","last_unstale":"2023-09-23T20:48:58.058059+0000","last_undegraded":"2023-09-23T20:48:58.058059+0000","last_fullsized":"2023-09-23T20:48:58.058059+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:27.906490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055087+0000","last_change":"2023-09-23T20:48:07.525099+0000","last_active":"2023-09-23T20:48:58.055087+0000","last_peered":"2023-09-23T20:48:58.055087+0000","last_clean":"2023-09-23T20:48:58.055087+0000","last_became_active":"2023-09-23T20:48:07.524868+0000","last_became_peered":"2023-09-23T20:48:07.524868+0000","last_unstale":"2023-09-23T20:48:58.055087+0000","last_undegraded":"2023-09-23T20:48:58.055087+0000","last_fullsized":"2023-09-23T20:48:58.055087+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:19.340586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140203+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:48:58.140203+0000","last_peered":"2023-09-23T20:48:58.140203+0000","last_clean":"2023-09-23T20:48:58.140203+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:48:58.140203+0000","last_undegraded":"2023-09-23T20:48:58.140203+0000","last_fullsized":"2023-09-23T20:48:58.140203+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595558+0000","last_change":"2023-09-23T20:46:05.099313+0000","last_active":"2023-09-23T20:49:02.595558+0000","last_peered":"2023-09-23T20:49:02.595558+0000","last_clean":"2023-09-23T20:49:02.595558+0000","last_became_active":"2023-09-23T20:46:05.099055+0000","last_became_peered":"2023-09-23T20:46:05.099055+0000","last_unstale":"2023-09-23T20:49:02.595558+0000","last_undegraded":"2023-09-23T20:49:02.595558+0000","last_fullsized":"2023-09-23T20:49:02.595558+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:51.820608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058022+0000","last_change":"2023-09-23T20:30:22.107118+0000","last_active":"2023-09-23T20:48:58.058022+0000","last_peered":"2023-09-23T20:48:58.058022+0000","last_clean":"2023-09-23T20:48:58.058022+0000","last_became_active":"2023-09-23T20:30:22.106839+0000","last_became_peered":"2023-09-23T20:30:22.106839+0000","last_unstale":"2023-09-23T20:48:58.058022+0000","last_undegraded":"2023-09-23T20:48:58.058022+0000","last_fullsized":"2023-09-23T20:48:58.058022+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:02.981333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058390+0000","last_change":"2023-09-23T20:45:00.007712+0000","last_active":"2023-09-23T20:48:58.058390+0000","last_peered":"2023-09-23T20:48:58.058390+0000","last_clean":"2023-09-23T20:48:58.058390+0000","last_became_active":"2023-09-23T20:45:00.007399+0000","last_became_peered":"2023-09-23T20:45:00.007399+0000","last_unstale":"2023-09-23T20:48:58.058390+0000","last_undegraded":"2023-09-23T20:48:58.058390+0000","last_fullsized":"2023-09-23T20:48:58.058390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:48.842612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139939+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:48:58.139939+0000","last_peered":"2023-09-23T20:48:58.139939+0000","last_clean":"2023-09-23T20:48:58.139939+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:48:58.139939+0000","last_undegraded":"2023-09-23T20:48:58.139939+0000","last_fullsized":"2023-09-23T20:48:58.139939+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055261+0000","last_change":"2023-09-23T20:48:07.540014+0000","last_active":"2023-09-23T20:48:58.055261+0000","last_peered":"2023-09-23T20:48:58.055261+0000","last_clean":"2023-09-23T20:48:58.055261+0000","last_became_active":"2023-09-23T20:48:07.539722+0000","last_became_peered":"2023-09-23T20:48:07.539722+0000","last_unstale":"2023-09-23T20:48:58.055261+0000","last_undegraded":"2023-09-23T20:48:58.055261+0000","last_fullsized":"2023-09-23T20:48:58.055261+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.937232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140140+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:48:58.140140+0000","last_peered":"2023-09-23T20:48:58.140140+0000","last_clean":"2023-09-23T20:48:58.140140+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:48:58.140140+0000","last_undegraded":"2023-09-23T20:48:58.140140+0000","last_fullsized":"2023-09-23T20:48:58.140140+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058777+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:48:58.058777+0000","last_peered":"2023-09-23T20:48:58.058777+0000","last_clean":"2023-09-23T20:48:58.058777+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:48:58.058777+0000","last_undegraded":"2023-09-23T20:48:58.058777+0000","last_fullsized":"2023-09-23T20:48:58.058777+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139702+0000","last_change":"2023-09-23T20:30:22.110586+0000","last_active":"2023-09-23T20:48:58.139702+0000","last_peered":"2023-09-23T20:48:58.139702+0000","last_clean":"2023-09-23T20:48:58.139702+0000","last_became_active":"2023-09-23T20:30:22.110248+0000","last_became_peered":"2023-09-23T20:30:22.110248+0000","last_unstale":"2023-09-23T20:48:58.139702+0000","last_undegraded":"2023-09-23T20:48:58.139702+0000","last_fullsized":"2023-09-23T20:48:58.139702+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:29.867944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140341+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:48:58.140341+0000","last_peered":"2023-09-23T20:48:58.140341+0000","last_clean":"2023-09-23T20:48:58.140341+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:48:58.140341+0000","last_undegraded":"2023-09-23T20:48:58.140341+0000","last_fullsized":"2023-09-23T20:48:58.140341+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060100+0000","last_change":"2023-09-23T20:40:53.862653+0000","last_active":"2023-09-23T20:48:58.060100+0000","last_peered":"2023-09-23T20:48:58.060100+0000","last_clean":"2023-09-23T20:48:58.060100+0000","last_became_active":"2023-09-23T20:40:53.862259+0000","last_became_peered":"2023-09-23T20:40:53.862259+0000","last_unstale":"2023-09-23T20:48:58.060100+0000","last_undegraded":"2023-09-23T20:48:58.060100+0000","last_fullsized":"2023-09-23T20:48:58.060100+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:28.626871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055685+0000","last_change":"2023-09-23T20:48:07.557905+0000","last_active":"2023-09-23T20:48:58.055685+0000","last_peered":"2023-09-23T20:48:58.055685+0000","last_clean":"2023-09-23T20:48:58.055685+0000","last_became_active":"2023-09-23T20:48:07.557568+0000","last_became_peered":"2023-09-23T20:48:07.557568+0000","last_unstale":"2023-09-23T20:48:58.055685+0000","last_undegraded":"2023-09-23T20:48:58.055685+0000","last_fullsized":"2023-09-23T20:48:58.055685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:14.229704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140303+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:48:58.140303+0000","last_peered":"2023-09-23T20:48:58.140303+0000","last_clean":"2023-09-23T20:48:58.140303+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:48:58.140303+0000","last_undegraded":"2023-09-23T20:48:58.140303+0000","last_fullsized":"2023-09-23T20:48:58.140303+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058595+0000","last_change":"2023-09-23T20:30:22.772475+0000","last_active":"2023-09-23T20:48:58.058595+0000","last_peered":"2023-09-23T20:48:58.058595+0000","last_clean":"2023-09-23T20:48:58.058595+0000","last_became_active":"2023-09-23T20:30:22.772159+0000","last_became_peered":"2023-09-23T20:30:22.772159+0000","last_unstale":"2023-09-23T20:48:58.058595+0000","last_undegraded":"2023-09-23T20:48:58.058595+0000","last_fullsized":"2023-09-23T20:48:58.058595+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:08.373446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140167+0000","last_change":"2023-09-23T20:45:00.003948+0000","last_active":"2023-09-23T20:48:58.140167+0000","last_peered":"2023-09-23T20:48:58.140167+0000","last_clean":"2023-09-23T20:48:58.140167+0000","last_became_active":"2023-09-23T20:45:00.003639+0000","last_became_peered":"2023-09-23T20:45:00.003639+0000","last_unstale":"2023-09-23T20:48:58.140167+0000","last_undegraded":"2023-09-23T20:48:58.140167+0000","last_fullsized":"2023-09-23T20:48:58.140167+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:59.619696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057894+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.057894+0000","last_peered":"2023-09-23T20:48:58.057894+0000","last_clean":"2023-09-23T20:48:58.057894+0000","last_became_active":"2023-09-23T20:48:07.544029+0000","last_became_peered":"2023-09-23T20:48:07.544029+0000","last_unstale":"2023-09-23T20:48:58.057894+0000","last_undegraded":"2023-09-23T20:48:58.057894+0000","last_fullsized":"2023-09-23T20:48:58.057894+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:52.671448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140012+0000","last_change":"2023-09-23T20:30:22.113582+0000","last_active":"2023-09-23T20:48:58.140012+0000","last_peered":"2023-09-23T20:48:58.140012+0000","last_clean":"2023-09-23T20:48:58.140012+0000","last_became_active":"2023-09-23T20:30:22.113291+0000","last_became_peered":"2023-09-23T20:30:22.113291+0000","last_unstale":"2023-09-23T20:48:58.140012+0000","last_undegraded":"2023-09-23T20:48:58.140012+0000","last_fullsized":"2023-09-23T20:48:58.140012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:19.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060226+0000","last_change":"2023-09-23T20:46:05.097625+0000","last_active":"2023-09-23T20:48:58.060226+0000","last_peered":"2023-09-23T20:48:58.060226+0000","last_clean":"2023-09-23T20:48:58.060226+0000","last_became_active":"2023-09-23T20:46:05.097296+0000","last_became_peered":"2023-09-23T20:46:05.097296+0000","last_unstale":"2023-09-23T20:48:58.060226+0000","last_undegraded":"2023-09-23T20:48:58.060226+0000","last_fullsized":"2023-09-23T20:48:58.060226+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:34.564149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058440+0000","last_change":"2023-09-23T20:48:07.522410+0000","last_active":"2023-09-23T20:48:58.058440+0000","last_peered":"2023-09-23T20:48:58.058440+0000","last_clean":"2023-09-23T20:48:58.058440+0000","last_became_active":"2023-09-23T20:48:07.522058+0000","last_became_peered":"2023-09-23T20:48:07.522058+0000","last_unstale":"2023-09-23T20:48:58.058440+0000","last_undegraded":"2023-09-23T20:48:58.058440+0000","last_fullsized":"2023-09-23T20:48:58.058440+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:24.844663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140085+0000","last_change":"2023-09-23T20:45:00.002738+0000","last_active":"2023-09-23T20:48:58.140085+0000","last_peered":"2023-09-23T20:48:58.140085+0000","last_clean":"2023-09-23T20:48:58.140085+0000","last_became_active":"2023-09-23T20:45:00.002403+0000","last_became_peered":"2023-09-23T20:45:00.002403+0000","last_unstale":"2023-09-23T20:48:58.140085+0000","last_undegraded":"2023-09-23T20:48:58.140085+0000","last_fullsized":"2023-09-23T20:48:58.140085+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:54.551205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054983+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:48:58.054983+0000","last_peered":"2023-09-23T20:48:58.054983+0000","last_clean":"2023-09-23T20:48:58.054983+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:48:58.054983+0000","last_undegraded":"2023-09-23T20:48:58.054983+0000","last_fullsized":"2023-09-23T20:48:58.054983+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057922+0000","last_change":"2023-09-23T20:48:07.520873+0000","last_active":"2023-09-23T20:48:58.057922+0000","last_peered":"2023-09-23T20:48:58.057922+0000","last_clean":"2023-09-23T20:48:58.057922+0000","last_became_active":"2023-09-23T20:48:07.520517+0000","last_became_peered":"2023-09-23T20:48:07.520517+0000","last_unstale":"2023-09-23T20:48:58.057922+0000","last_undegraded":"2023-09-23T20:48:58.057922+0000","last_fullsized":"2023-09-23T20:48:58.057922+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:43.626732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060090+0000","last_change":"2023-09-23T20:45:00.009049+0000","last_active":"2023-09-23T20:48:58.060090+0000","last_peered":"2023-09-23T20:48:58.060090+0000","last_clean":"2023-09-23T20:48:58.060090+0000","last_became_active":"2023-09-23T20:45:00.008700+0000","last_became_peered":"2023-09-23T20:45:00.008700+0000","last_unstale":"2023-09-23T20:48:58.060090+0000","last_undegraded":"2023-09-23T20:48:58.060090+0000","last_fullsized":"2023-09-23T20:48:58.060090+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:52.333134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057799+0000","last_change":"2023-09-23T20:46:05.106502+0000","last_active":"2023-09-23T20:48:58.057799+0000","last_peered":"2023-09-23T20:48:58.057799+0000","last_clean":"2023-09-23T20:48:58.057799+0000","last_became_active":"2023-09-23T20:46:05.106134+0000","last_became_peered":"2023-09-23T20:46:05.106134+0000","last_unstale":"2023-09-23T20:48:58.057799+0000","last_undegraded":"2023-09-23T20:48:58.057799+0000","last_fullsized":"2023-09-23T20:48:58.057799+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:39.257351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057632+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:48:58.057632+0000","last_peered":"2023-09-23T20:48:58.057632+0000","last_clean":"2023-09-23T20:48:58.057632+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:48:58.057632+0000","last_undegraded":"2023-09-23T20:48:58.057632+0000","last_fullsized":"2023-09-23T20:48:58.057632+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058202+0000","last_change":"2023-09-23T20:45:00.008536+0000","last_active":"2023-09-23T20:48:58.058202+0000","last_peered":"2023-09-23T20:48:58.058202+0000","last_clean":"2023-09-23T20:48:58.058202+0000","last_became_active":"2023-09-23T20:45:00.008220+0000","last_became_peered":"2023-09-23T20:45:00.008220+0000","last_unstale":"2023-09-23T20:48:58.058202+0000","last_undegraded":"2023-09-23T20:48:58.058202+0000","last_fullsized":"2023-09-23T20:48:58.058202+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:04.017031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054990+0000","last_change":"2023-09-23T20:48:07.540484+0000","last_active":"2023-09-23T20:48:58.054990+0000","last_peered":"2023-09-23T20:48:58.054990+0000","last_clean":"2023-09-23T20:48:58.054990+0000","last_became_active":"2023-09-23T20:48:07.540219+0000","last_became_peered":"2023-09-23T20:48:07.540219+0000","last_unstale":"2023-09-23T20:48:58.054990+0000","last_undegraded":"2023-09-23T20:48:58.054990+0000","last_fullsized":"2023-09-23T20:48:58.054990+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:54.792744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139793+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:48:58.139793+0000","last_peered":"2023-09-23T20:48:58.139793+0000","last_clean":"2023-09-23T20:48:58.139793+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:48:58.139793+0000","last_undegraded":"2023-09-23T20:48:58.139793+0000","last_fullsized":"2023-09-23T20:48:58.139793+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595372+0000","last_change":"2023-09-23T20:45:00.001830+0000","last_active":"2023-09-23T20:49:02.595372+0000","last_peered":"2023-09-23T20:49:02.595372+0000","last_clean":"2023-09-23T20:49:02.595372+0000","last_became_active":"2023-09-23T20:45:00.001532+0000","last_became_peered":"2023-09-23T20:45:00.001532+0000","last_unstale":"2023-09-23T20:49:02.595372+0000","last_undegraded":"2023-09-23T20:49:02.595372+0000","last_fullsized":"2023-09-23T20:49:02.595372+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:17.499694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058858+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:48:58.058858+0000","last_peered":"2023-09-23T20:48:58.058858+0000","last_clean":"2023-09-23T20:48:58.058858+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:48:58.058858+0000","last_undegraded":"2023-09-23T20:48:58.058858+0000","last_fullsized":"2023-09-23T20:48:58.058858+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595237+0000","last_change":"2023-09-23T20:45:00.004588+0000","last_active":"2023-09-23T20:49:02.595237+0000","last_peered":"2023-09-23T20:49:02.595237+0000","last_clean":"2023-09-23T20:49:02.595237+0000","last_became_active":"2023-09-23T20:45:00.004185+0000","last_became_peered":"2023-09-23T20:45:00.004185+0000","last_unstale":"2023-09-23T20:49:02.595237+0000","last_undegraded":"2023-09-23T20:49:02.595237+0000","last_fullsized":"2023-09-23T20:49:02.595237+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:47.406920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.059029+0000","last_change":"2023-09-23T20:45:00.007439+0000","last_active":"2023-09-23T20:48:58.059029+0000","last_peered":"2023-09-23T20:48:58.059029+0000","last_clean":"2023-09-23T20:48:58.059029+0000","last_became_active":"2023-09-23T20:45:00.007046+0000","last_became_peered":"2023-09-23T20:45:00.007046+0000","last_unstale":"2023-09-23T20:48:58.059029+0000","last_undegraded":"2023-09-23T20:48:58.059029+0000","last_fullsized":"2023-09-23T20:48:58.059029+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:42.363434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058490+0000","last_change":"2023-09-23T20:40:53.853109+0000","last_active":"2023-09-23T20:48:58.058490+0000","last_peered":"2023-09-23T20:48:58.058490+0000","last_clean":"2023-09-23T20:48:58.058490+0000","last_became_active":"2023-09-23T20:40:53.852788+0000","last_became_peered":"2023-09-23T20:40:53.852788+0000","last_unstale":"2023-09-23T20:48:58.058490+0000","last_undegraded":"2023-09-23T20:48:58.058490+0000","last_fullsized":"2023-09-23T20:48:58.058490+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:55.251424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058273+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:48:58.058273+0000","last_peered":"2023-09-23T20:48:58.058273+0000","last_clean":"2023-09-23T20:48:58.058273+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:48:58.058273+0000","last_undegraded":"2023-09-23T20:48:58.058273+0000","last_fullsized":"2023-09-23T20:48:58.058273+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139764+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:48:58.139764+0000","last_peered":"2023-09-23T20:48:58.139764+0000","last_clean":"2023-09-23T20:48:58.139764+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:48:58.139764+0000","last_undegraded":"2023-09-23T20:48:58.139764+0000","last_fullsized":"2023-09-23T20:48:58.139764+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.268923+0000","last_change":"2023-09-23T20:46:05.075939+0000","last_active":"2023-09-23T20:48:58.268923+0000","last_peered":"2023-09-23T20:48:58.268923+0000","last_clean":"2023-09-23T20:48:58.268923+0000","last_became_active":"2023-09-23T20:46:05.075634+0000","last_became_peered":"2023-09-23T20:46:05.075634+0000","last_unstale":"2023-09-23T20:48:58.268923+0000","last_undegraded":"2023-09-23T20:48:58.268923+0000","last_fullsized":"2023-09-23T20:48:58.268923+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:46.867950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.596359+0000","last_change":"2023-09-23T20:45:00.002611+0000","last_active":"2023-09-23T20:49:02.596359+0000","last_peered":"2023-09-23T20:49:02.596359+0000","last_clean":"2023-09-23T20:49:02.596359+0000","last_became_active":"2023-09-23T20:45:00.002331+0000","last_became_peered":"2023-09-23T20:45:00.002331+0000","last_unstale":"2023-09-23T20:49:02.596359+0000","last_undegraded":"2023-09-23T20:49:02.596359+0000","last_fullsized":"2023-09-23T20:49:02.596359+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:19.968130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058681+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:48:58.058681+0000","last_peered":"2023-09-23T20:48:58.058681+0000","last_clean":"2023-09-23T20:48:58.058681+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:48:58.058681+0000","last_undegraded":"2023-09-23T20:48:58.058681+0000","last_fullsized":"2023-09-23T20:48:58.058681+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":503,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139669+0000","last_change":"2023-09-23T20:40:53.853903+0000","last_active":"2023-09-23T20:48:58.139669+0000","last_peered":"2023-09-23T20:48:58.139669+0000","last_clean":"2023-09-23T20:48:58.139669+0000","last_became_active":"2023-09-23T20:40:53.852537+0000","last_became_peered":"2023-09-23T20:40:53.852537+0000","last_unstale":"2023-09-23T20:48:58.139669+0000","last_undegraded":"2023-09-23T20:48:58.139669+0000","last_fullsized":"2023-09-23T20:48:58.139669+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:49.454779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055032+0000","last_change":"2023-09-23T20:48:07.521001+0000","last_active":"2023-09-23T20:48:58.055032+0000","last_peered":"2023-09-23T20:48:58.055032+0000","last_clean":"2023-09-23T20:48:58.055032+0000","last_became_active":"2023-09-23T20:48:07.520697+0000","last_became_peered":"2023-09-23T20:48:07.520697+0000","last_unstale":"2023-09-23T20:48:58.055032+0000","last_undegraded":"2023-09-23T20:48:58.055032+0000","last_fullsized":"2023-09-23T20:48:58.055032+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:21.120614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060569+0000","last_change":"2023-09-23T20:30:22.772777+0000","last_active":"2023-09-23T20:48:58.060569+0000","last_peered":"2023-09-23T20:48:58.060569+0000","last_clean":"2023-09-23T20:48:58.060569+0000","last_became_active":"2023-09-23T20:30:22.772481+0000","last_became_peered":"2023-09-23T20:30:22.772481+0000","last_unstale":"2023-09-23T20:48:58.060569+0000","last_undegraded":"2023-09-23T20:48:58.060569+0000","last_fullsized":"2023-09-23T20:48:58.060569+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:53.724607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139632+0000","last_change":"2023-09-23T20:45:00.001911+0000","last_active":"2023-09-23T20:48:58.139632+0000","last_peered":"2023-09-23T20:48:58.139632+0000","last_clean":"2023-09-23T20:48:58.139632+0000","last_became_active":"2023-09-23T20:45:00.001531+0000","last_became_peered":"2023-09-23T20:45:00.001531+0000","last_unstale":"2023-09-23T20:48:58.139632+0000","last_undegraded":"2023-09-23T20:48:58.139632+0000","last_fullsized":"2023-09-23T20:48:58.139632+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:17.927950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.074369+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:49:02.074369+0000","last_peered":"2023-09-23T20:49:02.074369+0000","last_clean":"2023-09-23T20:49:02.074369+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:49:02.074369+0000","last_undegraded":"2023-09-23T20:49:02.074369+0000","last_fullsized":"2023-09-23T20:49:02.074369+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076247+0000","last_change":"2023-09-23T20:46:05.077316+0000","last_active":"2023-09-23T20:49:02.076247+0000","last_peered":"2023-09-23T20:49:02.076247+0000","last_clean":"2023-09-23T20:49:02.076247+0000","last_became_active":"2023-09-23T20:46:05.077044+0000","last_became_peered":"2023-09-23T20:46:05.077044+0000","last_unstale":"2023-09-23T20:49:02.076247+0000","last_undegraded":"2023-09-23T20:49:02.076247+0000","last_fullsized":"2023-09-23T20:49:02.076247+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:56.517854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140070+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:48:58.140070+0000","last_peered":"2023-09-23T20:48:58.140070+0000","last_clean":"2023-09-23T20:48:58.140070+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:48:58.140070+0000","last_undegraded":"2023-09-23T20:48:58.140070+0000","last_fullsized":"2023-09-23T20:48:58.140070+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":87,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595415+0000","last_change":"2023-09-23T20:46:05.079434+0000","last_active":"2023-09-23T20:49:02.595415+0000","last_peered":"2023-09-23T20:49:02.595415+0000","last_clean":"2023-09-23T20:49:02.595415+0000","last_became_active":"2023-09-23T20:46:05.079111+0000","last_became_peered":"2023-09-23T20:46:05.079111+0000","last_unstale":"2023-09-23T20:49:02.595415+0000","last_undegraded":"2023-09-23T20:49:02.595415+0000","last_fullsized":"2023-09-23T20:49:02.595415+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.869457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072216+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:49:02.072216+0000","last_peered":"2023-09-23T20:49:02.072216+0000","last_clean":"2023-09-23T20:49:02.072216+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:49:02.072216+0000","last_undegraded":"2023-09-23T20:49:02.072216+0000","last_fullsized":"2023-09-23T20:49:02.072216+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055650+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:48:58.055650+0000","last_peered":"2023-09-23T20:48:58.055650+0000","last_clean":"2023-09-23T20:48:58.055650+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:48:58.055650+0000","last_undegraded":"2023-09-23T20:48:58.055650+0000","last_fullsized":"2023-09-23T20:48:58.055650+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595600+0000","last_change":"2023-09-23T20:45:00.005180+0000","last_active":"2023-09-23T20:49:02.595600+0000","last_peered":"2023-09-23T20:49:02.595600+0000","last_clean":"2023-09-23T20:49:02.595600+0000","last_became_active":"2023-09-23T20:45:00.004791+0000","last_became_peered":"2023-09-23T20:45:00.004791+0000","last_unstale":"2023-09-23T20:49:02.595600+0000","last_undegraded":"2023-09-23T20:49:02.595600+0000","last_fullsized":"2023-09-23T20:49:02.595600+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:37.690560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055025+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:48:58.055025+0000","last_peered":"2023-09-23T20:48:58.055025+0000","last_clean":"2023-09-23T20:48:58.055025+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:48:58.055025+0000","last_undegraded":"2023-09-23T20:48:58.055025+0000","last_fullsized":"2023-09-23T20:48:58.055025+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060682+0000","last_change":"2023-09-23T20:46:05.073849+0000","last_active":"2023-09-23T20:48:58.060682+0000","last_peered":"2023-09-23T20:48:58.060682+0000","last_clean":"2023-09-23T20:48:58.060682+0000","last_became_active":"2023-09-23T20:46:05.073536+0000","last_became_peered":"2023-09-23T20:46:05.073536+0000","last_unstale":"2023-09-23T20:48:58.060682+0000","last_undegraded":"2023-09-23T20:48:58.060682+0000","last_fullsized":"2023-09-23T20:48:58.060682+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.086216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076148+0000","last_change":"2023-09-23T20:45:00.005077+0000","last_active":"2023-09-23T20:49:02.076148+0000","last_peered":"2023-09-23T20:49:02.076148+0000","last_clean":"2023-09-23T20:49:02.076148+0000","last_became_active":"2023-09-23T20:45:00.004383+0000","last_became_peered":"2023-09-23T20:45:00.004383+0000","last_unstale":"2023-09-23T20:49:02.076148+0000","last_undegraded":"2023-09-23T20:49:02.076148+0000","last_fullsized":"2023-09-23T20:49:02.076148+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:45.178382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057732+0000","last_change":"2023-09-23T20:30:22.131210+0000","last_active":"2023-09-23T20:48:58.057732+0000","last_peered":"2023-09-23T20:48:58.057732+0000","last_clean":"2023-09-23T20:48:58.057732+0000","last_became_active":"2023-09-23T20:30:22.130879+0000","last_became_peered":"2023-09-23T20:30:22.130879+0000","last_unstale":"2023-09-23T20:48:58.057732+0000","last_undegraded":"2023-09-23T20:48:58.057732+0000","last_fullsized":"2023-09-23T20:48:58.057732+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:10.733014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":99,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140129+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:48:58.140129+0000","last_peered":"2023-09-23T20:48:58.140129+0000","last_clean":"2023-09-23T20:48:58.140129+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:48:58.140129+0000","last_undegraded":"2023-09-23T20:48:58.140129+0000","last_fullsized":"2023-09-23T20:48:58.140129+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139585+0000","last_change":"2023-09-23T20:48:07.539824+0000","last_active":"2023-09-23T20:48:58.139585+0000","last_peered":"2023-09-23T20:48:58.139585+0000","last_clean":"2023-09-23T20:48:58.139585+0000","last_became_active":"2023-09-23T20:48:07.539530+0000","last_became_peered":"2023-09-23T20:48:07.539530+0000","last_unstale":"2023-09-23T20:48:58.139585+0000","last_undegraded":"2023-09-23T20:48:58.139585+0000","last_fullsized":"2023-09-23T20:48:58.139585+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:21.997036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058551+0000","last_change":"2023-09-23T20:45:00.008268+0000","last_active":"2023-09-23T20:48:58.058551+0000","last_peered":"2023-09-23T20:48:58.058551+0000","last_clean":"2023-09-23T20:48:58.058551+0000","last_became_active":"2023-09-23T20:45:00.006393+0000","last_became_peered":"2023-09-23T20:45:00.006393+0000","last_unstale":"2023-09-23T20:48:58.058551+0000","last_undegraded":"2023-09-23T20:48:58.058551+0000","last_fullsized":"2023-09-23T20:48:58.058551+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:41.255405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058217+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.058217+0000","last_peered":"2023-09-23T20:48:58.058217+0000","last_clean":"2023-09-23T20:48:58.058217+0000","last_became_active":"2023-09-23T20:48:07.544066+0000","last_became_peered":"2023-09-23T20:48:07.544066+0000","last_unstale":"2023-09-23T20:48:58.058217+0000","last_undegraded":"2023-09-23T20:48:58.058217+0000","last_fullsized":"2023-09-23T20:48:58.058217+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:28.905486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058955+0000","last_change":"2023-09-23T20:30:22.104198+0000","last_active":"2023-09-23T20:48:58.058955+0000","last_peered":"2023-09-23T20:48:58.058955+0000","last_clean":"2023-09-23T20:48:58.058955+0000","last_became_active":"2023-09-23T20:30:22.103970+0000","last_became_peered":"2023-09-23T20:30:22.103970+0000","last_unstale":"2023-09-23T20:48:58.058955+0000","last_undegraded":"2023-09-23T20:48:58.058955+0000","last_fullsized":"2023-09-23T20:48:58.058955+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:32.820350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058740+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:48:58.058740+0000","last_peered":"2023-09-23T20:48:58.058740+0000","last_clean":"2023-09-23T20:48:58.058740+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:48:58.058740+0000","last_undegraded":"2023-09-23T20:48:58.058740+0000","last_fullsized":"2023-09-23T20:48:58.058740+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055468+0000","last_change":"2023-09-23T20:48:07.521311+0000","last_active":"2023-09-23T20:48:58.055468+0000","last_peered":"2023-09-23T20:48:58.055468+0000","last_clean":"2023-09-23T20:48:58.055468+0000","last_became_active":"2023-09-23T20:48:07.521030+0000","last_became_peered":"2023-09-23T20:48:07.521030+0000","last_unstale":"2023-09-23T20:48:58.055468+0000","last_undegraded":"2023-09-23T20:48:58.055468+0000","last_fullsized":"2023-09-23T20:48:58.055468+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.849053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055021+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:48:58.055021+0000","last_peered":"2023-09-23T20:48:58.055021+0000","last_clean":"2023-09-23T20:48:58.055021+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:48:58.055021+0000","last_undegraded":"2023-09-23T20:48:58.055021+0000","last_fullsized":"2023-09-23T20:48:58.055021+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072229+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:49:02.072229+0000","last_peered":"2023-09-23T20:49:02.072229+0000","last_clean":"2023-09-23T20:49:02.072229+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:49:02.072229+0000","last_undegraded":"2023-09-23T20:49:02.072229+0000","last_fullsized":"2023-09-23T20:49:02.072229+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057625+0000","last_change":"2023-09-23T20:45:00.004734+0000","last_active":"2023-09-23T20:48:58.057625+0000","last_peered":"2023-09-23T20:48:58.057625+0000","last_clean":"2023-09-23T20:48:58.057625+0000","last_became_active":"2023-09-23T20:45:00.004413+0000","last_became_peered":"2023-09-23T20:45:00.004413+0000","last_unstale":"2023-09-23T20:48:58.057625+0000","last_undegraded":"2023-09-23T20:48:58.057625+0000","last_fullsized":"2023-09-23T20:48:58.057625+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:57.454649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201533,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":2180,"kb_used_omap":0,"kb_used_meta":296384,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":2232320,"data_stored":1659199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":303497216},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":4.4119999999999999,"15min":2.488},"min":{"1min":0.371,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.7990000000000004,"5min":225.56999999999999,"15min":225.56999999999999},"last":0.80700000000000005},{"interface":"front","average":{"1min":1.4730000000000001,"5min":4.6440000000000001,"15min":2.8570000000000002},"min":{"1min":0.36599999999999999,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":8.1549999999999994,"5min":225.61099999999999,"15min":225.61099999999999},"last":0.71199999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.132,"5min":3.2690000000000001,"15min":1.867},"min":{"1min":0.377,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":128.97999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":10.537000000000001,"5min":6.306,"15min":2.8900000000000001},"min":{"1min":0.432,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":128.81200000000001,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.78100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":4.1689999999999996,"15min":3.4820000000000002},"min":{"1min":0.40000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.2249999999999996,"5min":142.321,"15min":142.321},"last":0.748},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.3140000000000001,"15min":2.6619999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.8149999999999999,"5min":142.29300000000001,"15min":142.29300000000001},"last":0.73299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.209,"5min":10.209,"15min":10.209},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":138.345,"5min":138.345,"15min":138.345},"last":0.76200000000000001},{"interface":"front","average":{"1min":9.423,"5min":9.423,"15min":9.423},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":133.76400000000001,"5min":133.76400000000001,"15min":133.76400000000001},"last":0.69299999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.361,"15min":1.6379999999999999},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.216},"max":{"1min":3.4319999999999999,"5min":9.1400000000000006,"15min":103.541},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.2949999999999999,"15min":1.75},"min":{"1min":0.30199999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.843,"5min":8.4649999999999999,"15min":107.514},"last":0.64600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.498,"15min":1.8580000000000001},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":6.6260000000000003,"5min":9.4440000000000008,"15min":67.180999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.181,"5min":1.7250000000000001,"15min":1.77},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":7.9109999999999996,"5min":13.156000000000001,"15min":67.135000000000005},"last":0.66700000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.679,"15min":1.5780000000000001},"min":{"1min":0.41999999999999998,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":5.0220000000000002,"5min":10.509,"15min":16.032},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.1830000000000001,"15min":1.3280000000000001},"min":{"1min":0.41399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9269999999999996,"5min":8.4350000000000005,"15min":11.672000000000001},"last":0.79600000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463170,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":255164,"kb_used_data":2164,"kb_used_omap":0,"kb_used_meta":252992,"kb_avail":93489988,"statfs":{"total":95995035648,"available":95733747712,"internally_reserved":0,"allocated":2215936,"data_stored":1658846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":259063808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":7.2859999999999996,"5min":3.4649999999999999,"15min":2.4359999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.20399999999999999},"max":{"1min":92.164000000000001,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":7.2859999999999996,"5min":3.04,"15min":2.431},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":92.138000000000005,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.45100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.5110000000000001,"5min":2.907,"15min":2.073},"min":{"1min":0.30399999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":96.623999999999995,"5min":96.623999999999995,"15min":96.623999999999995},"last":0.54600000000000004},{"interface":"front","average":{"1min":6.5579999999999998,"5min":2.8090000000000002,"15min":1.839},"min":{"1min":0.30399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":96.254000000000005,"5min":96.254000000000005,"15min":96.254000000000005},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.3319999999999999,"15min":2.3029999999999999},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.7009999999999996,"5min":8.9849999999999994,"15min":8.9849999999999994},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.5209999999999999,"5min":2.6269999999999998,"15min":2.6070000000000002},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":10.59,"5min":10.59,"15min":10.59},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.125,"5min":6.125,"15min":6.125},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":79.668000000000006,"5min":79.668000000000006,"15min":79.668000000000006},"last":0.60099999999999998},{"interface":"front","average":{"1min":5.9980000000000002,"5min":5.9980000000000002,"15min":5.9980000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":79.700000000000003,"5min":79.700000000000003,"15min":79.700000000000003},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.2709999999999999,"15min":1.341},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":9.0709999999999997,"5min":9.0709999999999997,"15min":16.247},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.756,"15min":1.6479999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.27000000000000002},"max":{"1min":9.8650000000000002,"5min":9.8650000000000002,"15min":17.163},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.556,"15min":1.518},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.26000000000000001},"max":{"1min":9.0340000000000007,"5min":9.1969999999999992,"15min":11.484999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.214,"5min":1.7490000000000001,"15min":1.631},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":9.8949999999999996,"5min":9.8949999999999996,"15min":17.321000000000002},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:49:29 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":1.5589999999999999,"15min":1.4259999999999999},"min":{"1min":0.40200000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2590000000000003,"5min":9.3249999999999993,"15min":19.451000000000001},"last":0.378},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.605,"15min":1.6299999999999999},"min":{"1min":0.40000000000000002,"5min":0.36099999999999999,"15min":0.32100000000000001},"max":{"1min":9.7959999999999994,"5min":9.7959999999999994,"15min":16.295999999999999},"last":0.57799999999999996}]}]},{"osd":1,"up_from":114,"seq":489626271804,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236760,"kb_used_data":1744,"kb_used_omap":4,"kb_used_meta":235003,"kb_avail":93508392,"statfs":{"total":95995035648,"available":95752593408,"internally_reserved":0,"allocated":1785856,"data_stored":1086600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":240643982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":3.206,"5min":2.4340000000000002,"15min":2.1160000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.191},"max":{"1min":48.350999999999999,"5min":48.350999999999999,"15min":48.350999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":4.7350000000000003,"5min":2.698,"15min":2.048},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":52.018000000000001,"5min":52.018000000000001,"15min":52.018000000000001},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":2.0350000000000001,"15min":2.1789999999999998},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.7479999999999993,"5min":11.593999999999999,"15min":11.593999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.613,"15min":2.855},"min":{"1min":0.29499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.465999999999999,"5min":14.153,"15min":14.153},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.73,"15min":2.73},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.4969999999999999,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.04,"5min":2.04,"15min":2.04},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.5630000000000006,"5min":8.5630000000000006,"15min":8.5630000000000006},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.663,"15min":1.349},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.439,"5min":10.439,"15min":10.439},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.496,"5min":1.431,"15min":1.1870000000000001},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":10.34,"5min":10.34,"15min":10.34},"last":0.71399999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":2.4950000000000001,"15min":1.853},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.5410000000000004,"5min":10.177,"15min":10.177},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.415,"5min":1.635,"15min":1.268},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.3890000000000002,"5min":6.8680000000000003,"15min":6.8680000000000003},"last":0.58999999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.9990000000000001,"15min":1.7010000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.25,"5min":10.25,"15min":10.25},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.008,"5min":1.581,"15min":1.3149999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.4489999999999998,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":0.63800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.871,"15min":1.4790000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.3499999999999996,"5min":7.4480000000000004,"15min":7.4480000000000004},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.948,"5min":2.0569999999999999,"15min":1.911},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.0259999999999998,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.65700000000000003}]}]},{"osd":0,"up_from":92,"seq":395136991341,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":402740,"kb_used_data":2348,"kb_used_omap":5,"kb_used_meta":400378,"kb_avail":93342412,"statfs":{"total":95995035648,"available":95582629888,"internally_reserved":0,"allocated":2404352,"data_stored":1676026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":409987706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5780000000000003,"5min":3.2480000000000002,"15min":2.4420000000000002},"min":{"1min":0.20000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.140000000000001,"5min":64.140000000000001,"15min":64.140000000000001},"last":0.27800000000000002},{"interface":"front","average":{"1min":4.5860000000000003,"5min":3.3679999999999999,"15min":2.5270000000000001},"min":{"1min":0.20999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":59.570999999999998,"5min":59.570999999999998,"15min":59.570999999999998},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":2.319,"15min":2.0569999999999999},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.5839999999999996,"5min":49.652000000000001,"15min":49.652000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.8,"5min":4.0250000000000004,"15min":4.3360000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4720000000000004,"5min":49.595999999999997,"15min":49.595999999999997},"last":0.309}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":41.810000000000002,"5min":41.810000000000002,"15min":41.810000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":41.880000000000003,"5min":41.880000000000003,"15min":41.880000000000003},"last":0.372}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.9750000000000001,"15min":1.7589999999999999},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.3559999999999999,"5min":12.973000000000001,"15min":12.973000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.6850000000000001,"15min":2.1419999999999999},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":8.234,"5min":12.148,"15min":12.148},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":2.6040000000000001,"15min":2.1749999999999998},"min":{"1min":0.40600000000000003,"5min":0.30199999999999999,"15min":0.28799999999999998},"max":{"1min":5.6550000000000002,"5min":14.051,"15min":14.051},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.9410000000000001,"15min":2.2989999999999999},"min":{"1min":0.371,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.3119999999999994,"5min":12.332000000000001,"15min":12.332000000000001},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.1219999999999999,"15min":2.3580000000000001},"min":{"1min":0.39600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":7.8959999999999999,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":2.1659999999999999,"5min":2.2519999999999998,"15min":1.708},"min":{"1min":0.36299999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.1189999999999998,"5min":20.399999999999999,"15min":20.399999999999999},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":1.829,"15min":2.4550000000000001},"min":{"1min":0.46700000000000003,"5min":0.41299999999999998,"15min":0.35099999999999998},"max":{"1min":7.3780000000000001,"5min":12.23,"15min":12.23},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.9350000000000001,"5min":2.2149999999999999,"15min":1.7849999999999999},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.7430000000000003,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.56200000000000006}]}]},{"osd":2,"up_from":129,"seq":554050781231,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270952,"kb_used_data":1696,"kb_used_omap":7,"kb_used_meta":269240,"kb_avail":93474200,"statfs":{"total":95995035648,"available":95717580800,"internally_reserved":0,"allocated":1736704,"data_stored":1078264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":275701841},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.5070000000000001,"15min":2.9980000000000002},"min":{"1min":0.21299999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.5490000000000004,"5min":14.327,"15min":14.327},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.024,"5min":2.323,"15min":2.7589999999999999},"min":{"1min":0.217,"5min":0.186,"15min":0.186},"max":{"1min":6.2359999999999998,"5min":14.757999999999999,"15min":14.757999999999999},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":2.5880000000000001,"15min":2.8900000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.4039999999999999,"5min":13.023,"15min":13.023},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.3759999999999999,"15min":3.0019999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.0419999999999998,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:49:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.959,"5min":11.959,"15min":11.959},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.6539999999999999,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.819,"15min":1.9370000000000001},"min":{"1min":0.34100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.8440000000000003,"5min":7.8440000000000003,"15min":7.8440000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.411,"5min":2.3490000000000002,"15min":2.7109999999999999},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.317},"max":{"1min":6.2789999999999999,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.5630000000000002,"15min":3.0539999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.3760000000000003,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.374},{"interface":"front","average":{"1min":1.7509999999999999,"5min":2.5030000000000001,"15min":2.8610000000000002},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3630000000000004,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.7210000000000001,"15min":3.29},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.3449999999999998,"5min":7.476,"15min":7.476},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.1379999999999999,"5min":2.806,"15min":3.0790000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":9.5220000000000002,"5min":12.595000000000001,"15min":12.595000000000001},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.2320000000000002,"15min":2.5350000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.4909999999999997,"5min":6.4989999999999997,"15min":6.4989999999999997},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.669,"5min":2.5259999999999998,"15min":2.8839999999999999},"min":{"1min":0.41399999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.5979999999999999,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":0.50700000000000001}]}]},{"osd":3,"up_from":148,"seq":635655159831,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":135308,"kb_used_data":1668,"kb_used_omap":2,"kb_used_meta":133629,"kb_avail":93609844,"statfs":{"total":95995035648,"available":95856480256,"internally_reserved":0,"allocated":1708032,"data_stored":1075110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":136836149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":14.332000000000001,"5min":14.332000000000001,"15min":14.332000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":253.68100000000001,"5min":253.68100000000001,"15min":253.68100000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":13.208,"5min":13.208,"15min":13.208},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":253.46199999999999,"5min":253.46199999999999,"15min":253.46199999999999},"last":0.46700000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":10.417999999999999,"5min":10.417999999999999,"15min":10.417999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":172.77099999999999,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":9.4079999999999995,"5min":9.4079999999999995,"15min":9.4079999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":172.36600000000001,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.4619999999999997,"5min":5.4619999999999997,"15min":5.4619999999999997},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.51900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.4749999999999996,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.70999999999999996},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.2170000000000001,"15min":2.2170000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":16.408999999999999,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.76200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.5750000000000002,"5min":3.5750000000000002,"15min":3.5750000000000002},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.958,"5min":1.958,"15min":1.958},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":9.7780000000000005,"5min":9.7780000000000005,"15min":9.7780000000000005},"last":0.79400000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":1.972,"15min":1.972},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.7509999999999994,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.6419999999999999,"5min":2.6419999999999999,"15min":2.6419999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":16.451000000000001,"5min":16.451000000000001,"15min":16.451000000000001},"last":0.69899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.5640000000000001,"15min":1.5640000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.9450000000000003,"5min":4.9450000000000003,"15min":4.9450000000000003},"last":0.71899999999999997},{"interface":"front","average":{"1min":2.351,"5min":2.351,"15min":2.351},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.77300000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986443,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307912,"kb_used_data":1600,"kb_used_omap":0,"kb_used_meta":306304,"kb_avail":93437240,"statfs":{"total":95995035648,"available":95679733760,"internally_reserved":0,"allocated":1638400,"data_stored":1069084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":313655296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.52,"15min":1.839},"min":{"1min":0.216,"5min":0.216,"15min":0.214},"max":{"1min":8.0519999999999996,"5min":12.823,"15min":12.823},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.508,"5min":1.998,"15min":1.321},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.22600000000000001},"max":{"1min":4.3090000000000002,"5min":10.413,"15min":13.211},"last":0.315}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":8.4019999999999992,"5min":6.6639999999999997,"15min":3.23},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":125.35599999999999,"5min":286.31099999999998,"15min":286.31099999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":8.0459999999999994,"5min":6.7679999999999998,"15min":3.996},"min":{"1min":0.312,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":125.21899999999999,"5min":285.97000000000003,"15min":285.97000000000003},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.48,"15min":1.375},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.442},{"interface":"front","average":{"1min":1.988,"5min":2.347,"15min":2.3620000000000001},"min":{"1min":0.39300000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.9450000000000003,"5min":12.773999999999999,"15min":12.773999999999999},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":7.5990000000000002,"5min":7.5990000000000002,"15min":7.5990000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":107.285,"5min":107.285,"15min":107.285},"last":0.372},{"interface":"front","average":{"1min":7.1920000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":107.303,"5min":107.303,"15min":107.303},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.2390000000000001,"15min":1.258},"min":{"1min":0.32500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.2000000000000002,"5min":7.1740000000000004,"15min":17.713999999999999},"last":0.496},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.1879999999999999,"15min":1.331},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.891,"5min":9.0660000000000007,"15min":14.455},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.4630000000000001,"15min":1.5369999999999999},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.4109999999999996,"5min":16.271000000000001,"15min":21.920999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.234,"15min":1.345},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.28599999999999998},"max":{"1min":3.431,"5min":11.718,"15min":11.718},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.4750000000000001,"15min":1.6679999999999999},"min":{"1min":0.42499999999999999,"5min":0.315,"15min":0.31},"max":{"1min":6.0039999999999996,"5min":9.8390000000000004,"15min":21.792999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.321,"15min":1.47},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.317},"max":{"1min":3.306,"5min":7.8399999999999999,"15min":23.457000000000001},"last":0.55600000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724807,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":365368,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":363776,"kb_avail":93379784,"statfs":{"total":95995035648,"available":95620898816,"internally_reserved":0,"allocated":1622016,"data_stored":1064828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":372506624},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.4330000000000001,"15min":1.7669999999999999},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.7559999999999998,"5min":10.94,"15min":12.880000000000001},"last":0.503},{"interface":"front","average":{"1min":1.274,"5min":1.754,"15min":1.3859999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":10.028,"5min":10.955,"15min":10.955},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":4.5990000000000002,"15min":2.2130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":12.529999999999999,"5min":323.541,"15min":323.541},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.262,"5min":4.7270000000000003,"15min":2.5219999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.6900000000000004,"5min":324.072,"15min":324.072},"last":0.69599999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.137,"5min":2.415,"15min":2.423},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.585000000000001,"5min":13.394,"15min":13.394},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.827,"15min":1.663},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7430000000000003,"5min":14.263,"15min":14.263},"last":0.77700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.863,"15min":1.863},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.295,"5min":10.295,"15min":10.295},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.1080000000000005,"5min":8.1080000000000005,"15min":8.1080000000000005},"last":0.66300000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.90400000000000003,"15min":1.153},"min":{"1min":0.33900000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.71499999999999997,"5min":5.8529999999999998,"15min":9.8800000000000008},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.2509999999999999,"15min":1.444},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.23999999999999999},"max":{"1min":0.77400000000000002,"5min":8.3170000000000002,"15min":16.869},"last":0.74199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":1.325,"15min":1.544},"min":{"1min":0.316,"5min":0.28000000000000003,"15min":0.27000000000000002},"max":{"1min":6.1689999999999996,"5min":8.3010000000000002,"15min":17},"last":0.81000000000000005},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.2050000000000001,"15min":1.427},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.9870000000000001,"5min":9.6370000000000005,"15min":14.08},"last":0.72499999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:49:40 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.5960000000000001,"15min":1.6910000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.28199999999999997},"max":{"1min":2.9300000000000002,"5min":14.217000000000001,"15min":17.071999999999999},"last":0.873},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.96699999999999997,"15min":1.1259999999999999},"min":{"1min":0.40300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0499999999999998,"5min":5.601,"15min":8.6370000000000005},"last":0.79200000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:49:52.827 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:49:53.173 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:53 smithi123 ceph-mon[193903]: from='client.35109 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:53 smithi123 ceph-mon[193903]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:54.139 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:53 smithi121 ceph-mon[211526]: from='client.35109 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:54.139 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:53 smithi121 ceph-mon[211526]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:54.139 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:53 smithi121 ceph-mon[219709]: from='client.35109 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:54.139 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:53 smithi121 ceph-mon[219709]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:54.775 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:49:54.777 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:49:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:49:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:49:54] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2023-09-23T20:49:55.380 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":514,"stamp":"2023-09-23T20:49:54.639741+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2272776,"kb_used_data":14984,"kb_used_omap":20,"kb_used_meta":2257707,"kb_avail":747688440,"statfs":{"total":767960285184,"available":765632962560,"internally_reserved":0,"allocated":15343616,"data_stored":10367957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2311892622},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002670"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":141,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058647+0000","last_change":"2023-09-23T20:46:05.100324+0000","last_active":"2023-09-23T20:48:58.058647+0000","last_peered":"2023-09-23T20:48:58.058647+0000","last_clean":"2023-09-23T20:48:58.058647+0000","last_became_active":"2023-09-23T20:46:05.099981+0000","last_became_peered":"2023-09-23T20:46:05.099981+0000","last_unstale":"2023-09-23T20:48:58.058647+0000","last_undegraded":"2023-09-23T20:48:58.058647+0000","last_fullsized":"2023-09-23T20:48:58.058647+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:19.876971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055345+0000","last_change":"2023-09-23T20:48:07.517973+0000","last_active":"2023-09-23T20:48:58.055345+0000","last_peered":"2023-09-23T20:48:58.055345+0000","last_clean":"2023-09-23T20:48:58.055345+0000","last_became_active":"2023-09-23T20:48:07.517681+0000","last_became_peered":"2023-09-23T20:48:07.517681+0000","last_unstale":"2023-09-23T20:48:58.055345+0000","last_undegraded":"2023-09-23T20:48:58.055345+0000","last_fullsized":"2023-09-23T20:48:58.055345+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.789594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":87,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595140+0000","last_change":"2023-09-23T20:48:07.541333+0000","last_active":"2023-09-23T20:49:02.595140+0000","last_peered":"2023-09-23T20:49:02.595140+0000","last_clean":"2023-09-23T20:49:02.595140+0000","last_became_active":"2023-09-23T20:48:07.541014+0000","last_became_peered":"2023-09-23T20:48:07.541014+0000","last_unstale":"2023-09-23T20:49:02.595140+0000","last_undegraded":"2023-09-23T20:49:02.595140+0000","last_fullsized":"2023-09-23T20:49:02.595140+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:48.678651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":112,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140248+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:48:58.140248+0000","last_peered":"2023-09-23T20:48:58.140248+0000","last_clean":"2023-09-23T20:48:58.140248+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:48:58.140248+0000","last_undegraded":"2023-09-23T20:48:58.140248+0000","last_fullsized":"2023-09-23T20:48:58.140248+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054913+0000","last_change":"2023-09-23T20:48:07.540435+0000","last_active":"2023-09-23T20:48:58.054913+0000","last_peered":"2023-09-23T20:48:58.054913+0000","last_clean":"2023-09-23T20:48:58.054913+0000","last_became_active":"2023-09-23T20:48:07.540062+0000","last_became_peered":"2023-09-23T20:48:07.540062+0000","last_unstale":"2023-09-23T20:48:58.054913+0000","last_undegraded":"2023-09-23T20:48:58.054913+0000","last_fullsized":"2023-09-23T20:48:58.054913+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:19.449218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060678+0000","last_change":"2023-09-23T20:45:00.009034+0000","last_active":"2023-09-23T20:48:58.060678+0000","last_peered":"2023-09-23T20:48:58.060678+0000","last_clean":"2023-09-23T20:48:58.060678+0000","last_became_active":"2023-09-23T20:45:00.008706+0000","last_became_peered":"2023-09-23T20:45:00.008706+0000","last_unstale":"2023-09-23T20:48:58.060678+0000","last_undegraded":"2023-09-23T20:48:58.060678+0000","last_fullsized":"2023-09-23T20:48:58.060678+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:29.204761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140415+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:48:58.140415+0000","last_peered":"2023-09-23T20:48:58.140415+0000","last_clean":"2023-09-23T20:48:58.140415+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:48:58.140415+0000","last_undegraded":"2023-09-23T20:48:58.140415+0000","last_fullsized":"2023-09-23T20:48:58.140415+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054946+0000","last_change":"2023-09-23T20:48:07.533499+0000","last_active":"2023-09-23T20:48:58.054946+0000","last_peered":"2023-09-23T20:48:58.054946+0000","last_clean":"2023-09-23T20:48:58.054946+0000","last_became_active":"2023-09-23T20:48:07.520669+0000","last_became_peered":"2023-09-23T20:48:07.520669+0000","last_unstale":"2023-09-23T20:48:58.054946+0000","last_undegraded":"2023-09-23T20:48:58.054946+0000","last_fullsized":"2023-09-23T20:48:58.054946+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:55.109803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058059+0000","last_change":"2023-09-23T20:45:00.007125+0000","last_active":"2023-09-23T20:48:58.058059+0000","last_peered":"2023-09-23T20:48:58.058059+0000","last_clean":"2023-09-23T20:48:58.058059+0000","last_became_active":"2023-09-23T20:45:00.006403+0000","last_became_peered":"2023-09-23T20:45:00.006403+0000","last_unstale":"2023-09-23T20:48:58.058059+0000","last_undegraded":"2023-09-23T20:48:58.058059+0000","last_fullsized":"2023-09-23T20:48:58.058059+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:27.906490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055087+0000","last_change":"2023-09-23T20:48:07.525099+0000","last_active":"2023-09-23T20:48:58.055087+0000","last_peered":"2023-09-23T20:48:58.055087+0000","last_clean":"2023-09-23T20:48:58.055087+0000","last_became_active":"2023-09-23T20:48:07.524868+0000","last_became_peered":"2023-09-23T20:48:07.524868+0000","last_unstale":"2023-09-23T20:48:58.055087+0000","last_undegraded":"2023-09-23T20:48:58.055087+0000","last_fullsized":"2023-09-23T20:48:58.055087+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:19.340586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":126,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140203+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:48:58.140203+0000","last_peered":"2023-09-23T20:48:58.140203+0000","last_clean":"2023-09-23T20:48:58.140203+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:48:58.140203+0000","last_undegraded":"2023-09-23T20:48:58.140203+0000","last_fullsized":"2023-09-23T20:48:58.140203+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595558+0000","last_change":"2023-09-23T20:46:05.099313+0000","last_active":"2023-09-23T20:49:02.595558+0000","last_peered":"2023-09-23T20:49:02.595558+0000","last_clean":"2023-09-23T20:49:02.595558+0000","last_became_active":"2023-09-23T20:46:05.099055+0000","last_became_peered":"2023-09-23T20:46:05.099055+0000","last_unstale":"2023-09-23T20:49:02.595558+0000","last_undegraded":"2023-09-23T20:49:02.595558+0000","last_fullsized":"2023-09-23T20:49:02.595558+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:51.820608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":128,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058022+0000","last_change":"2023-09-23T20:30:22.107118+0000","last_active":"2023-09-23T20:48:58.058022+0000","last_peered":"2023-09-23T20:48:58.058022+0000","last_clean":"2023-09-23T20:48:58.058022+0000","last_became_active":"2023-09-23T20:30:22.106839+0000","last_became_peered":"2023-09-23T20:30:22.106839+0000","last_unstale":"2023-09-23T20:48:58.058022+0000","last_undegraded":"2023-09-23T20:48:58.058022+0000","last_fullsized":"2023-09-23T20:48:58.058022+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:02.981333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058390+0000","last_change":"2023-09-23T20:45:00.007712+0000","last_active":"2023-09-23T20:48:58.058390+0000","last_peered":"2023-09-23T20:48:58.058390+0000","last_clean":"2023-09-23T20:48:58.058390+0000","last_became_active":"2023-09-23T20:45:00.007399+0000","last_became_peered":"2023-09-23T20:45:00.007399+0000","last_unstale":"2023-09-23T20:48:58.058390+0000","last_undegraded":"2023-09-23T20:48:58.058390+0000","last_fullsized":"2023-09-23T20:48:58.058390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:48.842612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139939+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:48:58.139939+0000","last_peered":"2023-09-23T20:48:58.139939+0000","last_clean":"2023-09-23T20:48:58.139939+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:48:58.139939+0000","last_undegraded":"2023-09-23T20:48:58.139939+0000","last_fullsized":"2023-09-23T20:48:58.139939+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055261+0000","last_change":"2023-09-23T20:48:07.540014+0000","last_active":"2023-09-23T20:48:58.055261+0000","last_peered":"2023-09-23T20:48:58.055261+0000","last_clean":"2023-09-23T20:48:58.055261+0000","last_became_active":"2023-09-23T20:48:07.539722+0000","last_became_peered":"2023-09-23T20:48:07.539722+0000","last_unstale":"2023-09-23T20:48:58.055261+0000","last_undegraded":"2023-09-23T20:48:58.055261+0000","last_fullsized":"2023-09-23T20:48:58.055261+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.937232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140140+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:48:58.140140+0000","last_peered":"2023-09-23T20:48:58.140140+0000","last_clean":"2023-09-23T20:48:58.140140+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:48:58.140140+0000","last_undegraded":"2023-09-23T20:48:58.140140+0000","last_fullsized":"2023-09-23T20:48:58.140140+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058777+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:48:58.058777+0000","last_peered":"2023-09-23T20:48:58.058777+0000","last_clean":"2023-09-23T20:48:58.058777+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:48:58.058777+0000","last_undegraded":"2023-09-23T20:48:58.058777+0000","last_fullsized":"2023-09-23T20:48:58.058777+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":106,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139702+0000","last_change":"2023-09-23T20:30:22.110586+0000","last_active":"2023-09-23T20:48:58.139702+0000","last_peered":"2023-09-23T20:48:58.139702+0000","last_clean":"2023-09-23T20:48:58.139702+0000","last_became_active":"2023-09-23T20:30:22.110248+0000","last_became_peered":"2023-09-23T20:30:22.110248+0000","last_unstale":"2023-09-23T20:48:58.139702+0000","last_undegraded":"2023-09-23T20:48:58.139702+0000","last_fullsized":"2023-09-23T20:48:58.139702+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:29.867944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140341+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:48:58.140341+0000","last_peered":"2023-09-23T20:48:58.140341+0000","last_clean":"2023-09-23T20:48:58.140341+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:48:58.140341+0000","last_undegraded":"2023-09-23T20:48:58.140341+0000","last_fullsized":"2023-09-23T20:48:58.140341+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060100+0000","last_change":"2023-09-23T20:40:53.862653+0000","last_active":"2023-09-23T20:48:58.060100+0000","last_peered":"2023-09-23T20:48:58.060100+0000","last_clean":"2023-09-23T20:48:58.060100+0000","last_became_active":"2023-09-23T20:40:53.862259+0000","last_became_peered":"2023-09-23T20:40:53.862259+0000","last_unstale":"2023-09-23T20:48:58.060100+0000","last_undegraded":"2023-09-23T20:48:58.060100+0000","last_fullsized":"2023-09-23T20:48:58.060100+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:28.626871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055685+0000","last_change":"2023-09-23T20:48:07.557905+0000","last_active":"2023-09-23T20:48:58.055685+0000","last_peered":"2023-09-23T20:48:58.055685+0000","last_clean":"2023-09-23T20:48:58.055685+0000","last_became_active":"2023-09-23T20:48:07.557568+0000","last_became_peered":"2023-09-23T20:48:07.557568+0000","last_unstale":"2023-09-23T20:48:58.055685+0000","last_undegraded":"2023-09-23T20:48:58.055685+0000","last_fullsized":"2023-09-23T20:48:58.055685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:14.229704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140303+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:48:58.140303+0000","last_peered":"2023-09-23T20:48:58.140303+0000","last_clean":"2023-09-23T20:48:58.140303+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:48:58.140303+0000","last_undegraded":"2023-09-23T20:48:58.140303+0000","last_fullsized":"2023-09-23T20:48:58.140303+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058595+0000","last_change":"2023-09-23T20:30:22.772475+0000","last_active":"2023-09-23T20:48:58.058595+0000","last_peered":"2023-09-23T20:48:58.058595+0000","last_clean":"2023-09-23T20:48:58.058595+0000","last_became_active":"2023-09-23T20:30:22.772159+0000","last_became_peered":"2023-09-23T20:30:22.772159+0000","last_unstale":"2023-09-23T20:48:58.058595+0000","last_undegraded":"2023-09-23T20:48:58.058595+0000","last_fullsized":"2023-09-23T20:48:58.058595+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:08.373446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140167+0000","last_change":"2023-09-23T20:45:00.003948+0000","last_active":"2023-09-23T20:48:58.140167+0000","last_peered":"2023-09-23T20:48:58.140167+0000","last_clean":"2023-09-23T20:48:58.140167+0000","last_became_active":"2023-09-23T20:45:00.003639+0000","last_became_peered":"2023-09-23T20:45:00.003639+0000","last_unstale":"2023-09-23T20:48:58.140167+0000","last_undegraded":"2023-09-23T20:48:58.140167+0000","last_fullsized":"2023-09-23T20:48:58.140167+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:59.619696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057894+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.057894+0000","last_peered":"2023-09-23T20:48:58.057894+0000","last_clean":"2023-09-23T20:48:58.057894+0000","last_became_active":"2023-09-23T20:48:07.544029+0000","last_became_peered":"2023-09-23T20:48:07.544029+0000","last_unstale":"2023-09-23T20:48:58.057894+0000","last_undegraded":"2023-09-23T20:48:58.057894+0000","last_fullsized":"2023-09-23T20:48:58.057894+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:52.671448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140012+0000","last_change":"2023-09-23T20:30:22.113582+0000","last_active":"2023-09-23T20:48:58.140012+0000","last_peered":"2023-09-23T20:48:58.140012+0000","last_clean":"2023-09-23T20:48:58.140012+0000","last_became_active":"2023-09-23T20:30:22.113291+0000","last_became_peered":"2023-09-23T20:30:22.113291+0000","last_unstale":"2023-09-23T20:48:58.140012+0000","last_undegraded":"2023-09-23T20:48:58.140012+0000","last_fullsized":"2023-09-23T20:48:58.140012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:19.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060226+0000","last_change":"2023-09-23T20:46:05.097625+0000","last_active":"2023-09-23T20:48:58.060226+0000","last_peered":"2023-09-23T20:48:58.060226+0000","last_clean":"2023-09-23T20:48:58.060226+0000","last_became_active":"2023-09-23T20:46:05.097296+0000","last_became_peered":"2023-09-23T20:46:05.097296+0000","last_unstale":"2023-09-23T20:48:58.060226+0000","last_undegraded":"2023-09-23T20:48:58.060226+0000","last_fullsized":"2023-09-23T20:48:58.060226+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:34.564149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058440+0000","last_change":"2023-09-23T20:48:07.522410+0000","last_active":"2023-09-23T20:48:58.058440+0000","last_peered":"2023-09-23T20:48:58.058440+0000","last_clean":"2023-09-23T20:48:58.058440+0000","last_became_active":"2023-09-23T20:48:07.522058+0000","last_became_peered":"2023-09-23T20:48:07.522058+0000","last_unstale":"2023-09-23T20:48:58.058440+0000","last_undegraded":"2023-09-23T20:48:58.058440+0000","last_fullsized":"2023-09-23T20:48:58.058440+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:24.844663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":112,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072375+0000","last_change":"2023-09-23T20:45:00.002738+0000","last_active":"2023-09-23T20:49:02.072375+0000","last_peered":"2023-09-23T20:49:02.072375+0000","last_clean":"2023-09-23T20:49:02.072375+0000","last_became_active":"2023-09-23T20:45:00.002403+0000","last_became_peered":"2023-09-23T20:45:00.002403+0000","last_unstale":"2023-09-23T20:49:02.072375+0000","last_undegraded":"2023-09-23T20:49:02.072375+0000","last_fullsized":"2023-09-23T20:49:02.072375+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:54.551205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054983+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:48:58.054983+0000","last_peered":"2023-09-23T20:48:58.054983+0000","last_clean":"2023-09-23T20:48:58.054983+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:48:58.054983+0000","last_undegraded":"2023-09-23T20:48:58.054983+0000","last_fullsized":"2023-09-23T20:48:58.054983+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057922+0000","last_change":"2023-09-23T20:48:07.520873+0000","last_active":"2023-09-23T20:48:58.057922+0000","last_peered":"2023-09-23T20:48:58.057922+0000","last_clean":"2023-09-23T20:48:58.057922+0000","last_became_active":"2023-09-23T20:48:07.520517+0000","last_became_peered":"2023-09-23T20:48:07.520517+0000","last_unstale":"2023-09-23T20:48:58.057922+0000","last_undegraded":"2023-09-23T20:48:58.057922+0000","last_fullsized":"2023-09-23T20:48:58.057922+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:43.626732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060090+0000","last_change":"2023-09-23T20:45:00.009049+0000","last_active":"2023-09-23T20:48:58.060090+0000","last_peered":"2023-09-23T20:48:58.060090+0000","last_clean":"2023-09-23T20:48:58.060090+0000","last_became_active":"2023-09-23T20:45:00.008700+0000","last_became_peered":"2023-09-23T20:45:00.008700+0000","last_unstale":"2023-09-23T20:48:58.060090+0000","last_undegraded":"2023-09-23T20:48:58.060090+0000","last_fullsized":"2023-09-23T20:48:58.060090+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:52.333134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057799+0000","last_change":"2023-09-23T20:46:05.106502+0000","last_active":"2023-09-23T20:48:58.057799+0000","last_peered":"2023-09-23T20:48:58.057799+0000","last_clean":"2023-09-23T20:48:58.057799+0000","last_became_active":"2023-09-23T20:46:05.106134+0000","last_became_peered":"2023-09-23T20:46:05.106134+0000","last_unstale":"2023-09-23T20:48:58.057799+0000","last_undegraded":"2023-09-23T20:48:58.057799+0000","last_fullsized":"2023-09-23T20:48:58.057799+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:39.257351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057632+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:48:58.057632+0000","last_peered":"2023-09-23T20:48:58.057632+0000","last_clean":"2023-09-23T20:48:58.057632+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:48:58.057632+0000","last_undegraded":"2023-09-23T20:48:58.057632+0000","last_fullsized":"2023-09-23T20:48:58.057632+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058202+0000","last_change":"2023-09-23T20:45:00.008536+0000","last_active":"2023-09-23T20:48:58.058202+0000","last_peered":"2023-09-23T20:48:58.058202+0000","last_clean":"2023-09-23T20:48:58.058202+0000","last_became_active":"2023-09-23T20:45:00.008220+0000","last_became_peered":"2023-09-23T20:45:00.008220+0000","last_unstale":"2023-09-23T20:48:58.058202+0000","last_undegraded":"2023-09-23T20:48:58.058202+0000","last_fullsized":"2023-09-23T20:48:58.058202+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:04.017031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.054990+0000","last_change":"2023-09-23T20:48:07.540484+0000","last_active":"2023-09-23T20:48:58.054990+0000","last_peered":"2023-09-23T20:48:58.054990+0000","last_clean":"2023-09-23T20:48:58.054990+0000","last_became_active":"2023-09-23T20:48:07.540219+0000","last_became_peered":"2023-09-23T20:48:07.540219+0000","last_unstale":"2023-09-23T20:48:58.054990+0000","last_undegraded":"2023-09-23T20:48:58.054990+0000","last_fullsized":"2023-09-23T20:48:58.054990+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:54.792744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139793+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:48:58.139793+0000","last_peered":"2023-09-23T20:48:58.139793+0000","last_clean":"2023-09-23T20:48:58.139793+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:48:58.139793+0000","last_undegraded":"2023-09-23T20:48:58.139793+0000","last_fullsized":"2023-09-23T20:48:58.139793+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595372+0000","last_change":"2023-09-23T20:45:00.001830+0000","last_active":"2023-09-23T20:49:02.595372+0000","last_peered":"2023-09-23T20:49:02.595372+0000","last_clean":"2023-09-23T20:49:02.595372+0000","last_became_active":"2023-09-23T20:45:00.001532+0000","last_became_peered":"2023-09-23T20:45:00.001532+0000","last_unstale":"2023-09-23T20:49:02.595372+0000","last_undegraded":"2023-09-23T20:49:02.595372+0000","last_fullsized":"2023-09-23T20:49:02.595372+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:17.499694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058858+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:48:58.058858+0000","last_peered":"2023-09-23T20:48:58.058858+0000","last_clean":"2023-09-23T20:48:58.058858+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:48:58.058858+0000","last_undegraded":"2023-09-23T20:48:58.058858+0000","last_fullsized":"2023-09-23T20:48:58.058858+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595237+0000","last_change":"2023-09-23T20:45:00.004588+0000","last_active":"2023-09-23T20:49:02.595237+0000","last_peered":"2023-09-23T20:49:02.595237+0000","last_clean":"2023-09-23T20:49:02.595237+0000","last_became_active":"2023-09-23T20:45:00.004185+0000","last_became_peered":"2023-09-23T20:45:00.004185+0000","last_unstale":"2023-09-23T20:49:02.595237+0000","last_undegraded":"2023-09-23T20:49:02.595237+0000","last_fullsized":"2023-09-23T20:49:02.595237+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:47.406920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.059029+0000","last_change":"2023-09-23T20:45:00.007439+0000","last_active":"2023-09-23T20:48:58.059029+0000","last_peered":"2023-09-23T20:48:58.059029+0000","last_clean":"2023-09-23T20:48:58.059029+0000","last_became_active":"2023-09-23T20:45:00.007046+0000","last_became_peered":"2023-09-23T20:45:00.007046+0000","last_unstale":"2023-09-23T20:48:58.059029+0000","last_undegraded":"2023-09-23T20:48:58.059029+0000","last_fullsized":"2023-09-23T20:48:58.059029+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:42.363434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058490+0000","last_change":"2023-09-23T20:40:53.853109+0000","last_active":"2023-09-23T20:48:58.058490+0000","last_peered":"2023-09-23T20:48:58.058490+0000","last_clean":"2023-09-23T20:48:58.058490+0000","last_became_active":"2023-09-23T20:40:53.852788+0000","last_became_peered":"2023-09-23T20:40:53.852788+0000","last_unstale":"2023-09-23T20:48:58.058490+0000","last_undegraded":"2023-09-23T20:48:58.058490+0000","last_fullsized":"2023-09-23T20:48:58.058490+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:55.251424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058273+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:48:58.058273+0000","last_peered":"2023-09-23T20:48:58.058273+0000","last_clean":"2023-09-23T20:48:58.058273+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:48:58.058273+0000","last_undegraded":"2023-09-23T20:48:58.058273+0000","last_fullsized":"2023-09-23T20:48:58.058273+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139764+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:48:58.139764+0000","last_peered":"2023-09-23T20:48:58.139764+0000","last_clean":"2023-09-23T20:48:58.139764+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:48:58.139764+0000","last_undegraded":"2023-09-23T20:48:58.139764+0000","last_fullsized":"2023-09-23T20:48:58.139764+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.268923+0000","last_change":"2023-09-23T20:46:05.075939+0000","last_active":"2023-09-23T20:48:58.268923+0000","last_peered":"2023-09-23T20:48:58.268923+0000","last_clean":"2023-09-23T20:48:58.268923+0000","last_became_active":"2023-09-23T20:46:05.075634+0000","last_became_peered":"2023-09-23T20:46:05.075634+0000","last_unstale":"2023-09-23T20:48:58.268923+0000","last_undegraded":"2023-09-23T20:48:58.268923+0000","last_fullsized":"2023-09-23T20:48:58.268923+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:46.867950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.596359+0000","last_change":"2023-09-23T20:45:00.002611+0000","last_active":"2023-09-23T20:49:02.596359+0000","last_peered":"2023-09-23T20:49:02.596359+0000","last_clean":"2023-09-23T20:49:02.596359+0000","last_became_active":"2023-09-23T20:45:00.002331+0000","last_became_peered":"2023-09-23T20:45:00.002331+0000","last_unstale":"2023-09-23T20:49:02.596359+0000","last_undegraded":"2023-09-23T20:49:02.596359+0000","last_fullsized":"2023-09-23T20:49:02.596359+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:19.968130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058681+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:48:58.058681+0000","last_peered":"2023-09-23T20:48:58.058681+0000","last_clean":"2023-09-23T20:48:58.058681+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:48:58.058681+0000","last_undegraded":"2023-09-23T20:48:58.058681+0000","last_fullsized":"2023-09-23T20:48:58.058681+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":503,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139669+0000","last_change":"2023-09-23T20:40:53.853903+0000","last_active":"2023-09-23T20:48:58.139669+0000","last_peered":"2023-09-23T20:48:58.139669+0000","last_clean":"2023-09-23T20:48:58.139669+0000","last_became_active":"2023-09-23T20:40:53.852537+0000","last_became_peered":"2023-09-23T20:40:53.852537+0000","last_unstale":"2023-09-23T20:48:58.139669+0000","last_undegraded":"2023-09-23T20:48:58.139669+0000","last_fullsized":"2023-09-23T20:48:58.139669+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:49.454779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055032+0000","last_change":"2023-09-23T20:48:07.521001+0000","last_active":"2023-09-23T20:48:58.055032+0000","last_peered":"2023-09-23T20:48:58.055032+0000","last_clean":"2023-09-23T20:48:58.055032+0000","last_became_active":"2023-09-23T20:48:07.520697+0000","last_became_peered":"2023-09-23T20:48:07.520697+0000","last_unstale":"2023-09-23T20:48:58.055032+0000","last_undegraded":"2023-09-23T20:48:58.055032+0000","last_fullsized":"2023-09-23T20:48:58.055032+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:21.120614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060569+0000","last_change":"2023-09-23T20:30:22.772777+0000","last_active":"2023-09-23T20:48:58.060569+0000","last_peered":"2023-09-23T20:48:58.060569+0000","last_clean":"2023-09-23T20:48:58.060569+0000","last_became_active":"2023-09-23T20:30:22.772481+0000","last_became_peered":"2023-09-23T20:30:22.772481+0000","last_unstale":"2023-09-23T20:48:58.060569+0000","last_undegraded":"2023-09-23T20:48:58.060569+0000","last_fullsized":"2023-09-23T20:48:58.060569+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:53.724607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139632+0000","last_change":"2023-09-23T20:45:00.001911+0000","last_active":"2023-09-23T20:48:58.139632+0000","last_peered":"2023-09-23T20:48:58.139632+0000","last_clean":"2023-09-23T20:48:58.139632+0000","last_became_active":"2023-09-23T20:45:00.001531+0000","last_became_peered":"2023-09-23T20:45:00.001531+0000","last_unstale":"2023-09-23T20:48:58.139632+0000","last_undegraded":"2023-09-23T20:48:58.139632+0000","last_fullsized":"2023-09-23T20:48:58.139632+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:17.927950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.074369+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:49:02.074369+0000","last_peered":"2023-09-23T20:49:02.074369+0000","last_clean":"2023-09-23T20:49:02.074369+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:49:02.074369+0000","last_undegraded":"2023-09-23T20:49:02.074369+0000","last_fullsized":"2023-09-23T20:49:02.074369+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076247+0000","last_change":"2023-09-23T20:46:05.077316+0000","last_active":"2023-09-23T20:49:02.076247+0000","last_peered":"2023-09-23T20:49:02.076247+0000","last_clean":"2023-09-23T20:49:02.076247+0000","last_became_active":"2023-09-23T20:46:05.077044+0000","last_became_peered":"2023-09-23T20:46:05.077044+0000","last_unstale":"2023-09-23T20:49:02.076247+0000","last_undegraded":"2023-09-23T20:49:02.076247+0000","last_fullsized":"2023-09-23T20:49:02.076247+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:56.517854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140070+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:48:58.140070+0000","last_peered":"2023-09-23T20:48:58.140070+0000","last_clean":"2023-09-23T20:48:58.140070+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:48:58.140070+0000","last_undegraded":"2023-09-23T20:48:58.140070+0000","last_fullsized":"2023-09-23T20:48:58.140070+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":87,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595415+0000","last_change":"2023-09-23T20:46:05.079434+0000","last_active":"2023-09-23T20:49:02.595415+0000","last_peered":"2023-09-23T20:49:02.595415+0000","last_clean":"2023-09-23T20:49:02.595415+0000","last_became_active":"2023-09-23T20:46:05.079111+0000","last_became_peered":"2023-09-23T20:46:05.079111+0000","last_unstale":"2023-09-23T20:49:02.595415+0000","last_undegraded":"2023-09-23T20:49:02.595415+0000","last_fullsized":"2023-09-23T20:49:02.595415+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.869457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072216+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:49:02.072216+0000","last_peered":"2023-09-23T20:49:02.072216+0000","last_clean":"2023-09-23T20:49:02.072216+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:49:02.072216+0000","last_undegraded":"2023-09-23T20:49:02.072216+0000","last_fullsized":"2023-09-23T20:49:02.072216+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055650+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:48:58.055650+0000","last_peered":"2023-09-23T20:48:58.055650+0000","last_clean":"2023-09-23T20:48:58.055650+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:48:58.055650+0000","last_undegraded":"2023-09-23T20:48:58.055650+0000","last_fullsized":"2023-09-23T20:48:58.055650+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":114,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.595600+0000","last_change":"2023-09-23T20:45:00.005180+0000","last_active":"2023-09-23T20:49:02.595600+0000","last_peered":"2023-09-23T20:49:02.595600+0000","last_clean":"2023-09-23T20:49:02.595600+0000","last_became_active":"2023-09-23T20:45:00.004791+0000","last_became_peered":"2023-09-23T20:45:00.004791+0000","last_unstale":"2023-09-23T20:49:02.595600+0000","last_undegraded":"2023-09-23T20:49:02.595600+0000","last_fullsized":"2023-09-23T20:49:02.595600+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:37.690560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055025+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:48:58.055025+0000","last_peered":"2023-09-23T20:48:58.055025+0000","last_clean":"2023-09-23T20:48:58.055025+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:48:58.055025+0000","last_undegraded":"2023-09-23T20:48:58.055025+0000","last_fullsized":"2023-09-23T20:48:58.055025+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.060682+0000","last_change":"2023-09-23T20:46:05.073849+0000","last_active":"2023-09-23T20:48:58.060682+0000","last_peered":"2023-09-23T20:48:58.060682+0000","last_clean":"2023-09-23T20:48:58.060682+0000","last_became_active":"2023-09-23T20:46:05.073536+0000","last_became_peered":"2023-09-23T20:46:05.073536+0000","last_unstale":"2023-09-23T20:48:58.060682+0000","last_undegraded":"2023-09-23T20:48:58.060682+0000","last_fullsized":"2023-09-23T20:48:58.060682+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.086216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.076148+0000","last_change":"2023-09-23T20:45:00.005077+0000","last_active":"2023-09-23T20:49:02.076148+0000","last_peered":"2023-09-23T20:49:02.076148+0000","last_clean":"2023-09-23T20:49:02.076148+0000","last_became_active":"2023-09-23T20:45:00.004383+0000","last_became_peered":"2023-09-23T20:45:00.004383+0000","last_unstale":"2023-09-23T20:49:02.076148+0000","last_undegraded":"2023-09-23T20:49:02.076148+0000","last_fullsized":"2023-09-23T20:49:02.076148+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:45.178382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057732+0000","last_change":"2023-09-23T20:30:22.131210+0000","last_active":"2023-09-23T20:48:58.057732+0000","last_peered":"2023-09-23T20:48:58.057732+0000","last_clean":"2023-09-23T20:48:58.057732+0000","last_became_active":"2023-09-23T20:30:22.130879+0000","last_became_peered":"2023-09-23T20:30:22.130879+0000","last_unstale":"2023-09-23T20:48:58.057732+0000","last_undegraded":"2023-09-23T20:48:58.057732+0000","last_fullsized":"2023-09-23T20:48:58.057732+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:10.733014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":99,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.140129+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:48:58.140129+0000","last_peered":"2023-09-23T20:48:58.140129+0000","last_clean":"2023-09-23T20:48:58.140129+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:48:58.140129+0000","last_undegraded":"2023-09-23T20:48:58.140129+0000","last_fullsized":"2023-09-23T20:48:58.140129+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.139585+0000","last_change":"2023-09-23T20:48:07.539824+0000","last_active":"2023-09-23T20:48:58.139585+0000","last_peered":"2023-09-23T20:48:58.139585+0000","last_clean":"2023-09-23T20:48:58.139585+0000","last_became_active":"2023-09-23T20:48:07.539530+0000","last_became_peered":"2023-09-23T20:48:07.539530+0000","last_unstale":"2023-09-23T20:48:58.139585+0000","last_undegraded":"2023-09-23T20:48:58.139585+0000","last_fullsized":"2023-09-23T20:48:58.139585+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:21.997036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058551+0000","last_change":"2023-09-23T20:45:00.008268+0000","last_active":"2023-09-23T20:48:58.058551+0000","last_peered":"2023-09-23T20:48:58.058551+0000","last_clean":"2023-09-23T20:48:58.058551+0000","last_became_active":"2023-09-23T20:45:00.006393+0000","last_became_peered":"2023-09-23T20:45:00.006393+0000","last_unstale":"2023-09-23T20:48:58.058551+0000","last_undegraded":"2023-09-23T20:48:58.058551+0000","last_fullsized":"2023-09-23T20:48:58.058551+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:41.255405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058217+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:48:58.058217+0000","last_peered":"2023-09-23T20:48:58.058217+0000","last_clean":"2023-09-23T20:48:58.058217+0000","last_became_active":"2023-09-23T20:48:07.544066+0000","last_became_peered":"2023-09-23T20:48:07.544066+0000","last_unstale":"2023-09-23T20:48:58.058217+0000","last_undegraded":"2023-09-23T20:48:58.058217+0000","last_fullsized":"2023-09-23T20:48:58.058217+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:28.905486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":108,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058955+0000","last_change":"2023-09-23T20:30:22.104198+0000","last_active":"2023-09-23T20:48:58.058955+0000","last_peered":"2023-09-23T20:48:58.058955+0000","last_clean":"2023-09-23T20:48:58.058955+0000","last_became_active":"2023-09-23T20:30:22.103970+0000","last_became_peered":"2023-09-23T20:30:22.103970+0000","last_unstale":"2023-09-23T20:48:58.058955+0000","last_undegraded":"2023-09-23T20:48:58.058955+0000","last_fullsized":"2023-09-23T20:48:58.058955+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:32.820350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.058740+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:48:58.058740+0000","last_peered":"2023-09-23T20:48:58.058740+0000","last_clean":"2023-09-23T20:48:58.058740+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:48:58.058740+0000","last_undegraded":"2023-09-23T20:48:58.058740+0000","last_fullsized":"2023-09-23T20:48:58.058740+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":118,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055468+0000","last_change":"2023-09-23T20:48:07.521311+0000","last_active":"2023-09-23T20:48:58.055468+0000","last_peered":"2023-09-23T20:48:58.055468+0000","last_clean":"2023-09-23T20:48:58.055468+0000","last_became_active":"2023-09-23T20:48:07.521030+0000","last_became_peered":"2023-09-23T20:48:07.521030+0000","last_unstale":"2023-09-23T20:48:58.055468+0000","last_undegraded":"2023-09-23T20:48:58.055468+0000","last_fullsized":"2023-09-23T20:48:58.055468+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.849053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.055021+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:48:58.055021+0000","last_peered":"2023-09-23T20:48:58.055021+0000","last_clean":"2023-09-23T20:48:58.055021+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:48:58.055021+0000","last_undegraded":"2023-09-23T20:48:58.055021+0000","last_fullsized":"2023-09-23T20:48:58.055021+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":115,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-09-23T20:49:02.072229+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:49:02.072229+0000","last_peered":"2023-09-23T20:49:02.072229+0000","last_clean":"2023-09-23T20:49:02.072229+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:49:02.072229+0000","last_undegraded":"2023-09-23T20:49:02.072229+0000","last_fullsized":"2023-09-23T20:49:02.072229+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-09-23T20:48:58.057625+0000","last_change":"2023-09-23T20:45:00.004734+0000","last_active":"2023-09-23T20:48:58.057625+0000","last_peered":"2023-09-23T20:48:58.057625+0000","last_clean":"2023-09-23T20:48:58.057625+0000","last_became_active":"2023-09-23T20:45:00.004413+0000","last_became_peered":"2023-09-23T20:45:00.004413+0000","last_unstale":"2023-09-23T20:48:58.057625+0000","last_undegraded":"2023-09-23T20:48:58.057625+0000","last_fullsized":"2023-09-23T20:48:58.057625+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:57.454649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201534,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":2180,"kb_used_omap":0,"kb_used_meta":296384,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":2232320,"data_stored":1659199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":303497216},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":4.4119999999999999,"15min":2.488},"min":{"1min":0.371,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.7990000000000004,"5min":225.56999999999999,"15min":225.56999999999999},"last":0.436},{"interface":"front","average":{"1min":1.4730000000000001,"5min":4.6440000000000001,"15min":2.8570000000000002},"min":{"1min":0.36599999999999999,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":8.1549999999999994,"5min":225.61099999999999,"15min":225.61099999999999},"last":0.57099999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.132,"5min":3.2690000000000001,"15min":1.867},"min":{"1min":0.377,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":128.97999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.495},{"interface":"front","average":{"1min":10.537000000000001,"5min":6.306,"15min":2.8900000000000001},"min":{"1min":0.432,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":128.81200000000001,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":4.1689999999999996,"15min":3.4820000000000002},"min":{"1min":0.40000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.2249999999999996,"5min":142.321,"15min":142.321},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.3140000000000001,"15min":2.6619999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.8149999999999999,"5min":142.29300000000001,"15min":142.29300000000001},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":10.209,"5min":10.209,"15min":10.209},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":138.345,"5min":138.345,"15min":138.345},"last":0.38500000000000001},{"interface":"front","average":{"1min":9.423,"5min":9.423,"15min":9.423},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":133.76400000000001,"5min":133.76400000000001,"15min":133.76400000000001},"last":0.40699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.361,"15min":1.6379999999999999},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.216},"max":{"1min":3.4319999999999999,"5min":9.1400000000000006,"15min":103.541},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.2949999999999999,"15min":1.75},"min":{"1min":0.30199999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.843,"5min":8.4649999999999999,"15min":107.514},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.498,"15min":1.8580000000000001},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":6.6260000000000003,"5min":9.4440000000000008,"15min":67.180999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.181,"5min":1.7250000000000001,"15min":1.77},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":7.9109999999999996,"5min":13.156000000000001,"15min":67.135000000000005},"last":0.625}]},{"osd":6,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.679,"15min":1.5780000000000001},"min":{"1min":0.41999999999999998,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":5.0220000000000002,"5min":10.509,"15min":16.032},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.1830000000000001,"15min":1.3280000000000001},"min":{"1min":0.41399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9269999999999996,"5min":8.4350000000000005,"15min":11.672000000000001},"last":0.55500000000000005}]}]},{"osd":6,"up_from":47,"seq":201863463171,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":255164,"kb_used_data":2164,"kb_used_omap":0,"kb_used_meta":252992,"kb_avail":93489988,"statfs":{"total":95995035648,"available":95733747712,"internally_reserved":0,"allocated":2215936,"data_stored":1658846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":259063808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":7.2859999999999996,"5min":3.4649999999999999,"15min":2.4359999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.20399999999999999},"max":{"1min":92.164000000000001,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.434},{"interface":"front","average":{"1min":7.2859999999999996,"5min":3.04,"15min":2.431},"min":{"1min":0.24099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":92.138000000000005,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.5110000000000001,"5min":2.907,"15min":2.073},"min":{"1min":0.30399999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":96.623999999999995,"5min":96.623999999999995,"15min":96.623999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":6.5579999999999998,"5min":2.8090000000000002,"15min":1.839},"min":{"1min":0.30399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":96.254000000000005,"5min":96.254000000000005,"15min":96.254000000000005},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.3319999999999999,"15min":2.3029999999999999},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.7009999999999996,"5min":8.9849999999999994,"15min":8.9849999999999994},"last":0.625},{"interface":"front","average":{"1min":2.5209999999999999,"5min":2.6269999999999998,"15min":2.6070000000000002},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":10.59,"5min":10.59,"15min":10.59},"last":0.58599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":6.125,"5min":6.125,"15min":6.125},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":79.668000000000006,"5min":79.668000000000006,"15min":79.668000000000006},"last":0.66000000000000003},{"interface":"front","average":{"1min":5.9980000000000002,"5min":5.9980000000000002,"15min":5.9980000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":79.700000000000003,"5min":79.700000000000003,"15min":79.700000000000003},"last":0.71299999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.2709999999999999,"15min":1.341},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.255},"max":{"1min":9.0709999999999997,"5min":9.0709999999999997,"15min":16.247},"last":0.69399999999999995},{"interface":"front","average":{"1min":2.1850000000000001,"5min":1.756,"15min":1.6479999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.27000000000000002},"max":{"1min":9.8650000000000002,"5min":9.8650000000000002,"15min":17.163},"last":0.74099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.556,"15min":1.518},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.26000000000000001},"max":{"1min":9.0340000000000007,"5min":9.1969999999999992,"15min":11.484999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":2.214,"5min":1.7490000000000001,"15min":1.631},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":9.8949999999999996,"5min":9.8949999999999996,"15min":17.321000000000002},"last":0.78100000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:49:29 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":1.5589999999999999,"15min":1.4259999999999999},"min":{"1min":0.40200000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2590000000000003,"5min":9.3249999999999993,"15min":19.451000000000001},"last":0.77300000000000002},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.605,"15min":1.6299999999999999},"min":{"1min":0.40000000000000002,"5min":0.36099999999999999,"15min":0.32100000000000001},"max":{"1min":9.7959999999999994,"5min":9.7959999999999994,"15min":16.295999999999999},"last":0.80900000000000005}]}]},{"osd":1,"up_from":114,"seq":489626271805,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236760,"kb_used_data":1744,"kb_used_omap":4,"kb_used_meta":235003,"kb_avail":93508392,"statfs":{"total":95995035648,"available":95752593408,"internally_reserved":0,"allocated":1785856,"data_stored":1086600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":240643982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":3.206,"5min":2.4340000000000002,"15min":2.1160000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.191},"max":{"1min":48.350999999999999,"5min":48.350999999999999,"15min":48.350999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":4.7350000000000003,"5min":2.698,"15min":2.048},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":52.018000000000001,"5min":52.018000000000001,"15min":52.018000000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":2.0350000000000001,"15min":2.1789999999999998},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":8.7479999999999993,"5min":11.593999999999999,"15min":11.593999999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.8160000000000001,"5min":2.613,"15min":2.855},"min":{"1min":0.29499999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.465999999999999,"5min":14.153,"15min":14.153},"last":0.29099999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.73,"15min":2.73},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.4969999999999999,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.04,"5min":2.04,"15min":2.04},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.5630000000000006,"5min":8.5630000000000006,"15min":8.5630000000000006},"last":0.32000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.663,"15min":1.349},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.439,"5min":10.439,"15min":10.439},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.496,"5min":1.431,"15min":1.1870000000000001},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":10.34,"5min":10.34,"15min":10.34},"last":0.51200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":2.4950000000000001,"15min":1.853},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.5410000000000004,"5min":10.177,"15min":10.177},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.415,"5min":1.635,"15min":1.268},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.3890000000000002,"5min":6.8680000000000003,"15min":6.8680000000000003},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.984,"5min":1.9990000000000001,"15min":1.7010000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.25,"5min":10.25,"15min":10.25},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.008,"5min":1.581,"15min":1.3149999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.4489999999999998,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:49:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":1.871,"15min":1.4790000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.3499999999999996,"5min":7.4480000000000004,"15min":7.4480000000000004},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.948,"5min":2.0569999999999999,"15min":1.911},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.0259999999999998,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.52100000000000002}]}]},{"osd":0,"up_from":92,"seq":395136991342,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":402740,"kb_used_data":2348,"kb_used_omap":5,"kb_used_meta":400378,"kb_avail":93342412,"statfs":{"total":95995035648,"available":95582629888,"internally_reserved":0,"allocated":2404352,"data_stored":1676026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":409987706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5780000000000003,"5min":3.2480000000000002,"15min":2.4420000000000002},"min":{"1min":0.20000000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.140000000000001,"5min":64.140000000000001,"15min":64.140000000000001},"last":0.249},{"interface":"front","average":{"1min":4.5860000000000003,"5min":3.3679999999999999,"15min":2.5270000000000001},"min":{"1min":0.20999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":59.570999999999998,"5min":59.570999999999998,"15min":59.570999999999998},"last":0.26700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":2.319,"15min":2.0569999999999999},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.5839999999999996,"5min":49.652000000000001,"15min":49.652000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.8,"5min":4.0250000000000004,"15min":4.3360000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4720000000000004,"5min":49.595999999999997,"15min":49.595999999999997},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.4089999999999998,"5min":3.4089999999999998,"15min":3.4089999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":41.810000000000002,"5min":41.810000000000002,"15min":41.810000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":41.880000000000003,"5min":41.880000000000003,"15min":41.880000000000003},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.9750000000000001,"15min":1.7589999999999999},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.3559999999999999,"5min":12.973000000000001,"15min":12.973000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.6850000000000001,"15min":2.1419999999999999},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":8.234,"5min":12.148,"15min":12.148},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":2.6040000000000001,"15min":2.1749999999999998},"min":{"1min":0.40600000000000003,"5min":0.30199999999999999,"15min":0.28799999999999998},"max":{"1min":5.6550000000000002,"5min":14.051,"15min":14.051},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.9410000000000001,"15min":2.2989999999999999},"min":{"1min":0.371,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.3119999999999994,"5min":12.332000000000001,"15min":12.332000000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.1219999999999999,"15min":2.3580000000000001},"min":{"1min":0.39600000000000002,"5min":0.307,"15min":0.307},"max":{"1min":7.8959999999999999,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.1659999999999999,"5min":2.2519999999999998,"15min":1.708},"min":{"1min":0.36299999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":9.1189999999999998,"5min":20.399999999999999,"15min":20.399999999999999},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":1.829,"15min":2.4550000000000001},"min":{"1min":0.46700000000000003,"5min":0.41299999999999998,"15min":0.35099999999999998},"max":{"1min":7.3780000000000001,"5min":12.23,"15min":12.23},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.9350000000000001,"5min":2.2149999999999999,"15min":1.7849999999999999},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.7430000000000003,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.501}]}]},{"osd":2,"up_from":129,"seq":554050781232,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270952,"kb_used_data":1696,"kb_used_omap":7,"kb_used_meta":269240,"kb_avail":93474200,"statfs":{"total":95995035648,"available":95717580800,"internally_reserved":0,"allocated":1736704,"data_stored":1078264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":275701841},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.5070000000000001,"15min":2.9980000000000002},"min":{"1min":0.21299999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.5490000000000004,"5min":14.327,"15min":14.327},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.024,"5min":2.323,"15min":2.7589999999999999},"min":{"1min":0.217,"5min":0.186,"15min":0.186},"max":{"1min":6.2359999999999998,"5min":14.757999999999999,"15min":14.757999999999999},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":2.5880000000000001,"15min":2.8900000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.4039999999999999,"5min":13.023,"15min":13.023},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.3759999999999999,"15min":3.0019999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.0419999999999998,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:49:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.959,"5min":11.959,"15min":11.959},"last":0.435},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.6539999999999999,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":1.819,"15min":1.9370000000000001},"min":{"1min":0.34100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.8440000000000003,"5min":7.8440000000000003,"15min":7.8440000000000003},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.411,"5min":2.3490000000000002,"15min":2.7109999999999999},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.317},"max":{"1min":6.2789999999999999,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.65000000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":2.5630000000000002,"15min":3.0539999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.3760000000000003,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.7509999999999999,"5min":2.5030000000000001,"15min":2.8610000000000002},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3630000000000004,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":2.7210000000000001,"15min":3.29},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.3449999999999998,"5min":7.476,"15min":7.476},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.1379999999999999,"5min":2.806,"15min":3.0790000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":9.5220000000000002,"5min":12.595000000000001,"15min":12.595000000000001},"last":0.69499999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":2.2320000000000002,"15min":2.5350000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.4909999999999997,"5min":6.4989999999999997,"15min":6.4989999999999997},"last":0.624},{"interface":"front","average":{"1min":1.669,"5min":2.5259999999999998,"15min":2.8839999999999999},"min":{"1min":0.41399999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.5979999999999999,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":0.70299999999999996}]}]},{"osd":3,"up_from":148,"seq":635655159831,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":135308,"kb_used_data":1668,"kb_used_omap":2,"kb_used_meta":133629,"kb_avail":93609844,"statfs":{"total":95995035648,"available":95856480256,"internally_reserved":0,"allocated":1708032,"data_stored":1075110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":136836149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":14.332000000000001,"5min":14.332000000000001,"15min":14.332000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":253.68100000000001,"5min":253.68100000000001,"15min":253.68100000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":13.208,"5min":13.208,"15min":13.208},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":253.46199999999999,"5min":253.46199999999999,"15min":253.46199999999999},"last":0.46700000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":10.417999999999999,"5min":10.417999999999999,"15min":10.417999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":172.77099999999999,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":9.4079999999999995,"5min":9.4079999999999995,"15min":9.4079999999999995},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":172.36600000000001,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":5.4619999999999997,"5min":5.4619999999999997,"15min":5.4619999999999997},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.51900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.4749999999999996,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.70999999999999996},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.2170000000000001,"15min":2.2170000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":16.408999999999999,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.76200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.5750000000000002,"5min":3.5750000000000002,"15min":3.5750000000000002},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.958,"5min":1.958,"15min":1.958},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":9.7780000000000005,"5min":9.7780000000000005,"15min":9.7780000000000005},"last":0.79400000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":1.972,"15min":1.972},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.7509999999999994,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.6419999999999999,"5min":2.6419999999999999,"15min":2.6419999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":16.451000000000001,"5min":16.451000000000001,"15min":16.451000000000001},"last":0.69899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":1.5640000000000001,"15min":1.5640000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.9450000000000003,"5min":4.9450000000000003,"15min":4.9450000000000003},"last":0.71899999999999997},{"interface":"front","average":{"1min":2.351,"5min":2.351,"15min":2.351},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":11.561999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.77300000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986444,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307912,"kb_used_data":1600,"kb_used_omap":0,"kb_used_meta":306304,"kb_avail":93437240,"statfs":{"total":95995035648,"available":95679733760,"internally_reserved":0,"allocated":1638400,"data_stored":1069084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":313655296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.52,"15min":1.839},"min":{"1min":0.216,"5min":0.216,"15min":0.214},"max":{"1min":8.0519999999999996,"5min":12.823,"15min":12.823},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.508,"5min":1.998,"15min":1.321},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.22600000000000001},"max":{"1min":4.3090000000000002,"5min":10.413,"15min":13.211},"last":0.315}]},{"osd":1,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":8.4019999999999992,"5min":6.6639999999999997,"15min":3.23},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":125.35599999999999,"5min":286.31099999999998,"15min":286.31099999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":8.0459999999999994,"5min":6.7679999999999998,"15min":3.996},"min":{"1min":0.312,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":125.21899999999999,"5min":285.97000000000003,"15min":285.97000000000003},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":1.48,"15min":1.375},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.442},{"interface":"front","average":{"1min":1.988,"5min":2.347,"15min":2.3620000000000001},"min":{"1min":0.39300000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.9450000000000003,"5min":12.773999999999999,"15min":12.773999999999999},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:49:08 2023","interfaces":[{"interface":"back","average":{"1min":7.5990000000000002,"5min":7.5990000000000002,"15min":7.5990000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":107.285,"5min":107.285,"15min":107.285},"last":0.372},{"interface":"front","average":{"1min":7.1920000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":107.303,"5min":107.303,"15min":107.303},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.2390000000000001,"15min":1.258},"min":{"1min":0.32500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.2000000000000002,"5min":7.1740000000000004,"15min":17.713999999999999},"last":0.496},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.1879999999999999,"15min":1.331},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.891,"5min":9.0660000000000007,"15min":14.455},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.4630000000000001,"15min":1.5369999999999999},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.4109999999999996,"5min":16.271000000000001,"15min":21.920999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.234,"15min":1.345},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.28599999999999998},"max":{"1min":3.431,"5min":11.718,"15min":11.718},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.4750000000000001,"15min":1.6679999999999999},"min":{"1min":0.42499999999999999,"5min":0.315,"15min":0.31},"max":{"1min":6.0039999999999996,"5min":9.8390000000000004,"15min":21.792999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.321,"15min":1.47},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.317},"max":{"1min":3.306,"5min":7.8399999999999999,"15min":23.457000000000001},"last":0.55600000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724808,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":365368,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":363776,"kb_avail":93379784,"statfs":{"total":95995035648,"available":95620898816,"internally_reserved":0,"allocated":1622016,"data_stored":1064828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":372506624},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.4330000000000001,"15min":1.7669999999999999},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.7559999999999998,"5min":10.94,"15min":12.880000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.274,"5min":1.754,"15min":1.3859999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":10.028,"5min":10.955,"15min":10.955},"last":0.61699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":4.5990000000000002,"15min":2.2130000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":12.529999999999999,"5min":323.541,"15min":323.541},"last":0.503},{"interface":"front","average":{"1min":1.262,"5min":4.7270000000000003,"15min":2.5219999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.6900000000000004,"5min":324.072,"15min":324.072},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":3.137,"5min":2.415,"15min":2.423},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":12.585000000000001,"5min":13.394,"15min":13.394},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.827,"15min":1.663},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7430000000000003,"5min":14.263,"15min":14.263},"last":0.54400000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.863,"15min":1.863},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.295,"5min":10.295,"15min":10.295},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.1080000000000005,"5min":8.1080000000000005,"15min":8.1080000000000005},"last":0.59999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.90400000000000003,"15min":1.153},"min":{"1min":0.33900000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.71499999999999997,"5min":5.8529999999999998,"15min":9.8800000000000008},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.2509999999999999,"15min":1.444},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.23999999999999999},"max":{"1min":0.77400000000000002,"5min":8.3170000000000002,"15min":16.869},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:49:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":1.325,"15min":1.544},"min":{"1min":0.316,"5min":0.28000000000000003,"15min":0.27000000000000002},"max":{"1min":6.1689999999999996,"5min":8.3010000000000002,"15min":17},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.2050000000000001,"15min":1.427},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.9870000000000001,"5min":9.6370000000000005,"15min":14.08},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:49:40 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.5960000000000001,"15min":1.6910000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.28199999999999997},"max":{"1min":2.9300000000000002,"5min":14.217000000000001,"15min":17.071999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.96699999999999997,"15min":1.1259999999999999},"min":{"1min":0.40300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0499999999999998,"5min":5.601,"15min":8.6370000000000005},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:49:55.387 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:49:55.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T20:49:55.734 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:55.969 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:55 smithi121 ceph-mon[211526]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:55.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:55 smithi121 ceph-mon[219709]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:55 smithi123 ceph-mon[193903]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:56.974 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:56 smithi121 ceph-mon[211526]: from='client.35115 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:56.975 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:56 smithi121 ceph-mon[219709]: from='client.35115 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:56 smithi123 ceph-mon[193903]: from='client.35115 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:57.372 INFO:teuthology.orchestra.run.smithi121.stdout:epoch 163 2023-09-23T20:49:57.372 INFO:teuthology.orchestra.run.smithi121.stdout:fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 2023-09-23T20:49:57.372 INFO:teuthology.orchestra.run.smithi121.stdout:created 2023-09-23T20:23:56.454135+0000 2023-09-23T20:49:57.372 INFO:teuthology.orchestra.run.smithi121.stdout:modified 2023-09-23T20:49:02.066000+0000 2023-09-23T20:49:57.372 INFO:teuthology.orchestra.run.smithi121.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:crush_version 22 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:full_ratio 0.95 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:backfillfull_ratio 0.9 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:nearfull_ratio 0.85 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:require_min_compat_client luminous 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:min_compat_client luminous 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:require_osd_release quincy 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:stretch_mode_enabled false 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 64 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T20:49:57.373 INFO:teuthology.orchestra.run.smithi121.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 69 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:max_osd 8 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 up in weight 1 up_from 92 up_thru 153 down_at 89 last_clean_interval [8,88) [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724] [v2:172.21.15.121:6804/3792083724,v1:172.21.15.121:6805/3792083724] exists,up 9367e9c4-5ec4-4c89-a35d-16375c1fee90 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 up in weight 1 up_from 114 up_thru 161 down_at 107 last_clean_interval [13,106) [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709] [v2:172.21.15.121:6812/1737064709,v1:172.21.15.121:6813/1737064709] exists,up 798606f1-4cc7-4397-9f99-1f3d8259eba1 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 up in weight 1 up_from 129 up_thru 148 down_at 124 last_clean_interval [18,123) [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913] [v2:172.21.15.121:6820/2050616913,v1:172.21.15.121:6821/2050616913] exists,up 03ecb78d-0bc7-43d1-8c25-1c6051697c2f 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 up in weight 1 up_from 148 up_thru 159 down_at 142 last_clean_interval [26,141) [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631] [v2:172.21.15.121:6828/3257078631,v1:172.21.15.121:6829/3257078631] exists,up e6711d54-f7b7-4876-924e-e9b4c58f9e2f 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 up in weight 1 up_from 31 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785] [v2:172.21.15.123:6802/4231264785,v1:172.21.15.123:6803/4231264785] exists,up 84bd1437-f644-42b0-8bc8-6a3e1814ad9d 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 up in weight 1 up_from 39 up_thru 156 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602] [v2:172.21.15.123:6810/3299108602,v1:172.21.15.123:6811/3299108602] exists,up eaf000e3-5147-4aa8-a0d3-65c6a38bd803 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 up in weight 1 up_from 47 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390] [v2:172.21.15.123:6818/3633151390,v1:172.21.15.123:6819/3633151390] exists,up 29ce2c8b-96a4-4b71-97b2-c307f944ea0c 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 up in weight 1 up_from 55 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382] [v2:172.21.15.123:6826/4186046382,v1:172.21.15.123:6827/4186046382] exists,up 69e6dd32-d849-40d2-bef3-a0cf635c0346 2023-09-23T20:49:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:pg_upmap_items 4.4 [1,5] 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:pg_upmap_items 4.a [1,2] 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:pg_upmap_items 4.f [1,2] 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/961670864 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:0/2952381553 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/2000763757 expires 2023-09-24T20:33:20.814708+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/3020663534 expires 2023-09-24T20:24:24.560243+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/4233508401 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/199329623 expires 2023-09-24T20:29:24.862767+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6800/1448037439 expires 2023-09-24T20:33:20.814708+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6800/232072490 expires 2023-09-24T20:24:49.573230+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:0/2412578238 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.375 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6800/3124261962 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:0/3564595757 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/2517434125 expires 2023-09-24T20:24:49.573230+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6801/232072490 expires 2023-09-24T20:24:49.573230+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6801/40316177 expires 2023-09-24T20:29:24.862767+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6801/1448037439 expires 2023-09-24T20:33:20.814708+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/3148587077 expires 2023-09-24T20:24:49.573230+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6800/40316177 expires 2023-09-24T20:29:24.862767+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/167205692 expires 2023-09-24T20:33:20.814708+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:0/623964166 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/3933818352 expires 2023-09-24T20:33:20.814708+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:0/1025991692 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:6834/1638010485 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.376 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:6835/1638010485 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/1167207050 expires 2023-09-24T20:29:24.862767+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6801/1535070110 expires 2023-09-24T20:24:24.560243+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/3188450480 expires 2023-09-24T20:24:49.573230+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/2776087744 expires 2023-09-24T20:29:24.862767+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/2713351150 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/156999919 expires 2023-09-24T20:33:20.814708+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/985594636 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/1290253295 expires 2023-09-24T20:29:24.862767+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/3404394206 expires 2023-09-24T20:24:24.560243+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6801/3124261962 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:6800/1535070110 expires 2023-09-24T20:24:24.560243+0000 2023-09-23T20:49:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/719574644 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.378 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.123:0/1669790000 expires 2023-09-24T20:34:36.437609+0000 2023-09-23T20:49:57.378 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/1502674050 expires 2023-09-24T20:24:24.560243+0000 2023-09-23T20:49:57.378 INFO:teuthology.orchestra.run.smithi121.stdout:blocklist 172.21.15.121:0/85909652 expires 2023-09-24T20:35:42.236940+0000 2023-09-23T20:49:57.969 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:57 smithi121 ceph-mon[211526]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:57.969 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3800147895' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:49:57.969 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:57.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:57 smithi121 ceph-mon[219709]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:57.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3800147895' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:49:57.970 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:57.971 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 92 up_thru 153 down_at 89 last_clean_interval [8,88) [v2:172.21.15.121:6802/3792083724,v1:172.21.15.121:6803/3792083724] [v2:172.21.15.121:6804/3792083724,v1:172.21.15.121:6805/3792083724] exists,up 9367e9c4-5ec4-4c89-a35d-16375c1fee90', 'osd.1 up in weight 1 up_from 114 up_thru 161 down_at 107 last_clean_interval [13,106) [v2:172.21.15.121:6810/1737064709,v1:172.21.15.121:6811/1737064709] [v2:172.21.15.121:6812/1737064709,v1:172.21.15.121:6813/1737064709] exists,up 798606f1-4cc7-4397-9f99-1f3d8259eba1', 'osd.2 up in weight 1 up_from 129 up_thru 148 down_at 124 last_clean_interval [18,123) [v2:172.21.15.121:6818/2050616913,v1:172.21.15.121:6819/2050616913] [v2:172.21.15.121:6820/2050616913,v1:172.21.15.121:6821/2050616913] exists,up 03ecb78d-0bc7-43d1-8c25-1c6051697c2f', 'osd.3 up in weight 1 up_from 148 up_thru 159 down_at 142 last_clean_interval [26,141) [v2:172.21.15.121:6826/3257078631,v1:172.21.15.121:6827/3257078631] [v2:172.21.15.121:6828/3257078631,v1:172.21.15.121:6829/3257078631] exists,up e6711d54-f7b7-4876-924e-e9b4c58f9e2f', 'osd.4 up in weight 1 up_from 31 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6800/4231264785,v1:172.21.15.123:6801/4231264785] [v2:172.21.15.123:6802/4231264785,v1:172.21.15.123:6803/4231264785] exists,up 84bd1437-f644-42b0-8bc8-6a3e1814ad9d', 'osd.5 up in weight 1 up_from 39 up_thru 156 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6808/3299108602,v1:172.21.15.123:6809/3299108602] [v2:172.21.15.123:6810/3299108602,v1:172.21.15.123:6811/3299108602] exists,up eaf000e3-5147-4aa8-a0d3-65c6a38bd803', 'osd.6 up in weight 1 up_from 47 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6816/3633151390,v1:172.21.15.123:6817/3633151390] [v2:172.21.15.123:6818/3633151390,v1:172.21.15.123:6819/3633151390] exists,up 29ce2c8b-96a4-4b71-97b2-c307f944ea0c', 'osd.7 up in weight 1 up_from 55 up_thru 148 down_at 0 last_clean_interval [0,0) [v2:172.21.15.123:6824/4186046382,v1:172.21.15.123:6825/4186046382] [v2:172.21.15.123:6826/4186046382,v1:172.21.15.123:6827/4186046382] exists,up 69e6dd32-d849-40d2-bef3-a0cf635c0346'] 2023-09-23T20:49:57.971 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T20:49:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:57 smithi123 ceph-mon[193903]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:49:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3800147895' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:49:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:58.317 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:59.824 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:49:59.824 INFO:teuthology.orchestra.run.smithi121.stdout: "name": "mon.a", 2023-09-23T20:49:59.824 INFO:teuthology.orchestra.run.smithi121.stdout: "cluster": "ceph", 2023-09-23T20:49:59.824 INFO:teuthology.orchestra.run.smithi121.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T20:49:59.825 INFO:teuthology.orchestra.run.smithi121.stdout: "admin_socket_mode": "", 2023-09-23T20:49:59.825 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T20:49:59.825 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_client_required": "cephx, none", 2023-09-23T20:49:59.825 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_cluster_required": "cephx", 2023-09-23T20:49:59.825 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_debug": "true", 2023-09-23T20:49:59.825 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T20:49:59.825 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_service_required": "cephx", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "auth_supported": "", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_aio": "true", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_aio_reap_max": "16", 2023-09-23T20:49:59.826 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_async_discard": "false", 2023-09-23T20:49:59.827 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_block_size": "4096", 2023-09-23T20:49:59.827 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_debug_aio": "false", 2023-09-23T20:49:59.827 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T20:49:59.827 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T20:49:59.827 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T20:49:59.827 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_enable_discard": "false", 2023-09-23T20:49:59.827 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_flock_retry": "3", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_inject_crash": "0", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_ioring": "false", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_ioring_hipri": "false", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T20:49:59.828 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bdev_type": "", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_allocator": "hybrid", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_buffered_io": "true", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T20:49:59.829 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_replay_recovery": "false", 2023-09-23T20:49:59.830 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluefs_sync_write": "false", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_allocator": "hybrid", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T20:49:59.831 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_create": "true", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_db_create": "false", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_db_path": "", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_db_size": "0", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_path": "", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_size": "107374182400", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_wal_create": "false", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_wal_path": "", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T20:49:59.832 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_bluefs": "true", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_autotune": "true", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T20:49:59.833 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_size": "0", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cache_type": "2q", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_clone_cow": "true", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T20:49:59.834 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_mode": "none", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_freelist": "false", 2023-09-23T20:49:59.835 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T20:49:59.836 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T20:49:59.836 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T20:49:59.836 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T20:49:59.836 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T20:49:59.836 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T20:49:59.836 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T20:49:59.836 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T20:49:59.837 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fail_eio": "false", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T20:49:59.838 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T20:49:59.839 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T20:49:59.840 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T20:49:59.840 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T20:49:59.840 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T20:49:59.840 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T20:49:59.840 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T20:49:59.840 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_max_blob_size": "0", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T20:49:59.841 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T20:49:59.842 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T20:49:59.842 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T20:49:59.842 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T20:49:59.842 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T20:49:59.842 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T20:49:59.842 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T20:49:59.842 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T20:49:59.843 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T20:49:59.843 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T20:49:59.843 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T20:49:59.843 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T20:49:59.843 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T20:49:59.843 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T20:49:59.843 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_spdk_mem": "512", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T20:49:59.844 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_tracing": "false", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T20:49:59.845 INFO:teuthology.orchestra.run.smithi121.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T20:49:59.846 INFO:teuthology.orchestra.run.smithi121.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "cephx_cluster_require_version": "2", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "cephx_require_signatures": "false", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "cephx_require_version": "2", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "cephx_service_require_signatures": "false", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "cephx_service_require_version": "2", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "cephx_sign_messages": "true", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "chdir": "", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_acl_type": "", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_asio_thread_count": "2", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_cache_mid": "0.750000", 2023-09-23T20:49:59.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_cache_size": "16384", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_caps_release_delay": "5", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_check_pool_perm": "true", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_debug_force_sync_read": "false", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_debug_getattr_caps": "false", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_die_on_failed_remount": "false", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_dirsize_rbytes": "true", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_force_lazyio": "false", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_fs": "", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T20:49:59.848 INFO:teuthology.orchestra.run.smithi121.stdout: "client_inject_release_failure": "false", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_max_inline_size": "4096", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_mds_namespace": "", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_metadata": "", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_mount_gid": "-1", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_mount_timeout": "300", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_mount_uid": "-1", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_mountpoint": "/", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_notify_timeout": "10", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_oc": "true", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T20:49:59.849 INFO:teuthology.orchestra.run.smithi121.stdout: "client_oc_max_objects": "1000", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_oc_size": "209715200", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_permissions": "true", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_quota": "true", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_quota_df": "true", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_readahead_max_bytes": "0", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_readahead_max_periods": "4", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_readahead_min": "131072", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_reconnect_stale": "false", 2023-09-23T20:49:59.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_shutdown_timeout": "30", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_snapdir": ".snap", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_tick_interval": "1", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_trace": "", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_use_faked_inos": "false", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_use_random_mds": "false", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "clog_to_graylog": "false", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "clog_to_graylog_port": "12201", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "clog_to_monitors": "default=true", 2023-09-23T20:49:59.851 INFO:teuthology.orchestra.run.smithi121.stdout: "clog_to_syslog": "false", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "clog_to_syslog_level": "info", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "cluster_addr": "-", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "cluster_network": "", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "cluster_network_interface": "", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "compressor_zlib_isal": "false", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "compressor_zlib_level": "5", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "compressor_zstd_level": "1", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T20:49:59.852 INFO:teuthology.orchestra.run.smithi121.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "crimson_seastar_smp": "1", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "crush_location": "", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "crush_location_hook": "", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "crush_location_hook_timeout": "10", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "daemonize": "false", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "daos_pool": "tank", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T20:49:59.853 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_alienstore": "0/5", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_asok": "1/5", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_asok_assert_abort": "false", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_auth": "1/5", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_bdev": "1/3", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_bluefs": "1/5", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_bluestore": "1/5", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_buffer": "0/1", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T20:49:59.854 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_cephsqlite": "0/5", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_client": "0/5", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_compressor": "1/5", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_context": "0/1", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_crush": "1/1", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_crypto": "1/5", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_cyanstore": "0/5", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T20:49:59.855 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_dpdk": "1/5", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_eventtrace": "1/5", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_filer": "0/1", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_filestore": "1/3", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_finisher": "1/1", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_fuse": "1/5", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_javaclient": "1/5", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_journal": "1/3", 2023-09-23T20:49:59.856 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_journaler": "0/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_kstore": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_leveldb": "4/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_lockdep": "0/1", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mclock": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mds": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mds_balancer": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mds_locker": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mds_log": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mds_migrator": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_memstore": "1/5", 2023-09-23T20:49:59.857 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mgr": "2/5", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mgrc": "1/5", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_mon": "20/20", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_monc": "0/10", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_ms": "1/1", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_none": "0/5", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_objclass": "0/5", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_objectcacher": "0/5", 2023-09-23T20:49:59.858 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_objecter": "0/1", 2023-09-23T20:49:59.859 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_optracker": "0/5", 2023-09-23T20:49:59.859 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_osd": "1/5", 2023-09-23T20:49:59.859 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_paxos": "20/20", 2023-09-23T20:49:59.859 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_perfcounter": "1/5", 2023-09-23T20:49:59.859 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_prioritycache": "1/5", 2023-09-23T20:49:59.859 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rados": "0/5", 2023-09-23T20:49:59.859 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rbd": "0/5", 2023-09-23T20:49:59.860 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T20:49:59.860 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T20:49:59.860 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rbd_replay": "0/5", 2023-09-23T20:49:59.860 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_refs": "0/0", 2023-09-23T20:49:59.860 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_reserver": "1/1", 2023-09-23T20:49:59.860 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rgw": "1/5", 2023-09-23T20:49:59.860 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rgw_access": "1/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rgw_flight": "1/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rgw_sync": "1/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_rocksdb": "4/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore": "0/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_backref": "0/5", 2023-09-23T20:49:59.861 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_cache": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_device": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_epm": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_journal": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_lba": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_odata": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_omap": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_onode": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_t": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_seastore_tm": "0/5", 2023-09-23T20:49:59.862 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_striper": "0/1", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_test": "0/5", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_throttle": "1/1", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_timer": "0/1", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "debug_tp": "0/5", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "device_failure_prediction_mode": "none", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "err_to_graylog": "false", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "err_to_journald": "false", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "err_to_stderr": "true", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "err_to_syslog": "false", 2023-09-23T20:49:59.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event_tracing": "false", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "exporter_http_port": "9926", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "exporter_prio_limit": "5", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "exporter_sort_metrics": "true", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "exporter_stats_period": "5", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "fake_statfs_for_testing": "0", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "fatal_signal_handlers": "true", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "filer_max_purge_ops": "10", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "filer_max_truncate_ops": "128", 2023-09-23T20:49:59.864 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T20:49:59.865 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_blackhole": "false", 2023-09-23T20:49:59.865 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T20:49:59.865 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_btrfs_snap": "true", 2023-09-23T20:49:59.865 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_caller_concurrency": "10", 2023-09-23T20:49:59.865 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T20:49:59.865 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T20:49:59.865 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T20:49:59.866 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_debug_omap_check": "false", 2023-09-23T20:49:59.866 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T20:49:59.866 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_debug_verify_split": "false", 2023-09-23T20:49:59.866 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_dump_file": "", 2023-09-23T20:49:59.866 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T20:49:59.866 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T20:49:59.866 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_fadvise": "true", 2023-09-23T20:49:59.867 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_fail_eio": "true", 2023-09-23T20:49:59.867 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T20:49:59.867 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_fd_cache_size": "128", 2023-09-23T20:49:59.867 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_fiemap": "false", 2023-09-23T20:49:59.867 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T20:49:59.867 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T20:49:59.868 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T20:49:59.868 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_inject_stall": "0", 2023-09-23T20:49:59.868 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_journal_parallel": "false", 2023-09-23T20:49:59.868 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_journal_trailing": "false", 2023-09-23T20:49:59.868 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_journal_writeahead": "false", 2023-09-23T20:49:59.868 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_kill_at": "0", 2023-09-23T20:49:59.868 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T20:49:59.869 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T20:49:59.869 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T20:49:59.869 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T20:49:59.869 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T20:49:59.869 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T20:49:59.869 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T20:49:59.869 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T20:49:59.870 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T20:49:59.870 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T20:49:59.870 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T20:49:59.870 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T20:49:59.870 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T20:49:59.870 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T20:49:59.871 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_merge_threshold": "-10", 2023-09-23T20:49:59.871 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T20:49:59.871 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_odsync_write": "false", 2023-09-23T20:49:59.871 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T20:49:59.871 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_omap_backend_path": "", 2023-09-23T20:49:59.871 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T20:49:59.871 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_op_threads": "2", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_punch_hole": "false", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T20:49:59.872 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_queue_max_ops": "50", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_seek_data_hole": "false", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_sloppy_crc": "false", 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T20:49:59.874 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_splice": "false", 2023-09-23T20:49:59.874 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_split_multiple": "2", 2023-09-23T20:49:59.874 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_split_rand_factor": "20", 2023-09-23T20:49:59.874 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_update_to": "1000", 2023-09-23T20:49:59.874 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T20:49:59.874 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T20:49:59.874 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T20:49:59.875 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T20:49:59.876 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T20:49:59.876 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T20:49:59.876 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_xfs_extsize": "false", 2023-09-23T20:49:59.876 INFO:teuthology.orchestra.run.smithi121.stdout: "filestore_zfs_snap": "false", 2023-09-23T20:49:59.876 INFO:teuthology.orchestra.run.smithi121.stdout: "fio_dir": "/tmp/fio", 2023-09-23T20:49:59.876 INFO:teuthology.orchestra.run.smithi121.stdout: "fsid": "ffb133f0-5a4e-11ee-8db4-212e2dc638e7", 2023-09-23T20:49:59.876 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_allow_other": "true", 2023-09-23T20:49:59.877 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T20:49:59.877 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_big_writes": "true", 2023-09-23T20:49:59.877 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_debug": "false", 2023-09-23T20:49:59.877 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_default_permissions": "false", 2023-09-23T20:49:59.877 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_disable_pagecache": "false", 2023-09-23T20:49:59.877 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_max_write": "0", 2023-09-23T20:49:59.878 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_multithreaded": "true", 2023-09-23T20:49:59.878 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_require_active_mds": "true", 2023-09-23T20:49:59.878 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_set_user_groups": "true", 2023-09-23T20:49:59.878 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_splice_move": "true", 2023-09-23T20:49:59.878 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_splice_read": "true", 2023-09-23T20:49:59.878 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_splice_write": "true", 2023-09-23T20:49:59.879 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T20:49:59.879 INFO:teuthology.orchestra.run.smithi121.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T20:49:59.879 INFO:teuthology.orchestra.run.smithi121.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T20:49:59.879 INFO:teuthology.orchestra.run.smithi121.stdout: "gss_target_name": "ceph", 2023-09-23T20:49:59.879 INFO:teuthology.orchestra.run.smithi121.stdout: "heartbeat_file": "", 2023-09-23T20:49:59.879 INFO:teuthology.orchestra.run.smithi121.stdout: "heartbeat_inject_failure": "0", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "heartbeat_interval": "5", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "host": "smithi121", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T20:49:59.880 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T20:49:59.881 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T20:49:59.881 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T20:49:59.881 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T20:49:59.881 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T20:49:59.881 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T20:49:59.881 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T20:49:59.882 INFO:teuthology.orchestra.run.smithi121.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T20:49:59.882 INFO:teuthology.orchestra.run.smithi121.stdout: "inject_early_sigterm": "false", 2023-09-23T20:49:59.882 INFO:teuthology.orchestra.run.smithi121.stdout: "jaeger_agent_port": "6799", 2023-09-23T20:49:59.882 INFO:teuthology.orchestra.run.smithi121.stdout: "jaeger_tracing_enable": "false", 2023-09-23T20:49:59.882 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_aio": "true", 2023-09-23T20:49:59.883 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_align_min_size": "65536", 2023-09-23T20:49:59.883 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_block_align": "true", 2023-09-23T20:49:59.883 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_block_size": "4096", 2023-09-23T20:49:59.883 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_dio": "true", 2023-09-23T20:49:59.883 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_discard": "false", 2023-09-23T20:49:59.883 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_force_aio": "false", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_ignore_corruption": "false", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_max_write_entries": "100", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_replay_from": "0", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T20:49:59.884 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_write_header_frequency": "0", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "journal_zero_on_create": "false", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "journaler_prefetch_periods": "10", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "journaler_prezero_periods": "5", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "journaler_write_head_interval": "15", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "key": "", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "keyfile": "", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_backend": "rocksdb", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_max_bytes": "67108864", 2023-09-23T20:49:59.885 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_max_ops": "512", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_onode_map_size": "1024", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "kstore_sync_transaction": "false", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_block_size": "65536", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_bloom_size": "0", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_cache_size": "536870912", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_compression": "false", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_log": "", 2023-09-23T20:49:59.886 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_max_open_files": "0", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_paranoid": "false", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "librados_thread_count": "2", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "lockdep": "false", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "lockdep_force_backtrace": "false", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "log_coarse_timestamps": "true", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "log_flush_on_exit": "false", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "log_graylog_port": "12201", 2023-09-23T20:49:59.887 INFO:teuthology.orchestra.run.smithi121.stdout: "log_max_new": "1000", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_max_recent": "10000", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_stderr_prefix": "", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_to_file": "true", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_to_graylog": "false", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_to_journald": "false", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_to_stderr": "false", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "log_to_syslog": "false", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_action_on_write_error": "1", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_alternate_name_max": "8192", 2023-09-23T20:49:59.888 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_asio_thread_count": "2", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_export_pin": "true", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_interval": "10", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_max": "-1", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_max_until": "-1", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_merge_size": "50", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T20:49:59.889 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_mode": "0", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_split_bits": "3", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_split_size": "10000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T20:49:59.890 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache_mid": "0.700000", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache_trim_interval": "1", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T20:49:59.891 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_debug_auth_pins": "false", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_debug_frag": "false", 2023-09-23T20:49:59.892 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_debug_scatterstat": "false", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_debug_subtrees": "false", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_default_dir_hash": "2", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_defer_session_stale": "true", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dir_max_entries": "0", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dir_prefetch": "true", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T20:49:59.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_early_reply": "true", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_enable_op_tracker": "true", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_enforce_unique_name": "true", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T20:49:59.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T20:49:59.895 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T20:49:59.895 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T20:49:59.895 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T20:49:59.895 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T20:49:59.895 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T20:49:59.895 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T20:49:59.895 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T20:49:59.896 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T20:49:59.896 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_inject_health_dummy": "false", 2023-09-23T20:49:59.896 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T20:49:59.896 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T20:49:59.896 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T20:49:59.896 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T20:49:59.896 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T20:49:59.897 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_join_fs": "", 2023-09-23T20:49:59.897 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_journal_format": "1", 2023-09-23T20:49:59.897 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_create_at": "0", 2023-09-23T20:49:59.897 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_export_at": "0", 2023-09-23T20:49:59.897 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_import_at": "0", 2023-09-23T20:49:59.897 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_journal_at": "0", 2023-09-23T20:49:59.897 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T20:49:59.898 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T20:49:59.898 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_link_at": "0", 2023-09-23T20:49:59.898 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T20:49:59.898 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_openc_at": "0", 2023-09-23T20:49:59.898 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_rename_at": "0", 2023-09-23T20:49:59.898 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T20:49:59.898 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T20:49:59.899 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log_max_events": "-1", 2023-09-23T20:49:59.899 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log_max_segments": "128", 2023-09-23T20:49:59.899 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log_pause": "false", 2023-09-23T20:49:59.899 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log_segment_size": "0", 2023-09-23T20:49:59.899 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T20:49:59.899 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T20:49:59.899 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T20:49:59.900 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T20:49:59.900 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_completed_requests": "100000", 2023-09-23T20:49:59.900 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_export_size": "20971520", 2023-09-23T20:49:59.900 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_file_recover": "32", 2023-09-23T20:49:59.900 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_purge_files": "64", 2023-09-23T20:49:59.900 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_purge_ops": "8192", 2023-09-23T20:49:59.900 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T20:49:59.901 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T20:49:59.901 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T20:49:59.901 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T20:49:59.901 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_metrics_update_interval": "2", 2023-09-23T20:49:59.901 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_min_caps_per_client": "100", 2023-09-23T20:49:59.901 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T20:49:59.901 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T20:49:59.902 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_numa_node": "-1", 2023-09-23T20:49:59.902 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T20:49:59.902 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T20:49:59.902 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_op_history_duration": "600", 2023-09-23T20:49:59.902 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_op_history_size": "20", 2023-09-23T20:49:59.902 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_op_log_threshold": "5", 2023-09-23T20:49:59.902 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_ping_grace": "15", 2023-09-23T20:49:59.903 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_ping_interval": "5", 2023-09-23T20:49:59.903 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T20:49:59.903 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T20:49:59.903 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_recall_max_caps": "30000", 2023-09-23T20:49:59.903 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T20:49:59.903 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T20:49:59.903 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T20:49:59.904 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T20:49:59.904 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T20:49:59.904 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_replay_interval": "1.000000", 2023-09-23T20:49:59.904 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T20:49:59.904 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T20:49:59.904 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_root_ino_gid": "0", 2023-09-23T20:49:59.905 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_root_ino_uid": "0", 2023-09-23T20:49:59.905 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T20:49:59.905 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T20:49:59.905 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T20:49:59.905 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T20:49:59.906 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T20:49:59.906 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T20:49:59.906 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T20:49:59.906 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T20:49:59.906 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T20:49:59.906 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_shutdown_check": "0", 2023-09-23T20:49:59.907 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_skip_ino": "0", 2023-09-23T20:49:59.907 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T20:49:59.907 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T20:49:59.907 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_snap_min_uid": "0", 2023-09-23T20:49:59.907 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_snap_rstat": "false", 2023-09-23T20:49:59.907 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T20:49:59.907 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_symlink_recovery": "true", 2023-09-23T20:49:59.908 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T20:49:59.908 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_thrash_exports": "0", 2023-09-23T20:49:59.908 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_thrash_fragments": "0", 2023-09-23T20:49:59.908 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_tick_interval": "5.000000", 2023-09-23T20:49:59.908 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_valgrind_exit": "false", 2023-09-23T20:49:59.908 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_verify_backtrace": "1", 2023-09-23T20:49:59.908 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_verify_scatter": "false", 2023-09-23T20:49:59.909 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T20:49:59.909 INFO:teuthology.orchestra.run.smithi121.stdout: "mds_wipe_sessions": "false", 2023-09-23T20:49:59.909 INFO:teuthology.orchestra.run.smithi121.stdout: "mempool_debug": "false", 2023-09-23T20:49:59.909 INFO:teuthology.orchestra.run.smithi121.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T20:49:59.909 INFO:teuthology.orchestra.run.smithi121.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "memstore_page_set": "false", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "memstore_page_size": "65536", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_client_bytes": "134217728", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_client_messages": "512", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T20:49:59.910 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_disabled_modules": "", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_mds_messages": "128", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_mon_messages": "128", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_osd_messages": "8192", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_pool": "true", 2023-09-23T20:49:59.911 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_standby_modules": "true", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_stats_period": "5", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_stats_threshold": "5", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_tick_period": "2", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_allow_pool_delete": "true", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T20:49:59.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_bytes": "104857600", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T20:49:59.913 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T20:49:59.914 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_compact_on_start": "false", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_compact_on_trim": "true", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T20:49:59.915 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_cpu_threads": "4", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_data_avail_crit": "5", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_data_avail_warn": "5", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_dump_json": "false", 2023-09-23T20:49:59.916 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_extra_checks": "false", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_election_default_strategy": "1", 2023-09-23T20:49:59.917 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_election_timeout": "5.000000", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_enable_op_tracker": "true", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_fake_pool_delete": "false", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_force_quorum_join": "false", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_health_log_update_period": "5", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_health_max_detail": "50", 2023-09-23T20:49:59.918 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_health_to_clog": "true", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_host": "[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] [v2:172.21.15.121:3301/0,v1:172.21.15.121:6790/0]", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_host_override": "", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_initial_members": "", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_lease": "5.000000", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T20:49:59.919 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_log_full_interval": "50", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_log_max": "10000", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_log_max_summary": "50", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_log_epochs": "500", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_osd": "10000", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T20:49:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_memory_autotune": "true", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_memory_target": "2147483648", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mgr_digest_period": "5", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T20:49:59.921 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_op_complaint_time": "30", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_op_history_duration": "600", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_op_history_size": "20", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_op_log_threshold": "5", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T20:49:59.922 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_cache_size": "500", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T20:49:59.923 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T20:49:59.924 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_report_timeout": "900", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T20:49:59.925 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T20:49:59.926 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T20:49:59.927 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T20:49:59.927 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T20:49:59.927 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_reweight_max_osds": "4", 2023-09-23T20:49:59.927 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T20:49:59.927 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T20:49:59.927 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T20:49:59.927 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T20:49:59.928 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T20:49:59.928 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_scrub_interval": "86400", 2023-09-23T20:49:59.928 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_scrub_max_keys": "100", 2023-09-23T20:49:59.928 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_scrub_timeout": "300", 2023-09-23T20:49:59.928 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_session_timeout": "300", 2023-09-23T20:49:59.928 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_smart_report_timeout": "5", 2023-09-23T20:49:59.929 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T20:49:59.929 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T20:49:59.929 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T20:49:59.929 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_stretch_pool_size": "4", 2023-09-23T20:49:59.929 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T20:49:59.930 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T20:49:59.930 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_sync_debug": "false", 2023-09-23T20:49:59.930 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T20:49:59.930 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T20:49:59.930 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T20:49:59.930 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T20:49:59.930 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T20:49:59.931 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T20:49:59.931 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_tick_interval": "5", 2023-09-23T20:49:59.931 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T20:49:59.931 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T20:49:59.931 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T20:49:59.931 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T20:49:59.932 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T20:49:59.932 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T20:49:59.932 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T20:49:59.932 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T20:49:59.932 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T20:49:59.932 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T20:49:59.932 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T20:49:59.933 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T20:49:59.933 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T20:49:59.933 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_older_version": "true", 2023-09-23T20:49:59.933 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T20:49:59.933 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T20:49:59.934 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T20:49:59.934 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T20:49:59.934 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T20:49:59.934 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T20:49:59.934 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T20:49:59.934 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T20:49:59.935 INFO:teuthology.orchestra.run.smithi121.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T20:49:59.935 INFO:teuthology.orchestra.run.smithi121.stdout: "monmap": "", 2023-09-23T20:49:59.935 INFO:teuthology.orchestra.run.smithi121.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T20:49:59.935 INFO:teuthology.orchestra.run.smithi121.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T20:49:59.935 INFO:teuthology.orchestra.run.smithi121.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T20:49:59.935 INFO:teuthology.orchestra.run.smithi121.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T20:49:59.935 INFO:teuthology.orchestra.run.smithi121.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "motr_tracing_enabled": "false", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_op_threads": "3", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_cm": "false", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_device_name": "", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T20:49:59.936 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_sl": "3", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_rdma_type": "ib", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_async_reap_threshold": "5", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_before_connect": "false", 2023-09-23T20:49:59.937 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_ipv4": "true", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_ipv6": "false", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_msgr1": "true", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_msgr2": "true", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_port_max": "7568", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_port_min": "6800", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_retry_count": "3", 2023-09-23T20:49:59.938 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_bind_retry_delay": "5", 2023-09-23T20:49:59.939 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_blackhole_client": "false", 2023-09-23T20:49:59.939 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_blackhole_mds": "false", 2023-09-23T20:49:59.939 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_blackhole_mgr": "false", 2023-09-23T20:49:59.939 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_blackhole_mon": "false", 2023-09-23T20:49:59.939 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_blackhole_osd": "false", 2023-09-23T20:49:59.939 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_client_mode": "crc secure", 2023-09-23T20:49:59.939 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T20:49:59.940 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_cluster_type": "", 2023-09-23T20:49:59.940 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_compress_secure": "false", 2023-09-23T20:49:59.940 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T20:49:59.940 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T20:49:59.940 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_crc_data": "true", 2023-09-23T20:49:59.940 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_crc_header": "true", 2023-09-23T20:49:59.941 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T20:49:59.941 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_die_on_bug": "true", 2023-09-23T20:49:59.941 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_die_on_old_message": "true", 2023-09-23T20:49:59.941 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T20:49:59.941 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T20:49:59.941 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T20:49:59.942 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T20:49:59.942 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T20:49:59.942 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T20:49:59.942 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T20:49:59.942 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T20:49:59.942 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T20:49:59.943 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_hugepages": "", 2023-09-23T20:49:59.943 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T20:49:59.943 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T20:49:59.943 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_lro": "true", 2023-09-23T20:49:59.943 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T20:49:59.943 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T20:49:59.944 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_pmd": "", 2023-09-23T20:49:59.944 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_port_id": "0", 2023-09-23T20:49:59.944 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T20:49:59.944 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T20:49:59.944 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_dump_on_send": "false", 2023-09-23T20:49:59.944 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T20:49:59.944 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T20:49:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T20:49:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_inject_delay_type": "", 2023-09-23T20:49:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T20:49:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_inject_network_congestion": "0", 2023-09-23T20:49:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_inject_socket_failures": "0", 2023-09-23T20:49:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T20:49:59.945 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_max_accept_failures": "4", 2023-09-23T20:49:59.946 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_max_backoff": "15.000000", 2023-09-23T20:49:59.946 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T20:49:59.946 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T20:49:59.946 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T20:49:59.946 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T20:49:59.947 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_osd_compress_mode": "none", 2023-09-23T20:49:59.947 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T20:49:59.947 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T20:49:59.947 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_pq_min_cost": "65536", 2023-09-23T20:49:59.947 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_public_type": "", 2023-09-23T20:49:59.947 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_service_mode": "crc secure", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_tcp_nodelay": "true", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "ms_type": "async+posix", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "no_config_file": "false", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T20:49:59.948 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_inflight_ops": "1024", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "objecter_timeout": "10.000000", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "objectstore_blackhole": "false", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "openssl_engine_opts": "", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T20:49:59.949 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_agent_max_ops": "4", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_agent_slop": "0.020000", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_asio_thread_count": "2", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_backfill_scan_max": "512", 2023-09-23T20:49:59.950 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_backfill_scan_min": "64", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_backoff_on_peering": "false", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_beacon_report_interval": "300", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bench_duration": "30", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_blkin_trace_all": "false", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T20:49:59.951 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_class_update_on_start": "true", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_client_message_cap": "256", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_client_op_priority": "63", 2023-09-23T20:49:59.952 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_client_watch_timeout": "30", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_command_max_records": "256", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_compact_on_start": "false", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_crush_update_on_start": "true", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T20:49:59.953 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_op_order": "false", 2023-09-23T20:49:59.954 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_shutdown": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T20:49:59.955 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T20:49:59.956 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_default_notify_timeout": "30", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_enable_op_tracker": "true", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T20:49:59.957 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_fast_info": "true", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_fast_shutdown": "true", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_function_tracing": "false", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_heartbeat_grace": "20", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_heartbeat_interval": "6", 2023-09-23T20:49:59.958 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_heartbeat_stale": "600", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T20:49:59.959 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_journal_size": "5120", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_kill_backfill_at": "0", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_map_cache_size": "50", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_map_dedup": "true", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_map_message_max": "40", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_attr_name_len": "100", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_attr_size": "0", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_backfills": "1", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_markdown_count": "5", 2023-09-23T20:49:59.960 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_markdown_period": "600", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_object_name_len": "2048", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_object_size": "134217728", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_pgls": "1024", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_push_cost": "8388608", 2023-09-23T20:49:59.961 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_push_objects": "10", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_scrubs": "1", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_max_write_size": "90", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T20:49:59.962 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_profile": "balanced", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T20:49:59.963 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_memory_base": "805306368", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_memory_target": "4294967296", 2023-09-23T20:49:59.964 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_memory_target_autotune": "false", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_min_recovery_priority": "0", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mon_report_interval": "5", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_num_cache_shards": "32", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T20:49:59.965 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_numa_node": "-1", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_objecter_finishers": "1", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_objectstore": "bluestore", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_objectstore_fuse": "false", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_objectstore_tracing": "false", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_history_duration": "600", 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_history_size": "20", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_log_threshold": "5", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_num_shards": "0", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T20:49:59.967 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_op_thread_timeout": "15", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_open_classes_on_start": "true", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_os_flags": "0", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_peering_op_priority": "255", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_delete_priority": "5", 2023-09-23T20:49:59.968 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_crimson": "false", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T20:49:59.970 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_flags": "0", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_min_size": "0", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_size": "3", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_default_type": "replicated", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T20:49:59.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_cost": "20971520", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_max_active": "0", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_op_priority": "3", 2023-09-23T20:49:59.972 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_priority": "5", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_repair_during_recovery": "false", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T20:49:59.973 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_cost": "52428800", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_end_hour": "0", 2023-09-23T20:49:59.974 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_priority": "5", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T20:49:59.975 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_skip_data_digest": "false", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_smart_report_timeout": "5", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_snap_trim_priority": "5", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T20:49:59.976 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_target_transaction_size": "30", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T20:49:59.977 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_tracing": "false", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_use_stale_snap": "false", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_kill_at": "0", 2023-09-23T20:49:59.978 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_max_join_drift": "10", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_min": "500", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_min_wait": "0.050000", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_service_trim_max": "500", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_service_trim_min": "250", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_stash_full_interval": "25", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_trim_max": "500", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "paxos_trim_min": "250", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "perf": "true", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "pid_file": "", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T20:49:59.979 INFO:teuthology.orchestra.run.smithi121.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T20:49:59.980 INFO:teuthology.orchestra.run.smithi121.stdout: "public_addr": "-", 2023-09-23T20:49:59.980 INFO:teuthology.orchestra.run.smithi121.stdout: "public_addrv": "", 2023-09-23T20:49:59.980 INFO:teuthology.orchestra.run.smithi121.stdout: "public_bind_addr": "-", 2023-09-23T20:49:59.980 INFO:teuthology.orchestra.run.smithi121.stdout: "public_network": "172.21.0.0/20", 2023-09-23T20:49:59.980 INFO:teuthology.orchestra.run.smithi121.stdout: "public_network_interface": "", 2023-09-23T20:49:59.980 INFO:teuthology.orchestra.run.smithi121.stdout: "qat_compressor_enabled": "false", 2023-09-23T20:49:59.980 INFO:teuthology.orchestra.run.smithi121.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T20:49:59.981 INFO:teuthology.orchestra.run.smithi121.stdout: "rados_mon_op_timeout": "0", 2023-09-23T20:49:59.981 INFO:teuthology.orchestra.run.smithi121.stdout: "rados_osd_op_timeout": "0", 2023-09-23T20:49:59.981 INFO:teuthology.orchestra.run.smithi121.stdout: "rados_tracing": "false", 2023-09-23T20:49:59.981 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_atime_update_interval": "60", 2023-09-23T20:49:59.981 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T20:49:59.981 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T20:49:59.981 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache": "true", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_policy": "writearound", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_size": "33554432", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T20:49:59.982 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_compression_hint": "none", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_clone_format": "auto", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_data_pool": "", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_features": "61", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_format": "2", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_map_options": "", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_order": "22", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_pool": "rbd", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T20:49:59.983 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_stripe_count": "0", 2023-09-23T20:49:59.985 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T20:49:59.985 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T20:49:59.985 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T20:49:59.985 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T20:49:59.985 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_io_scheduler": "simple", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T20:49:59.986 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T20:49:59.987 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T20:49:59.987 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_order": "24", 2023-09-23T20:49:59.987 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_pool": "", 2023-09-23T20:49:59.987 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_journal_splay_width": "4", 2023-09-23T20:49:59.987 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T20:49:59.987 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T20:49:59.988 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T20:49:59.988 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T20:49:59.988 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T20:49:59.988 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T20:49:59.988 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T20:49:59.988 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T20:49:59.989 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T20:49:59.989 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T20:49:59.989 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T20:49:59.989 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T20:49:59.989 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T20:49:59.989 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T20:49:59.990 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T20:49:59.990 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T20:49:59.990 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T20:49:59.990 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T20:49:59.990 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T20:49:59.990 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T20:49:59.990 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T20:49:59.991 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T20:49:59.991 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T20:49:59.991 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T20:49:59.991 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T20:49:59.991 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T20:49:59.991 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T20:49:59.991 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T20:49:59.992 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T20:49:59.992 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T20:49:59.992 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T20:49:59.992 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T20:49:59.992 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T20:49:59.992 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T20:49:59.992 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T20:49:59.993 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T20:49:59.993 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_op_threads": "1", 2023-09-23T20:49:59.993 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T20:49:59.993 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T20:49:59.993 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T20:49:59.993 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T20:49:59.993 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_plugins": "", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T20:49:59.994 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T20:49:59.995 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T20:49:59.996 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T20:49:59.996 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T20:49:59.996 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T20:49:59.996 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T20:49:59.996 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T20:49:59.996 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T20:49:59.996 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_tracing": "false", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_validate_names": "true", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "rbd_validate_pool": "true", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "restapi_base_url": "", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "restapi_log_level": "", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_admin_entry": "admin", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_backend_store": "rados", 2023-09-23T20:49:59.997 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_barbican_url": "", 2023-09-23T20:49:59.998 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_beast_enable_async": "true", 2023-09-23T20:49:59.998 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T20:49:59.998 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T20:49:59.998 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T20:49:59.998 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T20:49:59.998 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T20:49:59.999 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T20:49:59.999 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T20:49:59.999 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_cache_enabled": "true", 2023-09-23T20:49:59.999 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T20:49:59.999 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T20:49:59.999 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_config_store": "rados", 2023-09-23T20:49:59.999 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_content_length_compat": "false", 2023-09-23T20:50:00.000 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T20:50:00.000 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T20:50:00.000 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T20:50:00.000 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_cross_domain_policy": "", 2023-09-23T20:50:00.000 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T20:50:00.000 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T20:50:00.001 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T20:50:00.001 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T20:50:00.001 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T20:50:00.001 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T20:50:00.001 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T20:50:00.002 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T20:50:00.002 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T20:50:00.002 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T20:50:00.002 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T20:50:00.002 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T20:50:00.002 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T20:50:00.003 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T20:50:00.003 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T20:50:00.003 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T20:50:00.003 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T20:50:00.003 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T20:50:00.004 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T20:50:00.004 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T20:50:00.004 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T20:50:00.004 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T20:50:00.004 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T20:50:00.004 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T20:50:00.005 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T20:50:00.005 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T20:50:00.005 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T20:50:00.005 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T20:50:00.005 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T20:50:00.005 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T20:50:00.005 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T20:50:00.006 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T20:50:00.007 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T20:50:00.007 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T20:50:00.007 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T20:50:00.007 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T20:50:00.007 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T20:50:00.007 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T20:50:00.007 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T20:50:00.008 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T20:50:00.008 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T20:50:00.008 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T20:50:00.008 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T20:50:00.008 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T20:50:00.008 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T20:50:00.009 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data_log_window": "30", 2023-09-23T20:50:00.009 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T20:50:00.009 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T20:50:00.009 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T20:50:00.009 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T20:50:00.009 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T20:50:00.009 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T20:50:00.010 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dns_name": "", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dns_s3website_name": "", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T20:50:00.011 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enable_ops_log": "false", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enable_static_website": "false", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enable_usage_log": "false", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_expose_bucket": "false", 2023-09-23T20:50:00.012 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_extended_http_attrs": "", 2023-09-23T20:50:00.013 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_filter": "none", 2023-09-23T20:50:00.013 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T20:50:00.013 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T20:50:00.013 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T20:50:00.013 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T20:50:00.013 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T20:50:00.013 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_max_objs": "32", 2023-09-23T20:50:00.014 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T20:50:00.014 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T20:50:00.014 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T20:50:00.014 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T20:50:00.014 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T20:50:00.014 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T20:50:00.014 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T20:50:00.015 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T20:50:00.015 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T20:50:00.015 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_init_timeout": "300", 2023-09-23T20:50:00.015 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T20:50:00.015 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T20:50:00.015 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T20:50:00.016 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T20:50:00.016 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T20:50:00.016 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_password": "", 2023-09-23T20:50:00.016 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T20:50:00.016 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_project": "", 2023-09-23T20:50:00.016 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T20:50:00.017 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_token": "", 2023-09-23T20:50:00.017 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T20:50:00.017 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_admin_user": "", 2023-09-23T20:50:00.017 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_api_version": "2", 2023-09-23T20:50:00.017 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T20:50:00.017 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T20:50:00.017 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T20:50:00.018 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T20:50:00.018 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T20:50:00.018 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T20:50:00.018 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T20:50:00.018 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T20:50:00.018 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T20:50:00.018 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T20:50:00.019 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_url": "", 2023-09-23T20:50:00.019 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T20:50:00.019 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T20:50:00.019 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T20:50:00.019 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lc_max_objs": "32", 2023-09-23T20:50:00.020 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T20:50:00.020 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lc_max_worker": "3", 2023-09-23T20:50:00.020 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T20:50:00.020 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T20:50:00.020 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T20:50:00.020 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_log_http_headers": "", 2023-09-23T20:50:00.021 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_attr_size": "0", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T20:50:00.022 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_listing_results": "1000", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_notify_retries": "10", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T20:50:00.023 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T20:50:00.024 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_num_control_oids": "8", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_numa_node": "-1", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_op_tracing": "false", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_opa_token": "", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_opa_url": "", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ops_log_rados": "false", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T20:50:00.026 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_print_continue": "true", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T20:50:00.027 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_rados_tracing": "false", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_realm": "", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_realm_id": "", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_region": "", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_request_uri": "", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T20:50:00.028 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_resolve_cname": "false", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_run_sync_thread": "true", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T20:50:00.029 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_script_uri": "", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_service_provider_name": "", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sts_client_id": "", 2023-09-23T20:50:00.030 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sts_client_secret": "", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sts_entry": "sts", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sts_key": "sts", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_auth_url": "", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_custom_header": "", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_need_stats": "true", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_tenant_name": "", 2023-09-23T20:50:00.031 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_url": "", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_lease_period": "120", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T20:50:00.032 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_thread_pool_size": "512", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_torrent_comment": "", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_torrent_createby": "", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_torrent_encoding": "", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_torrent_flag": "false", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_torrent_origin": "", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_torrent_tracker": "", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T20:50:00.033 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_usage_max_shards": "32", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_use_opa_authz": "false", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T20:50:00.034 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_user_unique_email": "true", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_verify_ssl": "true", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_zone": "", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_zone_id": "", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_zonegroup": "", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_zonegroup_id": "", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T20:50:00.035 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_block_size": "4096", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T20:50:00.036 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_partition_filters": "false", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_perf": "false", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "run_dir": "/var/run/ceph", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_block_create": "true", 2023-09-23T20:50:00.037 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_device_size": "53687091200", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_main_device_type": "SSD", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T20:50:00.038 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "seastore_segment_size": "67108864", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "set_keepcaps": "false", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "setgroup": "ceph", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "setuser": "ceph", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "setuser_match_path": "", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "thp": "false", 2023-09-23T20:50:00.039 INFO:teuthology.orchestra.run.smithi121.stdout: "threadpool_default_timeout": "60", 2023-09-23T20:50:00.040 INFO:teuthology.orchestra.run.smithi121.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T20:50:00.040 INFO:teuthology.orchestra.run.smithi121.stdout: "throttler_perf_counter": "true" 2023-09-23T20:50:00.040 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:50:00.041 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:49:59 smithi123 ceph-mon[193903]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:00.041 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:49:59 smithi121 ceph-mon[211526]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:00.041 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:49:59 smithi121 ceph-mon[219709]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:00.445 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T20:50:00.793 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:00.978 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:00 smithi121 ceph-mon[211526]: overall HEALTH_OK 2023-09-23T20:50:00.979 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:00 smithi121 ceph-mon[219709]: overall HEALTH_OK 2023-09-23T20:50:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:00 smithi123 ceph-mon[193903]: overall HEALTH_OK 2023-09-23T20:50:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:01 smithi123 ceph-mon[193903]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:02.014 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:01 smithi121 ceph-mon[211526]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:02.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:01 smithi121 ceph-mon[219709]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:02.243 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a: {} 2023-09-23T20:50:02.254 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b: {} 2023-09-23T20:50:02.263 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c: {} 2023-09-23T20:50:02.264 INFO:teuthology.orchestra.run.smithi121.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:50:02.265 INFO:teuthology.orchestra.run.smithi121.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:50:02.265 INFO:teuthology.orchestra.run.smithi121.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:50:02.862 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T20:50:02.862 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T20:50:02.862 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:50:02.863 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:50:02.863 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T20:50:02.863 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T20:50:02.863 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T20:50:02.864 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:50:02.864 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:50:02.864 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T20:50:02.864 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:02.864 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T20:50:02.865 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:50:02.865 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T20:50:02.865 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:50:02.866 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T20:50:02.866 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:50:02.936 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1219163517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:50:02.987 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:02.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:02.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:03.092 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:03.123 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1219163517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:50:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1219163517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:50:03.169 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:03.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:03.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:03.275 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:03.314 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:03.318 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:03.357 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:03.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:03.369 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:03.372 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:03.462 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:03.490 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:03.597 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:03.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:03.700 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:03.728 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:03.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:03 smithi121 ceph-mon[211526]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:03.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:03 smithi121 ceph-mon[219709]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:03 smithi123 ceph-mon[193903]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:04.475 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:04.582 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:04.589 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:04.613 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:04.660 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:04.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:04.765 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:04.793 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:04.838 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:04.946 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:04.973 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.014 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:05.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:05.122 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:05.148 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:50:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:50:04] "GET /metrics HTTP/1.1" 200 33972 "" "Prometheus/2.43.0" 2023-09-23T20:50:05.189 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:05.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:05.295 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:05.322 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.364 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:05.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:05.469 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:05.494 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.534 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:05.641 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:05.666 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.707 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:05.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:05.813 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:05.845 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.903 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:05 smithi123 ceph-mon[193903]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:06.013 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:06.040 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:06.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:05 smithi121 ceph-mon[211526]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:06.062 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:05 smithi121 ceph-mon[219709]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:06.081 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:06.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:06.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:06.187 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:06.216 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:06.810 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:06.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:06.913 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:06.942 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:06.999 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:07.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:07.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:07.074 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_4' created 2023-09-23T20:50:07.103 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:07.135 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:07.183 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:07.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:07.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:07.208 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:50:07.208 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:50:07.209 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T20:50:07.209 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:50:07.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/637976374' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:50:07.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:07 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:50:07.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:07 smithi121 ceph-mon[211526]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:07.239 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/637976374' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:50:07.240 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:07 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:50:07.240 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:07 smithi121 ceph-mon[219709]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:07.289 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:07.322 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/637976374' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:50:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:07 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:50:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:07 smithi123 ceph-mon[193903]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:08.080 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:50:08.201 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:08.202 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T20:50:08.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:08.277 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:50:08.304 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:08.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.334 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:50:08.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T20:50:08.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.335 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1420433043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:08.337 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1420433043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.341 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: osdmap e164: 8 total, 8 up, 8 in 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: failed to encode map e164 with expected crc 2023-09-23T20:50:08.342 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1420433043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:08.386 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:08.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:08.493 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:08.528 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.582 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:08.586 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:50:08.608 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:50:08.629 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:50:08.649 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:50:08.669 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:50:08.686 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:08.689 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:50:08.708 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:50:08.720 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.728 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:50:08.770 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:08.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:08.902 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.944 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:08.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:09.050 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:09.224 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:09.250 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.291 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:09.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:09.396 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1420433043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/514994126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[219709]: pgmap v523: 89 pgs: 1 creating+activating, 3 creating+peering, 11 unknown, 74 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1420433043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/514994126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:09.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:09 smithi121 ceph-mon[211526]: pgmap v523: 89 pgs: 1 creating+activating, 3 creating+peering, 11 unknown, 74 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:09.419 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:09.419 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.419 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1420433043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:09.419 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: osdmap e165: 8 total, 8 up, 8 in 2023-09-23T20:50:09.419 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.420 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.420 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: failed to encode map e165 with expected crc 2023-09-23T20:50:09.420 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/514994126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:09.420 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:09 smithi123 ceph-mon[193903]: pgmap v523: 89 pgs: 1 creating+activating, 3 creating+peering, 11 unknown, 74 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:09.422 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.463 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:09.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:09.570 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:09.597 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.744 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:09.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:09.798 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:09.848 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:09.849 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:09.874 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.915 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:09.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:10.022 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:10.048 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.090 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:10.197 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:10.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:10 smithi123 ceph-mon[193903]: failed to encode map e165 with expected crc 2023-09-23T20:50:10.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:10 smithi123 ceph-mon[193903]: failed to encode map e166 with expected crc 2023-09-23T20:50:10.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:10 smithi123 ceph-mon[193903]: failed to encode map e166 with expected crc 2023-09-23T20:50:10.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:10 smithi123 ceph-mon[193903]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T20:50:10.222 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.264 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:10.370 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:10.390 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[219709]: failed to encode map e165 with expected crc 2023-09-23T20:50:10.390 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[219709]: failed to encode map e166 with expected crc 2023-09-23T20:50:10.390 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[219709]: failed to encode map e166 with expected crc 2023-09-23T20:50:10.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[219709]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T20:50:10.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[211526]: failed to encode map e165 with expected crc 2023-09-23T20:50:10.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[211526]: failed to encode map e166 with expected crc 2023-09-23T20:50:10.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[211526]: failed to encode map e166 with expected crc 2023-09-23T20:50:10.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:10 smithi121 ceph-mon[211526]: osdmap e166: 8 total, 8 up, 8 in 2023-09-23T20:50:10.400 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.496 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:10.599 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:10.626 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.666 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:10.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:10.772 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:10.799 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.877 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:10.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:10.980 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:11.006 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.046 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:11.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:11.154 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:11.182 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.364 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:11 smithi121 ceph-mon[211526]: failed to encode map e166 with expected crc 2023-09-23T20:50:11.364 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:11 smithi121 ceph-mon[211526]: failed to encode map e166 with expected crc 2023-09-23T20:50:11.364 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:11 smithi121 ceph-mon[211526]: pgmap v525: 89 pgs: 1 creating+activating, 3 creating+peering, 85 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:11.365 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:11 smithi121 ceph-mon[219709]: failed to encode map e166 with expected crc 2023-09-23T20:50:11.365 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:11 smithi121 ceph-mon[219709]: failed to encode map e166 with expected crc 2023-09-23T20:50:11.365 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:11 smithi121 ceph-mon[219709]: pgmap v525: 89 pgs: 1 creating+activating, 3 creating+peering, 85 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:11.402 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:11.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:11 smithi123 ceph-mon[193903]: failed to encode map e166 with expected crc 2023-09-23T20:50:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:11 smithi123 ceph-mon[193903]: failed to encode map e166 with expected crc 2023-09-23T20:50:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:11 smithi123 ceph-mon[193903]: pgmap v525: 89 pgs: 1 creating+activating, 3 creating+peering, 85 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:11.498 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:50:11.506 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:11.571 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.733 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:11.768 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:50:11.837 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:11.865 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.909 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:11.995 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:12.014 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:12.038 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:12.038 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:12.041 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.083 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:12.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:12.105 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T20:50:12.188 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:12.219 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/210642438' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:12.315 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:12.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/210642438' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:12.317 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/210642438' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:12.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:12.419 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:12.445 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.485 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:12.592 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:12.618 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.660 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:12.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:12.767 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:12.793 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.834 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:12.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:12.940 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:12.966 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.006 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:13.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:13.113 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:13.138 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.179 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:13.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:13.287 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:13.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/210642438' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[211526]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[211526]: pgmap v527: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/210642438' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[219709]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T20:50:13.314 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:13.315 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:13.315 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:13.315 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:13.315 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:13 smithi121 ceph-mon[219709]: pgmap v527: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:13.316 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.365 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:13.373 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:50:13.469 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/210642438' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:50:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:13 smithi123 ceph-mon[193903]: osdmap e167: 8 total, 8 up, 8 in 2023-09-23T20:50:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:13 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:13 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:13 smithi123 ceph-mon[193903]: pgmap v527: 89 pgs: 1 creating+activating, 88 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:13.495 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.536 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:13.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:13.643 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:13.672 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.876 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:13.916 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:13.980 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:14.006 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:14.036 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:14.048 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:14.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:14.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:14.154 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:14.184 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:14.216 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T20:50:14.217 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:50:14.217 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:50:14.219 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:50:14.219 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:50:14.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:14 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:14.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:14 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:14.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:14 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:14.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:14 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:14.454 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T20:50:14.455 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:50:14.455 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_293770 2023-09-23T20:50:14.455 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:50:14.455 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:50:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:14 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:14 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:14.744 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:14.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:14.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:14.849 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:14.882 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:15.094 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:50:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:50:14] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.43.0" 2023-09-23T20:50:15.125 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:50:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[211526]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[211526]: pgmap v528: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/396750866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[219709]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[219709]: pgmap v528: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/396750866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:15.455 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 1993 1977 123.556 123.562 0.00335543 0.00807761 2023-09-23T20:50:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:15 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:15 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:15 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:15 smithi123 ceph-mon[193903]: failed to encode map e167 with expected crc 2023-09-23T20:50:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:15 smithi123 ceph-mon[193903]: pgmap v528: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-09-23T20:50:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/396750866' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:15.489 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:15.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:15.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:15.600 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:15.649 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:15.738 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:15.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:15.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:15.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:15.915 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:16.114 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:16.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[211526]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T20:50:16.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:16.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:16.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:16 smithi121 ceph-mon[219709]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T20:50:16.458 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 5590 5574 174.089 224.812 0.00395197 0.00573256 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: failed to encode map e168 with expected crc 2023-09-23T20:50:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:16.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:16 smithi123 ceph-mon[193903]: osdmap e168: 8 total, 8 up, 8 in 2023-09-23T20:50:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:17 smithi121 ceph-mon[211526]: pgmap v530: 89 pgs: 89 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-09-23T20:50:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:17 smithi121 ceph-mon[219709]: pgmap v530: 89 pgs: 89 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-09-23T20:50:17.456 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 15 7449 7434 154.812 116.25 0.00426227 0.00645486 2023-09-23T20:50:17.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:17 smithi123 ceph-mon[193903]: pgmap v530: 89 pgs: 89 active+clean; 36 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-09-23T20:50:17.721 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:17.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:17.826 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:17.861 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:18.456 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 9851 9835 153.621 150.062 0.00522438 0.00650189 2023-09-23T20:50:18.650 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:50:18.702 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:50:18.724 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:50:18.743 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:50:18.762 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:50:18.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:50:18.815 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138235 7.6 7:65e4897b:::benchmark_data_smithi121_234442_object27928:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.124153+0000", 2023-09-23T20:50:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.71235653700001, 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2796559160000001, 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138235 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124153+0000", 2023-09-23T20:50:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124153+0000", 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124152+0000", 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124160+0000", 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5029999999999997e-06 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124171+0000", 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0913999999999999e-05 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124179+0000", 2023-09-23T20:50:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.678e-06 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124231+0000", 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1907000000000003e-05 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124243+0000", 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2257e-05 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.392005+0000", 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2677619739999999 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.392113+0000", 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010784 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.403703+0000", 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011590324000000001 2023-09-23T20:50:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.403809+0000", 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106048 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138245 7.6 7:6d2ee90e:::benchmark_data_smithi121_234442_object27938:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.124192+0000", 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.712317461, 2023-09-23T20:50:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2959034380000001, 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138245 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124192+0000", 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124192+0000", 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124192+0000", 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124195+0000", 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7839999999999999e-06 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124201+0000", 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2439999999999998e-06 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124206+0000", 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3619999999999997e-06 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124293+0000", 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6792000000000005e-05 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.124303+0000", 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4550000000000001e-06 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.392442+0000", 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.268139286 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.392537+0000", 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5203000000000003e-05 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420007+0000", 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027469693999999999 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420096+0000", 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9058e-05 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138262 7.6 7:67585cd0:::benchmark_data_smithi121_234442_object27955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.131345+0000", 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.70516511, 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2727730209999999, 2023-09-23T20:50:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138262 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131345+0000", 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131345+0000", 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131344+0000", 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131347+0000", 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7700000000000002e-06 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131353+0000", 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.84e-06 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131358+0000", 2023-09-23T20:50:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.147e-06 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.139859+0000", 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085006449999999994 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.139870+0000", 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1565e-05 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.394920+0000", 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2550495749999999 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.395056+0000", 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013606600000000001 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.395188+0000", 2023-09-23T20:50:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000132225 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.403649+0000", 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0084607599999999995 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.403843+0000", 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019439899999999999 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.403966+0000", 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012282800000000001 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404007+0000", 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0896000000000003e-05 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404118+0000", 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110713 2023-09-23T20:50:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138263 7.6 7:6411e0c9:::benchmark_data_smithi121_234442_object27956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.131367+0000", 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.70514317499999, 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.272987468, 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.860 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138263 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131367+0000", 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131367+0000", 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.861 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131366+0000", 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131369+0000", 2023-09-23T20:50:18.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5490000000000001e-06 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131375+0000", 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3659999999999997e-06 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.131381+0000", 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6579999999999999e-06 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.139933+0000", 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085525229999999994 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.139941+0000", 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7680000000000008e-06 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.397381+0000", 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2574393779999999 2023-09-23T20:50:18.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.397481+0000", 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010008799999999999 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.397558+0000", 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7480999999999999e-05 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.865 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.403659+0000", 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061003960000000001 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404159+0000", 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000500812 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404202+0000", 2023-09-23T20:50:18.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2225000000000003e-05 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404252+0000", 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0101e-05 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404354+0000", 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000102452 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.867 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138268 7.6 7:6aaffb7c:::benchmark_data_smithi121_234442_object27961:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.145323+0000", 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.69118662, 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.275013113, 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138268 2023-09-23T20:50:18.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145323+0000", 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145323+0000", 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145322+0000", 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145333+0000", 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1028e-05 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145351+0000", 2023-09-23T20:50:18.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8742000000000001e-05 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145362+0000", 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0637e-05 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145421+0000", 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8995e-05 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.145434+0000", 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2976000000000001e-05 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.401436+0000", 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2560016090000001 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.401521+0000", 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5603e-05 2023-09-23T20:50:18.872 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.401620+0000", 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9146000000000002e-05 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.403912+0000", 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002291618 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.407115+0000", 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003202973 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420203+0000", 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013087889 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420248+0000", 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4671e-05 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420336+0000", 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8796999999999997e-05 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138279 7.6 7:6be9279d:::benchmark_data_smithi121_234442_object27972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.160721+0000", 2023-09-23T20:50:18.875 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.67578855599999, 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2598222269999999, 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138279 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.160721+0000", 2023-09-23T20:50:18.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.160721+0000", 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.160720+0000", 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.160728+0000", 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6890000000000004e-06 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.160745+0000", 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7039999999999999e-05 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.160752+0000", 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8520000000000004e-06 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.173532+0000", 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012779509 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.173546+0000", 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3750999999999999e-05 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404393+0000", 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2308472799999999 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404497+0000", 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000103838 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.404600+0000", 2023-09-23T20:50:18.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010313 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.413425+0000", 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088249220000000007 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420146+0000", 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067211789999999999 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420428+0000", 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000281475 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420466+0000", 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8834999999999997e-05 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420544+0000", 2023-09-23T20:50:18.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7206999999999994e-05 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138297 7.6 7:6e2b6412:::benchmark_data_smithi121_234442_object27990:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.182685+0000", 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.65382495099999, 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2380372099999999, 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.883 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138297 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182685+0000", 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182685+0000", 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182683+0000", 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182691+0000", 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0739999999999993e-06 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182708+0000", 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6923e-05 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182717+0000", 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7730000000000005e-06 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182810+0000", 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3457999999999999e-05 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.182825+0000", 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4406000000000001e-05 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.407143+0000", 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2243177949999999 2023-09-23T20:50:18.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.407274+0000", 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000131767 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.407394+0000", 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119839 2023-09-23T20:50:18.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.416038+0000", 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086439580000000002 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420385+0000", 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004347119 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420593+0000", 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020776799999999999 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420637+0000", 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4159000000000002e-05 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.420722+0000", 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4846999999999996e-05 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.34248.0:138306 7.6 7:6520040c:::benchmark_data_smithi121_234442_object27999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e99)", 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:42:32.200548+0000", 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 466.63596158799999, 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.234073491, 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.891 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.34248", 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 138306 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200548+0000", 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200548+0000", 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200546+0000", 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200558+0000", 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1463e-05 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200573+0000", 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5838999999999999e-05 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200584+0000", 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0763e-05 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200642+0000", 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7941e-05 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:32.200654+0000", 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2184000000000001e-05 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.410759+0000", 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.210104378 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.410856+0000", 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7499999999999998e-05 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.410960+0000", 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010387000000000001 2023-09-23T20:50:18.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.419737+0000", 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0087765919999999997 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.434420+0000", 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014683188999999999 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.434466+0000", 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6072999999999997e-05 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.434526+0000", 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0316999999999999e-05 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:42:33.434622+0000", 2023-09-23T20:50:18.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5482000000000005e-05 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19260 8.1 8:8112e7df:::benchmark_data_smithi121_252781_object19259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997064+0000", 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 318.83944624399999, 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2529551620000001, 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:50:18.899 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19260 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997064+0000", 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997064+0000", 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997062+0000", 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997089+0000", 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6701999999999999e-05 2023-09-23T20:50:18.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997107+0000", 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7991000000000001e-05 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997119+0000", 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2067e-05 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997395+0000", 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000276218 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997403+0000", 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6769999999999995e-06 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.249935+0000", 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2525322729999999 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.250019+0000", 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3875000000000006e-05 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19270 8.1 8:84a81d50:::benchmark_data_smithi121_252781_object19269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997136+0000", 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 318.83937415399998, 2023-09-23T20:50:18.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.266036414, 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19270 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997136+0000", 2023-09-23T20:50:18.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997136+0000", 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997135+0000", 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997164+0000", 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9624999999999998e-05 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997176+0000", 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1945e-05 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997183+0000", 2023-09-23T20:50:18.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2249999999999997e-06 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997416+0000", 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000233079 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997419+0000", 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6839999999999998e-06 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.250714+0000", 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.253294525 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.250891+0000", 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017732699999999999 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.251012+0000", 2023-09-23T20:50:18.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000120912 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.255523+0000", 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045112390000000002 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.256773+0000", 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012498940000000001 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.262221+0000", 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054475269999999998 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.262296+0000", 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4915999999999999e-05 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.263172+0000", 2023-09-23T20:50:18.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000876383 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19271 8.1 8:851b3fc2:::benchmark_data_smithi121_252781_object19270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997189+0000", 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 318.83932112000002, 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2667984699999999, 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.912 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19271 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997189+0000", 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997189+0000", 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997188+0000", 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997213+0000", 2023-09-23T20:50:18.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5196000000000001e-05 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997221+0000", 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7589999999999997e-06 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997226+0000", 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4149999999999998e-06 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997430+0000", 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000203432 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997433+0000", 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3859999999999999e-06 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.252000+0000", 2023-09-23T20:50:18.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2545671190000001 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.252140+0000", 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014005199999999999 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.252329+0000", 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000189106 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.256650+0000", 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004320885 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.256819+0000", 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168794 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.263233+0000", 2023-09-23T20:50:18.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064139009999999996 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.263307+0000", 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3503000000000004e-05 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.263987+0000", 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00068071400000000002 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.919 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19277 8.1 8:88aa0c5d:::benchmark_data_smithi121_252781_object19276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997231+0000", 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 318.839278691, 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.267411233, 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.920 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19277 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997231+0000", 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997231+0000", 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997231+0000", 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997250+0000", 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9451000000000001e-05 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997256+0000", 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.207e-06 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997262+0000", 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5389999999999998e-06 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997446+0000", 2023-09-23T20:50:18.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018423100000000001 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997452+0000", 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.801e-06 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.253250+0000", 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.255797958 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.253412+0000", 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016198800000000001 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.253571+0000", 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015911000000000001 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.256733+0000", 2023-09-23T20:50:18.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031617540000000001 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.258025+0000", 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012920869999999999 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.264055+0000", 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0060301419999999996 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.264109+0000", 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4418e-05 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.264642+0000", 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00053297100000000003 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19278 8.1 8:842dafa6:::benchmark_data_smithi121_252781_object19277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997279+0000", 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 318.839231305, 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.267984131, 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19278 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.928 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997279+0000", 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997279+0000", 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997278+0000", 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997298+0000", 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9559999999999999e-05 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997306+0000", 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.14e-06 2023-09-23T20:50:18.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997311+0000", 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5140000000000001e-06 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997468+0000", 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000157114 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997474+0000", 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.913e-06 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.254448+0000", 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.256973422 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.254599+0000", 2023-09-23T20:50:18.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015122399999999999 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.254776+0000", 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017681899999999999 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.258239+0000", 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034635690000000001 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.259023+0000", 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00078313199999999997 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.264771+0000", 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057481930000000004 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.264821+0000", 2023-09-23T20:50:18.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0098e-05 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265263+0000", 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00044190400000000002 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19294 8.1 8:8216d95b:::benchmark_data_smithi121_252781_object19293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:50:18.935 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 318.83919385799999, 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2685024899999999, 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19294 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:50:18.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997334+0000", 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7994000000000001e-05 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997339+0000", 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997344+0000", 2023-09-23T20:50:18.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2279999999999998e-06 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997493+0000", 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014906399999999999 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997500+0000", 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2759999999999997e-06 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.255558+0000", 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2580583510000001 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.255704+0000", 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000145931 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.255845+0000", 2023-09-23T20:50:18.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014099100000000001 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.258428+0000", 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002583049 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.259031+0000", 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00060272199999999998 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265363+0000", 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063324569999999997 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265422+0000", 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8442000000000001e-05 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265819+0000", 2023-09-23T20:50:18.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039688400000000002 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95051 8.e 8:737847a6:::benchmark_data_smithi121_252781_object95050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.060916+0000", 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 254.77559433100001, 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2772001780000002, 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.943 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95051 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060916+0000", 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060916+0000", 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060915+0000", 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060935+0000", 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9998000000000001e-05 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060942+0000", 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1359999999999999e-06 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060948+0000", 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4689999999999996e-06 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070165+0000", 2023-09-23T20:50:18.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092176889999999994 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070178+0000", 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2893e-05 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070395+0000", 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0002171209999999 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070408+0000", 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2425e-05 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.321120+0000", 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.250712338 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.321280+0000", 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016026500000000001 2023-09-23T20:50:18.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.321453+0000", 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017232799999999999 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329458+0000", 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080051189999999998 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329467+0000", 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6409999999999997e-06 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.336925+0000", 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074573410000000001 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.337008+0000", 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3193000000000001e-05 2023-09-23T20:50:18.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.338116+0000", 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0011077330000000001 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95053 8.e 8:7ad97c2c:::benchmark_data_smithi121_252781_object95052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 254.775557448, 2023-09-23T20:50:18.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2798300080000002, 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95053 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060972+0000", 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0196000000000001e-05 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060992+0000", 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0049999999999999e-05 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.061000+0000", 2023-09-23T20:50:18.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3749999999999997e-06 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070202+0000", 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092024120000000001 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070210+0000", 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8509999999999999e-06 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070431+0000", 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0002214380000001 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070440+0000", 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2379999999999997e-06 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.326020+0000", 2023-09-23T20:50:18.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.255580801 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.326163+0000", 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014299500000000001 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.326332+0000", 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168989 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333147+0000", 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068145599999999999 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.338206+0000", 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050593840000000001 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.339743+0000", 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001536765 2023-09-23T20:50:18.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.339804+0000", 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0646000000000002e-05 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.340782+0000", 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00097870100000000001 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.959 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95061 8.e 8:74014da4:::benchmark_data_smithi121_252781_object95060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.077670+0000", 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 254.75884022599999, 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.270089563, 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95061 2023-09-23T20:50:18.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077670+0000", 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077670+0000", 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077668+0000", 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077698+0000", 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9169000000000001e-05 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077712+0000", 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4589999999999999e-05 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077720+0000", 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.14e-06 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077774+0000", 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3924000000000001e-05 2023-09-23T20:50:18.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077787+0000", 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.296e-05 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070461+0000", 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.992673625 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070469+0000", 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9300000000000003e-06 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329493+0000", 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2590239009999999 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329630+0000", 2023-09-23T20:50:18.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000137701 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329759+0000", 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000128572 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.336891+0000", 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071319629999999998 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.340812+0000", 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0039211530000000001 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.346685+0000", 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0058733609999999997 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.346742+0000", 2023-09-23T20:50:18.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6841e-05 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.347759+0000", 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001016996 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95065 8.e 8:75a28b09:::benchmark_data_smithi121_252781_object95064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:50:18.968 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.082096+0000", 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 254.754414346, 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2666930719999998, 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95065 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082096+0000", 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082096+0000", 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082095+0000", 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:50:18.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082112+0000", 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7521e-05 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082124+0000", 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1742999999999999e-05 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082132+0000", 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9570000000000002e-06 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082182+0000", 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0015999999999997e-05 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082191+0000", 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7900000000000005e-06 2023-09-23T20:50:18.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070491+0000", 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98829999700000004 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070498+0000", 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5549999999999997e-06 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333194+0000", 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2626954370000001 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333342+0000", 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148679 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333508+0000", 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000165308 2023-09-23T20:50:18.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.341077+0000", 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075696369999999997 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.343593+0000", 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025160489999999998 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.347824+0000", 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042309499999999998 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.347898+0000", 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3291e-05 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.348789+0000", 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089107699999999999 2023-09-23T20:50:18.976 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:186211 8.c 8:3aecfc72:::benchmark_data_smithi121_252781_object77036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:06.495533+0000", 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 132.34097725199999, 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1146059930000001, 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.977 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 186211 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495533+0000", 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495533+0000", 2023-09-23T20:50:18.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495532+0000", 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495535+0000", 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.869e-06 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495540+0000", 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4280000000000004e-06 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495545+0000", 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5820000000000003e-06 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495659+0000", 2023-09-23T20:50:18.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114418 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495791+0000", 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013153700000000001 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 7", 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495928+0000", 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013725 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.504256+0000", 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0083274019999999994 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.510300+0000", 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0060443040000000003 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.510331+0000", 2023-09-23T20:50:18.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0778000000000002e-05 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.512602+0000", 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.00227154 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.513103+0000", 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050099699999999999 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600139+0000", 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.087035502 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600301+0000", 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000162118 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.610027+0000", 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097265790000000008 2023-09-23T20:50:18.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.610139+0000", 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000111156 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:186219 8.c 8:3b0c0d24:::benchmark_data_smithi121_252781_object77044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:06.514378+0000", 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 132.322132337, 2023-09-23T20:50:18.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0982041680000001, 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 186219 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514378+0000", 2023-09-23T20:50:18.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514378+0000", 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514376+0000", 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.987 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514383+0000", 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.584e-06 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514395+0000", 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2095e-05 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514404+0000", 2023-09-23T20:50:18.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0799999999999995e-06 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514455+0000", 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0735000000000001e-05 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514467+0000", 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2680000000000001e-05 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.513156+0000", 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99868875800000001 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.513168+0000", 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1637000000000001e-05 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600577+0000", 2023-09-23T20:50:18.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0874094480000001 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600669+0000", 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1785000000000003e-05 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600751+0000", 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2384000000000003e-05 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.606048+0000", 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052968579999999998 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.608509+0000", 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002460907 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.612298+0000", 2023-09-23T20:50:18.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037888959999999999 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.612357+0000", 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8764999999999997e-05 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.612582+0000", 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00022486899999999999 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.993 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.994 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.994 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:18.994 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:18.994 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:50:18.994 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:18.995 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3803236433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:18.995 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:50:18.995 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:18.995 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.995 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.996 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.996 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.996 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.996 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.996 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.996 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:50:18.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:18.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3803236433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:18.997 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:18.997 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3803236433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:19.088 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:19.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:19.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:19.256 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:19.364 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:19.457 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 12323 12307 153.794 154.5 0.00837073 0.00649406 2023-09-23T20:50:19.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:19 smithi121 ceph-mon[219709]: pgmap v531: 89 pgs: 89 active+clean; 197 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s wr, 392 op/s 2023-09-23T20:50:19.899 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:50:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:19 smithi121 ceph-mon[211526]: pgmap v531: 89 pgs: 89 active+clean; 197 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s wr, 392 op/s 2023-09-23T20:50:19.995 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:19 smithi123 ceph-mon[193903]: pgmap v531: 89 pgs: 89 active+clean; 197 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s wr, 392 op/s 2023-09-23T20:50:20.456 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 14596 14580 151.837 142.062 0.00377375 0.00657911 2023-09-23T20:50:21.456 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 16982 16966 151.448 149.125 0.00674981 0.00659683 2023-09-23T20:50:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:21 smithi121 ceph-mon[211526]: pgmap v532: 89 pgs: 89 active+clean; 676 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:50:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:21 smithi121 ceph-mon[219709]: pgmap v532: 89 pgs: 89 active+clean; 676 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:50:21.997 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:21 smithi123 ceph-mon[193903]: pgmap v532: 89 pgs: 89 active+clean; 676 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:50:22.090 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:50:22.204 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:22.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:22.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:22.317 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:22.398 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:22.456 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 18942 18926 147.829 122.5 0.00473989 0.00675639 2023-09-23T20:50:22.613 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:22.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:22.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:22.721 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:22.746 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:22.804 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:22.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:22.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:23.024 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:23.094 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:23.131 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:23.250 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:50:23.260 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:50:23.283 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:23.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:23.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:23.393 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:23.438 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:23.458 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 21142 21126 146.651 137.5 0.00713833 0.00681363 2023-09-23T20:50:23.528 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:23.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:23.635 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:23.690 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:23.712 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:23 smithi121 ceph-mon[211526]: pgmap v533: 89 pgs: 89 active+clean; 811 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T20:50:23.772 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:23.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:23.879 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:23.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:23 smithi121 ceph-mon[219709]: pgmap v533: 89 pgs: 89 active+clean; 811 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T20:50:23.934 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:23 smithi123 ceph-mon[193903]: pgmap v533: 89 pgs: 89 active+clean; 811 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T20:50:24.050 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:24.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:24.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:24.160 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:24.220 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:24.354 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:50:24.458 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 23057 23041 143.954 119.688 0.00533571 0.00694304 2023-09-23T20:50:24.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:24 smithi123 ceph-mon[193903]: from='client.45116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:25.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:50:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:50:24] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.43.0" 2023-09-23T20:50:25.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:24 smithi121 ceph-mon[219709]: from='client.45116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:25.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:24 smithi121 ceph-mon[211526]: from='client.45116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:25.458 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 24777 24761 140.64 107.5 0.0109116 0.00710524 2023-09-23T20:50:25.621 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:25.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:25.653 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":533,"stamp":"2023-09-23T20:50:22.646773+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":850073504,"num_objects":12986,"num_object_clones":0,"num_object_copies":38958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12986,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":13262,"num_write_kb":832013,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13178,"ondisk_log_size":13178,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4200472,"kb_used_data":1931288,"kb_used_omap":20,"kb_used_meta":2269099,"kb_avail":745760744,"statfs":{"total":767960285184,"available":763659001856,"internally_reserved":0,"allocated":1977638912,"data_stored":1972587157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2323558030},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5,0,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":849477632,"num_objects":12962,"num_object_clones":0,"num_object_copies":38886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12962,"num_write_kb":829568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002842"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":146,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746341+0000","last_change":"2023-09-23T20:46:05.100324+0000","last_active":"2023-09-23T20:50:12.746341+0000","last_peered":"2023-09-23T20:50:12.746341+0000","last_clean":"2023-09-23T20:50:12.746341+0000","last_became_active":"2023-09-23T20:46:05.099981+0000","last_became_peered":"2023-09-23T20:46:05.099981+0000","last_unstale":"2023-09-23T20:50:12.746341+0000","last_undegraded":"2023-09-23T20:50:12.746341+0000","last_fullsized":"2023-09-23T20:50:12.746341+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:19.876971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.099130+0000","last_change":"2023-09-23T20:48:07.517973+0000","last_active":"2023-09-23T20:50:12.099130+0000","last_peered":"2023-09-23T20:50:12.099130+0000","last_clean":"2023-09-23T20:50:12.099130+0000","last_became_active":"2023-09-23T20:48:07.517681+0000","last_became_peered":"2023-09-23T20:48:07.517681+0000","last_unstale":"2023-09-23T20:50:12.099130+0000","last_undegraded":"2023-09-23T20:50:12.099130+0000","last_fullsized":"2023-09-23T20:50:12.099130+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.789594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":92,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.217011+0000","last_change":"2023-09-23T20:48:07.541333+0000","last_active":"2023-09-23T20:50:13.217011+0000","last_peered":"2023-09-23T20:50:13.217011+0000","last_clean":"2023-09-23T20:50:13.217011+0000","last_became_active":"2023-09-23T20:48:07.541014+0000","last_became_peered":"2023-09-23T20:48:07.541014+0000","last_unstale":"2023-09-23T20:50:13.217011+0000","last_undegraded":"2023-09-23T20:50:13.217011+0000","last_fullsized":"2023-09-23T20:50:13.217011+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:48.678651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102157+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:50:12.102157+0000","last_peered":"2023-09-23T20:50:12.102157+0000","last_clean":"2023-09-23T20:50:12.102157+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:50:12.102157+0000","last_undegraded":"2023-09-23T20:50:12.102157+0000","last_fullsized":"2023-09-23T20:50:12.102157+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.098894+0000","last_change":"2023-09-23T20:48:07.540435+0000","last_active":"2023-09-23T20:50:12.098894+0000","last_peered":"2023-09-23T20:50:12.098894+0000","last_clean":"2023-09-23T20:50:12.098894+0000","last_became_active":"2023-09-23T20:48:07.540062+0000","last_became_peered":"2023-09-23T20:48:07.540062+0000","last_unstale":"2023-09-23T20:50:12.098894+0000","last_undegraded":"2023-09-23T20:50:12.098894+0000","last_fullsized":"2023-09-23T20:50:12.098894+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:19.449218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.647653+0000","last_change":"2023-09-23T20:45:00.009034+0000","last_active":"2023-09-23T20:50:13.647653+0000","last_peered":"2023-09-23T20:50:13.647653+0000","last_clean":"2023-09-23T20:50:13.647653+0000","last_became_active":"2023-09-23T20:45:00.008706+0000","last_became_peered":"2023-09-23T20:45:00.008706+0000","last_unstale":"2023-09-23T20:50:13.647653+0000","last_undegraded":"2023-09-23T20:50:13.647653+0000","last_fullsized":"2023-09-23T20:50:13.647653+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:29.204761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":115,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102002+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:50:12.102002+0000","last_peered":"2023-09-23T20:50:12.102002+0000","last_clean":"2023-09-23T20:50:12.102002+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:50:12.102002+0000","last_undegraded":"2023-09-23T20:50:12.102002+0000","last_fullsized":"2023-09-23T20:50:12.102002+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":106,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.098865+0000","last_change":"2023-09-23T20:48:07.533499+0000","last_active":"2023-09-23T20:50:12.098865+0000","last_peered":"2023-09-23T20:50:12.098865+0000","last_clean":"2023-09-23T20:50:12.098865+0000","last_became_active":"2023-09-23T20:48:07.520669+0000","last_became_peered":"2023-09-23T20:48:07.520669+0000","last_unstale":"2023-09-23T20:50:12.098865+0000","last_undegraded":"2023-09-23T20:50:12.098865+0000","last_fullsized":"2023-09-23T20:50:12.098865+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:55.109803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":128,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107409+0000","last_change":"2023-09-23T20:45:00.007125+0000","last_active":"2023-09-23T20:50:12.107409+0000","last_peered":"2023-09-23T20:50:12.107409+0000","last_clean":"2023-09-23T20:50:12.107409+0000","last_became_active":"2023-09-23T20:45:00.006403+0000","last_became_peered":"2023-09-23T20:45:00.006403+0000","last_unstale":"2023-09-23T20:50:12.107409+0000","last_undegraded":"2023-09-23T20:50:12.107409+0000","last_fullsized":"2023-09-23T20:50:12.107409+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:27.906490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.098877+0000","last_change":"2023-09-23T20:48:07.525099+0000","last_active":"2023-09-23T20:50:12.098877+0000","last_peered":"2023-09-23T20:50:12.098877+0000","last_clean":"2023-09-23T20:50:12.098877+0000","last_became_active":"2023-09-23T20:48:07.524868+0000","last_became_peered":"2023-09-23T20:48:07.524868+0000","last_unstale":"2023-09-23T20:50:12.098877+0000","last_undegraded":"2023-09-23T20:50:12.098877+0000","last_fullsized":"2023-09-23T20:50:12.098877+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:19.340586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":131,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.076057+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:50:14.076057+0000","last_peered":"2023-09-23T20:50:14.076057+0000","last_clean":"2023-09-23T20:50:14.076057+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:50:14.076057+0000","last_undegraded":"2023-09-23T20:50:14.076057+0000","last_fullsized":"2023-09-23T20:50:14.076057+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.216805+0000","last_change":"2023-09-23T20:46:05.099313+0000","last_active":"2023-09-23T20:50:13.216805+0000","last_peered":"2023-09-23T20:50:13.216805+0000","last_clean":"2023-09-23T20:50:13.216805+0000","last_became_active":"2023-09-23T20:46:05.099055+0000","last_became_peered":"2023-09-23T20:46:05.099055+0000","last_unstale":"2023-09-23T20:50:13.216805+0000","last_undegraded":"2023-09-23T20:50:13.216805+0000","last_fullsized":"2023-09-23T20:50:13.216805+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:51.820608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":133,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746605+0000","last_change":"2023-09-23T20:30:22.107118+0000","last_active":"2023-09-23T20:50:12.746605+0000","last_peered":"2023-09-23T20:50:12.746605+0000","last_clean":"2023-09-23T20:50:12.746605+0000","last_became_active":"2023-09-23T20:30:22.106839+0000","last_became_peered":"2023-09-23T20:30:22.106839+0000","last_unstale":"2023-09-23T20:50:12.746605+0000","last_undegraded":"2023-09-23T20:50:12.746605+0000","last_fullsized":"2023-09-23T20:50:12.746605+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:02.981333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746525+0000","last_change":"2023-09-23T20:45:00.007712+0000","last_active":"2023-09-23T20:50:12.746525+0000","last_peered":"2023-09-23T20:50:12.746525+0000","last_clean":"2023-09-23T20:50:12.746525+0000","last_became_active":"2023-09-23T20:45:00.007399+0000","last_became_peered":"2023-09-23T20:45:00.007399+0000","last_unstale":"2023-09-23T20:50:12.746525+0000","last_undegraded":"2023-09-23T20:50:12.746525+0000","last_fullsized":"2023-09-23T20:50:12.746525+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:48.842612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":130,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.075333+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:50:14.075333+0000","last_peered":"2023-09-23T20:50:14.075333+0000","last_clean":"2023-09-23T20:50:14.075333+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:50:14.075333+0000","last_undegraded":"2023-09-23T20:50:14.075333+0000","last_fullsized":"2023-09-23T20:50:14.075333+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":106,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.099222+0000","last_change":"2023-09-23T20:48:07.540014+0000","last_active":"2023-09-23T20:50:12.099222+0000","last_peered":"2023-09-23T20:50:12.099222+0000","last_clean":"2023-09-23T20:50:12.099222+0000","last_became_active":"2023-09-23T20:48:07.539722+0000","last_became_peered":"2023-09-23T20:48:07.539722+0000","last_unstale":"2023-09-23T20:50:12.099222+0000","last_undegraded":"2023-09-23T20:50:12.099222+0000","last_fullsized":"2023-09-23T20:50:12.099222+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.937232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":116,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.481839+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:50:12.481839+0000","last_peered":"2023-09-23T20:50:12.481839+0000","last_clean":"2023-09-23T20:50:12.481839+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:50:12.481839+0000","last_undegraded":"2023-09-23T20:50:12.481839+0000","last_fullsized":"2023-09-23T20:50:12.481839+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107389+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:50:12.107389+0000","last_peered":"2023-09-23T20:50:12.107389+0000","last_clean":"2023-09-23T20:50:12.107389+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:50:12.107389+0000","last_undegraded":"2023-09-23T20:50:12.107389+0000","last_fullsized":"2023-09-23T20:50:12.107389+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":111,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.075769+0000","last_change":"2023-09-23T20:30:22.110586+0000","last_active":"2023-09-23T20:50:14.075769+0000","last_peered":"2023-09-23T20:50:14.075769+0000","last_clean":"2023-09-23T20:50:14.075769+0000","last_became_active":"2023-09-23T20:30:22.110248+0000","last_became_peered":"2023-09-23T20:30:22.110248+0000","last_unstale":"2023-09-23T20:50:14.075769+0000","last_undegraded":"2023-09-23T20:50:14.075769+0000","last_fullsized":"2023-09-23T20:50:14.075769+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:29.867944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":116,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102310+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:50:12.102310+0000","last_peered":"2023-09-23T20:50:12.102310+0000","last_clean":"2023-09-23T20:50:12.102310+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:50:12.102310+0000","last_undegraded":"2023-09-23T20:50:12.102310+0000","last_fullsized":"2023-09-23T20:50:12.102310+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":136,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.647241+0000","last_change":"2023-09-23T20:40:53.862653+0000","last_active":"2023-09-23T20:50:13.647241+0000","last_peered":"2023-09-23T20:50:13.647241+0000","last_clean":"2023-09-23T20:50:13.647241+0000","last_became_active":"2023-09-23T20:40:53.862259+0000","last_became_peered":"2023-09-23T20:40:53.862259+0000","last_unstale":"2023-09-23T20:50:13.647241+0000","last_undegraded":"2023-09-23T20:50:13.647241+0000","last_fullsized":"2023-09-23T20:50:13.647241+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:28.626871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":106,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.099344+0000","last_change":"2023-09-23T20:48:07.557905+0000","last_active":"2023-09-23T20:50:12.099344+0000","last_peered":"2023-09-23T20:50:12.099344+0000","last_clean":"2023-09-23T20:50:12.099344+0000","last_became_active":"2023-09-23T20:48:07.557568+0000","last_became_peered":"2023-09-23T20:48:07.557568+0000","last_unstale":"2023-09-23T20:50:12.099344+0000","last_undegraded":"2023-09-23T20:50:12.099344+0000","last_fullsized":"2023-09-23T20:50:12.099344+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:14.229704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"168'1139","reported_seq":1152,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:22.442283+0000","last_change":"2023-09-23T20:50:08.077384+0000","last_active":"2023-09-23T20:50:22.442283+0000","last_peered":"2023-09-23T20:50:22.442283+0000","last_clean":"2023-09-23T20:50:22.442283+0000","last_became_active":"2023-09-23T20:50:08.077095+0000","last_became_peered":"2023-09-23T20:50:08.077095+0000","last_unstale":"2023-09-23T20:50:22.442283+0000","last_undegraded":"2023-09-23T20:50:22.442283+0000","last_fullsized":"2023-09-23T20:50:22.442283+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":1139,"log_dups_size":0,"ondisk_log_size":1139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:34.442024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74645504,"num_objects":1139,"num_object_clones":0,"num_object_copies":3417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1139,"num_write_kb":72896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":129,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.075397+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:50:14.075397+0000","last_peered":"2023-09-23T20:50:14.075397+0000","last_clean":"2023-09-23T20:50:14.075397+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:50:14.075397+0000","last_undegraded":"2023-09-23T20:50:14.075397+0000","last_fullsized":"2023-09-23T20:50:14.075397+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":110,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107087+0000","last_change":"2023-09-23T20:30:22.772475+0000","last_active":"2023-09-23T20:50:12.107087+0000","last_peered":"2023-09-23T20:50:12.107087+0000","last_clean":"2023-09-23T20:50:12.107087+0000","last_became_active":"2023-09-23T20:30:22.772159+0000","last_became_peered":"2023-09-23T20:30:22.772159+0000","last_unstale":"2023-09-23T20:50:12.107087+0000","last_undegraded":"2023-09-23T20:50:12.107087+0000","last_fullsized":"2023-09-23T20:50:12.107087+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:08.373446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"168'603","reported_seq":616,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.262478+0000","last_change":"2023-09-23T20:50:08.083329+0000","last_active":"2023-09-23T20:50:18.262478+0000","last_peered":"2023-09-23T20:50:18.262478+0000","last_clean":"2023-09-23T20:50:18.262478+0000","last_became_active":"2023-09-23T20:50:08.082984+0000","last_became_peered":"2023-09-23T20:50:08.082984+0000","last_unstale":"2023-09-23T20:50:18.262478+0000","last_undegraded":"2023-09-23T20:50:18.262478+0000","last_fullsized":"2023-09-23T20:50:18.262478+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":603,"log_dups_size":0,"ondisk_log_size":603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.393867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39518208,"num_objects":603,"num_object_clones":0,"num_object_copies":1809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":603,"num_write_kb":38592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":130,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.076005+0000","last_change":"2023-09-23T20:45:00.003948+0000","last_active":"2023-09-23T20:50:14.076005+0000","last_peered":"2023-09-23T20:50:14.076005+0000","last_clean":"2023-09-23T20:50:14.076005+0000","last_became_active":"2023-09-23T20:45:00.003639+0000","last_became_peered":"2023-09-23T20:45:00.003639+0000","last_unstale":"2023-09-23T20:50:14.076005+0000","last_undegraded":"2023-09-23T20:50:14.076005+0000","last_fullsized":"2023-09-23T20:50:14.076005+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:59.619696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746381+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:50:12.746381+0000","last_peered":"2023-09-23T20:50:12.746381+0000","last_clean":"2023-09-23T20:50:12.746381+0000","last_became_active":"2023-09-23T20:48:07.544029+0000","last_became_peered":"2023-09-23T20:48:07.544029+0000","last_unstale":"2023-09-23T20:50:12.746381+0000","last_undegraded":"2023-09-23T20:50:12.746381+0000","last_fullsized":"2023-09-23T20:50:12.746381+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:52.671448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"168'652","reported_seq":664,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.677101+0000","last_change":"2023-09-23T20:50:08.078801+0000","last_active":"2023-09-23T20:50:18.677101+0000","last_peered":"2023-09-23T20:50:18.677101+0000","last_clean":"2023-09-23T20:50:18.677101+0000","last_became_active":"2023-09-23T20:50:08.078386+0000","last_became_peered":"2023-09-23T20:50:08.078386+0000","last_unstale":"2023-09-23T20:50:18.677101+0000","last_undegraded":"2023-09-23T20:50:18.677101+0000","last_fullsized":"2023-09-23T20:50:18.677101+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":652,"log_dups_size":0,"ondisk_log_size":652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:47.134724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42729472,"num_objects":652,"num_object_clones":0,"num_object_copies":1956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":652,"num_write_kb":41728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":112,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.075948+0000","last_change":"2023-09-23T20:30:22.113582+0000","last_active":"2023-09-23T20:50:14.075948+0000","last_peered":"2023-09-23T20:50:14.075948+0000","last_clean":"2023-09-23T20:50:14.075948+0000","last_became_active":"2023-09-23T20:30:22.113291+0000","last_became_peered":"2023-09-23T20:30:22.113291+0000","last_unstale":"2023-09-23T20:50:14.075948+0000","last_undegraded":"2023-09-23T20:50:14.075948+0000","last_fullsized":"2023-09-23T20:50:14.075948+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:19.431152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":137,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.647185+0000","last_change":"2023-09-23T20:46:05.097625+0000","last_active":"2023-09-23T20:50:13.647185+0000","last_peered":"2023-09-23T20:50:13.647185+0000","last_clean":"2023-09-23T20:50:13.647185+0000","last_became_active":"2023-09-23T20:46:05.097296+0000","last_became_peered":"2023-09-23T20:46:05.097296+0000","last_unstale":"2023-09-23T20:50:13.647185+0000","last_undegraded":"2023-09-23T20:50:13.647185+0000","last_fullsized":"2023-09-23T20:50:13.647185+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:34.564149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"168'630","reported_seq":642,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.705152+0000","last_change":"2023-09-23T20:50:08.648424+0000","last_active":"2023-09-23T20:50:18.705152+0000","last_peered":"2023-09-23T20:50:18.705152+0000","last_clean":"2023-09-23T20:50:18.705152+0000","last_became_active":"2023-09-23T20:50:08.648031+0000","last_became_peered":"2023-09-23T20:50:08.648031+0000","last_unstale":"2023-09-23T20:50:18.705152+0000","last_undegraded":"2023-09-23T20:50:18.705152+0000","last_fullsized":"2023-09-23T20:50:18.705152+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":630,"log_dups_size":0,"ondisk_log_size":630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:22.291407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41287680,"num_objects":630,"num_object_clones":0,"num_object_copies":1890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":630,"num_write_kb":40320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":128,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107370+0000","last_change":"2023-09-23T20:48:07.522410+0000","last_active":"2023-09-23T20:50:12.107370+0000","last_peered":"2023-09-23T20:50:12.107370+0000","last_clean":"2023-09-23T20:50:12.107370+0000","last_became_active":"2023-09-23T20:48:07.522058+0000","last_became_peered":"2023-09-23T20:48:07.522058+0000","last_unstale":"2023-09-23T20:50:12.107370+0000","last_undegraded":"2023-09-23T20:50:12.107370+0000","last_fullsized":"2023-09-23T20:50:12.107370+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:24.844663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":117,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102011+0000","last_change":"2023-09-23T20:45:00.002738+0000","last_active":"2023-09-23T20:50:12.102011+0000","last_peered":"2023-09-23T20:50:12.102011+0000","last_clean":"2023-09-23T20:50:12.102011+0000","last_became_active":"2023-09-23T20:45:00.002403+0000","last_became_peered":"2023-09-23T20:45:00.002403+0000","last_unstale":"2023-09-23T20:50:12.102011+0000","last_undegraded":"2023-09-23T20:50:12.102011+0000","last_fullsized":"2023-09-23T20:50:12.102011+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:54.551205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"168'903","reported_seq":916,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:20.569350+0000","last_change":"2023-09-23T20:50:08.711285+0000","last_active":"2023-09-23T20:50:20.569350+0000","last_peered":"2023-09-23T20:50:20.569350+0000","last_clean":"2023-09-23T20:50:20.569350+0000","last_became_active":"2023-09-23T20:50:08.710920+0000","last_became_peered":"2023-09-23T20:50:08.710920+0000","last_unstale":"2023-09-23T20:50:20.569350+0000","last_undegraded":"2023-09-23T20:50:20.569350+0000","last_fullsized":"2023-09-23T20:50:20.569350+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":903,"log_dups_size":0,"ondisk_log_size":903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:37.014886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59179008,"num_objects":903,"num_object_clones":0,"num_object_copies":2709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":903,"num_write_kb":57792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":130,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:15.114207+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:50:15.114207+0000","last_peered":"2023-09-23T20:50:15.114207+0000","last_clean":"2023-09-23T20:50:15.114207+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:50:15.114207+0000","last_undegraded":"2023-09-23T20:50:15.114207+0000","last_fullsized":"2023-09-23T20:50:15.114207+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746572+0000","last_change":"2023-09-23T20:48:07.520873+0000","last_active":"2023-09-23T20:50:12.746572+0000","last_peered":"2023-09-23T20:50:12.746572+0000","last_clean":"2023-09-23T20:50:12.746572+0000","last_became_active":"2023-09-23T20:48:07.520517+0000","last_became_peered":"2023-09-23T20:48:07.520517+0000","last_unstale":"2023-09-23T20:50:12.746572+0000","last_undegraded":"2023-09-23T20:50:12.746572+0000","last_fullsized":"2023-09-23T20:50:12.746572+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:43.626732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"168'579","reported_seq":592,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.270437+0000","last_change":"2023-09-23T20:50:08.711551+0000","last_active":"2023-09-23T20:50:18.270437+0000","last_peered":"2023-09-23T20:50:18.270437+0000","last_clean":"2023-09-23T20:50:18.270437+0000","last_became_active":"2023-09-23T20:50:08.710920+0000","last_became_peered":"2023-09-23T20:50:08.710920+0000","last_unstale":"2023-09-23T20:50:18.270437+0000","last_undegraded":"2023-09-23T20:50:18.270437+0000","last_fullsized":"2023-09-23T20:50:18.270437+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":579,"log_dups_size":0,"ondisk_log_size":579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:22.013843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37945344,"num_objects":579,"num_object_clones":0,"num_object_copies":1737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":579,"num_write_kb":37056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.647373+0000","last_change":"2023-09-23T20:45:00.009049+0000","last_active":"2023-09-23T20:50:13.647373+0000","last_peered":"2023-09-23T20:50:13.647373+0000","last_clean":"2023-09-23T20:50:13.647373+0000","last_became_active":"2023-09-23T20:45:00.008700+0000","last_became_peered":"2023-09-23T20:45:00.008700+0000","last_unstale":"2023-09-23T20:50:13.647373+0000","last_undegraded":"2023-09-23T20:50:13.647373+0000","last_fullsized":"2023-09-23T20:50:13.647373+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:52.333134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746540+0000","last_change":"2023-09-23T20:46:05.106502+0000","last_active":"2023-09-23T20:50:12.746540+0000","last_peered":"2023-09-23T20:50:12.746540+0000","last_clean":"2023-09-23T20:50:12.746540+0000","last_became_active":"2023-09-23T20:46:05.106134+0000","last_became_peered":"2023-09-23T20:46:05.106134+0000","last_unstale":"2023-09-23T20:50:12.746540+0000","last_undegraded":"2023-09-23T20:50:12.746540+0000","last_fullsized":"2023-09-23T20:50:12.746540+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:39.257351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"168'624","reported_seq":636,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.660291+0000","last_change":"2023-09-23T20:50:08.077812+0000","last_active":"2023-09-23T20:50:18.660291+0000","last_peered":"2023-09-23T20:50:18.660291+0000","last_clean":"2023-09-23T20:50:18.660291+0000","last_became_active":"2023-09-23T20:50:08.077458+0000","last_became_peered":"2023-09-23T20:50:08.077458+0000","last_unstale":"2023-09-23T20:50:18.660291+0000","last_undegraded":"2023-09-23T20:50:18.660291+0000","last_fullsized":"2023-09-23T20:50:18.660291+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":624,"log_dups_size":0,"ondisk_log_size":624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40894464,"num_objects":624,"num_object_clones":0,"num_object_copies":1872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":624,"num_write_kb":39936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":128,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746213+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:50:12.746213+0000","last_peered":"2023-09-23T20:50:12.746213+0000","last_clean":"2023-09-23T20:50:12.746213+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:50:12.746213+0000","last_undegraded":"2023-09-23T20:50:12.746213+0000","last_fullsized":"2023-09-23T20:50:12.746213+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746734+0000","last_change":"2023-09-23T20:45:00.008536+0000","last_active":"2023-09-23T20:50:12.746734+0000","last_peered":"2023-09-23T20:50:12.746734+0000","last_clean":"2023-09-23T20:50:12.746734+0000","last_became_active":"2023-09-23T20:45:00.008220+0000","last_became_peered":"2023-09-23T20:45:00.008220+0000","last_unstale":"2023-09-23T20:50:12.746734+0000","last_undegraded":"2023-09-23T20:50:12.746734+0000","last_fullsized":"2023-09-23T20:50:12.746734+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:04.017031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"168'874","reported_seq":887,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:20.565982+0000","last_change":"2023-09-23T20:50:08.647934+0000","last_active":"2023-09-23T20:50:20.565982+0000","last_peered":"2023-09-23T20:50:20.565982+0000","last_clean":"2023-09-23T20:50:20.565982+0000","last_became_active":"2023-09-23T20:50:08.647621+0000","last_became_peered":"2023-09-23T20:50:08.647621+0000","last_unstale":"2023-09-23T20:50:20.565982+0000","last_undegraded":"2023-09-23T20:50:20.565982+0000","last_fullsized":"2023-09-23T20:50:20.565982+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":874,"log_dups_size":0,"ondisk_log_size":874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.046520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57278464,"num_objects":874,"num_object_clones":0,"num_object_copies":2622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":874,"num_write_kb":55936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":108,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:15.114705+0000","last_change":"2023-09-23T20:48:07.540484+0000","last_active":"2023-09-23T20:50:15.114705+0000","last_peered":"2023-09-23T20:50:15.114705+0000","last_clean":"2023-09-23T20:50:15.114705+0000","last_became_active":"2023-09-23T20:48:07.540219+0000","last_became_peered":"2023-09-23T20:48:07.540219+0000","last_unstale":"2023-09-23T20:50:15.114705+0000","last_undegraded":"2023-09-23T20:50:15.114705+0000","last_fullsized":"2023-09-23T20:50:15.114705+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:54.792744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":119,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102044+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:50:12.102044+0000","last_peered":"2023-09-23T20:50:12.102044+0000","last_clean":"2023-09-23T20:50:12.102044+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:50:12.102044+0000","last_undegraded":"2023-09-23T20:50:12.102044+0000","last_fullsized":"2023-09-23T20:50:12.102044+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"168'650","reported_seq":662,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.691287+0000","last_change":"2023-09-23T20:50:08.078863+0000","last_active":"2023-09-23T20:50:18.691287+0000","last_peered":"2023-09-23T20:50:18.691287+0000","last_clean":"2023-09-23T20:50:18.691287+0000","last_became_active":"2023-09-23T20:50:08.078560+0000","last_became_peered":"2023-09-23T20:50:08.078560+0000","last_unstale":"2023-09-23T20:50:18.691287+0000","last_undegraded":"2023-09-23T20:50:18.691287+0000","last_fullsized":"2023-09-23T20:50:18.691287+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":650,"log_dups_size":0,"ondisk_log_size":650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:41.488151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42598400,"num_objects":650,"num_object_clones":0,"num_object_copies":1950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":650,"num_write_kb":41600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":125,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.217261+0000","last_change":"2023-09-23T20:45:00.001830+0000","last_active":"2023-09-23T20:50:13.217261+0000","last_peered":"2023-09-23T20:50:13.217261+0000","last_clean":"2023-09-23T20:50:13.217261+0000","last_became_active":"2023-09-23T20:45:00.001532+0000","last_became_peered":"2023-09-23T20:45:00.001532+0000","last_unstale":"2023-09-23T20:50:13.217261+0000","last_undegraded":"2023-09-23T20:50:13.217261+0000","last_fullsized":"2023-09-23T20:50:13.217261+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:17.499694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107462+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:50:12.107462+0000","last_peered":"2023-09-23T20:50:12.107462+0000","last_clean":"2023-09-23T20:50:12.107462+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:50:12.107462+0000","last_undegraded":"2023-09-23T20:50:12.107462+0000","last_fullsized":"2023-09-23T20:50:12.107462+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.217192+0000","last_change":"2023-09-23T20:45:00.004588+0000","last_active":"2023-09-23T20:50:13.217192+0000","last_peered":"2023-09-23T20:50:13.217192+0000","last_clean":"2023-09-23T20:50:13.217192+0000","last_became_active":"2023-09-23T20:45:00.004185+0000","last_became_peered":"2023-09-23T20:45:00.004185+0000","last_unstale":"2023-09-23T20:50:13.217192+0000","last_undegraded":"2023-09-23T20:50:13.217192+0000","last_fullsized":"2023-09-23T20:50:13.217192+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:47.406920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"168'1213","reported_seq":1226,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:22.474778+0000","last_change":"2023-09-23T20:50:08.076706+0000","last_active":"2023-09-23T20:50:22.474778+0000","last_peered":"2023-09-23T20:50:22.474778+0000","last_clean":"2023-09-23T20:50:22.474778+0000","last_became_active":"2023-09-23T20:50:08.076360+0000","last_became_peered":"2023-09-23T20:50:08.076360+0000","last_unstale":"2023-09-23T20:50:22.474778+0000","last_undegraded":"2023-09-23T20:50:22.474778+0000","last_fullsized":"2023-09-23T20:50:22.474778+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":1213,"log_dups_size":0,"ondisk_log_size":1213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.596097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79495168,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1213,"num_write_kb":77632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":132,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107226+0000","last_change":"2023-09-23T20:45:00.007439+0000","last_active":"2023-09-23T20:50:12.107226+0000","last_peered":"2023-09-23T20:50:12.107226+0000","last_clean":"2023-09-23T20:50:12.107226+0000","last_became_active":"2023-09-23T20:45:00.007046+0000","last_became_peered":"2023-09-23T20:45:00.007046+0000","last_unstale":"2023-09-23T20:50:12.107226+0000","last_undegraded":"2023-09-23T20:50:12.107226+0000","last_fullsized":"2023-09-23T20:50:12.107226+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:42.363434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":128,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746416+0000","last_change":"2023-09-23T20:40:53.853109+0000","last_active":"2023-09-23T20:50:12.746416+0000","last_peered":"2023-09-23T20:50:12.746416+0000","last_clean":"2023-09-23T20:50:12.746416+0000","last_became_active":"2023-09-23T20:40:53.852788+0000","last_became_peered":"2023-09-23T20:40:53.852788+0000","last_unstale":"2023-09-23T20:50:12.746416+0000","last_undegraded":"2023-09-23T20:50:12.746416+0000","last_fullsized":"2023-09-23T20:50:12.746416+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:55.251424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":137,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107271+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:50:12.107271+0000","last_peered":"2023-09-23T20:50:12.107271+0000","last_clean":"2023-09-23T20:50:12.107271+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:50:12.107271+0000","last_undegraded":"2023-09-23T20:50:12.107271+0000","last_fullsized":"2023-09-23T20:50:12.107271+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"168'656","reported_seq":668,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.690369+0000","last_change":"2023-09-23T20:50:08.080121+0000","last_active":"2023-09-23T20:50:18.690369+0000","last_peered":"2023-09-23T20:50:18.690369+0000","last_clean":"2023-09-23T20:50:18.690369+0000","last_became_active":"2023-09-23T20:50:08.079818+0000","last_became_peered":"2023-09-23T20:50:08.079818+0000","last_unstale":"2023-09-23T20:50:18.690369+0000","last_undegraded":"2023-09-23T20:50:18.690369+0000","last_fullsized":"2023-09-23T20:50:18.690369+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":656,"log_dups_size":0,"ondisk_log_size":656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:03.610431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42991616,"num_objects":656,"num_object_clones":0,"num_object_copies":1968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":656,"num_write_kb":41984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":125,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.224183+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:50:13.224183+0000","last_peered":"2023-09-23T20:50:13.224183+0000","last_clean":"2023-09-23T20:50:13.224183+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:50:13.224183+0000","last_undegraded":"2023-09-23T20:50:13.224183+0000","last_fullsized":"2023-09-23T20:50:13.224183+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":92,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.216805+0000","last_change":"2023-09-23T20:46:05.075939+0000","last_active":"2023-09-23T20:50:13.216805+0000","last_peered":"2023-09-23T20:50:13.216805+0000","last_clean":"2023-09-23T20:50:13.216805+0000","last_became_active":"2023-09-23T20:46:05.075634+0000","last_became_peered":"2023-09-23T20:46:05.075634+0000","last_unstale":"2023-09-23T20:50:13.216805+0000","last_undegraded":"2023-09-23T20:50:13.216805+0000","last_fullsized":"2023-09-23T20:50:13.216805+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:46.867950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.217549+0000","last_change":"2023-09-23T20:45:00.002611+0000","last_active":"2023-09-23T20:50:13.217549+0000","last_peered":"2023-09-23T20:50:13.217549+0000","last_clean":"2023-09-23T20:50:13.217549+0000","last_became_active":"2023-09-23T20:45:00.002331+0000","last_became_peered":"2023-09-23T20:45:00.002331+0000","last_unstale":"2023-09-23T20:50:13.217549+0000","last_undegraded":"2023-09-23T20:50:13.217549+0000","last_fullsized":"2023-09-23T20:50:13.217549+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:19.968130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"168'944","reported_seq":955,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:20.571840+0000","last_change":"2023-09-23T20:50:08.078559+0000","last_active":"2023-09-23T20:50:20.571840+0000","last_peered":"2023-09-23T20:50:20.571840+0000","last_clean":"2023-09-23T20:50:20.571840+0000","last_became_active":"2023-09-23T20:50:08.078294+0000","last_became_peered":"2023-09-23T20:50:08.078294+0000","last_unstale":"2023-09-23T20:50:20.571840+0000","last_undegraded":"2023-09-23T20:50:20.571840+0000","last_fullsized":"2023-09-23T20:50:20.571840+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":944,"log_dups_size":0,"ondisk_log_size":944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:11.281483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61865984,"num_objects":944,"num_object_clones":0,"num_object_copies":2832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":944,"num_write_kb":60416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":132,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107203+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:50:12.107203+0000","last_peered":"2023-09-23T20:50:12.107203+0000","last_clean":"2023-09-23T20:50:12.107203+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:50:12.107203+0000","last_undegraded":"2023-09-23T20:50:12.107203+0000","last_fullsized":"2023-09-23T20:50:12.107203+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":508,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.075314+0000","last_change":"2023-09-23T20:40:53.853903+0000","last_active":"2023-09-23T20:50:14.075314+0000","last_peered":"2023-09-23T20:50:14.075314+0000","last_clean":"2023-09-23T20:50:14.075314+0000","last_became_active":"2023-09-23T20:40:53.852537+0000","last_became_peered":"2023-09-23T20:40:53.852537+0000","last_unstale":"2023-09-23T20:50:14.075314+0000","last_undegraded":"2023-09-23T20:50:14.075314+0000","last_fullsized":"2023-09-23T20:50:14.075314+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:49.454779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":84,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.099023+0000","last_change":"2023-09-23T20:48:07.521001+0000","last_active":"2023-09-23T20:50:12.099023+0000","last_peered":"2023-09-23T20:50:12.099023+0000","last_clean":"2023-09-23T20:50:12.099023+0000","last_became_active":"2023-09-23T20:48:07.520697+0000","last_became_peered":"2023-09-23T20:48:07.520697+0000","last_unstale":"2023-09-23T20:50:12.099023+0000","last_undegraded":"2023-09-23T20:50:12.099023+0000","last_fullsized":"2023-09-23T20:50:12.099023+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:21.120614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.647587+0000","last_change":"2023-09-23T20:30:22.772777+0000","last_active":"2023-09-23T20:50:13.647587+0000","last_peered":"2023-09-23T20:50:13.647587+0000","last_clean":"2023-09-23T20:50:13.647587+0000","last_became_active":"2023-09-23T20:30:22.772481+0000","last_became_peered":"2023-09-23T20:30:22.772481+0000","last_unstale":"2023-09-23T20:50:13.647587+0000","last_undegraded":"2023-09-23T20:50:13.647587+0000","last_fullsized":"2023-09-23T20:50:13.647587+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:53.724607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"168'1199","reported_seq":1212,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:22.482502+0000","last_change":"2023-09-23T20:50:08.711909+0000","last_active":"2023-09-23T20:50:22.482502+0000","last_peered":"2023-09-23T20:50:22.482502+0000","last_clean":"2023-09-23T20:50:22.482502+0000","last_became_active":"2023-09-23T20:50:08.711573+0000","last_became_peered":"2023-09-23T20:50:08.711573+0000","last_unstale":"2023-09-23T20:50:22.482502+0000","last_undegraded":"2023-09-23T20:50:22.482502+0000","last_fullsized":"2023-09-23T20:50:22.482502+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":1199,"log_dups_size":0,"ondisk_log_size":1199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:17.475943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78577664,"num_objects":1199,"num_object_clones":0,"num_object_copies":3597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1199,"num_write_kb":76736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":149,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.075435+0000","last_change":"2023-09-23T20:45:00.001911+0000","last_active":"2023-09-23T20:50:14.075435+0000","last_peered":"2023-09-23T20:50:14.075435+0000","last_clean":"2023-09-23T20:50:14.075435+0000","last_became_active":"2023-09-23T20:45:00.001531+0000","last_became_peered":"2023-09-23T20:45:00.001531+0000","last_unstale":"2023-09-23T20:50:14.075435+0000","last_undegraded":"2023-09-23T20:50:14.075435+0000","last_fullsized":"2023-09-23T20:50:14.075435+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:17.927950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":116,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.101601+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:50:12.101601+0000","last_peered":"2023-09-23T20:50:12.101601+0000","last_clean":"2023-09-23T20:50:12.101601+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:50:12.101601+0000","last_undegraded":"2023-09-23T20:50:12.101601+0000","last_fullsized":"2023-09-23T20:50:12.101601+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102233+0000","last_change":"2023-09-23T20:46:05.077316+0000","last_active":"2023-09-23T20:50:12.102233+0000","last_peered":"2023-09-23T20:50:12.102233+0000","last_clean":"2023-09-23T20:50:12.102233+0000","last_became_active":"2023-09-23T20:46:05.077044+0000","last_became_peered":"2023-09-23T20:46:05.077044+0000","last_unstale":"2023-09-23T20:50:12.102233+0000","last_undegraded":"2023-09-23T20:50:12.102233+0000","last_fullsized":"2023-09-23T20:50:12.102233+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:56.517854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"168'660","reported_seq":672,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.674513+0000","last_change":"2023-09-23T20:50:08.083097+0000","last_active":"2023-09-23T20:50:18.674513+0000","last_peered":"2023-09-23T20:50:18.674513+0000","last_clean":"2023-09-23T20:50:18.674513+0000","last_became_active":"2023-09-23T20:50:08.082761+0000","last_became_peered":"2023-09-23T20:50:08.082761+0000","last_unstale":"2023-09-23T20:50:18.674513+0000","last_undegraded":"2023-09-23T20:50:18.674513+0000","last_fullsized":"2023-09-23T20:50:18.674513+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":660,"log_dups_size":0,"ondisk_log_size":660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:29.125314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43253760,"num_objects":660,"num_object_clones":0,"num_object_copies":1980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":660,"num_write_kb":42240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":132,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:14.075348+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:50:14.075348+0000","last_peered":"2023-09-23T20:50:14.075348+0000","last_clean":"2023-09-23T20:50:14.075348+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:50:14.075348+0000","last_undegraded":"2023-09-23T20:50:14.075348+0000","last_fullsized":"2023-09-23T20:50:14.075348+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":92,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.216855+0000","last_change":"2023-09-23T20:46:05.079434+0000","last_active":"2023-09-23T20:50:13.216855+0000","last_peered":"2023-09-23T20:50:13.216855+0000","last_clean":"2023-09-23T20:50:13.216855+0000","last_became_active":"2023-09-23T20:46:05.079111+0000","last_became_peered":"2023-09-23T20:46:05.079111+0000","last_unstale":"2023-09-23T20:50:13.216855+0000","last_undegraded":"2023-09-23T20:50:13.216855+0000","last_fullsized":"2023-09-23T20:50:13.216855+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.869457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":102,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.101894+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:50:12.101894+0000","last_peered":"2023-09-23T20:50:12.101894+0000","last_clean":"2023-09-23T20:50:12.101894+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:50:12.101894+0000","last_undegraded":"2023-09-23T20:50:12.101894+0000","last_fullsized":"2023-09-23T20:50:12.101894+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"168'677","reported_seq":689,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:18.667452+0000","last_change":"2023-09-23T20:50:08.078428+0000","last_active":"2023-09-23T20:50:18.667452+0000","last_peered":"2023-09-23T20:50:18.667452+0000","last_clean":"2023-09-23T20:50:18.667452+0000","last_became_active":"2023-09-23T20:50:08.078116+0000","last_became_peered":"2023-09-23T20:50:08.078116+0000","last_unstale":"2023-09-23T20:50:18.667452+0000","last_undegraded":"2023-09-23T20:50:18.667452+0000","last_fullsized":"2023-09-23T20:50:18.667452+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":677,"log_dups_size":0,"ondisk_log_size":677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:06.919475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44367872,"num_objects":677,"num_object_clones":0,"num_object_copies":2031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":677,"num_write_kb":43328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":130,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:15.114344+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:50:15.114344+0000","last_peered":"2023-09-23T20:50:15.114344+0000","last_clean":"2023-09-23T20:50:15.114344+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:50:15.114344+0000","last_undegraded":"2023-09-23T20:50:15.114344+0000","last_fullsized":"2023-09-23T20:50:15.114344+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":119,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.217490+0000","last_change":"2023-09-23T20:45:00.005180+0000","last_active":"2023-09-23T20:50:13.217490+0000","last_peered":"2023-09-23T20:50:13.217490+0000","last_clean":"2023-09-23T20:50:13.217490+0000","last_became_active":"2023-09-23T20:45:00.004791+0000","last_became_peered":"2023-09-23T20:45:00.004791+0000","last_unstale":"2023-09-23T20:50:13.217490+0000","last_undegraded":"2023-09-23T20:50:13.217490+0000","last_fullsized":"2023-09-23T20:50:13.217490+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:37.690560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.098765+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:50:12.098765+0000","last_peered":"2023-09-23T20:50:12.098765+0000","last_clean":"2023-09-23T20:50:12.098765+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:50:12.098765+0000","last_undegraded":"2023-09-23T20:50:12.098765+0000","last_fullsized":"2023-09-23T20:50:12.098765+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"168'959","reported_seq":971,"reported_epoch":168,"state":"active+clean","last_fresh":"2023-09-23T20:50:20.568695+0000","last_change":"2023-09-23T20:50:08.083439+0000","last_active":"2023-09-23T20:50:20.568695+0000","last_peered":"2023-09-23T20:50:20.568695+0000","last_clean":"2023-09-23T20:50:20.568695+0000","last_became_active":"2023-09-23T20:50:08.083061+0000","last_became_peered":"2023-09-23T20:50:08.083061+0000","last_unstale":"2023-09-23T20:50:20.568695+0000","last_undegraded":"2023-09-23T20:50:20.568695+0000","last_fullsized":"2023-09-23T20:50:20.568695+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":959,"log_dups_size":0,"ondisk_log_size":959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:41.569707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62849024,"num_objects":959,"num_object_clones":0,"num_object_copies":2877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":959,"num_write_kb":61376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":132,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:13.647173+0000","last_change":"2023-09-23T20:46:05.073849+0000","last_active":"2023-09-23T20:50:13.647173+0000","last_peered":"2023-09-23T20:50:13.647173+0000","last_clean":"2023-09-23T20:50:13.647173+0000","last_became_active":"2023-09-23T20:46:05.073536+0000","last_became_peered":"2023-09-23T20:46:05.073536+0000","last_unstale":"2023-09-23T20:50:13.647173+0000","last_undegraded":"2023-09-23T20:50:13.647173+0000","last_fullsized":"2023-09-23T20:50:13.647173+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.086216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":120,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102086+0000","last_change":"2023-09-23T20:45:00.005077+0000","last_active":"2023-09-23T20:50:12.102086+0000","last_peered":"2023-09-23T20:50:12.102086+0000","last_clean":"2023-09-23T20:50:12.102086+0000","last_became_active":"2023-09-23T20:45:00.004383+0000","last_became_peered":"2023-09-23T20:45:00.004383+0000","last_unstale":"2023-09-23T20:50:12.102086+0000","last_undegraded":"2023-09-23T20:50:12.102086+0000","last_fullsized":"2023-09-23T20:50:12.102086+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:45.178382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":110,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746273+0000","last_change":"2023-09-23T20:30:22.131210+0000","last_active":"2023-09-23T20:50:12.746273+0000","last_peered":"2023-09-23T20:50:12.746273+0000","last_clean":"2023-09-23T20:50:12.746273+0000","last_became_active":"2023-09-23T20:30:22.130879+0000","last_became_peered":"2023-09-23T20:30:22.130879+0000","last_unstale":"2023-09-23T20:50:12.746273+0000","last_undegraded":"2023-09-23T20:50:12.746273+0000","last_fullsized":"2023-09-23T20:50:12.746273+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:10.733014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":104,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.102082+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:50:12.102082+0000","last_peered":"2023-09-23T20:50:12.102082+0000","last_clean":"2023-09-23T20:50:12.102082+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:50:12.102082+0000","last_undegraded":"2023-09-23T20:50:12.102082+0000","last_fullsized":"2023-09-23T20:50:12.102082+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":141,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.101441+0000","last_change":"2023-09-23T20:48:07.539824+0000","last_active":"2023-09-23T20:50:12.101441+0000","last_peered":"2023-09-23T20:50:12.101441+0000","last_clean":"2023-09-23T20:50:12.101441+0000","last_became_active":"2023-09-23T20:48:07.539530+0000","last_became_peered":"2023-09-23T20:48:07.539530+0000","last_unstale":"2023-09-23T20:50:12.101441+0000","last_undegraded":"2023-09-23T20:50:12.101441+0000","last_fullsized":"2023-09-23T20:50:12.101441+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:21.997036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":130,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107383+0000","last_change":"2023-09-23T20:45:00.008268+0000","last_active":"2023-09-23T20:50:12.107383+0000","last_peered":"2023-09-23T20:50:12.107383+0000","last_clean":"2023-09-23T20:50:12.107383+0000","last_became_active":"2023-09-23T20:45:00.006393+0000","last_became_peered":"2023-09-23T20:45:00.006393+0000","last_unstale":"2023-09-23T20:50:12.107383+0000","last_undegraded":"2023-09-23T20:50:12.107383+0000","last_fullsized":"2023-09-23T20:50:12.107383+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:41.255405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746465+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:50:12.746465+0000","last_peered":"2023-09-23T20:50:12.746465+0000","last_clean":"2023-09-23T20:50:12.746465+0000","last_became_active":"2023-09-23T20:48:07.544066+0000","last_became_peered":"2023-09-23T20:48:07.544066+0000","last_unstale":"2023-09-23T20:50:12.746465+0000","last_undegraded":"2023-09-23T20:50:12.746465+0000","last_fullsized":"2023-09-23T20:50:12.746465+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:28.905486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":113,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.107495+0000","last_change":"2023-09-23T20:30:22.104198+0000","last_active":"2023-09-23T20:50:12.107495+0000","last_peered":"2023-09-23T20:50:12.107495+0000","last_clean":"2023-09-23T20:50:12.107495+0000","last_became_active":"2023-09-23T20:30:22.103970+0000","last_became_peered":"2023-09-23T20:30:22.103970+0000","last_unstale":"2023-09-23T20:50:12.107495+0000","last_undegraded":"2023-09-23T20:50:12.107495+0000","last_fullsized":"2023-09-23T20:50:12.107495+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:32.820350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":144,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746434+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:50:12.746434+0000","last_peered":"2023-09-23T20:50:12.746434+0000","last_clean":"2023-09-23T20:50:12.746434+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:50:12.746434+0000","last_undegraded":"2023-09-23T20:50:12.746434+0000","last_fullsized":"2023-09-23T20:50:12.746434+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":124,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.099211+0000","last_change":"2023-09-23T20:48:07.521311+0000","last_active":"2023-09-23T20:50:12.099211+0000","last_peered":"2023-09-23T20:50:12.099211+0000","last_clean":"2023-09-23T20:50:12.099211+0000","last_became_active":"2023-09-23T20:48:07.521030+0000","last_became_peered":"2023-09-23T20:48:07.521030+0000","last_unstale":"2023-09-23T20:50:12.099211+0000","last_undegraded":"2023-09-23T20:50:12.099211+0000","last_fullsized":"2023-09-23T20:50:12.099211+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.849053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":138,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.098972+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:50:12.098972+0000","last_peered":"2023-09-23T20:50:12.098972+0000","last_clean":"2023-09-23T20:50:12.098972+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:50:12.098972+0000","last_undegraded":"2023-09-23T20:50:12.098972+0000","last_fullsized":"2023-09-23T20:50:12.098972+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":120,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.101886+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:50:12.101886+0000","last_peered":"2023-09-23T20:50:12.101886+0000","last_clean":"2023-09-23T20:50:12.101886+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:50:12.101886+0000","last_undegraded":"2023-09-23T20:50:12.101886+0000","last_fullsized":"2023-09-23T20:50:12.101886+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":127,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:50:12.746214+0000","last_change":"2023-09-23T20:45:00.004734+0000","last_active":"2023-09-23T20:50:12.746214+0000","last_peered":"2023-09-23T20:50:12.746214+0000","last_clean":"2023-09-23T20:50:12.746214+0000","last_became_active":"2023-09-23T20:45:00.004413+0000","last_became_peered":"2023-09-23T20:45:00.004413+0000","last_unstale":"2023-09-23T20:50:12.746214+0000","last_undegraded":"2023-09-23T20:50:12.746214+0000","last_fullsized":"2023-09-23T20:50:12.746214+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:57.454649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":849477632,"num_objects":12962,"num_object_clones":0,"num_object_copies":38886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12962,"num_write_kb":829568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2272854016,"data_stored":2272854016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12962,"ondisk_log_size":12962,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201539,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":614964,"kb_used_data":313708,"kb_used_omap":0,"kb_used_meta":301248,"kb_avail":93130188,"statfs":{"total":95995035648,"available":95365312512,"internally_reserved":0,"allocated":321236992,"data_stored":320663242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308477952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":3.7429999999999999,"15min":2.427},"min":{"1min":0.311,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.80700000000000005,"5min":225.56999999999999,"15min":225.56999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":3.9630000000000001,"15min":2.7639999999999998},"min":{"1min":0.40999999999999998,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":0.71199999999999997,"5min":225.61099999999999,"15min":225.61099999999999},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":3.1379999999999999,"15min":1.8240000000000001},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.73699999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":6.1840000000000002,"15min":2.8490000000000002},"min":{"1min":0.39800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78100000000000003,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":3.6480000000000001,"15min":3.3079999999999998},"min":{"1min":0.35199999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.748,"5min":142.321,"15min":142.321},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":2.9540000000000002,"15min":2.5419999999999998},"min":{"1min":0.42399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.73299999999999998,"5min":142.29300000000001,"15min":142.29300000000001},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":8.2699999999999996,"15min":9.5619999999999994},"min":{"1min":0.36899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76500000000000001,"5min":138.345,"15min":138.345},"last":31.631},{"interface":"front","average":{"1min":0.51900000000000002,"5min":7.6420000000000003,"15min":8.8290000000000006},"min":{"1min":0.36399999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72399999999999998,"5min":133.76400000000001,"15min":133.76400000000001},"last":31.617000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.361,"15min":1.6379999999999999},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.216},"max":{"1min":3.4319999999999999,"5min":9.1400000000000006,"15min":103.541},"last":0.317},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.2949999999999999,"15min":1.75},"min":{"1min":0.30199999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.843,"5min":8.4649999999999999,"15min":107.514},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.498,"15min":1.8580000000000001},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":6.6260000000000003,"5min":9.4440000000000008,"15min":67.180999999999997},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.181,"5min":1.7250000000000001,"15min":1.77},"min":{"1min":0.39200000000000002,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":7.9109999999999996,"5min":13.156000000000001,"15min":67.135000000000005},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:49:39 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.679,"15min":1.5780000000000001},"min":{"1min":0.41999999999999998,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":5.0220000000000002,"5min":10.509,"15min":16.032},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.1830000000000001,"15min":1.3280000000000001},"min":{"1min":0.41399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9269999999999996,"5min":8.4350000000000005,"15min":11.672000000000001},"last":0.34000000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463176,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387364,"kb_used_data":124252,"kb_used_omap":0,"kb_used_meta":263104,"kb_avail":93357788,"statfs":{"total":95995035648,"available":95598374912,"internally_reserved":0,"allocated":127234048,"data_stored":126676329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":269418496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":3.2400000000000002,"15min":2.3479999999999999},"min":{"1min":0.40200000000000002,"5min":0.22900000000000001,"15min":0.20399999999999999},"max":{"1min":0.72399999999999998,"5min":92.164000000000001,"15min":92.164000000000001},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.839,"15min":2.327},"min":{"1min":0.376,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.71399999999999997,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.48399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":2.6840000000000002,"15min":1.9990000000000001},"min":{"1min":0.40899999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.77000000000000002,"5min":96.623999999999995,"15min":96.623999999999995},"last":1.806},{"interface":"front","average":{"1min":0.54500000000000004,"5min":2.6469999999999998,"15min":1.7849999999999999},"min":{"1min":0.36399999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.78500000000000003,"5min":96.254000000000005,"15min":96.254000000000005},"last":0.29299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.9790000000000001,"15min":2.1850000000000001},"min":{"1min":0.34100000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.82699999999999996,"5min":8.9849999999999994,"15min":8.9849999999999994},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.214,"15min":2.4700000000000002},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.68799999999999994,"5min":10.59,"15min":10.59},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":5.008,"15min":5.7519999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.73899999999999999,"5min":79.668000000000006,"15min":79.668000000000006},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":4.9210000000000003,"15min":5.6390000000000002},"min":{"1min":0.495,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73099999999999998,"5min":79.700000000000003,"15min":79.700000000000003},"last":0.371}]},{"osd":4,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.175,"15min":1.181},"min":{"1min":0.35199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.71799999999999997,"5min":9.0709999999999997,"15min":9.0709999999999997},"last":1.252},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.651,"15min":1.429},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.27000000000000002},"max":{"1min":0.86099999999999999,"5min":9.8650000000000002,"15min":10.058999999999999},"last":0.65900000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.429,"15min":1.393},"min":{"1min":0.40300000000000002,"5min":0.28899999999999998,"15min":0.26000000000000001},"max":{"1min":0.77000000000000002,"5min":9.0340000000000007,"15min":11.484999999999999},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.6299999999999999,"15min":1.494},"min":{"1min":0.379,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":0.80200000000000005,"5min":9.8949999999999996,"15min":17.321000000000002},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:49:29 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":1.5589999999999999,"15min":1.4259999999999999},"min":{"1min":0.40200000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2590000000000003,"5min":9.3249999999999993,"15min":19.451000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.605,"15min":1.6299999999999999},"min":{"1min":0.40000000000000002,"5min":0.36099999999999999,"15min":0.32100000000000001},"max":{"1min":9.7959999999999994,"5min":9.7959999999999994,"15min":16.295999999999999},"last":0.46999999999999997}]}]},{"osd":1,"up_from":114,"seq":489626271810,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":482324,"kb_used_data":255500,"kb_used_omap":4,"kb_used_meta":226811,"kb_avail":93262828,"statfs":{"total":95995035648,"available":95501135872,"internally_reserved":0,"allocated":261632000,"data_stored":260914349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":232255374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":2.1110000000000002,"15min":2.008},"min":{"1min":0.216,"5min":0.191,"15min":0.191},"max":{"1min":1.6339999999999999,"5min":48.350999999999999,"15min":48.350999999999999},"last":0.373},{"interface":"front","average":{"1min":0.36599999999999999,"5min":2.4260000000000002,"15min":1.958},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":1.6679999999999999,"5min":52.018000000000001,"15min":52.018000000000001},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.6970000000000001,"15min":2.0670000000000002},"min":{"1min":0.24199999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":3.8599999999999999,"5min":11.593999999999999,"15min":11.593999999999999},"last":3.8599999999999999},{"interface":"front","average":{"1min":0.434,"5min":2.1040000000000001,"15min":2.6850000000000001},"min":{"1min":0.22500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.6040000000000001,"5min":14.153,"15min":14.153},"last":0.99199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":2.2799999999999998,"15min":2.5800000000000001},"min":{"1min":0.28299999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.0470000000000002,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.505,"5min":1.7330000000000001,"15min":1.9370000000000001},"min":{"1min":0.32000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.0059999999999998,"5min":8.5630000000000006,"15min":8.5630000000000006},"last":1.0680000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.5389999999999999,"15min":1.3069999999999999},"min":{"1min":0.39200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.85099999999999998,"5min":10.439,"15min":10.439},"last":0.96499999999999997},{"interface":"front","average":{"1min":0.57599999999999996,"5min":1.333,"15min":1.1539999999999999},"min":{"1min":0.40600000000000003,"5min":0.307,"15min":0.307},"max":{"1min":0.745,"5min":10.34,"15min":10.34},"last":3.6040000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":2.2970000000000002,"15min":1.7869999999999999},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.89100000000000001,"5min":10.177,"15min":10.177},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":1.5229999999999999,"15min":1.2310000000000001},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.74099999999999999,"5min":6.8680000000000003,"15min":6.8680000000000003},"last":3.4249999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.7989999999999999,"15min":1.635},"min":{"1min":0.41199999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.78900000000000003,"5min":10.25,"15min":10.25},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.4510000000000001,"15min":1.272},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69099999999999995,"5min":6.9029999999999996,"15min":6.9029999999999996},"last":3.4710000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.726,"15min":1.431},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.755,"5min":7.4480000000000004,"15min":7.4480000000000004},"last":3.5739999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.8069999999999999,"15min":1.827},"min":{"1min":0.39200000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.82199999999999995,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":1.0409999999999999}]}]},{"osd":0,"up_from":92,"seq":395136991348,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":695280,"kb_used_data":276840,"kb_used_omap":5,"kb_used_meta":418426,"kb_avail":93049872,"statfs":{"total":95995035648,"available":95283068928,"internally_reserved":0,"allocated":283484160,"data_stored":282737439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":428468858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":2.9420000000000002,"15min":2.3399999999999999},"min":{"1min":0.23200000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.5150000000000001,"5min":64.140000000000001,"15min":64.140000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.496,"5min":3.0459999999999998,"15min":2.4199999999999999},"min":{"1min":0.23599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":3.661,"5min":59.570999999999998,"15min":59.570999999999998},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":2.0379999999999998,"15min":1.9630000000000001},"min":{"1min":0.251,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.407,"5min":49.652000000000001,"15min":49.652000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.53000000000000003,"5min":3.2330000000000001,"15min":4.0720000000000001},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4399999999999999,"5min":49.595999999999997,"15min":49.595999999999997},"last":0.76100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":2.8029999999999999,"15min":3.2069999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.47199999999999998,"5min":41.810000000000002,"15min":41.810000000000002},"last":94.947999999999993},{"interface":"front","average":{"1min":0.58699999999999997,"5min":3.0379999999999998,"15min":3.4460000000000002},"min":{"1min":0.32600000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.9380000000000002,"5min":41.880000000000003,"15min":41.880000000000003},"last":94.382999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.621,"15min":1.7190000000000001},"min":{"1min":0.34499999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.8650000000000002,"5min":12.973000000000001,"15min":12.973000000000001},"last":0.623},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.3919999999999999,"15min":2.0209999999999999},"min":{"1min":0.39900000000000002,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":0.79700000000000004,"5min":8.234,"15min":12.148},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":2.2160000000000002,"15min":2.1110000000000002},"min":{"1min":0.41799999999999998,"5min":0.30199999999999999,"15min":0.28799999999999998},"max":{"1min":5.9059999999999997,"5min":14.051,"15min":14.051},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.637,"15min":2.1709999999999998},"min":{"1min":0.41799999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83899999999999997,"5min":12.332000000000001,"15min":12.332000000000001},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.8149999999999999,"15min":2.2280000000000002},"min":{"1min":0.42999999999999999,"5min":0.307,"15min":0.307},"max":{"1min":0.79000000000000004,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.73299999999999998,"5min":2.0030000000000001,"15min":1.6679999999999999},"min":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.5409999999999999,"5min":20.399999999999999,"15min":20.399999999999999},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:50:10 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.53,"15min":2.3100000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.35099999999999998},"max":{"1min":0.81799999999999995,"5min":8.3330000000000002,"15min":12.23},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.75800000000000001,"5min":1.9910000000000001,"15min":1.742},"min":{"1min":0.497,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":3.4910000000000001,"5min":20.422999999999998,"15min":20.422999999999998},"last":0.72199999999999998}]}]},{"osd":2,"up_from":129,"seq":554050781237,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357028,"kb_used_data":100060,"kb_used_omap":7,"kb_used_meta":256952,"kb_avail":93388124,"statfs":{"total":95995035648,"available":95629438976,"internally_reserved":0,"allocated":102461440,"data_stored":101784605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":263118929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:05 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":1.9279999999999999,"15min":2.8050000000000002},"min":{"1min":0.218,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.59599999999999997,"5min":14.327,"15min":14.327},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.35499999999999998,"5min":1.798,"15min":2.5840000000000001},"min":{"1min":0.27300000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.53700000000000003,"5min":14.757999999999999,"15min":14.757999999999999},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:50:05 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":2.0459999999999998,"15min":2.7090000000000001},"min":{"1min":0.21299999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.70899999999999996,"5min":13.023,"15min":13.023},"last":0.312},{"interface":"front","average":{"1min":0.35199999999999998,"5min":1.7829999999999999,"15min":2.8039999999999998},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.58199999999999996,"5min":8.4139999999999997,"15min":8.4139999999999997},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:50:09 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":1.4079999999999999,"15min":1.5840000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.55100000000000005,"5min":11.959,"15min":11.959},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.88700000000000001,"15min":0.97299999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.55400000000000005,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:50:05 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.5389999999999999,"15min":1.8440000000000001},"min":{"1min":0.40500000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.751,"5min":7.8440000000000003,"15min":7.8440000000000003},"last":1.3939999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.891,"15min":2.5590000000000002},"min":{"1min":0.45700000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.73999999999999999,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":1.4099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:50:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":2.0150000000000001,"15min":2.871},"min":{"1min":0.374,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.78200000000000003,"5min":13.164999999999999,"15min":13.164999999999999},"last":1.282},{"interface":"front","average":{"1min":0.55800000000000005,"5min":2.0059999999999998,"15min":2.6949999999999998},"min":{"1min":0.40300000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76800000000000002,"5min":8.2070000000000007,"15min":8.2070000000000007},"last":1.819}]},{"osd":6,"last update":"Sat Sep 23 20:50:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":2.1190000000000002,"15min":3.089},"min":{"1min":0.38200000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73199999999999998,"5min":7.476,"15min":7.476},"last":0.751},{"interface":"front","average":{"1min":0.60299999999999998,"5min":2.2829999999999999,"15min":2.9049999999999998},"min":{"1min":0.42199999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76200000000000001,"5min":12.595000000000001,"15min":12.595000000000001},"last":0.69999999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:50:05 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.8129999999999999,"15min":2.395},"min":{"1min":0.44600000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77100000000000002,"5min":6.4989999999999997,"15min":6.4989999999999997},"last":1.581},{"interface":"front","average":{"1min":0.57799999999999996,"5min":2.0289999999999999,"15min":2.718},"min":{"1min":0.42599999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.75,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":0.442}]}]},{"osd":3,"up_from":148,"seq":635655159837,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":539272,"kb_used_data":405632,"kb_used_omap":2,"kb_used_meta":133629,"kb_avail":93205880,"statfs":{"total":95995035648,"available":95442821120,"internally_reserved":0,"allocated":415367168,"data_stored":414715851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":136836149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":11.531000000000001,"15min":13.398},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":0.51900000000000002,"5min":253.68100000000001,"15min":253.68100000000001},"last":2.3370000000000002},{"interface":"front","average":{"1min":0.33900000000000002,"5min":10.634,"15min":12.35},"min":{"1min":0.22500000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.52800000000000002,"5min":253.46199999999999,"15min":253.46199999999999},"last":2.5249999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":8.407,"15min":9.7469999999999999},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":0.66700000000000004,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.36799999999999999,"5min":7.5999999999999996,"15min":8.8049999999999997},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.54200000000000004,"5min":172.36600000000001,"15min":172.36600000000001},"last":2.129}]},{"osd":2,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.90500000000000003,"15min":0.99299999999999999},"min":{"1min":0.27700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.54500000000000004,"5min":5.4619999999999997,"15min":5.4619999999999997},"last":2.2400000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.82999999999999996,"15min":0.90400000000000003},"min":{"1min":0.26600000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56100000000000005,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":3.8690000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.46,"15min":1.6080000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79200000000000004,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.881,"15min":2.105},"min":{"1min":0.40699999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76200000000000001,"5min":16.408999999999999,"15min":16.408999999999999},"last":2.2080000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.0529999999999999,"15min":1.1339999999999999},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71799999999999997,"5min":3.5750000000000002,"15min":3.5750000000000002},"last":2.601},{"interface":"front","average":{"1min":0.57299999999999995,"5min":1.681,"15min":1.865},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79400000000000004,"5min":9.7780000000000005,"15min":9.7780000000000005},"last":0.59399999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.694,"15min":1.879},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79500000000000004,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":0.625},{"interface":"front","average":{"1min":0.57399999999999995,"5min":2.2280000000000002,"15min":2.504},"min":{"1min":0.42999999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.748,"5min":16.451000000000001,"15min":16.451000000000001},"last":2.1739999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":1.363,"15min":1.4970000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.77800000000000002,"5min":4.9450000000000003,"15min":4.9450000000000003},"last":2.569},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.9970000000000001,"15min":2.2330000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81499999999999995,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.67700000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986449,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":425712,"kb_used_data":109480,"kb_used_omap":0,"kb_used_meta":316224,"kb_avail":93319440,"statfs":{"total":95995035648,"available":95559106560,"internally_reserved":0,"allocated":112107520,"data_stored":111537575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":323813376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.2469999999999999,"15min":1.7789999999999999},"min":{"1min":0.34599999999999997,"5min":0.216,"15min":0.214},"max":{"1min":0.73499999999999999,"5min":12.823,"15min":12.823},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.679,"15min":1.3029999999999999},"min":{"1min":0.30599999999999999,"5min":0.23799999999999999,"15min":0.22600000000000001},"max":{"1min":0.61799999999999999,"5min":8.6080000000000005,"15min":13.211},"last":0.91400000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":6.468,"15min":3.165},"min":{"1min":0.34300000000000003,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.67800000000000005,"5min":286.31099999999998,"15min":286.31099999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.49399999999999999,"5min":6.3440000000000003,"15min":3.855},"min":{"1min":0.39600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.66300000000000003,"5min":285.97000000000003,"15min":285.97000000000003},"last":1.079}]},{"osd":2,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.3120000000000001,"15min":1.319},"min":{"1min":0.36099999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.64100000000000001,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.753},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.9830000000000001,"15min":2.2410000000000001},"min":{"1min":0.33900000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.66100000000000003,"5min":12.773999999999999,"15min":12.773999999999999},"last":0.433}]},{"osd":3,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":6.181,"15min":7.1260000000000003},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72599999999999998,"5min":107.285,"15min":107.285},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.54200000000000004,"5min":5.8620000000000001,"15min":6.7480000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.75600000000000001,"5min":107.303,"15min":107.303},"last":0.72299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.2390000000000001,"15min":1.258},"min":{"1min":0.32500000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.2000000000000002,"5min":7.1740000000000004,"15min":17.713999999999999},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.1879999999999999,"15min":1.331},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.891,"5min":9.0660000000000007,"15min":14.455},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.45,"15min":1.2829999999999999},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70599999999999996,"5min":16.271000000000001,"15min":16.271000000000001},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.1890000000000001,"15min":1.2549999999999999},"min":{"1min":0.379,"5min":0.31900000000000001,"15min":0.28599999999999998},"max":{"1min":0.81899999999999995,"5min":11.718,"15min":11.718},"last":0.84899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:49:41 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.4750000000000001,"15min":1.6679999999999999},"min":{"1min":0.42499999999999999,"5min":0.315,"15min":0.31},"max":{"1min":6.0039999999999996,"5min":9.8390000000000004,"15min":21.792999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.321,"15min":1.47},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.317},"max":{"1min":3.306,"5min":7.8399999999999999,"15min":23.457000000000001},"last":0.98299999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724813,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":698528,"kb_used_data":345816,"kb_used_omap":0,"kb_used_meta":352704,"kb_avail":93046624,"statfs":{"total":95995035648,"available":95279742976,"internally_reserved":0,"allocated":354115584,"data_stored":353557767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":361168896},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.129,"15min":1.677},"min":{"1min":0.41899999999999998,"5min":0.20000000000000001,"15min":0.19400000000000001},"max":{"1min":1.2130000000000001,"5min":8.032,"15min":12.880000000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":1.365,"15min":1.3460000000000001},"min":{"1min":0.311,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":0.83299999999999996,"5min":10.028,"15min":10.955},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":4.4989999999999997,"15min":2.1789999999999998},"min":{"1min":0.34899999999999998,"5min":0.245,"15min":0.245},"max":{"1min":1.071,"5min":323.541,"15min":323.541},"last":0.312},{"interface":"front","average":{"1min":0.55900000000000005,"5min":4.5549999999999997,"15min":2.4649999999999999},"min":{"1min":0.34100000000000003,"5min":0.247,"15min":0.247},"max":{"1min":0.99399999999999999,"5min":324.072,"15min":324.072},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":2.04,"15min":2.298},"min":{"1min":0.435,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71499999999999997,"5min":13.394,"15min":13.394},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.625,"15min":1.595},"min":{"1min":0.38200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.0249999999999999,"5min":14.263,"15min":14.263},"last":0.27800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.5960000000000001,"15min":1.774},"min":{"1min":0.37,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.147,"5min":10.295,"15min":10.295},"last":198.33099999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.304,"15min":1.4199999999999999},"min":{"1min":0.38700000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.163,"5min":8.1080000000000005,"15min":8.1080000000000005},"last":198.47300000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.90400000000000003,"15min":1.153},"min":{"1min":0.33900000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.71499999999999997,"5min":5.8529999999999998,"15min":9.8800000000000008},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.2509999999999999,"15min":1.444},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.23999999999999999},"max":{"1min":0.77400000000000002,"5min":8.3170000000000002,"15min":16.869},"last":0.32100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.3100000000000001,"15min":1.3560000000000001},"min":{"1min":0.41299999999999998,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":0.81000000000000005,"5min":8.3010000000000002,"15min":17},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.1950000000000001,"15min":1.238},"min":{"1min":0.33400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.72499999999999998,"5min":9.6370000000000005,"15min":14.08},"last":0.48299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:49:40 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.5960000000000001,"15min":1.6910000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.28199999999999997},"max":{"1min":2.9300000000000002,"5min":14.217000000000001,"15min":17.071999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.96699999999999997,"15min":1.1259999999999999},"min":{"1min":0.40300000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.0499999999999998,"5min":5.601,"15min":8.6370000000000005},"last":0.42799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":311558144,"data_stored":311558144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":341835776,"data_stored":341835776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":110100480,"data_stored":110100480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":481951744,"data_stored":481951744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":132120576,"data_stored":132120576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":384303104,"data_stored":384303104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":126287872,"data_stored":126287872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":384696320,"data_stored":384696320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:50:25.659 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-09-23T20:50:25.730 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:25.772 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:25.830 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:25.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:25.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:25.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:25 smithi121 ceph-mon[219709]: pgmap v534: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T20:50:25.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:25 smithi121 ceph-mon[211526]: pgmap v534: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T20:50:25.938 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:25 smithi123 ceph-mon[193903]: pgmap v534: 89 pgs: 89 active+clean; 1.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T20:50:26.040 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:26.136 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:26.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:26.243 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:26.290 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:26.460 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 26760 26744 139.233 123.938 0.0101643 0.0071756 2023-09-23T20:50:27.004 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:27.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:27.110 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:27 smithi121 ceph-mon[211526]: pgmap v535: 89 pgs: 89 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T20:50:27.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:27 smithi121 ceph-mon[219709]: pgmap v535: 89 pgs: 89 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T20:50:27.173 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:27.223 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:27.289 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:27.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:27.394 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:27.422 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:27.460 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 28604 28588 137.388 115.25 0.00182208 0.00720464 2023-09-23T20:50:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:27 smithi123 ceph-mon[193903]: pgmap v535: 89 pgs: 89 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T20:50:27.705 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:27.842 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:27.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:27.955 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:28.024 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:28.098 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:28.098 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1281641762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.106 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:50:28.173 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:50:28.173 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:29820 9.2 e169/164 9:4cd1b3be:::benchmark_data_smithi121_293770_object29819:head v 169'1870, mlcod=169'1868)", 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:28.170646+0000", 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0025052550000000001, 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025299839999999999, 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:28.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170646+0000", 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170646+0000", 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:28.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170645+0000", 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170688+0000", 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2704000000000003e-05 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170709+0000", 2023-09-23T20:50:28.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1430999999999999e-05 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170719+0000", 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8479999999999996e-06 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170800+0000", 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0962000000000002e-05 2023-09-23T20:50:28.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170823+0000", 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2504e-05 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:29821 9.6 e169/164 9:6832a472:::benchmark_data_smithi121_293770_object29820:head v 169'1834, mlcod=169'1832)", 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:28.170506+0000", 2023-09-23T20:50:28.178 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0026455979999999999, 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027149190000000001, 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170506+0000", 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:28.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170506+0000", 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170505+0000", 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170789+0000", 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000284281 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170798+0000", 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5650000000000002e-06 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170803+0000", 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7690000000000002e-06 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:28.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170861+0000", 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7321999999999998e-05 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:28.170881+0000", 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.022e-05 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 2 2023-09-23T20:50:28.182 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:50:28.347 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:28.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:28.353 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:28.353 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1281641762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.453 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:28.460 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 30392 30376 135.557 111.75 0.0162916 0.00737134 2023-09-23T20:50:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1281641762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.490 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:28.589 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:28.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:28.699 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:28.757 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1281641762' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.106 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[219709]: pgmap v537: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-09-23T20:50:29.107 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:29.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:29.214 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:29.239 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:29.303 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:29.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:29.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:29.405 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:29.405 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1281641762' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:29.405 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T20:50:29.405 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.405 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:29 smithi121 ceph-mon[211526]: pgmap v537: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-09-23T20:50:29.415 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:29.457 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:29.460 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 32205 32189 134.073 113.312 0.00326733 0.00744499 2023-09-23T20:50:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1281641762' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: osdmap e169: 8 total, 8 up, 8 in 2023-09-23T20:50:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: failed to encode map e169 with expected crc 2023-09-23T20:50:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:29 smithi123 ceph-mon[193903]: pgmap v537: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-09-23T20:50:30.461 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 33725 33709 131.619 95 0.00687635 0.00759247 2023-09-23T20:50:30.532 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:30.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:30.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:30.635 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:30.746 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:50:30.914 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:30.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:31.021 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:31.096 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:31.462 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 35167 35151 129.178 90.125 0.00422158 0.00773591 2023-09-23T20:50:31.651 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:31.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:31.757 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:31.810 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:31.908 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:31 smithi121 ceph-mon[211526]: pgmap v538: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T20:50:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:31 smithi121 ceph-mon[219709]: pgmap v538: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T20:50:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:31.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:31.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:31 smithi123 ceph-mon[193903]: pgmap v538: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T20:50:32.016 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:32.060 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.133 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:32.209 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:50:32.240 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:32.255 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:50:32.295 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:50:32.326 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.335 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:50:32.378 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:50:32.404 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:32.420 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:50:32.462 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 36847 36831 127.835 105 0.0131864 0.00781713 2023-09-23T20:50:32.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:50:32.476 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:32.510 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:50:32.516 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:50:32.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:50:32.585 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:32.624 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.681 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:32.784 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1468314989' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:32.785 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1468314989' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:32.787 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:32.820 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.898 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:32.909 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1468314989' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:33.011 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:33.069 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:33.462 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 38534 38518 126.656 105.438 0.00572759 0.0078881 2023-09-23T20:50:33.758 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 1 pgp_num to 1 2023-09-23T20:50:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:33 smithi121 ceph-mon[211526]: pgmap v539: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T20:50:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1534855462' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:50:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:33 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:50:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:33 smithi121 ceph-mon[219709]: pgmap v539: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T20:50:34.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1534855462' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:50:34.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:33 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:50:34.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:33 smithi123 ceph-mon[193903]: pgmap v539: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-09-23T20:50:34.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1534855462' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:50:34.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:33 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:50:34.462 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:50:34.462641+0000 min lat: 0.0010359 max lat: 0.305445 avg lat: 0.00794901 2023-09-23T20:50:34.462 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:50:34.462 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 40250 40234 125.685 107.25 0.00504932 0.00794901 2023-09-23T20:50:35.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:50:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:50:34] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.43.0" 2023-09-23T20:50:35.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:50:35.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[211526]: pgmap v541: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:50:35.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:34 smithi121 ceph-mon[219709]: pgmap v541: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:50:35.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:50:35.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: osdmap e170: 8 total, 8 up, 8 in 2023-09-23T20:50:35.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: failed to encode map e170 with expected crc 2023-09-23T20:50:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:34 smithi123 ceph-mon[193903]: pgmap v541: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-09-23T20:50:35.462 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 41677 41661 123.947 89.1875 0.00515308 0.00806289 2023-09-23T20:50:35.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:35.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:35.699 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:35.795 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:35.882 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:35.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:35.940 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:35.988 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:36.035 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:36.106 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:36.106 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:36.123 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:36.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:36.232 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:36.299 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:36.464 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 43343 43327 123.035 104.125 0.00718451 0.00812363 2023-09-23T20:50:36.650 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:36.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:36.762 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:36.808 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.059 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:37.163 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:37.197 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.238 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:37.341 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:37.384 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:37 smithi123 ceph-mon[193903]: pgmap v542: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T20:50:37.386 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:37 smithi121 ceph-mon[219709]: pgmap v542: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T20:50:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:37 smithi121 ceph-mon[211526]: pgmap v542: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T20:50:37.454 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:37.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:37.464 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 44603 44587 121.11 78.75 0.00624735 0.00825332 2023-09-23T20:50:37.564 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:50:37.564 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:37.604 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.680 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:37.786 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:37.855 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.157 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:50:38.223 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:38.342 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:38.375 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.444 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:38.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:38.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 46119 46103 120 94.75 0.00627025 0.00832894 2023-09-23T20:50:38.551 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:38.643 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.768 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:38.798 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:38.909 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:38.991 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.040 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:39.205 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:39.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:39.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:39.324 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:39.378 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.465 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:39.466 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 47518 47502 118.698 87.4375 0.0107942 0.00841938 2023-09-23T20:50:39.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:39.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:39.571 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:39.658 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.736 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:50:39.857 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:39.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:39.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:39.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:39 smithi121 ceph-mon[211526]: pgmap v543: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T20:50:39.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/59795071' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:39.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:39.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:39 smithi121 ceph-mon[219709]: pgmap v543: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T20:50:39.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/59795071' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:39.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:39.967 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:39 smithi123 ceph-mon[193903]: pgmap v543: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T20:50:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/59795071' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:39.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:40.031 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:40.467 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 49144 49128 118.041 101.625 0.00833008 0.00846737 2023-09-23T20:50:40.602 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:50:40.602 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:50:40.602 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:50:40.603 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T20:50:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T20:50:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[211526]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T20:50:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:40 smithi121 ceph-mon[219709]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: osdmap e171: 8 total, 8 up, 8 in 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: failed to encode map e171 with expected crc 2023-09-23T20:50:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:40 smithi123 ceph-mon[193903]: failed to encode map e171 with expected crc 2023-09-23T20:50:41.467 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 50325 50309 116.403 73.8125 0.00651847 0.00858557 2023-09-23T20:50:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:41 smithi121 ceph-mon[211526]: pgmap v545: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T20:50:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:41 smithi121 ceph-mon[219709]: pgmap v545: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T20:50:41.986 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:41.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:41.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:42.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:42.164 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:42.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:41 smithi123 ceph-mon[193903]: pgmap v545: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T20:50:42.308 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:42.413 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:42.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:42.467 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 51633 51617 115.166 81.75 0.0107363 0.00867655 2023-09-23T20:50:42.521 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:42.568 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:42.640 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:42.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:42.749 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:42.771 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:42.771 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:42.820 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:43.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 53077 53061 114.305 90.25 0.00501243 0.0087448 2023-09-23T20:50:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[211526]: pgmap v546: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T20:50:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2739779218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[219709]: pgmap v546: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T20:50:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2739779218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:43 smithi123 ceph-mon[193903]: pgmap v546: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T20:50:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2739779218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:43.786 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:43.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:43.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:43.892 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:43.946 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:44.018 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:50:44.018 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:50:44.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:50:44.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:50:44.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:50:44.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:50:44.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:50:44.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:50:44.029 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:44.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:44.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:44.115 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:50:44.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:50:44.135 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:44.177 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:44.245 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:44.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:44.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:44.353 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:44.379 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:44.402 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:44 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:44.422 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:44.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:44.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 54122 54106 112.672 65.3125 0.0102661 0.00883446 2023-09-23T20:50:44.528 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:44.656 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:44 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:44.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:44 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:44.968 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:44.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:44.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:45.076 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:45.103 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:45.148 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:45.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:45.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:45.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:50:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:50:44] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.43.0" 2023-09-23T20:50:45.253 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:45.313 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:45.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 55264 55248 111.339 71.375 0.0124628 0.00897731 2023-09-23T20:50:45.471 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:45.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:45.577 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:45.645 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:45 smithi123 ceph-mon[193903]: pgmap v547: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2023-09-23T20:50:45.810 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:45.841 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:45.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:45.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:45.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:45 smithi121 ceph-mon[211526]: pgmap v547: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2023-09-23T20:50:45.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:45 smithi121 ceph-mon[219709]: pgmap v547: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.46k op/s 2023-09-23T20:50:45.947 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:45.986 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:46.069 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:46.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:46.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:46.094 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:46.176 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:46.214 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:46.335 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:46.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:46.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:46.448 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:46.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 56505 56489 110.283 77.5625 0.0119319 0.00906283 2023-09-23T20:50:46.520 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:46.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:46 smithi121 conmon[211495]: 2023-09-23T20:50:46.624+0000 7fdc50c66700 -1 mon.a@0(leader).osd e171 definitely_dead 0 2023-09-23T20:50:46.932 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:50:47.120 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.6. 2023-09-23T20:50:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2041759864' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:50:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:50:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[211526]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:50:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[211526]: pgmap v548: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:50:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2041759864' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:50:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:50:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[219709]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:50:47.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:47 smithi121 ceph-mon[219709]: pgmap v548: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:50:47.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 57398 57382 108.633 55.8125 0.00311556 0.00911124 2023-09-23T20:50:47.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2041759864' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:50:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:50:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:47 smithi123 ceph-mon[193903]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:50:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:47 smithi123 ceph-mon[193903]: pgmap v548: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:50:48.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:50:48.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[211526]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:50:48.103 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T20:50:48.103 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.103 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.103 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.103 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.103 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:48 smithi121 ceph-mon[219709]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 57398 57382 105.439 0 - 0.00911124 2023-09-23T20:50:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:50:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T20:50:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:48 smithi123 ceph-mon[193903]: failed to encode map e172 with expected crc 2023-09-23T20:50:48.731 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:48.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:48.835 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:48.865 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: pgmap v551: 89 pgs: 1 remapped+peering, 2 peering, 86 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 591 B/s, 0 objects/s recovering 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[211526]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: pgmap v551: 89 pgs: 1 remapped+peering, 2 peering, 86 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 591 B/s, 0 objects/s recovering 2023-09-23T20:50:49.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:49 smithi121 ceph-mon[219709]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T20:50:49.339 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:49.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:49.452 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:49.471 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 57572 57556 102.738 5.4375 0.0205963 0.00972683 2023-09-23T20:50:49.472 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:57560 9.2 e174/164 9:45c80ceb:::benchmark_data_smithi121_293770_object57559:head v 174'3605, mlcod=174'3603)", 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:49.461510+0000", 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.011211841, 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011247404000000001, 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:50:49.473 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461510+0000", 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461510+0000", 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:50:49.474 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461509+0000", 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461535+0000", 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5935000000000002e-05 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461550+0000", 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5608999999999999e-05 2023-09-23T20:50:49.475 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461560+0000", 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2890000000000003e-06 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461630+0000", 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9935e-05 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:50:49.476 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.461652+0000", 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2520000000000001e-05 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:50:49.477 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 1 2023-09-23T20:50:49.478 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:50:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: failed to encode map e173 with expected crc 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: pgmap v551: 89 pgs: 1 remapped+peering, 2 peering, 86 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 591 B/s, 0 objects/s recovering 2023-09-23T20:50:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:49 smithi123 ceph-mon[193903]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T20:50:49.578 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:49.785 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:49.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:49.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:49.894 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:49.942 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:50.040 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:50.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:50.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:50.122 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:50:50.146 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:50.173 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:50.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:50 smithi123 ceph-mon[193903]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:50 smithi123 ceph-mon[193903]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:50 smithi123 ceph-mon[193903]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:50 smithi123 ceph-mon[193903]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:50 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:50:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1622849449' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:50 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:50.279 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:50.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:50.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:50.385 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[219709]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[219709]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[219709]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[219709]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:50:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1622849449' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[211526]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[211526]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[211526]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[211526]: failed to encode map e174 with expected crc 2023-09-23T20:50:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:50:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1622849449' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:50.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:50 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:50.468 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 58202 58186 100.978 39.375 0.0167389 0.0098958 2023-09-23T20:50:50.477 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:50.637 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:50.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:50.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:50.713 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:50.741 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:50:50.755 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:50.792 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:50.856 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:50.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:50.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:50.966 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:50.992 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T20:50:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: pgmap v554: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 2 peering, 84 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 3481/172989 objects misplaced (2.012%); 625 KiB/s, 11 objects/s recovering 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:51.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: pgmap v554: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 2 peering, 84 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 3481/172989 objects misplaced (2.012%); 625 KiB/s, 11 objects/s recovering 2023-09-23T20:50:51.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:51 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:51.470 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 58944 58928 99.4997 46.375 0.0109811 0.0100441 2023-09-23T20:50:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:51.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T20:50:51.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: failed to encode map e175 with expected crc 2023-09-23T20:50:51.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:51.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: pgmap v554: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 2 peering, 84 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 3481/172989 objects misplaced (2.012%); 625 KiB/s, 11 objects/s recovering 2023-09-23T20:50:51.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:51 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:51.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:51.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:51.628 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:51.709 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:51.935 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:51.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:52.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:52.083 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:52.152 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:50:52.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:52.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[211526]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[211526]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[211526]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[211526]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[219709]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[219709]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[219709]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T20:50:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:52 smithi121 ceph-mon[219709]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.262 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:52.309 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:52.370 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:50:52.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:52 smithi123 ceph-mon[193903]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:52 smithi123 ceph-mon[193903]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:52 smithi123 ceph-mon[193903]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T20:50:52.377 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:52 smithi123 ceph-mon[193903]: failed to encode map e176 with expected crc 2023-09-23T20:50:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:52.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:52.470 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 59850 59834 98.3716 56.625 0.0402891 0.0101584 2023-09-23T20:50:52.479 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:52.529 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:53.081 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:50:53.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:53.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:53.194 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:53.199 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:53.234 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:53.289 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:53.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:53.392 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:53 smithi121 ceph-mon[211526]: pgmap v556: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 85 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s; 6974/174414 objects misplaced (3.999%); 1.8 MiB/s, 31 objects/s recovering 2023-09-23T20:50:53.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:53 smithi121 ceph-mon[219709]: pgmap v556: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 85 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s; 6974/174414 objects misplaced (3.999%); 1.8 MiB/s, 31 objects/s recovering 2023-09-23T20:50:53.470 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:53.471 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 60548 60532 96.9675 43.625 0.00675774 0.0103072 2023-09-23T20:50:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:53 smithi123 ceph-mon[193903]: pgmap v556: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 85 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s; 6974/174414 objects misplaced (3.999%); 1.8 MiB/s, 31 objects/s recovering 2023-09-23T20:50:53.574 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:50:53.592 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:50:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:53.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:53.696 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:53.841 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:54.193 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:50:54.194 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:50:54.194 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T20:50:54.194 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T20:50:54.242 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:54.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:54.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:54.350 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:54 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:50:54.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:54 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:50:54.454 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:54.470 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:50:54.471294+0000 min lat: 0.0010359 max lat: 2.15326 avg lat: 0.0104402 2023-09-23T20:50:54.471 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:50:54.471 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 61309 61293 95.7321 47.5625 0.0133918 0.0104402 2023-09-23T20:50:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:54 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:50:54.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:50:54.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:50:54.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:50:54.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:50:54.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:50:54.579 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:50:54.637 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:50:54.678 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:50:54.704 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:50:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:50:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:50:54] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.43.0" 2023-09-23T20:50:55.474 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 62308 62292 94.9114 62.4375 0.0122718 0.0105317 2023-09-23T20:50:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4238990272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:55 smithi123 ceph-mon[193903]: pgmap v557: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s; 3588/179820 objects degraded (1.995%); 6974/179820 objects misplaced (3.878%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:50:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4238990272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:55 smithi121 ceph-mon[219709]: pgmap v557: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s; 3588/179820 objects degraded (1.995%); 6974/179820 objects misplaced (3.878%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:50:55.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4238990272' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:55.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:55 smithi121 ceph-mon[211526]: pgmap v557: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 804 op/s; 3588/179820 objects degraded (1.995%); 6974/179820 objects misplaced (3.878%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:50:56.169 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:56.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:56.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:56.283 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:56.317 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:56.382 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:50:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:56.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:56.476 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 63251 63235 94.0529 58.9375 0.00621038 0.0106273 2023-09-23T20:50:56.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:56 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 3588/179820 objects degraded (1.995%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:50:56.490 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:56.565 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:56 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 3588/179820 objects degraded (1.995%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:50:56.565 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:56 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 3588/179820 objects degraded (1.995%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:50:56.571 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:56.986 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:56.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:56.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:57.096 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:57.198 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:57.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:57 smithi121 ceph-mon[211526]: pgmap v558: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 534 op/s; 3588/182937 objects degraded (1.961%); 6768/182937 objects misplaced (3.700%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:50:57.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:57 smithi121 ceph-mon[219709]: pgmap v558: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 534 op/s; 3588/182937 objects degraded (1.961%); 6768/182937 objects misplaced (3.700%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:50:57.476 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 64116 64100 93.123 54.0625 0.00468694 0.010732 2023-09-23T20:50:57.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:57 smithi123 ceph-mon[193903]: pgmap v558: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 534 op/s; 3588/182937 objects degraded (1.961%); 6768/182937 objects misplaced (3.700%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T20:50:57.568 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:57.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:57.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:57.683 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:57.717 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:57.743 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:57.743 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:57.749 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:57.749 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:57.770 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:57.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:57.875 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:57.997 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:58.129 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:58.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:58.235 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:58.270 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:58.334 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:50:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:58.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:58.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:58.443 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:58.476 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 65002 64986 92.2653 55.375 0.0217249 0.0108317 2023-09-23T20:50:58.511 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:58.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:50:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:59.111 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:50:59.280 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:50:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:59.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:59.388 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:59.446 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:50:59 smithi121 ceph-mon[219709]: pgmap v559: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 40 MiB/s wr, 635 op/s; 3588/187092 objects degraded (1.918%); 6585/187092 objects misplaced (3.520%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:50:59.447 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:50:59 smithi121 ceph-mon[211526]: pgmap v559: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 40 MiB/s wr, 635 op/s; 3588/187092 objects degraded (1.918%); 6585/187092 objects misplaced (3.520%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:50:59.448 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:50:59.476 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 65719 65703 91.2111 44.8125 0.0112775 0.0109585 2023-09-23T20:50:59.513 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:59.656 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:50:59.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:59.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:50:59 smithi123 ceph-mon[193903]: pgmap v559: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 40 MiB/s wr, 635 op/s; 3588/187092 objects degraded (1.918%); 6585/187092 objects misplaced (3.520%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:50:59.785 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:59.892 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:00.476 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 66440 66424 90.2081 45.0625 0.00768633 0.0110798 2023-09-23T20:51:01.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:00 smithi121 ceph-mon[219709]: pgmap v560: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s; 3588/195717 objects degraded (1.833%); 6361/195717 objects misplaced (3.250%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:51:01.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:00 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3588/195717 objects degraded (1.833%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:01.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:00 smithi121 ceph-mon[211526]: pgmap v560: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s; 3588/195717 objects degraded (1.833%); 6361/195717 objects misplaced (3.250%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:51:01.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:00 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3588/195717 objects degraded (1.833%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:01.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:00 smithi123 ceph-mon[193903]: pgmap v560: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s; 3588/195717 objects degraded (1.833%); 6361/195717 objects misplaced (3.250%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:51:01.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:00 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3588/195717 objects degraded (1.833%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:01.476 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 15 67398 67383 89.5642 59.9375 0.00555889 0.0111583 2023-09-23T20:51:02.477 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 68220 68204 88.7657 51.3125 0.0103731 0.0112606 2023-09-23T20:51:02.610 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:02.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:02.717 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:02.751 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:02.806 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:02.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:02.913 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:02.941 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:02.944 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:02.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:02.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:02.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:03.090 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:03.186 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:03.478 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 69024 69008 87.9798 50.25 0.0101438 0.0113612 2023-09-23T20:51:03.579 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:03.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:03.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:03.704 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:03.706 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[211526]: pgmap v561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 3588/198153 objects degraded (1.811%); 6175/198153 objects misplaced (3.116%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:51:03.706 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.706 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.707 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.730 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:03.778 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:03.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:03.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:03.884 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:03.888 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:03.919 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:03.966 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:03.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:03.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:03.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:03 smithi123 ceph-mon[193903]: pgmap v561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 3588/198153 objects degraded (1.811%); 6175/198153 objects misplaced (3.116%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:51:03.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.977 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[219709]: pgmap v561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 726 op/s; 3588/198153 objects degraded (1.811%); 6175/198153 objects misplaced (3.116%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T20:51:03.977 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.977 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:03.977 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:04.073 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:04.170 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:04.477 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 69771 69755 87.1542 46.6875 0.0207814 0.0114687 2023-09-23T20:51:04.942 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:51:04.951 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:04.954 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.6. 2023-09-23T20:51:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:04.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:04.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:04.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:04.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3895982309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:04.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:04.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:04 smithi121 conmon[211495]: 2023-09-23T20:51:04.826+0000 7fdc50c66700 -1 mon.a@0(leader).osd e176 definitely_dead 0 2023-09-23T20:51:04.969 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:51:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:51:04] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2023-09-23T20:51:04.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:04.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:04.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:04.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3895982309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:04.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:04.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:04.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:04 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:04.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:04 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:04.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:04 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:04.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3895982309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:04.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:05.070 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:05.096 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:05.139 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:05.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:05.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:05.245 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:05.279 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:05.478 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 70230 70214 86.0081 28.6875 0.00291803 0.0115162 2023-09-23T20:51:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: pgmap v562: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s; 3588/205068 objects degraded (1.750%); 6175/205068 objects misplaced (3.011%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:51:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2450055795' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:51:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:51:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:51:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:51:05.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: failed to encode map e177 with expected crc 2023-09-23T20:51:05.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: failed to encode map e177 with expected crc 2023-09-23T20:51:05.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: failed to encode map e177 with expected crc 2023-09-23T20:51:05.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: failed to encode map e177 with expected crc 2023-09-23T20:51:05.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:05 smithi123 ceph-mon[193903]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: pgmap v562: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s; 3588/205068 objects degraded (1.750%); 6175/205068 objects misplaced (3.011%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:51:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2450055795' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:51:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:51:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[211526]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: pgmap v562: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s; 3588/205068 objects degraded (1.750%); 6175/205068 objects misplaced (3.011%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2450055795' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:51:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:05 smithi121 ceph-mon[219709]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.291 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:06.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:06.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:06.397 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:06.413 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T20:51:06.462 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:06.478 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 70670 70654 84.882 27.5 0.00697159 0.0117754 2023-09-23T20:51:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e177 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:06.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3588/205068 objects degraded (1.750%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e177 with expected crc 2023-09-23T20:51:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:51:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3588/205068 objects degraded (1.750%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e177 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3588/205068 objects degraded (1.750%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:07.478 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 71630 71614 84.4126 60 0.0215188 0.0118421 2023-09-23T20:51:07.514 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:07.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:07.622 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:07.690 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:07.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:07.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:07.796 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:07.850 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:07 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:07 smithi123 ceph-mon[193903]: failed to encode map e178 with expected crc 2023-09-23T20:51:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:07 smithi123 ceph-mon[193903]: pgmap v565: 89 pgs: 1 remapped+peering, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 854 op/s; 3588/207621 objects degraded (1.728%); 3124/207621 objects misplaced (1.505%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:51:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:07 smithi123 ceph-mon[193903]: failed to encode map e179 with expected crc 2023-09-23T20:51:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:07 smithi123 ceph-mon[193903]: failed to encode map e179 with expected crc 2023-09-23T20:51:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:07 smithi123 ceph-mon[193903]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:51:08.090 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[211526]: failed to encode map e178 with expected crc 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[211526]: pgmap v565: 89 pgs: 1 remapped+peering, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 854 op/s; 3588/207621 objects degraded (1.728%); 3124/207621 objects misplaced (1.505%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[211526]: failed to encode map e179 with expected crc 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[211526]: failed to encode map e179 with expected crc 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[211526]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[219709]: failed to encode map e178 with expected crc 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[219709]: pgmap v565: 89 pgs: 1 remapped+peering, 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 53 MiB/s wr, 854 op/s; 3588/207621 objects degraded (1.728%); 3124/207621 objects misplaced (1.505%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:51:08.091 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[219709]: failed to encode map e179 with expected crc 2023-09-23T20:51:08.092 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[219709]: failed to encode map e179 with expected crc 2023-09-23T20:51:08.092 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:07 smithi121 ceph-mon[219709]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:51:08.185 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:08.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:08.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:08.342 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:08.447 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:08.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:08.478 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 72630 72614 84.0069 62.5 0.0128452 0.0118985 2023-09-23T20:51:08.559 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:08.619 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:08.698 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:08.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:08.741 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:51:08.777 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:51:08.809 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:08.813 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:51:08.857 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:51:08.907 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:51:08.916 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:08.949 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:51:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:08 smithi123 ceph-mon[193903]: failed to encode map e179 with expected crc 2023-09-23T20:51:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/63592688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:08.997 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:51:09.025 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:09.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:09.035 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:08 smithi121 ceph-mon[211526]: failed to encode map e179 with expected crc 2023-09-23T20:51:09.035 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/63592688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:09.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:08 smithi121 ceph-mon[219709]: failed to encode map e179 with expected crc 2023-09-23T20:51:09.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/63592688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:09.040 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:51:09.040 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:51:09.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b7ed88ab:::benchmark_data_smithi121_252781_object87596:head, version: 125'5492, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b7ed88ab:::benchmark_data_smithi121_252781_object87596:head@125'5492, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 208.43150049400001, 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17604561799999999, 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.041 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605345+0000", 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6996000000000001e-05 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605357+0000", 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2374e-05 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778323+0000", 2023-09-23T20:51:09.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.172965539 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778421+0000", 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8530999999999996e-05 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778437+0000", 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5136999999999999e-05 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.781334+0000", 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028975680000000001 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b02b4366:::benchmark_data_smithi121_252781_object90395:head, version: 125'5662, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b02b4366:::benchmark_data_smithi121_252781_object90395:head@125'5662, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 206.110785558, 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22766398199999999, 2023-09-23T20:51:09.045 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.925997+0000", 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999938 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926050+0000", 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2899999999999998e-05 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150250+0000", 2023-09-23T20:51:09.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22419977199999999 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150290+0000", 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9712000000000002e-05 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150341+0000", 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1696000000000001e-05 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150349+0000", 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7409999999999992e-06 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.153667+0000", 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033184040000000001 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b6dc2210:::benchmark_data_smithi121_252781_object93072:head, version: 127'5840, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b6dc2210:::benchmark_data_smithi121_252781_object93072:head@127'5840, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:51:09.049 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.664676958, 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16357189899999999, 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372135+0000", 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3283000000000001e-05 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.529427+0000", 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.157291876 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.529452+0000", 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5023000000000001e-05 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.531548+0000", 2023-09-23T20:51:09.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020965010000000002 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.531560+0000", 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1425e-05 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.535684+0000", 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041242800000000001 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.053 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172419 8.f 8:f3e775cc:::benchmark_data_smithi121_252781_object63244:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 201.91972952, 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0270669539999999, 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:09.054 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172419 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:51:09.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117058+0000", 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117065+0000", 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9940000000000003e-06 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117075+0000", 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8649999999999996e-06 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117083+0000", 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7399999999999993e-06 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117315+0000", 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00023133999999999999 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117325+0000", 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0008e-05 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144044+0000", 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.026719535 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.058 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144126+0000", 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2231000000000006e-05 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172440 8.f 8:fcfd2d26:::benchmark_data_smithi121_252781_object63265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 201.91969192900001, 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047514855, 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.059 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172440 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.060 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117099+0000", 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117106+0000", 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2210000000000002e-06 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117111+0000", 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2360000000000004e-06 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117343+0000", 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000232597 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117349+0000", 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1930000000000001e-06 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144377+0000", 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.027027844 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144494+0000", 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000117108 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144624+0000", 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000130677 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.155446+0000", 2023-09-23T20:51:09.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010821377 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164470+0000", 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090247639999999994 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164526+0000", 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5665999999999999e-05 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164612+0000", 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.577e-05 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172441 8.f 8:f67e9cac:::benchmark_data_smithi121_252781_object63266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 201.91967384599999, 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047671113, 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.066 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172441 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.068 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117118+0000", 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8109999999999999e-06 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117122+0000", 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7160000000000002e-06 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117127+0000", 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4100000000000001e-06 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.069 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117369+0000", 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024246900000000001 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117375+0000", 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3000000000000001e-06 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148014+0000", 2023-09-23T20:51:09.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.030639455 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148124+0000", 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110089 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148227+0000", 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010324599999999999 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.155452+0000", 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072244780000000003 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164664+0000", 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092118040000000005 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164708+0000", 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4780999999999997e-05 2023-09-23T20:51:09.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164786+0000", 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7847000000000002e-05 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.a 143/130 [PushOp(8:5b7183a5:::benchmark_data_smithi121_252781_object76859:head, version: 125'4857, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:5b7183a5:::benchmark_data_smithi121_252781_object76859:head@125'4857, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:48.626718+0000", 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 200.41007139800001, 2023-09-23T20:51:09.073 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10125018, 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626718+0000", 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626718+0000", 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.074 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626717+0000", 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626738+0000", 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1288999999999999e-05 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626749+0000", 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0528e-05 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626758+0000", 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8429999999999999e-06 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626810+0000", 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2621999999999999e-05 2023-09-23T20:51:09.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.724953+0000", 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.098142939999999998 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.727968+0000", 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003014507 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:174677 8.d e144/143 8:b7ddb6cb:::benchmark_data_smithi121_252781_object65502:head v 144'10966, mlcod=144'10963)", 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 197.62737767799999, 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.043301345999999998, 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.079 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409416+0000", 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2820000000000004e-06 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409424+0000", 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4670000000000004e-06 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409430+0000", 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3650000000000003e-06 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409472+0000", 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2267e-05 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409491+0000", 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9117999999999999e-05 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.413890+0000", 2023-09-23T20:51:09.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004399053 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452713+0000", 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038822397000000002 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:174680 8.f 8:fb6dc5c3:::benchmark_data_smithi121_252781_object65505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:51:09.082 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.412110+0000", 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 197.624679281, 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044728539999999997, 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 174680 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.083 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412110+0000", 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412110+0000", 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412108+0000", 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412116+0000", 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7719999999999994e-06 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412130+0000", 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3902e-05 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412139+0000", 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1519999999999998e-06 2023-09-23T20:51:09.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412206+0000", 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7294999999999999e-05 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412337+0000", 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013082600000000001 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412457+0000", 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011976000000000001 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456665+0000", 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044208285999999999 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456694+0000", 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9335999999999998e-05 2023-09-23T20:51:09.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456747+0000", 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2333999999999997e-05 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456838+0000", 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1529999999999999e-05 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.088 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:174681 8.f 8:fb2398b1:::benchmark_data_smithi121_252781_object65506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 197.624634684, 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.055719985999999999, 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 174681 2023-09-23T20:51:09.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412157+0000", 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.095e-06 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412163+0000", 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6460000000000004e-06 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412169+0000", 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1399999999999999e-06 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452744+0000", 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.040575356999999999 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452875+0000", 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013148599999999999 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452994+0000", 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000118662 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.463838+0000", 2023-09-23T20:51:09.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010843983 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.467748+0000", 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0039096160000000003 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.467792+0000", 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4431999999999997e-05 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.467874+0000", 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2099000000000001e-05 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:174679 8.d e144/143 8:be774cb5:::benchmark_data_smithi121_252781_object65504:head v 144'10967, mlcod=144'10964)", 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.414661+0000", 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 197.62212790800001, 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050115271000000003, 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.095 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414661+0000", 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414661+0000", 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414660+0000", 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414668+0000", 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1589999999999993e-06 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452693+0000", 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038024924000000002 2023-09-23T20:51:09.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452706+0000", 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2772000000000001e-05 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452744+0000", 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8118000000000003e-05 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452765+0000", 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1257999999999999e-05 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.464754+0000", 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011988421000000001 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.464776+0000", 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2609e-05 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.099 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:177178 8.f 8:f0621e84:::benchmark_data_smithi121_252781_object68003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 194.68297927399999, 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36433712000000001, 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:09.100 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 177178 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353808+0000", 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353817+0000", 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6180000000000001e-06 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353835+0000", 2023-09-23T20:51:09.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7601e-05 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353845+0000", 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0268000000000001e-05 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353898+0000", 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3173000000000001e-05 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.354023+0000", 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000124456 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.354142+0000", 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000118789 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.357799+0000", 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036574469999999999 2023-09-23T20:51:09.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718014+0000", 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36021456600000001 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718061+0000", 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7530000000000001e-05 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718147+0000", 2023-09-23T20:51:09.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5612999999999995e-05 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:177186 8.f 8:f634efb7:::benchmark_data_smithi121_252781_object68011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 194.670052148, 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.361305761, 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.106 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 177186 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366735+0000", 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366742+0000", 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8569999999999997e-06 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366756+0000", 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3405e-05 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366765+0000", 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7969999999999995e-06 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366817+0000", 2023-09-23T20:51:09.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2021000000000002e-05 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366923+0000", 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106098 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366996+0000", 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2354999999999994e-05 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.370015+0000", 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030194229999999998 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.727915+0000", 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35789987899999998 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.727963+0000", 2023-09-23T20:51:09.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7817e-05 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.728043+0000", 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9654999999999995e-05 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185557 8.4 e146/134 8:23b68925:::benchmark_data_smithi121_252781_object76382:head v 146'11606, mlcod=146'11603)", 2023-09-23T20:51:09.112 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 183.544340468, 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33023901100000003, 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492455+0000", 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5970000000000003e-06 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492464+0000", 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9279999999999999e-06 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492471+0000", 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4699999999999999e-06 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492508+0000", 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7024999999999999e-05 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492525+0000", 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6776e-05 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822566+0000", 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33004159199999999 2023-09-23T20:51:09.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822688+0000", 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121403 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185566 8.d e146/143 8:bb91d518:::benchmark_data_smithi121_252781_object76391:head v 146'11646, mlcod=146'11643)", 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 183.541768859, 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32767350499999998, 2023-09-23T20:51:09.117 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495025+0000", 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6550000000000003e-06 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495035+0000", 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.057e-05 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495043+0000", 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9699999999999999e-06 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495079+0000", 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6300999999999997e-05 2023-09-23T20:51:09.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495094+0000", 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4430000000000001e-05 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822604+0000", 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32751017900000001 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822694+0000", 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9649999999999997e-05 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185568 8.4 e146/134 8:2156ec6b:::benchmark_data_smithi121_252781_object76393:head v 146'11607, mlcod=146'11604)", 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 183.53614808399999, 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33877399600000002, 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.122 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500640+0000", 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500649+0000", 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0450000000000006e-06 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822380+0000", 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32173094000000002 2023-09-23T20:51:09.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822388+0000", 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1999999999999994e-06 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.824928+0000", 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025395029999999998 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.824942+0000", 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4331999999999999e-05 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.839373+0000", 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01443056 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.839415+0000", 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2320000000000001e-05 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.126 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185570 8.a e147/145 8:550a95bc:::benchmark_data_smithi121_252781_object76395:head v 147'11761, mlcod=146'11759)", 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 183.53503678499999, 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32342548900000001, 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501756+0000", 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9689999999999996e-06 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822281+0000", 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32052530800000001 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822294+0000", 2023-09-23T20:51:09.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2822e-05 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822388+0000", 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4215999999999994e-05 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822411+0000", 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3161999999999999e-05 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.825145+0000", 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027331389999999999 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.825178+0000", 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3142999999999998e-05 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:57315 9.0 e171/164 9:0328d5f3:::benchmark_data_smithi121_293770_object57314:head v 171'3588, mlcod=171'3586)", 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.087617+0000", 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 21.949171555, 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057652203999999999, 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087617+0000", 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087617+0000", 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087616+0000", 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087640+0000", 2023-09-23T20:51:09.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4063e-05 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087653+0000", 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2913e-05 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087660+0000", 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6959999999999999e-06 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087753+0000", 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3505999999999995e-05 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087776+0000", 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2350000000000001e-05 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.145270+0000", 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057493778000000002 2023-09-23T20:51:09.135 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:57324 9.2 e171/164 9:4fe6fb3c:::benchmark_data_smithi121_293770_object57323:head v 171'3588, mlcod=171'3586)", 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.096479+0000", 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 21.940310233000002, 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.059996267999999998, 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096479+0000", 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096479+0000", 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096478+0000", 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:51:09.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096507+0000", 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9575000000000001e-05 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142872+0000", 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.046364378999999997 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142882+0000", 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0198e-05 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143205+0000", 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00032271200000000001 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143220+0000", 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5903e-05 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156416+0000", 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013195231 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156475+0000", 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9321999999999999e-05 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.140 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:57334 9.6 e171/164 9:6e49c932:::benchmark_data_smithi121_293770_object57333:head v 171'3517, mlcod=171'3515)", 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 21.930748759, 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050547171000000002, 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106071+0000", 2023-09-23T20:51:09.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1495999999999998e-05 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142607+0000", 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036535521000000001 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142621+0000", 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4562000000000001e-05 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143462+0000", 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00084066100000000003 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143487+0000", 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4536e-05 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156547+0000", 2023-09-23T20:51:09.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013060824 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156587+0000", 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9947000000000002e-05 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:09.145 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:51:09.146 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:09.146 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:51:09.147 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.147 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.147 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.147 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.147 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.149 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.149 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.149 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:51:09.240 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:09.478 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 73713 73697 83.7102 67.6875 0.00884272 0.0119407 2023-09-23T20:51:09.763 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:09.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:09.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:09.879 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:09.933 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:10.018 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:10.019 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:09 smithi121 ceph-mon[211526]: pgmap v567: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 4 peering, 82 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 1669/210840 objects degraded (0.792%); 2.8 MiB/s, 46 objects/s recovering 2023-09-23T20:51:10.020 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:09 smithi121 ceph-mon[219709]: pgmap v567: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 4 peering, 82 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 1669/210840 objects degraded (0.792%); 2.8 MiB/s, 46 objects/s recovering 2023-09-23T20:51:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:10.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:10.121 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:10.145 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:09 smithi123 ceph-mon[193903]: pgmap v567: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 4 peering, 82 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 1669/210840 objects degraded (0.792%); 2.8 MiB/s, 46 objects/s recovering 2023-09-23T20:51:10.147 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:10.179 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:51:10.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:10.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:10.335 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:10.364 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:10.478 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 74604 74588 83.2098 55.6875 0.00590405 0.0120091 2023-09-23T20:51:10.837 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:10.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:10.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:10.942 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:10.989 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:10.990 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:51:11.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:11.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:11.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:11.195 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:11.261 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:11.313 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:11.344 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:11.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:11.414 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:51:11.414 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:51:11.453 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:11.479 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 75577 75561 82.8157 60.8125 0.0162661 0.012069 2023-09-23T20:51:11.491 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:11.563 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:11.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:11.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:11.682 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:11.730 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:11.804 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:11.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:11.910 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:11.977 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:11 smithi123 ceph-mon[193903]: pgmap v568: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 1671/219255 objects degraded (0.762%); 389/219255 objects misplaced (0.177%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T20:51:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1671/219255 objects degraded (0.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:11 smithi121 ceph-mon[211526]: pgmap v568: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 1671/219255 objects degraded (0.762%); 389/219255 objects misplaced (0.177%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T20:51:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1671/219255 objects degraded (0.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:11 smithi121 ceph-mon[219709]: pgmap v568: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 1671/219255 objects degraded (0.762%); 389/219255 objects misplaced (0.177%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T20:51:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1671/219255 objects degraded (0.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:12.255 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:12.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:12.367 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:12.414 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:12.480 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 76624 76608 82.5161 65.4375 0.00603313 0.012114 2023-09-23T20:51:12.540 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:12.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:12.653 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:12.710 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:12.807 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:12.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:12.917 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:12.953 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:51:12.968 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:12.968 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:12.972 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:13.067 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:13.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:13.156 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:13.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:13.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:13.169 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:13.174 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:13.217 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:13.280 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:13.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:13.389 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:13.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 77692 77676 82.2477 66.75 0.0083553 0.0121529 2023-09-23T20:51:13.496 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:14.001 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:14.010 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:13 smithi121 ceph-mon[211526]: pgmap v569: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s; 1671/222033 objects degraded (0.753%); 389/222033 objects misplaced (0.175%); 5.2 MiB/s, 85 objects/s recovering 2023-09-23T20:51:14.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:14.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:13 smithi121 ceph-mon[219709]: pgmap v569: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s; 1671/222033 objects degraded (0.753%); 389/222033 objects misplaced (0.175%); 5.2 MiB/s, 85 objects/s recovering 2023-09-23T20:51:14.111 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:14.181 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:13 smithi123 ceph-mon[193903]: pgmap v569: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s; 1671/222033 objects degraded (0.753%); 389/222033 objects misplaced (0.175%); 5.2 MiB/s, 85 objects/s recovering 2023-09-23T20:51:14.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:51:14.481380+0000 min lat: 0.0010359 max lat: 2.15326 avg lat: 0.0121984 2023-09-23T20:51:14.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:51:14.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 78719 78703 81.9467 64.1875 0.0193708 0.0121984 2023-09-23T20:51:14.802 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:51:15.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/909970553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:15.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:14 smithi121 ceph-mon[219709]: pgmap v570: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 1671/230511 objects degraded (0.725%); 389/230511 objects misplaced (0.169%); 3.3 MiB/s, 54 objects/s recovering 2023-09-23T20:51:15.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:51:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:51:14] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T20:51:15.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/909970553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:15.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:14 smithi121 ceph-mon[211526]: pgmap v570: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 1671/230511 objects degraded (0.725%); 389/230511 objects misplaced (0.169%); 3.3 MiB/s, 54 objects/s recovering 2023-09-23T20:51:15.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/909970553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:15.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:14 smithi123 ceph-mon[193903]: pgmap v570: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 1671/230511 objects degraded (0.725%); 389/230511 objects misplaced (0.169%); 3.3 MiB/s, 54 objects/s recovering 2023-09-23T20:51:15.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 79287 79271 81.1855 35.5 0.00302538 0.012223 2023-09-23T20:51:16.060 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:16.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:16.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[211526]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/909970553' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:51:16.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[211526]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:51:16.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[211526]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[211526]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.073 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:16.073 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[219709]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.073 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/909970553' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:51:16.073 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[219709]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:51:16.073 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[219709]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.073 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:15 smithi121 ceph-mon[219709]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:16.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:16.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:15 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:16.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:15 smithi123 ceph-mon[193903]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/909970553' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:51:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:15 smithi123 ceph-mon[193903]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:51:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:15 smithi123 ceph-mon[193903]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:15 smithi123 ceph-mon[193903]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.255 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:16.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 79990 79974 80.5848 43.9375 0.00907294 0.0124045 2023-09-23T20:51:16.579 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:16.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:16.690 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:16.697 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:16.785 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:16.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[211526]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[211526]: failed to encode map e181 with expected crc 2023-09-23T20:51:16.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[211526]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:51:16.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[211526]: failed to encode map e181 with expected crc 2023-09-23T20:51:16.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1671/230511 objects degraded (0.725%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:16.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[211526]: pgmap v573: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 1671/233814 objects degraded (0.715%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:51:16.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[219709]: failed to encode map e180 with expected crc 2023-09-23T20:51:16.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[219709]: failed to encode map e181 with expected crc 2023-09-23T20:51:16.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[219709]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:51:16.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[219709]: failed to encode map e181 with expected crc 2023-09-23T20:51:16.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1671/230511 objects degraded (0.725%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:16.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:16 smithi121 ceph-mon[219709]: pgmap v573: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 1671/233814 objects degraded (0.715%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:51:17.189 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:17.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:17.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:16 smithi123 ceph-mon[193903]: failed to encode map e180 with expected crc 2023-09-23T20:51:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:16 smithi123 ceph-mon[193903]: failed to encode map e181 with expected crc 2023-09-23T20:51:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:16 smithi123 ceph-mon[193903]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:51:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:16 smithi123 ceph-mon[193903]: failed to encode map e181 with expected crc 2023-09-23T20:51:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1671/230511 objects degraded (0.725%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:16 smithi123 ceph-mon[193903]: pgmap v573: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 1671/233814 objects degraded (0.715%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:51:17.296 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:17.443 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:17.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 80999 80983 80.3067 63.0625 0.0164158 0.0124466 2023-09-23T20:51:17.555 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:51:17.555 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:51:17.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.555 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:81049 9.8 9:194791fc:::benchmark_data_smithi121_293770_object81048:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T20:51:17.555 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:51:17.550606+0000", 2023-09-23T20:51:17.555 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0027765839999999999, 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028162059999999999, 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 81049 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:17.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550606+0000", 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550606+0000", 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550604+0000", 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:17.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550627+0000", 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2667000000000002e-05 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550639+0000", 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1952e-05 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550646+0000", 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.888e-06 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550696+0000", 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0791999999999998e-05 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550803+0000", 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106557 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550914+0000", 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0001109 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:81050 9.2 9:4e258c30:::benchmark_data_smithi121_293770_object81049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:51:17.550665+0000", 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0027172870000000001, 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002847601, 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 81050 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.561 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550665+0000", 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550665+0000", 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550664+0000", 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550685+0000", 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0248e-05 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550692+0000", 2023-09-23T20:51:17.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4070000000000004e-06 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550697+0000", 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2719999999999997e-06 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550752+0000", 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4515999999999998e-05 2023-09-23T20:51:17.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.550892+0000", 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139765 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.551040+0000", 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148293 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.45101.0:81032 9.2 e181/164)", 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:51:17.552064+0000", 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.001317765, 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0015207230000000001, 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:17.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552064+0000", 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552064+0000", 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552064+0000", 2023-09-23T20:51:17.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552068+0000", 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2719999999999999e-06 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552079+0000", 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0974e-05 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552085+0000", 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.297e-06 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.45101.0:81043 9.2 e181/164)", 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:51:17.552094+0000", 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0012884820000000001, 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0015378340000000001, 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:17.569 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552094+0000", 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552094+0000", 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552093+0000", 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552095+0000", 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.035e-06 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552101+0000", 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.186e-06 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.552105+0000", 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9949999999999999e-06 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:81032 9.2 9:4da76f5c:::benchmark_data_smithi121_293770_object81031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:51:17.529499+0000", 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.023882658000000001, 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024188584999999999, 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 81032 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:17.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529499+0000", 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529499+0000", 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:17.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529499+0000", 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529520+0000", 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0985e-05 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529527+0000", 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6079999999999998e-06 2023-09-23T20:51:17.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529534+0000", 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.054e-06 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529584+0000", 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0800999999999997e-05 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:17.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529722+0000", 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013755700000000001 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.529886+0000", 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000164489 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.535945+0000", 2023-09-23T20:51:17.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006058852 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.541683+0000", 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057375220000000001 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.553640+0000", 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011957160999999999 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.553693+0000", 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3131999999999999e-05 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:81043 9.2 9:44ac6b9d:::benchmark_data_smithi121_293770_object81042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:51:17.538101+0000", 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.015280907, 2023-09-23T20:51:17.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015698637000000001, 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 81043 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538101+0000", 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538101+0000", 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538101+0000", 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538122+0000", 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1195e-05 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538129+0000", 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.43e-06 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538135+0000", 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4729999999999999e-06 2023-09-23T20:51:17.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538181+0000", 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6048999999999999e-05 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538303+0000", 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012234600000000001 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:51:17.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.538431+0000", 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000127945 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:51:17.541691+0000", 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00325961 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:51:17.584 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 6 2023-09-23T20:51:17.585 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:51:17.648 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:17.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:17.754 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:17.821 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:17 smithi121 ceph-mon[211526]: failed to encode map e181 with expected crc 2023-09-23T20:51:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3902976127' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:17 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:17.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:17 smithi121 ceph-mon[219709]: failed to encode map e181 with expected crc 2023-09-23T20:51:17.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3902976127' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:17.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:17 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:18.137 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:51:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:17 smithi123 ceph-mon[193903]: failed to encode map e181 with expected crc 2023-09-23T20:51:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3902976127' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:17 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:18.335 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:18.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:18.441 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:18.470 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:18.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 81999 81983 80.0285 62.5 0.00975822 0.0124837 2023-09-23T20:51:18.733 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:51:19.123 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[219709]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:19.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[219709]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:51:19.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[219709]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[219709]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[219709]: pgmap v575: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 828/237936 objects degraded (0.348%); 492/237936 objects misplaced (0.207%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:51:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[211526]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[211526]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:51:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[211526]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[211526]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:19 smithi121 ceph-mon[211526]: pgmap v575: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 828/237936 objects degraded (0.348%); 492/237936 objects misplaced (0.207%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:51:19.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 82894 82878 79.6579 55.9375 0.00519853 0.0125492 2023-09-23T20:51:19.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:19 smithi123 ceph-mon[193903]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:19 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:19.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:19 smithi123 ceph-mon[193903]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:51:19.485 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:19 smithi123 ceph-mon[193903]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.485 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:19 smithi123 ceph-mon[193903]: failed to encode map e182 with expected crc 2023-09-23T20:51:19.485 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:19 smithi123 ceph-mon[193903]: pgmap v575: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 828/237936 objects degraded (0.348%); 492/237936 objects misplaced (0.207%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:51:19.555 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:19.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:19.663 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:19.706 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:19.766 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:19.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:19.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:19.872 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:19.897 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:20.077 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:51:20.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:51:20.292 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:20.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:20.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:20.414 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:20.448 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:20.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 83737 83721 79.2493 52.6875 0.0268316 0.0126121 2023-09-23T20:51:20.508 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:20.515 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:20.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:20.624 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:20.690 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:21.019 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:21.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:21.125 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:21.174 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:21.262 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:21.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:21.373 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:21.420 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:21.430 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:51:21.441 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:21.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:51:21.481 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 84694 84678 78.9592 59.8125 0.00805385 0.0126584 2023-09-23T20:51:21.491 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:21.492 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:21.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:21.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:21.526 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:51:21.567 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:51:21.596 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:21.608 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:51:21.642 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:21.651 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:51:21.682 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:51:21.720 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:51:21.722 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.722 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.722 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.722 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:51:21.746 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:21.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:21.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:21.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:21 smithi123 ceph-mon[193903]: pgmap v576: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 828/246924 objects degraded (0.335%); 492/246924 objects misplaced (0.199%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:51:21.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 828/246924 objects degraded (0.335%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:21.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2040998022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:21.855 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:21.907 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:21 smithi121 ceph-mon[211526]: pgmap v576: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 828/246924 objects degraded (0.335%); 492/246924 objects misplaced (0.199%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:51:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 828/246924 objects degraded (0.335%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2040998022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:21 smithi121 ceph-mon[219709]: pgmap v576: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 828/246924 objects degraded (0.335%); 492/246924 objects misplaced (0.199%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:51:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 828/246924 objects degraded (0.335%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:51:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2040998022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:22.482 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 85602 85586 78.6318 56.75 0.0121809 0.0127132 2023-09-23T20:51:22.931 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:22.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:22.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:23.035 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:23.110 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:23.482 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 86053 86037 77.9009 28.1875 0.00360629 0.0127149 2023-09-23T20:51:23.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: pgmap v577: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 828/249672 objects degraded (0.332%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:51:23.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:51:23.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[219709]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: pgmap v577: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 828/249672 objects degraded (0.332%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:51:23.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:51:23.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:23 smithi121 ceph-mon[211526]: failed to encode map e183 with expected crc 2023-09-23T20:51:23.950 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:23.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:23.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:24.057 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:24.128 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: pgmap v577: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 828/249672 objects degraded (0.332%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:51:24.128 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:51:24.129 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: failed to encode map e183 with expected crc 2023-09-23T20:51:24.129 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: failed to encode map e183 with expected crc 2023-09-23T20:51:24.129 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: failed to encode map e183 with expected crc 2023-09-23T20:51:24.129 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: failed to encode map e183 with expected crc 2023-09-23T20:51:24.129 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: failed to encode map e183 with expected crc 2023-09-23T20:51:24.129 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:23 smithi123 ceph-mon[193903]: failed to encode map e183 with expected crc 2023-09-23T20:51:24.129 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:24.220 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:24.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:24.333 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:24.400 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:24.482 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 86765 86749 77.4239 44.5 0.0076756 0.0129109 2023-09-23T20:51:25.134 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[219709]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.134 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[219709]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:51:25.134 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[219709]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[219709]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[219709]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[219709]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[211526]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[211526]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:51:25.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[211526]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[211526]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[211526]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:24 smithi121 ceph-mon[211526]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.136 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:51:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:51:24] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T20:51:25.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:24 smithi123 ceph-mon[193903]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:24 smithi123 ceph-mon[193903]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:51:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:24 smithi123 ceph-mon[193903]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:24 smithi123 ceph-mon[193903]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:24 smithi123 ceph-mon[193903]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:24 smithi123 ceph-mon[193903]: failed to encode map e184 with expected crc 2023-09-23T20:51:25.482 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 87626 87610 77.0914 53.8125 0.00390864 0.0129663 2023-09-23T20:51:25.593 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:51:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:25 smithi121 ceph-mon[211526]: pgmap v580: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2/256473 objects degraded (0.001%); 632/256473 objects misplaced (0.246%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:51:25.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:25 smithi121 ceph-mon[219709]: pgmap v580: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2/256473 objects degraded (0.001%); 632/256473 objects misplaced (0.246%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:51:25.956 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:25.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:26.061 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:26.129 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:26.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:25 smithi123 ceph-mon[193903]: pgmap v580: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2/256473 objects degraded (0.001%); 632/256473 objects misplaced (0.246%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:51:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:26.275 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:26.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:26.376 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:26.475 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:26.483 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 88443 88427 76.7294 51.0625 0.0199431 0.0130282 2023-09-23T20:51:26.647 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:26.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:26.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:26.761 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:26.798 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:26.869 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2/256473 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:26.869 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2821393885' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:26.869 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:26 smithi121 ceph-mon[211526]: pgmap v581: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 2/259245 objects degraded (0.001%); 632/259245 objects misplaced (0.244%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T20:51:26.869 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2/256473 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:26.870 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2821393885' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:26.870 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:26 smithi121 ceph-mon[219709]: pgmap v581: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 2/259245 objects degraded (0.001%); 632/259245 objects misplaced (0.244%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T20:51:26.883 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:26.959 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:51:26.960 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:51:27.097 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:27.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:27.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:27.140 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:51:27.141 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:51:27.206 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2/256473 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2821393885' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:26 smithi123 ceph-mon[193903]: pgmap v581: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 2/259245 objects degraded (0.001%); 632/259245 objects misplaced (0.244%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T20:51:27.249 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:27.292 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:27.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:27.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:27.434 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:27.483 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 89191 89175 76.3188 46.75 0.00986401 0.0130976 2023-09-23T20:51:28.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1985963977' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2821393885' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1985963977' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:28.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:28.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1985963977' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:51:28.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2821393885' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:28.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1985963977' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:51:28.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:51:28.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:28.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:28.174 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:28.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:28.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1985963977' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:51:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2821393885' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1985963977' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: failed to encode map e185 with expected crc 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:28.283 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:28.343 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:28.483 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 90002 89986 75.9725 50.6875 0.0119081 0.0131573 2023-09-23T20:51:28.629 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:51:28.791 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:28.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:28.901 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:28.948 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:29.044 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:29.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:29.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:29.149 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:29.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:28 smithi121 ceph-mon[211526]: failed to encode map e185 with expected crc 2023-09-23T20:51:29.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:28 smithi121 ceph-mon[211526]: pgmap v583: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 2/262797 objects degraded (0.001%); 78/262797 objects misplaced (0.030%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:51:29.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:28 smithi121 ceph-mon[219709]: failed to encode map e185 with expected crc 2023-09-23T20:51:29.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:28 smithi121 ceph-mon[219709]: pgmap v583: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 2/262797 objects degraded (0.001%); 78/262797 objects misplaced (0.030%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:51:29.179 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:29.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:28 smithi123 ceph-mon[193903]: failed to encode map e185 with expected crc 2023-09-23T20:51:29.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:28 smithi123 ceph-mon[193903]: pgmap v583: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 2/262797 objects degraded (0.001%); 78/262797 objects misplaced (0.030%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T20:51:29.483 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 90876 90860 75.6879 54.625 0.00584033 0.013207 2023-09-23T20:51:29.636 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:51:29.814 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:29.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:29.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:29.920 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:29.964 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:30.040 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:30.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:30.147 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:30.173 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:30.376 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:30.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:30.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:30.415 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:30.482 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:30.483 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 91419 91403 75.1387 33.9375 0.003514 0.0132191 2023-09-23T20:51:30.509 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:30.556 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:30.663 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:30.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:30.767 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:30.795 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:30.847 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:30.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:30.954 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:30.961 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:51:31.000 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:51:31.012 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:31.045 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:51:31.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[211526]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:51:31.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[211526]: failed to encode map e186 with expected crc 2023-09-23T20:51:31.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[211526]: failed to encode map e186 with expected crc 2023-09-23T20:51:31.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[211526]: pgmap v585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 2/270930 objects degraded (0.001%); 78/270930 objects misplaced (0.029%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:51:31.072 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[219709]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:51:31.072 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[219709]: failed to encode map e186 with expected crc 2023-09-23T20:51:31.072 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[219709]: failed to encode map e186 with expected crc 2023-09-23T20:51:31.072 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:30 smithi121 ceph-mon[219709]: pgmap v585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 2/270930 objects degraded (0.001%); 78/270930 objects misplaced (0.029%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:51:31.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:51:31.096 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:31.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:31.101 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:30 smithi123 ceph-mon[193903]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:51:31.102 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:30 smithi123 ceph-mon[193903]: failed to encode map e186 with expected crc 2023-09-23T20:51:31.102 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:30 smithi123 ceph-mon[193903]: failed to encode map e186 with expected crc 2023-09-23T20:51:31.102 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:30 smithi123 ceph-mon[193903]: pgmap v585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 2/270930 objects degraded (0.001%); 78/270930 objects misplaced (0.029%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:51:31.136 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:51:31.180 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:51:31.202 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:31.213 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.272 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:51:31.286 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:31.338 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:51:31.388 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:31.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:31.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:31.488 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 92001 91985 74.6303 36.375 0.0169186 0.0133949 2023-09-23T20:51:31.502 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:31.625 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:31.906 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:31.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1418974055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2/270930 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:51:31.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: failed to encode map e187 with expected crc 2023-09-23T20:51:31.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1418974055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:31.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2/270930 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:32.014 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:32.039 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: failed to encode map e187 with expected crc 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: failed to encode map e187 with expected crc 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: failed to encode map e187 with expected crc 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: failed to encode map e187 with expected crc 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: failed to encode map e187 with expected crc 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1418974055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2/270930 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:51:32.488 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 92714 92698 74.2449 44.5625 0.0214785 0.0134647 2023-09-23T20:51:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:32.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:32.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:33.074 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:33.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:32 smithi121 ceph-mon[211526]: pgmap v587: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 2/270930 objects degraded (0.001%); 78/270930 objects misplaced (0.029%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:51:33.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:32 smithi121 ceph-mon[219709]: pgmap v587: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 2/270930 objects degraded (0.001%); 78/270930 objects misplaced (0.029%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:51:33.141 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:32 smithi123 ceph-mon[193903]: pgmap v587: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 2/270930 objects degraded (0.001%); 78/270930 objects misplaced (0.029%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:51:33.462 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:33.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:33.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:33.488 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 93773 93757 74.1429 66.1875 0.0084881 0.0134826 2023-09-23T20:51:33.570 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:33.621 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:33.697 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:33.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:33.808 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:34.086 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:34.093 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:51:34.093 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194759 8.b 8:d1bf2064:::benchmark_data_smithi121_252781_object85584:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 193.91301343000001, 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0533581320000001, 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.094 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194759 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170899+0000", 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170910+0000", 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1206000000000001e-05 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170925+0000", 2023-09-23T20:51:34.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4596999999999999e-05 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170936+0000", 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1445e-05 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171104+0000", 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016754899999999999 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171116+0000", 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1989e-05 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224175+0000", 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0530593850000001 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224259+0000", 2023-09-23T20:51:34.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3837000000000004e-05 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194764 8.b 8:d2640955:::benchmark_data_smithi121_252781_object85589:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 193.912958651, 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.053600919, 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.099 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194764 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170955+0000", 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170961+0000", 2023-09-23T20:51:34.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2619999999999998e-06 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170968+0000", 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.345e-06 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170973+0000", 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3410000000000002e-06 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171165+0000", 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019139 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171173+0000", 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6260000000000001e-06 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224498+0000", 2023-09-23T20:51:34.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0533245570000001 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224557+0000", 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8866000000000002e-05 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194773 8.b 8:dae7dbf1:::benchmark_data_smithi121_252781_object85598:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:51:34.104 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 193.91293562000001, 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067740498, 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194773 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170978+0000", 2023-09-23T20:51:34.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170981+0000", 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.249e-06 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170986+0000", 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6590000000000004e-06 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170991+0000", 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7709999999999997e-06 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171203+0000", 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021228600000000001 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171210+0000", 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0199999999999997e-06 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224626+0000", 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0534161630000001 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:51:34.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224718+0000", 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1296000000000001e-05 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238636+0000", 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013917802 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238719+0000", 2023-09-23T20:51:34.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3721999999999996e-05 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194787 8.b 8:dd8b32fd:::benchmark_data_smithi121_252781_object85612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 193.91291896000001, 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067236472, 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.111 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194787 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170997+0000", 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2220000000000001e-06 2023-09-23T20:51:34.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171003+0000", 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4620000000000004e-06 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171008+0000", 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9250000000000003e-06 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171302+0000", 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000294331 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171311+0000", 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7250000000000003e-06 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224751+0000", 2023-09-23T20:51:34.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.053440331 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224842+0000", 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1292000000000006e-05 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224927+0000", 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5044999999999996e-05 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237978+0000", 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013050277000000001 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237987+0000", 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3090000000000001e-06 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238081+0000", 2023-09-23T20:51:34.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4129999999999995e-05 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238135+0000", 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4212999999999997e-05 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238232+0000", 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6571999999999996e-05 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.118 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194805 8.b 8:de52f621:::benchmark_data_smithi121_252781_object85630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 193.895820942, 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0503357229999999, 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.119 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194805 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188092+0000", 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188100+0000", 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.729e-06 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188113+0000", 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3016999999999999e-05 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188122+0000", 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3230000000000004e-06 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188182+0000", 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9775000000000003e-05 2023-09-23T20:51:34.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188194+0000", 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2452e-05 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227071+0000", 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.038876744 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227205+0000", 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013409199999999999 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227299+0000", 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3601e-05 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237991+0000", 2023-09-23T20:51:34.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010692421000000001 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238032+0000", 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0800000000000002e-05 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238293+0000", 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000260938 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238331+0000", 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8381999999999999e-05 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238429+0000", 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7999999999999997e-05 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194818 8.b 8:db0a1c54:::benchmark_data_smithi121_252781_object85643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 193.87796211099999, 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0410233019999999, 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194818 2023-09-23T20:51:34.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205951+0000", 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205957+0000", 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205975+0000", 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.766e-05 2023-09-23T20:51:34.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205984+0000", 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4099999999999997e-06 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.206051+0000", 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6910999999999997e-05 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.206063+0000", 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2242000000000001e-05 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230211+0000", 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.024147707 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230377+0000", 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000165635 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230528+0000", 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015100899999999999 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238042+0000", 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075139159999999998 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238773+0000", 2023-09-23T20:51:34.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00073170099999999997 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246813+0000", 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080395229999999998 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246870+0000", 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6894000000000001e-05 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246976+0000", 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000105787 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194827 8.b 8:d1592b5b:::benchmark_data_smithi121_252781_object85652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 193.86100700700001, 2023-09-23T20:51:34.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0286707319999999, 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194827 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:51:34.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222906+0000", 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222913+0000", 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3719999999999999e-06 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222926+0000", 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3294e-05 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222935+0000", 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4789999999999996e-06 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222996+0000", 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1631000000000007e-05 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.223009+0000", 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2109e-05 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.234721+0000", 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0117129 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.234865+0000", 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014378500000000001 2023-09-23T20:51:34.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.235011+0000", 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014604 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238786+0000", 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003774576 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238821+0000", 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5635e-05 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251421+0000", 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012599552 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251476+0000", 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5387999999999999e-05 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251578+0000", 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000101727 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.142 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216597 8.0 8:06510814:::benchmark_data_smithi121_252781_object107422:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.371975714, 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.054665387, 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216597 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711937+0000", 2023-09-23T20:51:34.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711944+0000", 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.25e-06 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711957+0000", 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2999e-05 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711966+0000", 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9719999999999998e-06 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.712019+0000", 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2799000000000001e-05 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.712028+0000", 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.003e-06 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766511+0000", 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.054482731 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766604+0000", 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3139000000000004e-05 2023-09-23T20:51:34.147 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216620 8.0 8:04259452:::benchmark_data_smithi121_252781_object107445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.348671494, 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0569858000000001, 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.148 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216620 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:51:34.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735241+0000", 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735249+0000", 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6429999999999995e-06 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.150 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735279+0000", 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9805e-05 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735289+0000", 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9820000000000003e-06 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735397+0000", 2023-09-23T20:51:34.151 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107877 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735410+0000", 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2962000000000001e-05 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766912+0000", 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.031502782 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.152 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.767011+0000", 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8770999999999993e-05 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.767472+0000", 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000460922 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785103+0000", 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017630777 2023-09-23T20:51:34.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787562+0000", 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024589770000000002 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792167+0000", 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046052189999999998 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.154 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792202+0000", 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5182000000000003e-05 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792229+0000", 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6404999999999999e-05 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.155 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216632 8.0 8:09792f9f:::benchmark_data_smithi121_252781_object107457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.156 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:51:34.156 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.338884471, 2023-09-23T20:51:34.156 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0473433670000001, 2023-09-23T20:51:34.156 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.157 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.157 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216632 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.158 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745028+0000", 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745036+0000", 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1319999999999994e-06 2023-09-23T20:51:34.159 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745053+0000", 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6854000000000002e-05 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745062+0000", 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4940000000000003e-06 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.160 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745125+0000", 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3314999999999999e-05 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745136+0000", 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0546e-05 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782661+0000", 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.037525241 2023-09-23T20:51:34.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782813+0000", 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015243099999999999 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782964+0000", 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000150925 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792128+0000", 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091633930000000006 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792279+0000", 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015110899999999999 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792323+0000", 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3788000000000002e-05 2023-09-23T20:51:34.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792350+0000", 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7915e-05 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792373+0000", 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2816e-05 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.164 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216637 8.0 8:0d71f1da:::benchmark_data_smithi121_252781_object107462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.329487062, 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0470203279999999, 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.165 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216637 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.166 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754426+0000", 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754433+0000", 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5839999999999997e-06 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.167 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754450+0000", 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7129000000000002e-05 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754458+0000", 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1689999999999992e-06 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754544+0000", 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5140000000000001e-05 2023-09-23T20:51:34.168 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754557+0000", 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3107e-05 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785135+0000", 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0305782050000001 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785220+0000", 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4860999999999999e-05 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785315+0000", 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5088999999999997e-05 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792290+0000", 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069754370000000001 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792437+0000", 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014619000000000001 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801364+0000", 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089271820000000005 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801415+0000", 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1214999999999997e-05 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801448+0000", 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.277e-05 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.172 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216642 8.0 8:0cd41a68:::benchmark_data_smithi121_252781_object107467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.32944073600001, 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047113078, 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216642 2023-09-23T20:51:34.173 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754473+0000", 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754476+0000", 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8870000000000001e-06 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754483+0000", 2023-09-23T20:51:34.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.985e-06 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754488+0000", 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.977e-06 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754584+0000", 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6200999999999996e-05 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.176 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754593+0000", 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5310000000000001e-06 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787595+0000", 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.033002559 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787703+0000", 2023-09-23T20:51:34.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107431 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787777+0000", 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4598000000000006e-05 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792296+0000", 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045180970000000004 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792476+0000", 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000180669 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801501+0000", 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090245069999999993 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801549+0000", 2023-09-23T20:51:34.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8742000000000002e-05 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801587+0000", 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7317000000000002e-05 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216645 8.0 8:049490ab:::benchmark_data_smithi121_252781_object107470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.180 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.32060554399999, 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.038903957, 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216645 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763307+0000", 2023-09-23T20:51:34.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763315+0000", 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0779999999999998e-06 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763331+0000", 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6860000000000001e-05 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.183 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763341+0000", 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4250000000000004e-06 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.765814+0000", 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024728929999999999 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.765825+0000", 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1243000000000001e-05 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.184 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.789907+0000", 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.024082245 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.790014+0000", 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106702 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.185 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.790102+0000", 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7682999999999994e-05 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792526+0000", 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024248820000000002 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801599+0000", 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090724640000000006 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.186 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802118+0000", 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00051864700000000001 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802173+0000", 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5547000000000002e-05 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802213+0000", 2023-09-23T20:51:34.187 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9702e-05 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216663 8.0 8:0ce516a0:::benchmark_data_smithi121_252781_object107488:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.298736332, 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.035871486, 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.188 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216663 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:51:34.189 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785177+0000", 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785184+0000", 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8469999999999998e-06 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.190 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785199+0000", 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5784999999999999e-05 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785209+0000", 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7880000000000001e-06 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785281+0000", 2023-09-23T20:51:34.191 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2002000000000006e-05 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785295+0000", 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3675e-05 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792572+0000", 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0072767949999999 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.192 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792688+0000", 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116554 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792802+0000", 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011395099999999999 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801610+0000", 2023-09-23T20:51:34.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088076600000000001 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.814161+0000", 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012550716 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.820977+0000", 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068159379999999997 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821010+0000", 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3442000000000003e-05 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821050+0000", 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9595999999999998e-05 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216676 8.0 8:0d0e61e5:::benchmark_data_smithi121_252781_object107501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.28041206699999, 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.017644091, 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216676 2023-09-23T20:51:34.196 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803507+0000", 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8500000000000002e-06 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803521+0000", 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4942e-05 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803529+0000", 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1300000000000003e-06 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803579+0000", 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0317e-05 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.199 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803589+0000", 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8409999999999995e-06 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795218+0000", 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99162894599999996 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.200 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795317+0000", 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9674999999999997e-05 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795433+0000", 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011584 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801615+0000", 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061815459999999996 2023-09-23T20:51:34.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.820910+0000", 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01929556 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821082+0000", 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017190599999999999 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821115+0000", 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2373000000000003e-05 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821146+0000", 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1846e-05 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.203 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216707 8.0 8:07b8eb28:::benchmark_data_smithi121_252781_object107532:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.842567+0000", 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 163.24134702800001, 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.003503821, 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216707 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.204 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842567+0000", 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842567+0000", 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842566+0000", 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842574+0000", 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9219999999999997e-06 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842587+0000", 2023-09-23T20:51:34.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.332e-05 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842595+0000", 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1089999999999998e-06 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842638+0000", 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2599999999999999e-05 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.207 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.842646+0000", 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4689999999999997e-06 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.805031+0000", 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96238472100000005 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.208 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.805144+0000", 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000113306 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.805262+0000", 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000117655 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.820871+0000", 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015609247 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.209 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.831973+0000", 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011102201000000001 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.845970+0000", 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013996312 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.846026+0000", 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6746999999999997e-05 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.846071+0000", 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4744999999999999e-05 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57331 9.1 9:8abe57f8:::benchmark_data_smithi121_293770_object57330:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:51:34.211 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 46.987724387999997, 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1261466759999998, 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57331 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.212 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096188+0000", 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096226+0000", 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7861999999999998e-05 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096239+0000", 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2727e-05 2023-09-23T20:51:34.214 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096289+0000", 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0037999999999998e-05 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.100438+0000", 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041489939999999996 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.215 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.100451+0000", 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2942000000000001e-05 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093896+0000", 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99344541600000003 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093910+0000", 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3175e-05 2023-09-23T20:51:34.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212636+0000", 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.118726068 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212786+0000", 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015021899999999999 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:51:34.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212927+0000", 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014154 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219297+0000", 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063694299999999997 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219311+0000", 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3786e-05 2023-09-23T20:51:34.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219344+0000", 2023-09-23T20:51:34.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3769000000000001e-05 2023-09-23T20:51:34.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219408+0000", 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3341000000000002e-05 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222337+0000", 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029289189999999999 2023-09-23T20:51:34.220 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57332 9.1 9:830fc147:::benchmark_data_smithi121_293770_object57331:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 46.982554354999998, 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1239288190000001, 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.221 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57332 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.222 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101359+0000", 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101380+0000", 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1217e-05 2023-09-23T20:51:34.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101396+0000", 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5953000000000001e-05 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101405+0000", 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7679999999999997e-06 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101456+0000", 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1297e-05 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101470+0000", 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4127e-05 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093968+0000", 2023-09-23T20:51:34.225 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.992497713 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093975+0000", 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5719999999999997e-06 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217098+0000", 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1231222729999999 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.226 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217225+0000", 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000127253 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217358+0000", 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013342700000000001 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222449+0000", 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050906379999999998 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222491+0000", 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1764999999999997e-05 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222529+0000", 2023-09-23T20:51:34.228 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8318999999999997e-05 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222585+0000", 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6302999999999999e-05 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225289+0000", 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027034759999999998 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.229 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57335 9.1 9:8de699c5:::benchmark_data_smithi121_293770_object57334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 46.982396066, 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1379105439999999, 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:34.230 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57335 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101541+0000", 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3251000000000002e-05 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101553+0000", 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2714999999999999e-05 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101561+0000", 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8099999999999998e-06 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101611+0000", 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9898000000000002e-05 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.234 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.234 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101618+0000", 2023-09-23T20:51:34.234 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3380000000000001e-06 2023-09-23T20:51:34.234 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.234 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094001+0000", 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99238338000000004 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094008+0000", 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7530000000000004e-06 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225332+0000", 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1313241270000001 2023-09-23T20:51:34.235 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225481+0000", 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000149269 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225643+0000", 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016234999999999999 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.236 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.230166+0000", 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045225370000000001 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231616+0000", 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0014502429999999999 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236427+0000", 2023-09-23T20:51:34.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048111459999999997 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236496+0000", 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8417999999999994e-05 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239429+0000", 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002933125 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.238 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57345 9.1 9:89ab31a2:::benchmark_data_smithi121_293770_object57344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 46.956114423999999, 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1243519640000001, 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:51:34.239 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57345 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:51:34.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127798+0000", 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127828+0000", 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9895000000000001e-05 2023-09-23T20:51:34.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127846+0000", 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8281e-05 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127857+0000", 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.081e-05 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127899+0000", 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2271999999999998e-05 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127908+0000", 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5469999999999997e-06 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094072+0000", 2023-09-23T20:51:34.243 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96616466300000003 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094083+0000", 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.098e-05 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227666+0000", 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.133582614 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:51:34.244 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227790+0000", 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012393300000000001 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227940+0000", 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000149989 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231704+0000", 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037640120000000002 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.245 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239483+0000", 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0077796499999999999 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249105+0000", 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096215469999999994 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:51:34.246 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249162+0000", 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7080999999999998e-05 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252152+0000", 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002989916 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:51:34.247 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:51:34.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:34.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:34.248 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:34.349 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:34.391 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:34.483 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:34.488 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:51:34.489444+0000 min lat: 0.0010359 max lat: 2.15326 avg lat: 0.013521 2023-09-23T20:51:34.489 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:51:34.489 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 94690 94674 73.9326 57.3125 0.00952562 0.013521 2023-09-23T20:51:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:34.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:34.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:34.598 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:51:34.624 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:34.667 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:34.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:34.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:34 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1390673614' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:34 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:34.770 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:34.844 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:51:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:51:34] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2023-09-23T20:51:34.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:34 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1390673614' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:34.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:34 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:34.845 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:34 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1390673614' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:34.845 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:34 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:34.847 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:35.362 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:51:35.431 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:35.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:35.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:35.491 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 95483 95467 73.6295 49.5625 0.0210013 0.0135768 2023-09-23T20:51:35.537 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:35.563 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:35.654 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:51:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:35 smithi123 ceph-mon[193903]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:35 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:35 smithi123 ceph-mon[193903]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:51:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:35 smithi123 ceph-mon[193903]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:35 smithi123 ceph-mon[193903]: pgmap v589: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 928 KiB/s, 13 objects/s recovering 2023-09-23T20:51:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:35 smithi123 ceph-mon[193903]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:35 smithi123 ceph-mon[193903]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[219709]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:35.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[219709]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[219709]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[219709]: pgmap v589: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 928 KiB/s, 13 objects/s recovering 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[219709]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[219709]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[211526]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[211526]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:51:35.841 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[211526]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.842 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[211526]: pgmap v589: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s; 928 KiB/s, 13 objects/s recovering 2023-09-23T20:51:35.842 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[211526]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.842 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:35 smithi121 ceph-mon[211526]: failed to encode map e188 with expected crc 2023-09-23T20:51:35.891 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:35.995 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:36.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:36.111 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:36.170 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:51:36.171 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:36.254 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:36.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:36.360 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:36.417 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:36.491 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 96277 96261 73.3368 49.625 0.0165397 0.0136306 2023-09-23T20:51:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:36 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/270930 objects degraded (0.001%), 1 pg degraded) 2023-09-23T20:51:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:36 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T20:51:36.823 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:36.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:36.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:36 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/270930 objects degraded (0.001%), 1 pg degraded) 2023-09-23T20:51:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:36 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T20:51:36.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:36 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/270930 objects degraded (0.001%), 1 pg degraded) 2023-09-23T20:51:36.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:36 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T20:51:36.935 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:37.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:37.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:37.150 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:37.190 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:37.297 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:37.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:37.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:37.401 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:37.491 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 97233 97217 73.1731 59.75 0.0209168 0.0136614 2023-09-23T20:51:37.535 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:37.542 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:37.680 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:37.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:37.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:37.791 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:37.851 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:37.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:37 smithi121 ceph-mon[211526]: pgmap v590: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s; 810 KiB/s, 11 objects/s recovering 2023-09-23T20:51:37.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:37 smithi121 ceph-mon[219709]: pgmap v590: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s; 810 KiB/s, 11 objects/s recovering 2023-09-23T20:51:37.925 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:37.928 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:37 smithi123 ceph-mon[193903]: pgmap v590: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s; 810 KiB/s, 11 objects/s recovering 2023-09-23T20:51:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:37.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:38.030 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:38.058 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:38.358 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:38.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:38.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:38.469 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:38.491 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 98032 98016 72.8965 49.9375 0.00577962 0.0136936 2023-09-23T20:51:38.497 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:38.667 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:51:38.882 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2390612500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:51:38.883 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:38 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:51:38.883 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2390612500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:51:38.883 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:38 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:51:38.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2390612500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:51:38.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:38 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:98516 9.c e189/164 9:31319440:::benchmark_data_smithi121_293770_object98515:head v 189'6137, mlcod=189'6135)", 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:39.013780+0000", 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0095728629999999992, 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0096008310000000006, 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:51:39.025 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.013780+0000", 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.013780+0000", 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.013778+0000", 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.013890+0000", 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011109500000000001 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:39.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.013913+0000", 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3047000000000001e-05 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.013921+0000", 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4600000000000003e-06 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.014028+0000", 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107396 2023-09-23T20:51:39.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.014052+0000", 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3232999999999999e-05 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:98517 9.c e189/164 9:3a64b11c:::benchmark_data_smithi121_293770_object98516:head v 189'6138, mlcod=189'6135)", 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:39.017433+0000", 2023-09-23T20:51:39.029 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0059199639999999998, 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0060203929999999998, 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017433+0000", 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017433+0000", 2023-09-23T20:51:39.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017432+0000", 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017472+0000", 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0661999999999998e-05 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017485+0000", 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2927e-05 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017492+0000", 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4829999999999996e-06 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:39.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017651+0000", 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015936300000000001 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.017674+0000", 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2535e-05 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:98519 9.c e189/164 9:3650c974:::benchmark_data_smithi121_293770_object98518:head v 189'6139, mlcod=189'6135)", 2023-09-23T20:51:39.033 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:39.020060+0000", 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0032921780000000002, 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034526919999999998, 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020060+0000", 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020060+0000", 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020060+0000", 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:39.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020143+0000", 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3778999999999999e-05 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020156+0000", 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2533e-05 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:39.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020163+0000", 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0960000000000003e-06 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020851+0000", 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00068788599999999997 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.020876+0000", 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4589000000000001e-05 2023-09-23T20:51:39.037 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:98521 9.c e189/164)", 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:39.022973+0000", 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.00037917400000000003, 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00059299699999999995, 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T20:51:39.038 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.022973+0000", 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.022973+0000", 2023-09-23T20:51:39.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.022973+0000", 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.023033+0000", 2023-09-23T20:51:39.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0294000000000002e-05 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.023050+0000", 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6693000000000001e-05 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:39.023056+0000", 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8020000000000003e-06 2023-09-23T20:51:39.041 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.042 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:39.042 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.042 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:39.042 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T20:51:39.042 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 4 2023-09-23T20:51:39.042 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:51:39.490 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:39.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 98923 98907 72.6903 55.6875 0.0106977 0.0137518 2023-09-23T20:51:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:39.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:39.601 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:39.713 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:51:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:51:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:39 smithi123 ceph-mon[193903]: pgmap v592: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 708 KiB/s, 10 objects/s recovering 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[219709]: pgmap v592: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 708 KiB/s, 10 objects/s recovering 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: failed to encode map e189 with expected crc 2023-09-23T20:51:39.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:39 smithi121 ceph-mon[211526]: pgmap v592: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 708 KiB/s, 10 objects/s recovering 2023-09-23T20:51:39.985 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:51:40.254 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:40.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:40.358 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:40.440 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:40.496 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 99727 99711 72.4294 50.25 0.0169236 0.0138016 2023-09-23T20:51:40.546 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:40.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:40.587 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:40.656 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:40.702 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:40 smithi123 ceph-mon[193903]: failed to encode map e189 with expected crc 2023-09-23T20:51:40.706 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:40.900 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:40.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:40.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:40 smithi121 ceph-mon[211526]: failed to encode map e189 with expected crc 2023-09-23T20:51:40.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:40 smithi121 ceph-mon[219709]: failed to encode map e189 with expected crc 2023-09-23T20:51:41.007 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:41.058 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:41.290 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:51:41.308 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:51:41.327 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:51:41.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:51:41.383 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:51:41.471 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:51:41.499 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 100471 100455 72.1286 46.5 0.0201106 0.0138597 2023-09-23T20:51:41.509 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:51:41.540 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:51:41.555 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:51:41.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:51:41.666 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:41.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:41.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:41.771 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:41.810 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:41 smithi121 ceph-mon[219709]: pgmap v593: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s; 696 KiB/s, 9 objects/s recovering 2023-09-23T20:51:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3788644871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:41.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3220521055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:41.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:41 smithi121 ceph-mon[211526]: pgmap v593: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s; 696 KiB/s, 9 objects/s recovering 2023-09-23T20:51:41.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3788644871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:41.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3220521055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:41 smithi123 ceph-mon[193903]: pgmap v593: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s; 696 KiB/s, 9 objects/s recovering 2023-09-23T20:51:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3788644871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3220521055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:42.100 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:42.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:42.112 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:51:42.207 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:42.268 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:42.500 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 101226 101210 71.8451 47.1875 0.0144286 0.013914 2023-09-23T20:51:42.579 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:42.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:42.665 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:51:42.666 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:51:42.686 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:42.777 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:43.315 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:43.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:43.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3788644871' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:43.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:51:43.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: pgmap v595: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s 2023-09-23T20:51:43.399 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3788644871' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:43.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: pgmap v595: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s 2023-09-23T20:51:43.401 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:43.420 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:43.424 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:51:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3788644871' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: failed to encode map e190 with expected crc 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:43.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: pgmap v595: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s 2023-09-23T20:51:43.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:51:43.504 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:43.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 102080 102064 71.634 53.375 0.0147722 0.0139553 2023-09-23T20:51:43.538 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:43.605 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:51:43.642 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:43.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:43.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:43.766 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:43.844 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:43.948 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:43.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:43.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:44.063 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:44.124 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:44.223 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:44.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:44.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:44.235 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:44.333 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:44.399 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:44.475 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:44.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.0085 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 102885 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 71.4411 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 36.228 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 224.812 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 1143 2023-09-23T20:51:44.505 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 579.648 2023-09-23T20:51:44.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 3597 2023-09-23T20:51:44.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T20:51:44.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0139938 2023-09-23T20:51:44.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.0388609 2023-09-23T20:51:44.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 2.15326 2023-09-23T20:51:44.506 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.0010359 2023-09-23T20:51:44.512 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:51:44.579 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:44.605 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:44.647 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:44.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:44.751 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:44.808 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:45.044 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:51:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:51:44] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T20:51:45.073 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:45.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:45.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:45.098 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:45.152 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:51:45.177 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:45.218 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:45.571 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:45.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:45.680 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:45.708 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:45.796 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:45.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:45.858 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:45 smithi123 ceph-mon[193903]: pgmap v596: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-09-23T20:51:45.913 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:46.009 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:45 smithi121 ceph-mon[211526]: pgmap v596: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-09-23T20:51:46.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:45 smithi121 ceph-mon[219709]: pgmap v596: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 914 op/s 2023-09-23T20:51:46.018 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:46.140 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:51:46.140 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:51:46.373 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:46.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:46.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:46.482 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:46.511 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:46.554 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:46.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:46.658 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:46.698 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:46.829 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:46.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:46.882 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:46.937 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:47.004 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:47.013 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:51:47.107 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/438433267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:47.107 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:47.107 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:47 smithi121 ceph-mon[211526]: pgmap v597: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 857 op/s 2023-09-23T20:51:47.117 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:47.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:47.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:51:47.228 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:47.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/438433267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:47.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:47.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:47 smithi123 ceph-mon[193903]: pgmap v597: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 857 op/s 2023-09-23T20:51:47.310 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:47.381 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/438433267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:47.381 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:51:47.381 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:47 smithi121 ceph-mon[219709]: pgmap v597: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 857 op/s 2023-09-23T20:51:47.558 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:48.109 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:48 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:48.109 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:48 smithi123 ceph-mon[193903]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.109 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:48.110 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:48 smithi123 ceph-mon[193903]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:51:48.110 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:48 smithi123 ceph-mon[193903]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.110 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:48 smithi123 ceph-mon[193903]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.120 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:48.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:48.237 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:48.356 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:48.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:48.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[219709]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:48.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[219709]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:51:48.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[219709]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.394 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[219709]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:48.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[211526]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:51:48.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[211526]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:51:48.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[211526]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:48 smithi121 ceph-mon[211526]: failed to encode map e191 with expected crc 2023-09-23T20:51:48.605 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:48.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:48.716 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:48.743 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:48.928 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:48.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:49.039 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:49.086 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:49.136 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:51:49.349 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/504380069' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.350 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:49 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.350 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:49 smithi121 ceph-mon[219709]: pgmap v599: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 775 op/s 2023-09-23T20:51:49.350 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/504380069' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.350 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:49 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.350 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:49 smithi121 ceph-mon[211526]: pgmap v599: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 775 op/s 2023-09-23T20:51:49.356 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:49.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:49.462 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/504380069' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:49 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:49 smithi123 ceph-mon[193903]: pgmap v599: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 775 op/s 2023-09-23T20:51:49.540 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:49.630 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:51:49.630 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:51:49.673 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:49.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:49.778 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:49.836 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:49.945 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:49.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:49.960 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:51:50.052 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:50.091 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:50.181 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:50.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:50.192 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:51:50.254 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[211526]: failed to encode map e192 with expected crc 2023-09-23T20:51:50.254 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[211526]: failed to encode map e192 with expected crc 2023-09-23T20:51:50.254 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:50.254 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[211526]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:51:50.255 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[219709]: failed to encode map e192 with expected crc 2023-09-23T20:51:50.255 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[219709]: failed to encode map e192 with expected crc 2023-09-23T20:51:50.255 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:50.255 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:50 smithi121 ceph-mon[219709]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:51:50.288 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:50.325 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:50.470 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:51:50.471 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:50 smithi123 ceph-mon[193903]: failed to encode map e192 with expected crc 2023-09-23T20:51:50.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:50 smithi123 ceph-mon[193903]: failed to encode map e192 with expected crc 2023-09-23T20:51:50.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:50 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:50.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:50 smithi123 ceph-mon[193903]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:51:50.484 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:50.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:50.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:50.493 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:51:50.522 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:51:50.560 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:51:50.592 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:50.608 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:51:50.648 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:51:50.660 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:50.692 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:51:50.726 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:51:50.760 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:51:50.784 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.785 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.785 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.785 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.785 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.785 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.785 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.785 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:51:50.786 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:50.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:50.850 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:50.893 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:50.967 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:51.110 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:51.143 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:51.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:51.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:51.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2453737203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:51.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:51 smithi123 ceph-mon[193903]: pgmap v601: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 838 op/s 2023-09-23T20:51:51.253 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:51:51.302 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2453737203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:51 smithi121 ceph-mon[211526]: pgmap v601: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 838 op/s 2023-09-23T20:51:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2453737203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:51 smithi121 ceph-mon[219709]: pgmap v601: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 838 op/s 2023-09-23T20:51:51.414 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:51:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:51.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:51:51.526 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:51.587 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:51.862 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:51.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:51.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:51.869 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:51.966 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:52.007 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:52.093 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:52.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:52.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:52.203 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:52.278 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:53.122 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:51:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:51:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T20:51:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T20:51:53.122 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:51:53.290 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:53.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:53.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:51:53.474 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:53.868 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:51:53.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:53.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:51:53.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:51:53.973 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:54.012 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:51:54.032 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:53 smithi123 ceph-mon[193903]: pgmap v602: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 582 op/s 2023-09-23T20:51:54.038 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:54.118 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:54.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:54.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:54.125 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:53 smithi121 ceph-mon[219709]: pgmap v602: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 582 op/s 2023-09-23T20:51:54.126 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:53 smithi121 ceph-mon[211526]: pgmap v602: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 582 op/s 2023-09-23T20:51:54.226 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:51:54.295 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:54.386 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:51:54.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:54.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:51:54.499 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:54.588 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:54.622 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:54.814 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:54.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:54.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:54.927 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:51:54.962 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:54.975 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:55.147 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:51:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:55.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:51:55.165 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:51:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:51:54] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T20:51:55.261 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:51:55.336 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:55.477 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:51:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:55.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:51:55.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:55.607 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:55.822 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:51:55.837 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:51:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[219709]: pgmap v603: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 4.9 MiB/s wr, 848 op/s 2023-09-23T20:51:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3583375158' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/767520138' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:55.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[211526]: pgmap v603: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 4.9 MiB/s wr, 848 op/s 2023-09-23T20:51:55.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3583375158' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/767520138' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:55.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:55.952 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:55.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:56.055 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:55 smithi123 ceph-mon[193903]: pgmap v603: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 4.9 MiB/s wr, 848 op/s 2023-09-23T20:51:56.055 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3583375158' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:56.055 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:56.055 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/767520138' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:56.055 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:51:56.065 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:56.095 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:56.649 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:51:56.649 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:51:56.649 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:19254 8.3 8:c32fabf0:::benchmark_data_smithi121_252781_object19253:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:44:59.997455+0000", 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 416.64826310799998, 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2634655770000001, 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.650 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 19254 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997455+0000", 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997455+0000", 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997453+0000", 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997700+0000", 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000246405 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997714+0000", 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.485e-05 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997725+0000", 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0322999999999999e-05 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998826+0000", 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011006989999999999 2023-09-23T20:51:56.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998835+0000", 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5009999999999993e-06 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.260835+0000", 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.261999836 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.260920+0000", 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5592000000000003e-05 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:19291 8.3 8:c64c6b63:::benchmark_data_smithi121_252781_object19290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:44:59.997734+0000", 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 416.64798402999998, 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2702351030000001, 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.655 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 19291 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997734+0000", 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.656 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997734+0000", 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997733+0000", 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997799+0000", 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5835999999999998e-05 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997810+0000", 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1031e-05 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997816+0000", 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9630000000000003e-06 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998860+0000", 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0010436289999999999 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998866+0000", 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2369999999999997e-06 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.261444+0000", 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.262578027 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.659 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.261578+0000", 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000133793 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.261706+0000", 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012807199999999999 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.263821+0000", 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021155929999999998 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.263872+0000", 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0754999999999998e-05 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.266921+0000", 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0030492079999999999 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.266985+0000", 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3360000000000003e-05 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.267969+0000", 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00098445600000000009 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:19300 8.3 8:cd31d4d8:::benchmark_data_smithi121_252781_object19299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 416.64789548599998, 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2736549619999999, 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 19300 2023-09-23T20:51:56.663 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997875+0000", 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3315e-05 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997884+0000", 2023-09-23T20:51:56.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1230000000000006e-06 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997891+0000", 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7789999999999998e-06 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998892+0000", 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0010008300000000001 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998898+0000", 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3430000000000001e-06 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.262620+0000", 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2637219449999999 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.262722+0000", 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010137900000000001 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.262886+0000", 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016391999999999999 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.264035+0000", 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001149033 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.264274+0000", 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00023902299999999999 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.270500+0000", 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062266190000000001 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.270562+0000", 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1509999999999999e-05 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.271477+0000", 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000915715 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.670 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95048 8.4 8:2454d6a3:::benchmark_data_smithi121_252781_object95047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 352.58158645100002, 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2752492069999999, 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95048 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064130+0000", 2023-09-23T20:51:56.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064169+0000", 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8658999999999997e-05 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064181+0000", 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2089999999999999e-05 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064188+0000", 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2400000000000001e-06 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064346+0000", 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015842800000000001 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064356+0000", 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.01e-05 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072299+0000", 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0079425179999999 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072311+0000", 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1701e-05 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.324944+0000", 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.252633109 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.325108+0000", 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016413699999999999 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:51:56.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.325270+0000", 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000162201 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333069+0000", 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077994290000000001 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333084+0000", 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4813e-05 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.338377+0000", 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0052931949999999997 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.338442+0000", 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4246999999999997e-05 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.339381+0000", 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00093899200000000004 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95052 8.d 8:bca20ec2:::benchmark_data_smithi121_252781_object95051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 352.58151801899999, 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2831711729999999, 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.679 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95052 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064199+0000", 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064234+0000", 2023-09-23T20:51:56.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4929999999999999e-05 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064242+0000", 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8450000000000003e-06 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064248+0000", 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8989999999999998e-06 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.682 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064373+0000", 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012531700000000001 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064383+0000", 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4509999999999998e-06 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072655+0000", 2023-09-23T20:51:56.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.008272584 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072666+0000", 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0275000000000001e-05 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336059+0000", 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2633933260000001 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336211+0000", 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000152299 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336390+0000", 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017826800000000001 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.340844+0000", 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044546439999999998 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.340887+0000", 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2695999999999999e-05 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346536+0000", 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056488220000000004 2023-09-23T20:51:56.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346600+0000", 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3899000000000006e-05 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.347371+0000", 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00077147299999999995 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95057 8.4 8:2a7799b6:::benchmark_data_smithi121_252781_object95056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 352.57506274100001, 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2767475730000002, 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95057 2023-09-23T20:51:56.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070654+0000", 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070684+0000", 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0334000000000001e-05 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070699+0000", 2023-09-23T20:51:56.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4211e-05 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070706+0000", 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7249999999999997e-06 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070765+0000", 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8112999999999997e-05 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.691 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070775+0000", 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0766e-05 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072335+0000", 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0015602290000001 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072343+0000", 2023-09-23T20:51:56.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0609999999999998e-06 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329624+0000", 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2572813840000001 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329748+0000", 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000123678 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329878+0000", 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013038500000000001 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333185+0000", 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033074290000000002 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333622+0000", 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043623000000000002 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346585+0000", 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012963585 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346641+0000", 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6227999999999998e-05 2023-09-23T20:51:56.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.347403+0000", 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00076131300000000001 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:172436 8.0 8:091633cd:::benchmark_data_smithi121_252781_object63261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:47:47.118851+0000", 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 249.52686739200001, 2023-09-23T20:51:56.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0583673920000001, 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 172436 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118851+0000", 2023-09-23T20:51:56.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118851+0000", 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118849+0000", 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118853+0000", 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8290000000000001e-06 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118862+0000", 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5210000000000002e-06 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118867+0000", 2023-09-23T20:51:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2259999999999996e-06 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.121949+0000", 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0030824630000000001 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.121957+0000", 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4499999999999998e-06 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.141454+0000", 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0194968390000001 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.144877+0000", 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003423129 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1", 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.145018+0000", 2023-09-23T20:51:56.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014086200000000001 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.157359+0000", 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012340894999999999 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177135+0000", 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019776065999999998 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177169+0000", 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4159999999999998e-05 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177218+0000", 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9178999999999998e-05 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:172444 8.8 8:1dfce91d:::benchmark_data_smithi121_252781_object63269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:47:47.118875+0000", 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 249.526843431, 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.058494244, 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 172444 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118875+0000", 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118875+0000", 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118874+0000", 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118877+0000", 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5569999999999998e-06 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118883+0000", 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1650000000000003e-06 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118887+0000", 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1359999999999999e-06 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.121980+0000", 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0030927149999999998 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.121985+0000", 2023-09-23T20:51:56.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4410000000000001e-06 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.151693+0000", 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.029708158 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.151806+0000", 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000112716 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 7", 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.151920+0000", 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113928 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.165278+0000", 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013357707 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177275+0000", 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011997173 2023-09-23T20:51:56.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177303+0000", 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8028999999999999e-05 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177369+0000", 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5949000000000002e-05 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.710 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186212 8.8 8:10804386:::benchmark_data_smithi121_252781_object77037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 230.15022802600001, 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1681942240000001, 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.711 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186212 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:51:56.712 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495489+0000", 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495493+0000", 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1779999999999999e-06 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495500+0000", 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7600000000000002e-06 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495504+0000", 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9720000000000003e-06 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500662+0000", 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051571189999999999 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500891+0000", 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022905000000000001 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 7", 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.501010+0000", 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119268 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509849+0000", 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088391219999999996 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509859+0000", 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0032e-05 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.716 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513758+0000", 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0038989819999999 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513773+0000", 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5116999999999999e-05 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602445+0000", 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0886714449999999 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602553+0000", 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010843199999999999 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663605+0000", 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.061052150999999999 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663684+0000", 2023-09-23T20:51:56.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9040000000000002e-05 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186226 8.8 8:109cf42b:::benchmark_data_smithi121_252781_object77051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 230.11609557599999, 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0872477360000001, 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.719 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186226 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529620+0000", 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529629+0000", 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8680000000000005e-06 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529649+0000", 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9874e-05 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529657+0000", 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6680000000000001e-06 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529724+0000", 2023-09-23T20:51:56.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7138000000000007e-05 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529737+0000", 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3083e-05 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513799+0000", 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98406165899999998 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513808+0000", 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8489999999999995e-06 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602617+0000", 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0888096490000001 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.724 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602726+0000", 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010896 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602846+0000", 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011963099999999999 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616554+0000", 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013708016 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616622+0000", 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7794999999999997e-05 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616656+0000", 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4397999999999998e-05 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616718+0000", 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1920000000000003e-05 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616870+0000", 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015218 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186232 8.8 8:18ce4fcc:::benchmark_data_smithi121_252781_object77057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:51:56.727 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 230.112799056, 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1309886809999998, 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186232 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532918+0000", 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:51:56.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532923+0000", 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7319999999999998e-06 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532936+0000", 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.293e-05 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532942+0000", 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6980000000000003e-06 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532993+0000", 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1447999999999999e-05 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532999+0000", 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3389999999999998e-06 2023-09-23T20:51:56.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513832+0000", 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98083224099999999 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513840+0000", 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1070000000000005e-06 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607582+0000", 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0937428 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607747+0000", 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016451500000000001 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.608823+0000", 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001075559 2023-09-23T20:51:56.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616564+0000", 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077410530000000003 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616924+0000", 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00036078199999999998 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663714+0000", 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.046789848000000002 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663776+0000", 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2114999999999997e-05 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663908+0000", 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013124799999999999 2023-09-23T20:51:56.735 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186215 8.0 8:00f6bf9b:::benchmark_data_smithi121_252781_object77040:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 229.13144115899999, 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.162298345, 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.736 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186215 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514275+0000", 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514284+0000", 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9570000000000008e-06 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514297+0000", 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2663000000000001e-05 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514305+0000", 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2970000000000004e-06 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514439+0000", 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013414199999999999 2023-09-23T20:51:56.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514449+0000", 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3139999999999993e-06 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649698+0000", 2023-09-23T20:51:56.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1352495979999999 2023-09-23T20:51:56.741 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.741 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.741 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:51:56.741 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649809+0000", 2023-09-23T20:51:56.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110911 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676465+0000", 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026655729999999999 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676575+0000", 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110305 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.742 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186222 8.0 8:0eef9874:::benchmark_data_smithi121_252781_object77047:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 229.13138356300001, 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1614129209999999, 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.743 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186222 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.744 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514339+0000", 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1680000000000003e-06 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.745 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514348+0000", 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0140000000000004e-06 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514353+0000", 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514474+0000", 2023-09-23T20:51:56.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121105 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514482+0000", 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4039999999999998e-06 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649859+0000", 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.135377012 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.747 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649954+0000", 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4914999999999995e-05 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.650033+0000", 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9120000000000001e-05 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663245+0000", 2023-09-23T20:51:56.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013211926000000001 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674215+0000", 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010970126 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675506+0000", 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001290914 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.749 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675559+0000", 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2643999999999999e-05 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675747+0000", 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000188815 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.750 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186230 8.0 8:041ac878:::benchmark_data_smithi121_252781_object77055:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 229.131358186, 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1616748569999999, 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186230 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.751 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514359+0000", 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514363+0000", 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8360000000000002e-06 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514370+0000", 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4899999999999997e-06 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514374+0000", 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6750000000000001e-06 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514504+0000", 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012982599999999999 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514512+0000", 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5959999999999998e-06 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654104+0000", 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.139592663 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654250+0000", 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014595400000000001 2023-09-23T20:51:56.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654394+0000", 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143442 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663683+0000", 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092891689999999999 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.756 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674249+0000", 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010566002 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675814+0000", 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015649539999999999 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675875+0000", 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1224999999999997e-05 2023-09-23T20:51:56.757 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676035+0000", 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015948000000000001 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186231 8.0 8:0bcd4c07:::benchmark_data_smithi121_252781_object77056:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:51:56.758 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 229.131337551, 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.168650363, 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186231 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:51:56.759 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514383+0000", 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514388+0000", 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5969999999999999e-06 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514393+0000", 2023-09-23T20:51:56.761 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.16e-06 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514533+0000", 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140841 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514540+0000", 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9890000000000003e-06 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659037+0000", 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.144496156 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659183+0000", 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014606600000000001 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659334+0000", 2023-09-23T20:51:56.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015159700000000001 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663874+0000", 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045394600000000004 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674301+0000", 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010427439 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682815+0000", 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085137149999999998 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682865+0000", 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0510999999999998e-05 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.683031+0000", 2023-09-23T20:51:56.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016547400000000001 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209245 8.8 8:1443b4f6:::benchmark_data_smithi121_252781_object100070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 195.917343746, 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.08464988, 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.766 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209245 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.767 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728373+0000", 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728379+0000", 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5089999999999999e-06 2023-09-23T20:51:56.768 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728395+0000", 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5710999999999999e-05 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728402+0000", 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7190000000000004e-06 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.769 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728795+0000", 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039319799999999999 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728805+0000", 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6690000000000001e-06 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757335+0000", 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0285302249999999 2023-09-23T20:51:56.770 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757427+0000", 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1818999999999996e-05 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757509+0000", 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1922000000000006e-05 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768732+0000", 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011223713 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812940+0000", 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.044207659000000003 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812972+0000", 2023-09-23T20:51:56.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1633e-05 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813001+0000", 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9300000000000001e-05 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813024+0000", 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3244999999999999e-05 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.773 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209248 8.8 8:136e166f:::benchmark_data_smithi121_252781_object100073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 195.91730728300001, 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.084736245, 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.774 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209248 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.775 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728410+0000", 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728413+0000", 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.756e-06 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.776 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728419+0000", 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3940000000000001e-06 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728423+0000", 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9400000000000004e-06 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728828+0000", 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00040448300000000002 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728835+0000", 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8469999999999998e-06 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761382+0000", 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0325472419999999 2023-09-23T20:51:56.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761496+0000", 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113714 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761583+0000", 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7459000000000004e-05 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.779 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.772112+0000", 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010529221 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812957+0000", 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040844889000000002 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813061+0000", 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000104036 2023-09-23T20:51:56.780 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813109+0000", 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7704999999999998e-05 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813147+0000", 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8046000000000001e-05 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.781 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209254 8.8 8:15bdd1a9:::benchmark_data_smithi121_252781_object100079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.740455+0000", 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 195.90526324800001, 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0728612630000001, 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.782 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209254 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740455+0000", 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740455+0000", 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740454+0000", 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740459+0000", 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9130000000000002e-06 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.784 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740471+0000", 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2014999999999999e-05 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740476+0000", 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9389999999999996e-06 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740521+0000", 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5689000000000003e-05 2023-09-23T20:51:56.785 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740535+0000", 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.381e-05 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.765266+0000", 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247304669999999 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.765359+0000", 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3289999999999996e-05 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.765433+0000", 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4147999999999995e-05 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.775265+0000", 2023-09-23T20:51:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0098324220000000004 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813215+0000", 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.037949825999999999 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813268+0000", 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3152999999999997e-05 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.788 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813294+0000", 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5103e-05 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813316+0000", 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2408e-05 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.789 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209264 8.8 8:1f8be86d:::benchmark_data_smithi121_252781_object100089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.752342+0000", 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 195.89337612700001, 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.061116331, 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209264 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.790 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752342+0000", 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752342+0000", 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752341+0000", 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752345+0000", 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9480000000000001e-06 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752355+0000", 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0171e-05 2023-09-23T20:51:56.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752360+0000", 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3150000000000002e-06 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752402+0000", 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2116999999999997e-05 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.793 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752408+0000", 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.981e-06 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768763+0000", 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.016355417 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768864+0000", 2023-09-23T20:51:56.794 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010038400000000001 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768936+0000", 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2478999999999995e-05 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.781881+0000", 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012944496 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.795 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813390+0000", 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031509523999999997 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813413+0000", 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2639000000000001e-05 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813438+0000", 2023-09-23T20:51:56.796 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5664999999999999e-05 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813458+0000", 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9808000000000001e-05 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209274 8.8 8:19434a6f:::benchmark_data_smithi121_252781_object100099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:51:56.797 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.764577+0000", 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 195.88114055200001, 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0490839320000001, 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209274 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:51:56.798 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.764577+0000", 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.764577+0000", 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:51:56.799 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.764576+0000", 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.764581+0000", 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8520000000000003e-06 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.764592+0000", 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1623999999999999e-05 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.764598+0000", 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.091e-06 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.765324+0000", 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00072648899999999998 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.765333+0000", 2023-09-23T20:51:56.801 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0990000000000005e-06 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.772144+0000", 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.006811253 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.772261+0000", 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000116097 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.772332+0000", 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1649000000000005e-05 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.784893+0000", 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012560445999999999 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813496+0000", 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.028603244 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813569+0000", 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3156000000000002e-05 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813621+0000", 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1548000000000002e-05 2023-09-23T20:51:56.804 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813661+0000", 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0774e-05 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:51:56.805 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:51:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:51:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[211526]: pgmap v605: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 2.0 MiB/s wr, 833 op/s 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:56 smithi121 ceph-mon[219709]: pgmap v605: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 2.0 MiB/s wr, 833 op/s 2023-09-23T20:51:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: failed to encode map e193 with expected crc 2023-09-23T20:51:56.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:56 smithi123 ceph-mon[193903]: pgmap v605: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 2.0 MiB/s wr, 833 op/s 2023-09-23T20:51:57.654 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:51:57.682 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:57.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:57.724 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:51:57.794 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:51:57.859 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:57.987 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:51:58.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:58.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:51:58.126 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:58.214 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:58.325 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:51:58.332 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:58.333 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:58.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:51:58.334 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:58.398 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:51:58.435 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:51:58.445 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:58.454 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:51:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:58.501 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:51:58.503 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:58.543 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:51:58.582 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:51:58.622 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:51:58.664 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:51:58.706 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:51:58.719 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:51:58.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.723 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.724 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.724 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.724 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.724 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:51:58.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:51:58.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:51:58.827 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:51:58.884 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:51:59.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2240283876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:59.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:51:59 smithi121 ceph-mon[211526]: pgmap v606: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:51:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2240283876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:51:59 smithi121 ceph-mon[219709]: pgmap v606: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:51:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2240283876' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:51:59.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:51:59 smithi123 ceph-mon[193903]: pgmap v606: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-09-23T20:51:59.722 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:52:00.275 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:00.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:00.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:00.383 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:00.445 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:00.544 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:00.544 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:00.622 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:00.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:00.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:00.651 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:00.727 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:00.785 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:00.895 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:00.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:00.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:01.001 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:01.068 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:01.243 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:01.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:01.310 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:52:01.348 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:01.432 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:01.689 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:52:01.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:01.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:01.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:01.843 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:01.888 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:01.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:01.931 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:52:01.991 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:02.014 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:01 smithi121 ceph-mon[211526]: pgmap v607: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T20:52:02.014 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4256501105' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:02.014 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:01 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:02.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:01 smithi121 ceph-mon[219709]: pgmap v607: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T20:52:02.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4256501105' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:02.015 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:01 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:02.016 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:02.058 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:02.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:02.146 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:52:02.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:01 smithi123 ceph-mon[193903]: pgmap v607: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T20:52:02.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4256501105' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:02.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:01 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:02.162 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:02.210 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:02.389 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:52:02.648 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:02.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:02.763 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:02.791 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:02.835 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:02.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:02.944 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:03.011 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:03.234 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:03.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:03.246 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:52:03.290 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:03 smithi123 ceph-mon[193903]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.291 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:03.291 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:03 smithi123 ceph-mon[193903]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:52:03.291 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:03 smithi123 ceph-mon[193903]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.291 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:03 smithi123 ceph-mon[193903]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.291 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:03 smithi123 ceph-mon[193903]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.291 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:03 smithi123 ceph-mon[193903]: pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T20:52:03.341 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:03.378 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:03.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[219709]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:03.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[219709]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:52:03.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[219709]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[219709]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[219709]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[219709]: pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T20:52:03.407 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[211526]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.408 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:03.408 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[211526]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:52:03.408 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[211526]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.408 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[211526]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.408 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[211526]: failed to encode map e194 with expected crc 2023-09-23T20:52:03.408 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:03 smithi121 ceph-mon[211526]: pgmap v609: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-09-23T20:52:03.454 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:03.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:03.558 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:03.608 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:03.653 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:03.752 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:52:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:03.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:03.859 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:03.922 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:03.980 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:03.980 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:04.021 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:04.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:04.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:04.125 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:04.213 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:52:04.216 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:04.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/551297410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:04.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:04.394 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/551297410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:04.394 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/551297410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:05.013 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:52:05.165 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:52:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:52:04] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.43.0" 2023-09-23T20:52:05.314 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:05.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:05.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:05.429 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:05.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:05.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T20:52:05.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:05.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:52:05.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:05.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:05 smithi123 ceph-mon[193903]: pgmap v611: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-09-23T20:52:05.504 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T20:52:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:52:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[211526]: pgmap v611: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: failed to encode map e195 with expected crc 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:05.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:05 smithi121 ceph-mon[219709]: pgmap v611: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-09-23T20:52:05.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:05.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:05.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:05.814 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:05.847 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:52:05.866 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:06.020 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:06.176 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:06.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:06.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:06.288 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:06.351 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:06.510 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:52:06.527 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:06.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:06.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:06.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:52:06.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:52:06.634 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:52:06.636 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:06.666 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:52:06.705 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:52:06.736 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:06.747 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:52:06.801 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:52:06.819 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:52:06.819 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:52:06.820 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:52:06.820 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:52:06.820 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:52:06.820 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:52:06.820 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:52:06.820 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:52:07.137 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2049734165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:07.138 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:07 smithi121 ceph-mon[211526]: pgmap v612: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T20:52:07.198 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:07.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:07.307 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:07.372 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:07.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2049734165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:07 smithi121 ceph-mon[219709]: pgmap v612: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T20:52:07.474 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2049734165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:07.474 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:07 smithi123 ceph-mon[193903]: pgmap v612: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-09-23T20:52:07.578 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:07.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:07.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:07.684 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:07.732 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:08.032 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:52:08.253 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:52:08.253 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:52:08.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162724 7.d 7:bce00e9a:::benchmark_data_smithi121_234442_object52417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.597343+0000", 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.65347799899996, 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.022867049, 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.254 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162724 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597343+0000", 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597343+0000", 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597342+0000", 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597348+0000", 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7780000000000004e-06 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597357+0000", 2023-09-23T20:52:08.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4020000000000008e-06 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597363+0000", 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7150000000000003e-06 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597409+0000", 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5683000000000003e-05 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597530+0000", 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012149799999999999 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.597653+0000", 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012325500000000001 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.604795+0000", 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0071415619999999997 2023-09-23T20:52:08.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.604817+0000", 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.19e-05 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.620144+0000", 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0153274290000001 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.620210+0000", 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5757000000000001e-05 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162758 7.d 7:b021cf25:::benchmark_data_smithi121_234442_object52451:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.652366+0000", 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.59845513100004, 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98482179999999997, 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.260 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162758 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652366+0000", 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652366+0000", 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652365+0000", 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652371+0000", 2023-09-23T20:52:08.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8360000000000003e-06 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652385+0000", 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3925e-05 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652392+0000", 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2210000000000002e-06 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652456+0000", 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4931999999999994e-05 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.652469+0000", 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2291e-05 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.620611+0000", 2023-09-23T20:52:08.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96814256499999995 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.620729+0000", 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117748 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.620860+0000", 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013051999999999999 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.630172+0000", 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.009312397 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.630201+0000", 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8547e-05 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637089+0000", 2023-09-23T20:52:08.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0068882170000000003 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637145+0000", 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5893999999999997e-05 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637188+0000", 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3003999999999997e-05 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.267 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162761 7.d 7:bf9696bf:::benchmark_data_smithi121_234442_object52454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.656577+0000", 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.59424412700002, 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98079633899999996, 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162761 2023-09-23T20:52:08.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656577+0000", 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656577+0000", 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656576+0000", 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656582+0000", 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0430000000000004e-06 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656598+0000", 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5888000000000001e-05 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656604+0000", 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656662+0000", 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8940999999999997e-05 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.656673+0000", 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0360000000000001e-05 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.624412+0000", 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96773918800000003 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.624494+0000", 2023-09-23T20:52:08.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1903000000000005e-05 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.624568+0000", 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4358999999999997e-05 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637035+0000", 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012466545000000001 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637235+0000", 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00020057899999999999 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637291+0000", 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5466000000000001e-05 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637336+0000", 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4987000000000001e-05 2023-09-23T20:52:08.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637373+0000", 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7317000000000002e-05 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162762 7.d 7:ba58a617:::benchmark_data_smithi121_234442_object52455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.657562+0000", 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.59325944099999, 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98727606000000001, 2023-09-23T20:52:08.275 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162762 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657562+0000", 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657562+0000", 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657561+0000", 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657565+0000", 2023-09-23T20:52:08.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.929e-06 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657575+0000", 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0611999999999999e-05 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657580+0000", 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9520000000000002e-06 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657617+0000", 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6465999999999999e-05 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.657624+0000", 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0620000000000003e-06 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.627400+0000", 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96977602100000004 2023-09-23T20:52:08.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.627510+0000", 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110473 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.627633+0000", 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000123051 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637241+0000", 2023-09-23T20:52:08.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0096074219999999991 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637412+0000", 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000171063 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.644730+0000", 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.007317802 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.644784+0000", 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4128000000000001e-05 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.644838+0000", 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3668999999999997e-05 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.282 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162774 7.d 7:bc704f04:::benchmark_data_smithi121_234442_object52467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.675136+0000", 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.57568534899997, 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97997332100000001, 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162774 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.675136+0000", 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.675136+0000", 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.675134+0000", 2023-09-23T20:52:08.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.675144+0000", 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4879999999999996e-06 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.675161+0000", 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7374e-05 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.675168+0000", 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3429999999999998e-06 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.676316+0000", 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011473220000000001 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.676325+0000", 2023-09-23T20:52:08.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5980000000000003e-06 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.630224+0000", 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95389869500000002 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.630342+0000", 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011769000000000001 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.630467+0000", 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012566300000000001 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637245+0000", 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0067776540000000001 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.644698+0000", 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0074531689999999999 2023-09-23T20:52:08.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655030+0000", 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010331809000000001 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655071+0000", 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0629e-05 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655109+0000", 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8277e-05 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162782 7.d 7:b68003fd:::benchmark_data_smithi121_234442_object52475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.684051+0000", 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.56676999299998, 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98743330200000001, 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.290 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162782 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684051+0000", 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684051+0000", 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684050+0000", 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684056+0000", 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9950000000000002e-06 2023-09-23T20:52:08.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684070+0000", 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4083000000000001e-05 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684076+0000", 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0950000000000001e-06 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684117+0000", 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1199000000000003e-05 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.684126+0000", 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7590000000000003e-06 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.633185+0000", 2023-09-23T20:52:08.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94905944099999995 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.633279+0000", 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3659000000000006e-05 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.633360+0000", 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1217999999999994e-05 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637417+0000", 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040565130000000003 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655005+0000", 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.017588606999999999 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.671385+0000", 2023-09-23T20:52:08.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016379411 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.671442+0000", 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6746000000000002e-05 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.671484+0000", 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2660999999999997e-05 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162804 7.d 7:b6af36a2:::benchmark_data_smithi121_234442_object52497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.715357+0000", 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.53546425699994, 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96041696399999998, 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162804 2023-09-23T20:52:08.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715357+0000", 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715357+0000", 2023-09-23T20:52:08.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715356+0000", 2023-09-23T20:52:08.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715362+0000", 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5459999999999997e-06 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715378+0000", 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5855e-05 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715384+0000", 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4880000000000004e-06 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715443+0000", 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8279999999999998e-05 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.715454+0000", 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0876000000000001e-05 2023-09-23T20:52:08.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637446+0000", 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92199202999999996 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637560+0000", 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011488 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.637693+0000", 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013239799999999999 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.644852+0000", 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0071590610000000004 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655168+0000", 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010316051999999999 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.675713+0000", 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020544514 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.675747+0000", 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4079999999999999e-05 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.675774+0000", 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7073e-05 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162811 7.d 7:b83177e2:::benchmark_data_smithi121_234442_object52504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.730913+0000", 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.51990835000004, 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94970387199999995, 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.307 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162811 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.730913+0000", 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.730913+0000", 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.730911+0000", 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.730917+0000", 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6949999999999997e-06 2023-09-23T20:52:08.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.730929+0000", 2023-09-23T20:52:08.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2177e-05 2023-09-23T20:52:08.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.730934+0000", 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0370000000000002e-06 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.730992+0000", 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7315000000000003e-05 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.731003+0000", 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1731e-05 2023-09-23T20:52:08.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.641084+0000", 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.91008057099999995 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.641198+0000", 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113925 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.641334+0000", 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013577700000000001 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.651798+0000", 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010464601 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.661870+0000", 2023-09-23T20:52:08.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010072057000000001 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.680531+0000", 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018660507 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.680579+0000", 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7920000000000002e-05 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.680616+0000", 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7790000000000002e-05 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162837 7.d 7:b9d91607:::benchmark_data_smithi121_234442_object52530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.770318+0000", 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.48050340400005, 2023-09-23T20:52:08.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.91708648100000001, 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162837 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770318+0000", 2023-09-23T20:52:08.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770318+0000", 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770316+0000", 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770322+0000", 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.181e-06 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770334+0000", 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2727e-05 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770340+0000", 2023-09-23T20:52:08.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7509999999999997e-06 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770403+0000", 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3021000000000005e-05 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.770414+0000", 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1139e-05 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.644889+0000", 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.87447459999999999 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.645010+0000", 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121357 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.645133+0000", 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012243200000000001 2023-09-23T20:52:08.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655178+0000", 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01004533 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.665106+0000", 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0099276869999999993 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687346+0000", 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022239990000000001 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687378+0000", 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2733e-05 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687404+0000", 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5738999999999999e-05 2023-09-23T20:52:08.322 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162846 7.d 7:b198f38f:::benchmark_data_smithi121_234442_object52539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.787395+0000", 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.46342624800002, 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90012329000000002, 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.323 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162846 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787395+0000", 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787395+0000", 2023-09-23T20:52:08.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787394+0000", 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787397+0000", 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1250000000000001e-06 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787410+0000", 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2656e-05 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787415+0000", 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7319999999999998e-06 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787479+0000", 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3937999999999997e-05 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.787490+0000", 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1528e-05 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.648476+0000", 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86098530100000004 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.648591+0000", 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115295 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.648723+0000", 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013170900000000001 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.661824+0000", 2023-09-23T20:52:08.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013101106 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.671533+0000", 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0097095570000000006 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687436+0000", 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015903092000000001 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687477+0000", 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0207999999999999e-05 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687518+0000", 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1498000000000001e-05 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162870 7.d 7:b190f487:::benchmark_data_smithi121_234442_object52563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.824157+0000", 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.42666419, 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86350722499999999, 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.331 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162870 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824157+0000", 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824157+0000", 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824156+0000", 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824162+0000", 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0410000000000003e-06 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824172+0000", 2023-09-23T20:52:08.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0093e-05 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824176+0000", 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6659999999999999e-06 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824227+0000", 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1002999999999999e-05 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824238+0000", 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0426e-05 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.651837+0000", 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82759958300000003 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.651919+0000", 2023-09-23T20:52:08.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1539999999999995e-05 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.652025+0000", 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000105856 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.661830+0000", 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0098046709999999992 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.675824+0000", 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013994685 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687576+0000", 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011752171 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687624+0000", 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7301e-05 2023-09-23T20:52:08.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687664+0000", 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0293000000000002e-05 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162872 7.d 7:ba129df5:::benchmark_data_smithi121_234442_object52565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.824335+0000", 2023-09-23T20:52:08.338 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.42648644400003, 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86346113000000002, 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162872 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824335+0000", 2023-09-23T20:52:08.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824335+0000", 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824334+0000", 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824337+0000", 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.79e-06 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824344+0000", 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1080000000000004e-06 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824348+0000", 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8380000000000004e-06 2023-09-23T20:52:08.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824385+0000", 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6909000000000002e-05 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.824391+0000", 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.037e-06 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655199+0000", 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83080853099999996 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655303+0000", 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010341899999999999 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.655409+0000", 2023-09-23T20:52:08.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010668 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.661879+0000", 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064697920000000003 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.675851+0000", 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013971963 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687718+0000", 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01186661 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687761+0000", 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3268e-05 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687796+0000", 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4616999999999997e-05 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162881 7.d 7:b9b5a619:::benchmark_data_smithi121_234442_object52574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.839917+0000", 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.41090420399996, 2023-09-23T20:52:08.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84799435599999995, 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162881 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.839917+0000", 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.839917+0000", 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.839915+0000", 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.839922+0000", 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2620000000000004e-06 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.839940+0000", 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8363999999999999e-05 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.839947+0000", 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.545e-06 2023-09-23T20:52:08.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.840003+0000", 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6107999999999999e-05 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.840013+0000", 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0169e-05 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.658491+0000", 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.818478539 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.658610+0000", 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011896999999999999 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.658734+0000", 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012326700000000001 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.671327+0000", 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012593638000000001 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.682023+0000", 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010695652 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687830+0000", 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058074210000000001 2023-09-23T20:52:08.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687876+0000", 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5379e-05 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.687911+0000", 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5431e-05 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.353 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.34248.0:162900 7.d 7:b3699d38:::benchmark_data_smithi121_234442_object52593:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e101)", 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:43:08.882069+0000", 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 539.36875232299997, 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.81025775300000002, 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.34248", 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3933526361", 2023-09-23T20:52:08.354 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 162900 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882069+0000", 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882069+0000", 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882067+0000", 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882075+0000", 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3529999999999997e-06 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882092+0000", 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7747000000000001e-05 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882100+0000", 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6860000000000006e-06 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882156+0000", 2023-09-23T20:52:08.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6449000000000001e-05 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:08.882169+0000", 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2055999999999999e-05 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.661906+0000", 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.779737605 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.661979+0000", 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3149e-05 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.662051+0000", 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1433999999999994e-05 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.671538+0000", 2023-09-23T20:52:08.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0094876639999999998 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.682060+0000", 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010521648 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.692233+0000", 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010172904999999999 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.692281+0000", 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7698000000000003e-05 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:43:09.692326+0000", 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5760999999999998e-05 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:79943 9.b e181/177 9:dd7b4559:::benchmark_data_smithi121_293770_object79942:head v 181'5056, mlcod=176'3847)", 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:16.443951+0000", 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 51.806870054999997, 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020189849999999999, 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.443951+0000", 2023-09-23T20:52:08.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.443951+0000", 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.443950+0000", 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.444075+0000", 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012445300000000001 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.444091+0000", 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5991e-05 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.444096+0000", 2023-09-23T20:52:08.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8270000000000003e-06 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.444265+0000", 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016883800000000001 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.444302+0000", 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6650000000000003e-05 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.464096+0000", 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019794055000000001 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:16.464141+0000", 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4855999999999999e-05 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:95365 9.b e188/183 9:d26c7f9a:::benchmark_data_smithi121_293770_object95364:head v 188'6077, mlcod=188'6073)", 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:35.372791+0000", 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 32.878029918999999, 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019152076000000001, 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.372791+0000", 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.372791+0000", 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.372790+0000", 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.377076+0000", 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042857640000000001 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.377094+0000", 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8451e-05 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.377101+0000", 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.686e-06 2023-09-23T20:52:08.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.377158+0000", 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7045e-05 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.377182+0000", 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3815000000000002e-05 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.391904+0000", 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014721758999999999 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:35.391943+0000", 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9610999999999997e-05 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:96023 9.1 e188/180 9:87c1d260:::benchmark_data_smithi121_293770_object96022:head v 188'5937, mlcod=188'5934)", 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:36.211976+0000", 2023-09-23T20:52:08.371 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 32.038845144, 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023274020999999999, 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.211976+0000", 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.211976+0000", 2023-09-23T20:52:08.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.211975+0000", 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.212004+0000", 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9533999999999999e-05 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.212021+0000", 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6259000000000001e-05 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.212028+0000", 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.182e-06 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.212085+0000", 2023-09-23T20:52:08.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6808000000000002e-05 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.212107+0000", 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2163000000000001e-05 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.235193+0000", 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023086512999999999 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.235250+0000", 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6582999999999997e-05 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:96163 9.0 e188/186 9:075f4833:::benchmark_data_smithi121_293770_object96162:head v 188'5995, mlcod=188'5993)", 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:51:36.366945+0000", 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 31.883875688, 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018932211000000001, 2023-09-23T20:52:08.376 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.366945+0000", 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.366945+0000", 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.366942+0000", 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999962 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.366988+0000", 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6727999999999997e-05 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.367006+0000", 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7666999999999999e-05 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.367013+0000", 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5230000000000001e-06 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.373309+0000", 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062966239999999998 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.373331+0000", 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1917999999999999e-05 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.385821+0000", 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012490322 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:51:36.385877+0000", 2023-09-23T20:52:08.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6093e-05 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:120741 9.b e195/183 9:df89bab6:::benchmark_data_smithi121_293770_object17853:head v 195'7690, mlcod=195'7684)", 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:07.636954+0000", 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.61386693199999998, 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019040567000000001, 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.381 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.636954+0000", 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.636954+0000", 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.636954+0000", 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.636960+0000", 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.636971+0000", 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0546e-05 2023-09-23T20:52:08.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.636975+0000", 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6269999999999999e-06 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.641194+0000", 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042188030000000001 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.644584+0000", 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033894189999999999 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.655956+0000", 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011372106 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.655995+0000", 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8964000000000002e-05 2023-09-23T20:52:08.385 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:120782 9.0 e195/186 9:0044dad4:::benchmark_data_smithi121_293770_object17894:head v 195'7590, mlcod=195'7587)", 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:07.687093+0000", 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.56372810600000001, 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01942725, 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:08.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.687093+0000", 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.687093+0000", 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.687092+0000", 2023-09-23T20:52:08.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.687100+0000", 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1790000000000008e-06 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.687114+0000", 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4334e-05 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:08.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.687121+0000", 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1750000000000002e-06 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.689389+0000", 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002267884 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.689409+0000", 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0927000000000001e-05 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.706479+0000", 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.017069602 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:07.706520+0000", 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1038999999999998e-05 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:08.390 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:08.391 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:52:08.668 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:08.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:08.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:08.773 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:08.804 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:08.847 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:09.058 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:09.171 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:09.231 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:52:09.267 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:09.385 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:09.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:09.505 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:09.593 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:09.804 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:09.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:09.924 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:09.957 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:09 smithi123 ceph-mon[193903]: pgmap v613: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T20:52:09.957 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:09.965 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:52:10.118 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:10.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:10.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:09 smithi121 ceph-mon[211526]: pgmap v613: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T20:52:10.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:09 smithi121 ceph-mon[219709]: pgmap v613: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-09-23T20:52:10.222 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:10.260 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:10.447 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:10.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:10.553 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:10.619 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:10.730 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:10.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:10.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:10.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:52:10.794 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:52:10.843 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:10.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2220464839' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:10.882 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:11.050 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2220464839' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:11.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2220464839' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:11.170 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:11.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:11.274 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:11.301 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:11.748 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:52:11.749 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:52:11.799 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:11.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:11.892 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:11.907 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:11.970 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:12.056 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: pgmap v614: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2220464839' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1696667290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.061 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:12.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:12.163 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: pgmap v614: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T20:52:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2220464839' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:52:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1696667290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: pgmap v614: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2220464839' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: failed to encode map e196 with expected crc 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1696667290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:12.170 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:52:12.228 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:12.876 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:52:12.876 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:52:12.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:123959 9.a e197/164 9:5beb19b4:::benchmark_data_smithi121_293770_object21071:head v 197'7744, mlcod=197'7742)", 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:12.874496+0000", 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.001114678, 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011550569999999999, 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874496+0000", 2023-09-23T20:52:12.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874496+0000", 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874495+0000", 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874503+0000", 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6310000000000003e-06 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874516+0000", 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3574000000000001e-05 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874523+0000", 2023-09-23T20:52:12.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7859999999999999e-06 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874649+0000", 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000125969 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874674+0000", 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5533000000000001e-05 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.880 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:123957 9.9 e197/164)", 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:12.874546+0000", 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.001064539, 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011617579999999999, 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:12.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874546+0000", 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874546+0000", 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874545+0000", 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:12.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874551+0000", 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2249999999999999e-06 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874560+0000", 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9070000000000002e-06 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.874565+0000", 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9409999999999998e-06 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:123950 9.e 9:78d0c90d:::benchmark_data_smithi121_293770_object21062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:12.861163+0000", 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.014447549000000001, 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014599884, 2023-09-23T20:52:12.884 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 123950 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861163+0000", 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861163+0000", 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861162+0000", 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861168+0000", 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7580000000000003e-06 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861185+0000", 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6419999999999999e-05 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861193+0000", 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7840000000000004e-06 2023-09-23T20:52:12.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861250+0000", 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7374000000000003e-05 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861391+0000", 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014060799999999999 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.861512+0000", 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121536 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.865388+0000", 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003876305 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.889 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:123953 9.c e197/164 9:3d785722:::benchmark_data_smithi121_293770_object21065:head v 197'7791, mlcod=197'7787)", 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:12.869808+0000", 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0058028289999999998, 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0060030559999999997, 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869808+0000", 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869808+0000", 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869807+0000", 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:12.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869814+0000", 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8430000000000004e-06 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869827+0000", 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3807e-05 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869834+0000", 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2269999999999998e-06 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869892+0000", 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8544000000000001e-05 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.869917+0000", 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4958999999999999e-05 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:123949 9.7 e197/164 9:e0896eea:::benchmark_data_smithi121_293770_object21061:head v 197'7680, mlcod=197'7678)", 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:12.870472+0000", 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0051388370000000003, 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053758590000000002, 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870472+0000", 2023-09-23T20:52:12.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870472+0000", 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870471+0000", 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870479+0000", 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2989999999999997e-06 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870493+0000", 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3940999999999999e-05 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:12.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870499+0000", 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5300000000000004e-06 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870581+0000", 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2023999999999999e-05 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.870602+0000", 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.12e-05 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.897 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop_reply(client.45101.0:123950 9.e e197/164)", 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:12.873930+0000", 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0016805920000000001, 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019503630000000001, 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.873930+0000", 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.873930+0000", 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.873929+0000", 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:52:12.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.873935+0000", 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5590000000000002e-06 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.873950+0000", 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4331999999999999e-05 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:52:12.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:12.873957+0000", 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0079999999999996e-06 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 6 2023-09-23T20:52:12.901 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:52:12.910 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:12 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:12.910 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:12.910 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:12 smithi123 ceph-mon[193903]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:52:12.910 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:12 smithi123 ceph-mon[193903]: failed to encode map e197 with expected crc 2023-09-23T20:52:12.910 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:12.910 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:12.910 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[211526]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:52:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[211526]: failed to encode map e197 with expected crc 2023-09-23T20:52:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[219709]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:52:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[219709]: failed to encode map e197 with expected crc 2023-09-23T20:52:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:13.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:13.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:13.478 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:13.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:13.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:13.586 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:13.614 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:13.822 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:13.824 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:52:13.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:13.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:13.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:13 smithi123 ceph-mon[193903]: failed to encode map e197 with expected crc 2023-09-23T20:52:13.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:13 smithi123 ceph-mon[193903]: failed to encode map e197 with expected crc 2023-09-23T20:52:13.946 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:13 smithi123 ceph-mon[193903]: pgmap v617: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T20:52:13.947 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:14.019 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.035 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:13 smithi121 ceph-mon[211526]: failed to encode map e197 with expected crc 2023-09-23T20:52:14.035 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:13 smithi121 ceph-mon[211526]: failed to encode map e197 with expected crc 2023-09-23T20:52:14.035 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:13 smithi121 ceph-mon[211526]: pgmap v617: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T20:52:14.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:13 smithi121 ceph-mon[219709]: failed to encode map e197 with expected crc 2023-09-23T20:52:14.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:13 smithi121 ceph-mon[219709]: failed to encode map e197 with expected crc 2023-09-23T20:52:14.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:13 smithi121 ceph-mon[219709]: pgmap v617: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-09-23T20:52:14.175 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:52:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:14.283 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:14.360 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.472 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:14.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:14.551 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:52:14.600 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:14.663 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.742 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:14.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:14.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:14.818 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:14.849 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:14.898 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:14.920 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:52:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:52:14] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-09-23T20:52:14.938 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:52:14.985 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:14.996 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:52:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:15.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:15.031 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:52:15.058 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:52:15.101 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:52:15.103 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:15.154 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:52:15.180 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:15.190 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:52:15.230 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:52:15.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:52:15.314 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:52:15.465 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:15.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:15.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:15.570 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:15.599 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:15.640 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:15.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:15.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:15.743 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:15.808 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:15.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:15 smithi123 ceph-mon[193903]: pgmap v618: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T20:52:15.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3898422401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:16.030 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:16.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:16.136 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:15 smithi121 ceph-mon[219709]: pgmap v618: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T20:52:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3898422401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:16.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:15 smithi121 ceph-mon[211526]: pgmap v618: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-09-23T20:52:16.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3898422401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:16.195 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:16.689 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:16.689 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:16.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:52:16.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:16.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:17.052 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:17.078 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:17.224 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:17.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:17.328 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:17.391 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:17.454 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:17.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[211526]: pgmap v619: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T20:52:17.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:17.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:17.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.602 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:17.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:17.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:17.701 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:52:17.707 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:17.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:17 smithi123 ceph-mon[193903]: pgmap v619: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T20:52:17.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:17.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:17.747 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[219709]: pgmap v619: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T20:52:17.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:17.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:17.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:17.782 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:17.884 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:52:17.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:17.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:17.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:18.054 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:18.137 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:18.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:18.245 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:18.289 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:52:18.296 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:52:18.303 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:52:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:18.413 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:18.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:18.517 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:18.569 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:19.161 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:52:19.515 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:19.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:19.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:19.623 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:19.719 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":619,"stamp":"2023-09-23T20:52:16.702850+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5306587064,"num_objects":80988,"num_object_clones":0,"num_object_copies":242964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80988,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":125108,"num_write_kb":6587086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3668,"num_bytes_recovered":241381416,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123524,"ondisk_log_size":123524,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18392240,"kb_used_data":15628080,"kb_used_omap":20,"kb_used_meta":2764011,"kb_avail":731568976,"statfs":{"total":767960285184,"available":749126631424,"internally_reserved":0,"allocated":16003153920,"data_stored":15997758826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2830347918},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,3,3,3,0,0,2,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374996992,"num_objects":-5722,"num_object_clones":0,"num_object_copies":-17166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5722,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.007074"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":175,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788822+0000","last_change":"2023-09-23T20:46:05.100324+0000","last_active":"2023-09-23T20:52:10.788822+0000","last_peered":"2023-09-23T20:52:10.788822+0000","last_clean":"2023-09-23T20:52:10.788822+0000","last_became_active":"2023-09-23T20:46:05.099981+0000","last_became_peered":"2023-09-23T20:46:05.099981+0000","last_unstale":"2023-09-23T20:52:10.788822+0000","last_undegraded":"2023-09-23T20:52:10.788822+0000","last_fullsized":"2023-09-23T20:52:10.788822+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:19.876971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766367+0000","last_change":"2023-09-23T20:48:07.517973+0000","last_active":"2023-09-23T20:52:10.766367+0000","last_peered":"2023-09-23T20:52:10.766367+0000","last_clean":"2023-09-23T20:52:10.766367+0000","last_became_active":"2023-09-23T20:48:07.517681+0000","last_became_peered":"2023-09-23T20:48:07.517681+0000","last_unstale":"2023-09-23T20:52:10.766367+0000","last_undegraded":"2023-09-23T20:52:10.766367+0000","last_fullsized":"2023-09-23T20:52:10.766367+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.789594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":122,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.152827+0000","last_change":"2023-09-23T20:48:07.541333+0000","last_active":"2023-09-23T20:52:12.152827+0000","last_peered":"2023-09-23T20:52:12.152827+0000","last_clean":"2023-09-23T20:52:12.152827+0000","last_became_active":"2023-09-23T20:48:07.541014+0000","last_became_peered":"2023-09-23T20:48:07.541014+0000","last_unstale":"2023-09-23T20:52:12.152827+0000","last_undegraded":"2023-09-23T20:52:12.152827+0000","last_fullsized":"2023-09-23T20:52:12.152827+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:48.678651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":147,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779427+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:52:10.779427+0000","last_peered":"2023-09-23T20:52:10.779427+0000","last_clean":"2023-09-23T20:52:10.779427+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:52:10.779427+0000","last_undegraded":"2023-09-23T20:52:10.779427+0000","last_fullsized":"2023-09-23T20:52:10.779427+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":154,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766311+0000","last_change":"2023-09-23T20:48:07.540435+0000","last_active":"2023-09-23T20:52:10.766311+0000","last_peered":"2023-09-23T20:52:10.766311+0000","last_clean":"2023-09-23T20:52:10.766311+0000","last_became_active":"2023-09-23T20:48:07.540062+0000","last_became_peered":"2023-09-23T20:48:07.540062+0000","last_unstale":"2023-09-23T20:52:10.766311+0000","last_undegraded":"2023-09-23T20:52:10.766311+0000","last_fullsized":"2023-09-23T20:52:10.766311+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:19.449218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":29,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.787962+0000","last_change":"2023-09-23T20:51:05.998368+0000","last_active":"2023-09-23T20:52:10.787962+0000","last_peered":"2023-09-23T20:52:10.787962+0000","last_clean":"2023-09-23T20:52:10.787962+0000","last_became_active":"2023-09-23T20:51:05.997978+0000","last_became_peered":"2023-09-23T20:51:05.997978+0000","last_unstale":"2023-09-23T20:52:10.787962+0000","last_undegraded":"2023-09-23T20:52:10.787962+0000","last_fullsized":"2023-09-23T20:52:10.787962+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:02.281720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":145,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779911+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:52:10.779911+0000","last_peered":"2023-09-23T20:52:10.779911+0000","last_clean":"2023-09-23T20:52:10.779911+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:52:10.779911+0000","last_undegraded":"2023-09-23T20:52:10.779911+0000","last_fullsized":"2023-09-23T20:52:10.779911+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":154,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766224+0000","last_change":"2023-09-23T20:51:06.005347+0000","last_active":"2023-09-23T20:52:10.766224+0000","last_peered":"2023-09-23T20:52:10.766224+0000","last_clean":"2023-09-23T20:52:10.766224+0000","last_became_active":"2023-09-23T20:51:06.005036+0000","last_became_peered":"2023-09-23T20:51:06.005036+0000","last_unstale":"2023-09-23T20:52:10.766224+0000","last_undegraded":"2023-09-23T20:52:10.766224+0000","last_fullsized":"2023-09-23T20:52:10.766224+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:19.359325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":157,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774412+0000","last_change":"2023-09-23T20:45:00.007125+0000","last_active":"2023-09-23T20:52:10.774412+0000","last_peered":"2023-09-23T20:52:10.774412+0000","last_clean":"2023-09-23T20:52:10.774412+0000","last_became_active":"2023-09-23T20:45:00.006403+0000","last_became_peered":"2023-09-23T20:45:00.006403+0000","last_unstale":"2023-09-23T20:52:10.774412+0000","last_undegraded":"2023-09-23T20:52:10.774412+0000","last_fullsized":"2023-09-23T20:52:10.774412+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:27.906490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":132,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.767459+0000","last_change":"2023-09-23T20:51:06.024444+0000","last_active":"2023-09-23T20:52:10.767459+0000","last_peered":"2023-09-23T20:52:10.767459+0000","last_clean":"2023-09-23T20:52:10.767459+0000","last_became_active":"2023-09-23T20:51:06.023727+0000","last_became_peered":"2023-09-23T20:51:06.023727+0000","last_unstale":"2023-09-23T20:52:10.767459+0000","last_undegraded":"2023-09-23T20:52:10.767459+0000","last_fullsized":"2023-09-23T20:52:10.767459+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:24.860774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":160,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.776737+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:52:10.776737+0000","last_peered":"2023-09-23T20:52:10.776737+0000","last_clean":"2023-09-23T20:52:10.776737+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:52:10.776737+0000","last_undegraded":"2023-09-23T20:52:10.776737+0000","last_fullsized":"2023-09-23T20:52:10.776737+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":141,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.783026+0000","last_change":"2023-09-23T20:51:06.002973+0000","last_active":"2023-09-23T20:52:10.783026+0000","last_peered":"2023-09-23T20:52:10.783026+0000","last_clean":"2023-09-23T20:52:10.783026+0000","last_became_active":"2023-09-23T20:51:06.001732+0000","last_became_peered":"2023-09-23T20:51:06.001732+0000","last_unstale":"2023-09-23T20:52:10.783026+0000","last_undegraded":"2023-09-23T20:52:10.783026+0000","last_fullsized":"2023-09-23T20:52:10.783026+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:44.025395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":202,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.792424+0000","last_change":"2023-09-23T20:51:05.966238+0000","last_active":"2023-09-23T20:52:10.792424+0000","last_peered":"2023-09-23T20:52:10.792424+0000","last_clean":"2023-09-23T20:52:10.792424+0000","last_became_active":"2023-09-23T20:51:05.959313+0000","last_became_peered":"2023-09-23T20:51:05.959313+0000","last_unstale":"2023-09-23T20:52:10.792424+0000","last_undegraded":"2023-09-23T20:52:10.792424+0000","last_fullsized":"2023-09-23T20:52:10.792424+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:02.228179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788631+0000","last_change":"2023-09-23T20:51:05.998598+0000","last_active":"2023-09-23T20:52:10.788631+0000","last_peered":"2023-09-23T20:52:10.788631+0000","last_clean":"2023-09-23T20:52:10.788631+0000","last_became_active":"2023-09-23T20:51:05.998180+0000","last_became_peered":"2023-09-23T20:51:05.998180+0000","last_unstale":"2023-09-23T20:52:10.788631+0000","last_undegraded":"2023-09-23T20:52:10.788631+0000","last_fullsized":"2023-09-23T20:52:10.788631+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:56.185992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.776669+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:52:10.776669+0000","last_peered":"2023-09-23T20:52:10.776669+0000","last_clean":"2023-09-23T20:52:10.776669+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:52:10.776669+0000","last_undegraded":"2023-09-23T20:52:10.776669+0000","last_fullsized":"2023-09-23T20:52:10.776669+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":136,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.767994+0000","last_change":"2023-09-23T20:48:07.540014+0000","last_active":"2023-09-23T20:52:10.767994+0000","last_peered":"2023-09-23T20:52:10.767994+0000","last_clean":"2023-09-23T20:52:10.767994+0000","last_became_active":"2023-09-23T20:48:07.539722+0000","last_became_peered":"2023-09-23T20:48:07.539722+0000","last_unstale":"2023-09-23T20:52:10.767994+0000","last_undegraded":"2023-09-23T20:52:10.767994+0000","last_fullsized":"2023-09-23T20:52:10.767994+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.937232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":145,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779171+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:52:10.779171+0000","last_peered":"2023-09-23T20:52:10.779171+0000","last_clean":"2023-09-23T20:52:10.779171+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:52:10.779171+0000","last_undegraded":"2023-09-23T20:52:10.779171+0000","last_fullsized":"2023-09-23T20:52:10.779171+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":156,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774942+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:52:10.774942+0000","last_peered":"2023-09-23T20:52:10.774942+0000","last_clean":"2023-09-23T20:52:10.774942+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:52:10.774942+0000","last_undegraded":"2023-09-23T20:52:10.774942+0000","last_fullsized":"2023-09-23T20:52:10.774942+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":160,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.776472+0000","last_change":"2023-09-23T20:51:05.958217+0000","last_active":"2023-09-23T20:52:10.776472+0000","last_peered":"2023-09-23T20:52:10.776472+0000","last_clean":"2023-09-23T20:52:10.776472+0000","last_became_active":"2023-09-23T20:51:05.957874+0000","last_became_peered":"2023-09-23T20:51:05.957874+0000","last_unstale":"2023-09-23T20:52:10.776472+0000","last_undegraded":"2023-09-23T20:52:10.776472+0000","last_fullsized":"2023-09-23T20:52:10.776472+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:13.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":146,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779566+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:52:10.779566+0000","last_peered":"2023-09-23T20:52:10.779566+0000","last_clean":"2023-09-23T20:52:10.779566+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:52:10.779566+0000","last_undegraded":"2023-09-23T20:52:10.779566+0000","last_fullsized":"2023-09-23T20:52:10.779566+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":39,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.786850+0000","last_change":"2023-09-23T20:51:06.038448+0000","last_active":"2023-09-23T20:52:10.786850+0000","last_peered":"2023-09-23T20:52:10.786850+0000","last_clean":"2023-09-23T20:52:10.786850+0000","last_became_active":"2023-09-23T20:51:06.005965+0000","last_became_peered":"2023-09-23T20:51:06.005965+0000","last_unstale":"2023-09-23T20:52:10.786850+0000","last_undegraded":"2023-09-23T20:52:10.786850+0000","last_fullsized":"2023-09-23T20:52:10.786850+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:13.397771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":136,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766878+0000","last_change":"2023-09-23T20:48:07.557905+0000","last_active":"2023-09-23T20:52:10.766878+0000","last_peered":"2023-09-23T20:52:10.766878+0000","last_clean":"2023-09-23T20:52:10.766878+0000","last_became_active":"2023-09-23T20:48:07.557568+0000","last_became_peered":"2023-09-23T20:48:07.557568+0000","last_unstale":"2023-09-23T20:52:10.766878+0000","last_undegraded":"2023-09-23T20:52:10.766878+0000","last_fullsized":"2023-09-23T20:52:10.766878+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:14.229704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"197'7563","reported_seq":7606,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.650258+0000","last_change":"2023-09-23T20:50:08.077384+0000","last_active":"2023-09-23T20:52:12.650258+0000","last_peered":"2023-09-23T20:52:12.650258+0000","last_clean":"2023-09-23T20:52:12.650258+0000","last_became_active":"2023-09-23T20:50:08.077095+0000","last_became_peered":"2023-09-23T20:50:08.077095+0000","last_unstale":"2023-09-23T20:52:12.650258+0000","last_undegraded":"2023-09-23T20:52:12.650258+0000","last_fullsized":"2023-09-23T20:52:12.650258+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7563,"log_dups_size":0,"ondisk_log_size":7563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:34.442024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331022336,"num_objects":5051,"num_object_clones":0,"num_object_copies":15153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7563,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":158,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.777609+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:52:10.777609+0000","last_peered":"2023-09-23T20:52:10.777609+0000","last_clean":"2023-09-23T20:52:10.777609+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:52:10.777609+0000","last_undegraded":"2023-09-23T20:52:10.777609+0000","last_fullsized":"2023-09-23T20:52:10.777609+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774222+0000","last_change":"2023-09-23T20:51:05.962659+0000","last_active":"2023-09-23T20:52:10.774222+0000","last_peered":"2023-09-23T20:52:10.774222+0000","last_clean":"2023-09-23T20:52:10.774222+0000","last_became_active":"2023-09-23T20:51:05.962048+0000","last_became_peered":"2023-09-23T20:51:05.962048+0000","last_unstale":"2023-09-23T20:52:10.774222+0000","last_undegraded":"2023-09-23T20:52:10.774222+0000","last_fullsized":"2023-09-23T20:52:10.774222+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:03.530084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"197'7824","reported_seq":7867,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.349612+0000","last_change":"2023-09-23T20:50:08.083329+0000","last_active":"2023-09-23T20:52:13.349612+0000","last_peered":"2023-09-23T20:52:13.349612+0000","last_clean":"2023-09-23T20:52:13.349612+0000","last_became_active":"2023-09-23T20:50:08.082984+0000","last_became_peered":"2023-09-23T20:50:08.082984+0000","last_unstale":"2023-09-23T20:52:13.349612+0000","last_undegraded":"2023-09-23T20:52:13.349612+0000","last_fullsized":"2023-09-23T20:52:13.349612+0000","mapping_epoch":164,"log_start":"168'500","ondisk_log_start":"168'500","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7324,"log_dups_size":0,"ondisk_log_size":7324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.393867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339607552,"num_objects":5182,"num_object_clones":0,"num_object_copies":15546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7824,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.776857+0000","last_change":"2023-09-23T20:45:00.003948+0000","last_active":"2023-09-23T20:52:10.776857+0000","last_peered":"2023-09-23T20:52:10.776857+0000","last_clean":"2023-09-23T20:52:10.776857+0000","last_became_active":"2023-09-23T20:45:00.003639+0000","last_became_peered":"2023-09-23T20:45:00.003639+0000","last_unstale":"2023-09-23T20:52:10.776857+0000","last_undegraded":"2023-09-23T20:52:10.776857+0000","last_fullsized":"2023-09-23T20:52:10.776857+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:59.619696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":173,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788234+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:52:10.788234+0000","last_peered":"2023-09-23T20:52:10.788234+0000","last_clean":"2023-09-23T20:52:10.788234+0000","last_became_active":"2023-09-23T20:48:07.544029+0000","last_became_peered":"2023-09-23T20:48:07.544029+0000","last_unstale":"2023-09-23T20:52:10.788234+0000","last_undegraded":"2023-09-23T20:52:10.788234+0000","last_fullsized":"2023-09-23T20:52:10.788234+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:52.671448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"197'7724","reported_seq":7765,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.813960+0000","last_change":"2023-09-23T20:50:08.078801+0000","last_active":"2023-09-23T20:52:13.813960+0000","last_peered":"2023-09-23T20:52:13.813960+0000","last_clean":"2023-09-23T20:52:13.813960+0000","last_became_active":"2023-09-23T20:50:08.078386+0000","last_became_peered":"2023-09-23T20:50:08.078386+0000","last_unstale":"2023-09-23T20:52:13.813960+0000","last_undegraded":"2023-09-23T20:52:13.813960+0000","last_fullsized":"2023-09-23T20:52:13.813960+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7724,"log_dups_size":0,"ondisk_log_size":7724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:47.134724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328990720,"num_objects":5020,"num_object_clones":0,"num_object_copies":15060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7724,"num_write_kb":407808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":169,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.776646+0000","last_change":"2023-09-23T20:51:05.967804+0000","last_active":"2023-09-23T20:52:10.776646+0000","last_peered":"2023-09-23T20:52:10.776646+0000","last_clean":"2023-09-23T20:52:10.776646+0000","last_became_active":"2023-09-23T20:51:05.959914+0000","last_became_peered":"2023-09-23T20:51:05.959914+0000","last_unstale":"2023-09-23T20:52:10.776646+0000","last_undegraded":"2023-09-23T20:52:10.776646+0000","last_fullsized":"2023-09-23T20:52:10.776646+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:03.210361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":29,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.787181+0000","last_change":"2023-09-23T20:51:05.957335+0000","last_active":"2023-09-23T20:52:10.787181+0000","last_peered":"2023-09-23T20:52:10.787181+0000","last_clean":"2023-09-23T20:52:10.787181+0000","last_became_active":"2023-09-23T20:51:05.956858+0000","last_became_peered":"2023-09-23T20:51:05.956858+0000","last_unstale":"2023-09-23T20:52:10.787181+0000","last_undegraded":"2023-09-23T20:52:10.787181+0000","last_fullsized":"2023-09-23T20:52:10.787181+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:31.085919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"197'7662","reported_seq":7703,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.783192+0000","last_change":"2023-09-23T20:50:08.648424+0000","last_active":"2023-09-23T20:52:13.783192+0000","last_peered":"2023-09-23T20:52:13.783192+0000","last_clean":"2023-09-23T20:52:13.783192+0000","last_became_active":"2023-09-23T20:50:08.648031+0000","last_became_peered":"2023-09-23T20:50:08.648031+0000","last_unstale":"2023-09-23T20:52:13.783192+0000","last_undegraded":"2023-09-23T20:52:13.783192+0000","last_fullsized":"2023-09-23T20:52:13.783192+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7662,"log_dups_size":0,"ondisk_log_size":7662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:22.291407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327548928,"num_objects":4998,"num_object_clones":0,"num_object_copies":14994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7662,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774500+0000","last_change":"2023-09-23T20:51:05.995979+0000","last_active":"2023-09-23T20:52:10.774500+0000","last_peered":"2023-09-23T20:52:10.774500+0000","last_clean":"2023-09-23T20:52:10.774500+0000","last_became_active":"2023-09-23T20:51:05.995591+0000","last_became_peered":"2023-09-23T20:51:05.995591+0000","last_unstale":"2023-09-23T20:52:10.774500+0000","last_undegraded":"2023-09-23T20:52:10.774500+0000","last_fullsized":"2023-09-23T20:52:10.774500+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:35.414294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":147,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779341+0000","last_change":"2023-09-23T20:45:00.002738+0000","last_active":"2023-09-23T20:52:10.779341+0000","last_peered":"2023-09-23T20:52:10.779341+0000","last_clean":"2023-09-23T20:52:10.779341+0000","last_became_active":"2023-09-23T20:45:00.002403+0000","last_became_peered":"2023-09-23T20:45:00.002403+0000","last_unstale":"2023-09-23T20:52:10.779341+0000","last_undegraded":"2023-09-23T20:52:10.779341+0000","last_fullsized":"2023-09-23T20:52:10.779341+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:54.551205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"197'7789","reported_seq":10695,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.734126+0000","last_change":"2023-09-23T20:51:15.811325+0000","last_active":"2023-09-23T20:52:15.734126+0000","last_peered":"2023-09-23T20:52:15.734126+0000","last_clean":"2023-09-23T20:52:15.734126+0000","last_became_active":"2023-09-23T20:51:15.805454+0000","last_became_peered":"2023-09-23T20:51:15.805454+0000","last_unstale":"2023-09-23T20:52:15.734126+0000","last_undegraded":"2023-09-23T20:52:15.734126+0000","last_fullsized":"2023-09-23T20:52:15.734126+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7789,"log_dups_size":0,"ondisk_log_size":7789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:13.254228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322240512,"num_objects":4917,"num_object_clones":0,"num_object_copies":14751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7789,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1418,"num_bytes_recovered":92930048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766162+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:52:10.766162+0000","last_peered":"2023-09-23T20:52:10.766162+0000","last_clean":"2023-09-23T20:52:10.766162+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:52:10.766162+0000","last_undegraded":"2023-09-23T20:52:10.766162+0000","last_fullsized":"2023-09-23T20:52:10.766162+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.792395+0000","last_change":"2023-09-23T20:51:06.001664+0000","last_active":"2023-09-23T20:52:10.792395+0000","last_peered":"2023-09-23T20:52:10.792395+0000","last_clean":"2023-09-23T20:52:10.792395+0000","last_became_active":"2023-09-23T20:51:06.001352+0000","last_became_peered":"2023-09-23T20:51:06.001352+0000","last_unstale":"2023-09-23T20:52:10.792395+0000","last_undegraded":"2023-09-23T20:52:10.792395+0000","last_fullsized":"2023-09-23T20:52:10.792395+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.688374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"197'7818","reported_seq":9547,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.326007+0000","last_change":"2023-09-23T20:51:30.833121+0000","last_active":"2023-09-23T20:52:13.326007+0000","last_peered":"2023-09-23T20:52:13.326007+0000","last_clean":"2023-09-23T20:52:13.326007+0000","last_became_active":"2023-09-23T20:51:30.828670+0000","last_became_peered":"2023-09-23T20:51:30.828670+0000","last_unstale":"2023-09-23T20:52:13.326007+0000","last_undegraded":"2023-09-23T20:52:13.326007+0000","last_fullsized":"2023-09-23T20:52:13.326007+0000","mapping_epoch":186,"log_start":"168'500","ondisk_log_start":"168'500","created":164,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7318,"log_dups_size":0,"ondisk_log_size":7318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:07.018027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332267520,"num_objects":5070,"num_object_clones":0,"num_object_copies":15210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7818,"num_write_kb":412416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":829,"num_bytes_recovered":54329344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":29,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.787267+0000","last_change":"2023-09-23T20:51:06.005884+0000","last_active":"2023-09-23T20:52:10.787267+0000","last_peered":"2023-09-23T20:52:10.787267+0000","last_clean":"2023-09-23T20:52:10.787267+0000","last_became_active":"2023-09-23T20:51:06.005399+0000","last_became_peered":"2023-09-23T20:51:06.005399+0000","last_unstale":"2023-09-23T20:52:10.787267+0000","last_undegraded":"2023-09-23T20:52:10.787267+0000","last_fullsized":"2023-09-23T20:52:10.787267+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:56.910456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":173,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788409+0000","last_change":"2023-09-23T20:46:05.106502+0000","last_active":"2023-09-23T20:52:10.788409+0000","last_peered":"2023-09-23T20:52:10.788409+0000","last_clean":"2023-09-23T20:52:10.788409+0000","last_became_active":"2023-09-23T20:46:05.106134+0000","last_became_peered":"2023-09-23T20:46:05.106134+0000","last_unstale":"2023-09-23T20:52:10.788409+0000","last_undegraded":"2023-09-23T20:52:10.788409+0000","last_fullsized":"2023-09-23T20:52:10.788409+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:39.257351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"197'7768","reported_seq":7808,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.787644+0000","last_change":"2023-09-23T20:50:08.077812+0000","last_active":"2023-09-23T20:52:13.787644+0000","last_peered":"2023-09-23T20:52:13.787644+0000","last_clean":"2023-09-23T20:52:13.787644+0000","last_became_active":"2023-09-23T20:50:08.077458+0000","last_became_peered":"2023-09-23T20:50:08.077458+0000","last_unstale":"2023-09-23T20:52:13.787644+0000","last_undegraded":"2023-09-23T20:52:13.787644+0000","last_fullsized":"2023-09-23T20:52:13.787644+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7768,"log_dups_size":0,"ondisk_log_size":7768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332922880,"num_objects":5080,"num_object_clones":0,"num_object_copies":15240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7768,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":157,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788124+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:52:10.788124+0000","last_peered":"2023-09-23T20:52:10.788124+0000","last_clean":"2023-09-23T20:52:10.788124+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:52:10.788124+0000","last_undegraded":"2023-09-23T20:52:10.788124+0000","last_fullsized":"2023-09-23T20:52:10.788124+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788949+0000","last_change":"2023-09-23T20:51:06.002223+0000","last_active":"2023-09-23T20:52:10.788949+0000","last_peered":"2023-09-23T20:52:10.788949+0000","last_clean":"2023-09-23T20:52:10.788949+0000","last_became_active":"2023-09-23T20:51:06.001904+0000","last_became_peered":"2023-09-23T20:51:06.001904+0000","last_unstale":"2023-09-23T20:52:10.788949+0000","last_undegraded":"2023-09-23T20:52:10.788949+0000","last_fullsized":"2023-09-23T20:52:10.788949+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:32.282239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"197'7907","reported_seq":7949,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.744236+0000","last_change":"2023-09-23T20:50:08.647934+0000","last_active":"2023-09-23T20:52:15.744236+0000","last_peered":"2023-09-23T20:52:15.744236+0000","last_clean":"2023-09-23T20:52:15.744236+0000","last_became_active":"2023-09-23T20:50:08.647621+0000","last_became_peered":"2023-09-23T20:50:08.647621+0000","last_unstale":"2023-09-23T20:52:15.744236+0000","last_undegraded":"2023-09-23T20:52:15.744236+0000","last_fullsized":"2023-09-23T20:52:15.744236+0000","mapping_epoch":164,"log_start":"168'200","ondisk_log_start":"168'200","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.046520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333381632,"num_objects":5087,"num_object_clones":0,"num_object_copies":15261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7907,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":137,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766157+0000","last_change":"2023-09-23T20:48:07.540484+0000","last_active":"2023-09-23T20:52:10.766157+0000","last_peered":"2023-09-23T20:52:10.766157+0000","last_clean":"2023-09-23T20:52:10.766157+0000","last_became_active":"2023-09-23T20:48:07.540219+0000","last_became_peered":"2023-09-23T20:48:07.540219+0000","last_unstale":"2023-09-23T20:52:10.766157+0000","last_undegraded":"2023-09-23T20:52:10.766157+0000","last_fullsized":"2023-09-23T20:52:10.766157+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:54.792744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":149,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.777043+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:52:10.777043+0000","last_peered":"2023-09-23T20:52:10.777043+0000","last_clean":"2023-09-23T20:52:10.777043+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:52:10.777043+0000","last_undegraded":"2023-09-23T20:52:10.777043+0000","last_fullsized":"2023-09-23T20:52:10.777043+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"197'7882","reported_seq":7923,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.811936+0000","last_change":"2023-09-23T20:50:08.078863+0000","last_active":"2023-09-23T20:52:13.811936+0000","last_peered":"2023-09-23T20:52:13.811936+0000","last_clean":"2023-09-23T20:52:13.811936+0000","last_became_active":"2023-09-23T20:50:08.078560+0000","last_became_peered":"2023-09-23T20:50:08.078560+0000","last_unstale":"2023-09-23T20:52:13.811936+0000","last_undegraded":"2023-09-23T20:52:13.811936+0000","last_fullsized":"2023-09-23T20:52:13.811936+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7882,"log_dups_size":0,"ondisk_log_size":7882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:41.488151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338690048,"num_objects":5168,"num_object_clones":0,"num_object_copies":15504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7882,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":155,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.782427+0000","last_change":"2023-09-23T20:45:00.001830+0000","last_active":"2023-09-23T20:52:10.782427+0000","last_peered":"2023-09-23T20:52:10.782427+0000","last_clean":"2023-09-23T20:52:10.782427+0000","last_became_active":"2023-09-23T20:45:00.001532+0000","last_became_peered":"2023-09-23T20:45:00.001532+0000","last_unstale":"2023-09-23T20:52:10.782427+0000","last_undegraded":"2023-09-23T20:52:10.782427+0000","last_fullsized":"2023-09-23T20:52:10.782427+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:17.499694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":174,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774985+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:52:10.774985+0000","last_peered":"2023-09-23T20:52:10.774985+0000","last_clean":"2023-09-23T20:52:10.774985+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:52:10.774985+0000","last_undegraded":"2023-09-23T20:52:10.774985+0000","last_fullsized":"2023-09-23T20:52:10.774985+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":108,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.782949+0000","last_change":"2023-09-23T20:45:00.004588+0000","last_active":"2023-09-23T20:52:10.782949+0000","last_peered":"2023-09-23T20:52:10.782949+0000","last_clean":"2023-09-23T20:52:10.782949+0000","last_became_active":"2023-09-23T20:45:00.004185+0000","last_became_peered":"2023-09-23T20:45:00.004185+0000","last_unstale":"2023-09-23T20:52:10.782949+0000","last_undegraded":"2023-09-23T20:52:10.782949+0000","last_fullsized":"2023-09-23T20:52:10.782949+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:47.406920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"197'7949","reported_seq":7991,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.649446+0000","last_change":"2023-09-23T20:50:08.076706+0000","last_active":"2023-09-23T20:52:12.649446+0000","last_peered":"2023-09-23T20:52:12.649446+0000","last_clean":"2023-09-23T20:52:12.649446+0000","last_became_active":"2023-09-23T20:50:08.076360+0000","last_became_peered":"2023-09-23T20:50:08.076360+0000","last_unstale":"2023-09-23T20:52:12.649446+0000","last_undegraded":"2023-09-23T20:52:12.649446+0000","last_fullsized":"2023-09-23T20:52:12.649446+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7949,"log_dups_size":0,"ondisk_log_size":7949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.596097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343867392,"num_objects":5247,"num_object_clones":0,"num_object_copies":15741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7949,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":181,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774661+0000","last_change":"2023-09-23T20:51:06.003661+0000","last_active":"2023-09-23T20:52:10.774661+0000","last_peered":"2023-09-23T20:52:10.774661+0000","last_clean":"2023-09-23T20:52:10.774661+0000","last_became_active":"2023-09-23T20:51:06.003277+0000","last_became_peered":"2023-09-23T20:51:06.003277+0000","last_unstale":"2023-09-23T20:52:10.774661+0000","last_undegraded":"2023-09-23T20:52:10.774661+0000","last_fullsized":"2023-09-23T20:52:10.774661+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:32.933372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":177,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788443+0000","last_change":"2023-09-23T20:51:06.033002+0000","last_active":"2023-09-23T20:52:10.788443+0000","last_peered":"2023-09-23T20:52:10.788443+0000","last_clean":"2023-09-23T20:52:10.788443+0000","last_became_active":"2023-09-23T20:51:06.032657+0000","last_became_peered":"2023-09-23T20:51:06.032657+0000","last_unstale":"2023-09-23T20:52:10.788443+0000","last_undegraded":"2023-09-23T20:52:10.788443+0000","last_fullsized":"2023-09-23T20:52:10.788443+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:07.293307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":166,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774405+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:52:10.774405+0000","last_peered":"2023-09-23T20:52:10.774405+0000","last_clean":"2023-09-23T20:52:10.774405+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:52:10.774405+0000","last_undegraded":"2023-09-23T20:52:10.774405+0000","last_fullsized":"2023-09-23T20:52:10.774405+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"197'7777","reported_seq":7817,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.808449+0000","last_change":"2023-09-23T20:50:08.080121+0000","last_active":"2023-09-23T20:52:13.808449+0000","last_peered":"2023-09-23T20:52:13.808449+0000","last_clean":"2023-09-23T20:52:13.808449+0000","last_became_active":"2023-09-23T20:50:08.079818+0000","last_became_peered":"2023-09-23T20:50:08.079818+0000","last_unstale":"2023-09-23T20:52:13.808449+0000","last_undegraded":"2023-09-23T20:52:13.808449+0000","last_fullsized":"2023-09-23T20:52:13.808449+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7777,"log_dups_size":0,"ondisk_log_size":7777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:03.610431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329842688,"num_objects":5033,"num_object_clones":0,"num_object_copies":15099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7777,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":154,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.776959+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:52:10.776959+0000","last_peered":"2023-09-23T20:52:10.776959+0000","last_clean":"2023-09-23T20:52:10.776959+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:52:10.776959+0000","last_undegraded":"2023-09-23T20:52:10.776959+0000","last_fullsized":"2023-09-23T20:52:10.776959+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.781403+0000","last_change":"2023-09-23T20:51:06.002709+0000","last_active":"2023-09-23T20:52:10.781403+0000","last_peered":"2023-09-23T20:52:10.781403+0000","last_clean":"2023-09-23T20:52:10.781403+0000","last_became_active":"2023-09-23T20:51:06.002425+0000","last_became_peered":"2023-09-23T20:51:06.002425+0000","last_unstale":"2023-09-23T20:52:10.781403+0000","last_undegraded":"2023-09-23T20:52:10.781403+0000","last_fullsized":"2023-09-23T20:52:10.781403+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:45.344651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":150,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.782570+0000","last_change":"2023-09-23T20:45:00.002611+0000","last_active":"2023-09-23T20:52:10.782570+0000","last_peered":"2023-09-23T20:52:10.782570+0000","last_clean":"2023-09-23T20:52:10.782570+0000","last_became_active":"2023-09-23T20:45:00.002331+0000","last_became_peered":"2023-09-23T20:45:00.002331+0000","last_unstale":"2023-09-23T20:52:10.782570+0000","last_undegraded":"2023-09-23T20:52:10.782570+0000","last_fullsized":"2023-09-23T20:52:10.782570+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:19.968130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"197'7742","reported_seq":7784,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.756410+0000","last_change":"2023-09-23T20:50:08.078559+0000","last_active":"2023-09-23T20:52:15.756410+0000","last_peered":"2023-09-23T20:52:15.756410+0000","last_clean":"2023-09-23T20:52:15.756410+0000","last_became_active":"2023-09-23T20:50:08.078294+0000","last_became_peered":"2023-09-23T20:50:08.078294+0000","last_unstale":"2023-09-23T20:52:15.756410+0000","last_undegraded":"2023-09-23T20:52:15.756410+0000","last_fullsized":"2023-09-23T20:52:15.756410+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7742,"log_dups_size":0,"ondisk_log_size":7742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:11.281483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318898176,"num_objects":4866,"num_object_clones":0,"num_object_copies":14598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7742,"num_write_kb":403456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":161,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774457+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:52:10.774457+0000","last_peered":"2023-09-23T20:52:10.774457+0000","last_clean":"2023-09-23T20:52:10.774457+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:52:10.774457+0000","last_undegraded":"2023-09-23T20:52:10.774457+0000","last_fullsized":"2023-09-23T20:52:10.774457+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":589,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.775330+0000","last_change":"2023-09-23T20:51:30.823817+0000","last_active":"2023-09-23T20:52:10.775330+0000","last_peered":"2023-09-23T20:52:10.775330+0000","last_clean":"2023-09-23T20:52:10.775330+0000","last_became_active":"2023-09-23T20:51:30.823372+0000","last_became_peered":"2023-09-23T20:51:30.823372+0000","last_unstale":"2023-09-23T20:52:10.775330+0000","last_undegraded":"2023-09-23T20:52:10.775330+0000","last_fullsized":"2023-09-23T20:52:10.775330+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:09.432277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":132,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.150730+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:52:12.150730+0000","last_peered":"2023-09-23T20:52:12.150730+0000","last_clean":"2023-09-23T20:52:12.150730+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:52:12.150730+0000","last_undegraded":"2023-09-23T20:52:12.150730+0000","last_fullsized":"2023-09-23T20:52:12.150730+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":29,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.787675+0000","last_change":"2023-09-23T20:51:05.957381+0000","last_active":"2023-09-23T20:52:10.787675+0000","last_peered":"2023-09-23T20:52:10.787675+0000","last_clean":"2023-09-23T20:52:10.787675+0000","last_became_active":"2023-09-23T20:51:05.956991+0000","last_became_peered":"2023-09-23T20:51:05.956991+0000","last_unstale":"2023-09-23T20:52:10.787675+0000","last_undegraded":"2023-09-23T20:52:10.787675+0000","last_fullsized":"2023-09-23T20:52:10.787675+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:20.613174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"197'7902","reported_seq":10753,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.618161+0000","last_change":"2023-09-23T20:51:23.778886+0000","last_active":"2023-09-23T20:52:12.618161+0000","last_peered":"2023-09-23T20:52:12.618161+0000","last_clean":"2023-09-23T20:52:12.618161+0000","last_became_active":"2023-09-23T20:51:23.771838+0000","last_became_peered":"2023-09-23T20:51:23.771838+0000","last_unstale":"2023-09-23T20:52:12.618161+0000","last_undegraded":"2023-09-23T20:52:12.618161+0000","last_fullsized":"2023-09-23T20:52:12.618161+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7902,"log_dups_size":0,"ondisk_log_size":7902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:54.532463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343801856,"num_objects":5246,"num_object_clones":0,"num_object_copies":15738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7902,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1391,"num_bytes_recovered":91160576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":178,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.775298+0000","last_change":"2023-09-23T20:45:00.001911+0000","last_active":"2023-09-23T20:52:10.775298+0000","last_peered":"2023-09-23T20:52:10.775298+0000","last_clean":"2023-09-23T20:52:10.775298+0000","last_became_active":"2023-09-23T20:45:00.001531+0000","last_became_peered":"2023-09-23T20:45:00.001531+0000","last_unstale":"2023-09-23T20:52:10.775298+0000","last_undegraded":"2023-09-23T20:52:10.775298+0000","last_fullsized":"2023-09-23T20:52:10.775298+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:17.927950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":146,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.780253+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:52:10.780253+0000","last_peered":"2023-09-23T20:52:10.780253+0000","last_clean":"2023-09-23T20:52:10.780253+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:52:10.780253+0000","last_undegraded":"2023-09-23T20:52:10.780253+0000","last_fullsized":"2023-09-23T20:52:10.780253+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":162,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779495+0000","last_change":"2023-09-23T20:46:05.077316+0000","last_active":"2023-09-23T20:52:10.779495+0000","last_peered":"2023-09-23T20:52:10.779495+0000","last_clean":"2023-09-23T20:52:10.779495+0000","last_became_active":"2023-09-23T20:46:05.077044+0000","last_became_peered":"2023-09-23T20:46:05.077044+0000","last_unstale":"2023-09-23T20:52:10.779495+0000","last_undegraded":"2023-09-23T20:52:10.779495+0000","last_fullsized":"2023-09-23T20:52:10.779495+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:56.517854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"197'7873","reported_seq":7914,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.754629+0000","last_change":"2023-09-23T20:50:08.083097+0000","last_active":"2023-09-23T20:52:13.754629+0000","last_peered":"2023-09-23T20:52:13.754629+0000","last_clean":"2023-09-23T20:52:13.754629+0000","last_became_active":"2023-09-23T20:50:08.082761+0000","last_became_peered":"2023-09-23T20:50:08.082761+0000","last_unstale":"2023-09-23T20:52:13.754629+0000","last_undegraded":"2023-09-23T20:52:13.754629+0000","last_fullsized":"2023-09-23T20:52:13.754629+0000","mapping_epoch":164,"log_start":"167'100","ondisk_log_start":"167'100","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7773,"log_dups_size":0,"ondisk_log_size":7773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:29.125314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333119488,"num_objects":5083,"num_object_clones":0,"num_object_copies":15249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7873,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":161,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.776682+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:52:10.776682+0000","last_peered":"2023-09-23T20:52:10.776682+0000","last_clean":"2023-09-23T20:52:10.776682+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:52:10.776682+0000","last_undegraded":"2023-09-23T20:52:10.776682+0000","last_fullsized":"2023-09-23T20:52:10.776682+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":122,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.782339+0000","last_change":"2023-09-23T20:46:05.079434+0000","last_active":"2023-09-23T20:52:10.782339+0000","last_peered":"2023-09-23T20:52:10.782339+0000","last_clean":"2023-09-23T20:52:10.782339+0000","last_became_active":"2023-09-23T20:46:05.079111+0000","last_became_peered":"2023-09-23T20:46:05.079111+0000","last_unstale":"2023-09-23T20:52:10.782339+0000","last_undegraded":"2023-09-23T20:52:10.782339+0000","last_fullsized":"2023-09-23T20:52:10.782339+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.869457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779201+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:52:10.779201+0000","last_peered":"2023-09-23T20:52:10.779201+0000","last_clean":"2023-09-23T20:52:10.779201+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:52:10.779201+0000","last_undegraded":"2023-09-23T20:52:10.779201+0000","last_fullsized":"2023-09-23T20:52:10.779201+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"197'7704","reported_seq":7746,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.753705+0000","last_change":"2023-09-23T20:50:08.078428+0000","last_active":"2023-09-23T20:52:13.753705+0000","last_peered":"2023-09-23T20:52:13.753705+0000","last_clean":"2023-09-23T20:52:13.753705+0000","last_became_active":"2023-09-23T20:50:08.078116+0000","last_became_peered":"2023-09-23T20:50:08.078116+0000","last_unstale":"2023-09-23T20:52:13.753705+0000","last_undegraded":"2023-09-23T20:52:13.753705+0000","last_fullsized":"2023-09-23T20:52:13.753705+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7704,"log_dups_size":0,"ondisk_log_size":7704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:06.919475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327745560,"num_objects":5002,"num_object_clones":0,"num_object_copies":15006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5002,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7704,"num_write_kb":406529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766954+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:52:10.766954+0000","last_peered":"2023-09-23T20:52:10.766954+0000","last_clean":"2023-09-23T20:52:10.766954+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:52:10.766954+0000","last_undegraded":"2023-09-23T20:52:10.766954+0000","last_fullsized":"2023-09-23T20:52:10.766954+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":167,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.782533+0000","last_change":"2023-09-23T20:51:05.997935+0000","last_active":"2023-09-23T20:52:10.782533+0000","last_peered":"2023-09-23T20:52:10.782533+0000","last_clean":"2023-09-23T20:52:10.782533+0000","last_became_active":"2023-09-23T20:51:05.997637+0000","last_became_peered":"2023-09-23T20:51:05.997637+0000","last_unstale":"2023-09-23T20:52:10.782533+0000","last_undegraded":"2023-09-23T20:52:10.782533+0000","last_fullsized":"2023-09-23T20:52:10.782533+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:27.645262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":157,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766080+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:52:10.766080+0000","last_peered":"2023-09-23T20:52:10.766080+0000","last_clean":"2023-09-23T20:52:10.766080+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:52:10.766080+0000","last_undegraded":"2023-09-23T20:52:10.766080+0000","last_fullsized":"2023-09-23T20:52:10.766080+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"197'7924","reported_seq":7966,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.717046+0000","last_change":"2023-09-23T20:50:08.083439+0000","last_active":"2023-09-23T20:52:15.717046+0000","last_peered":"2023-09-23T20:52:15.717046+0000","last_clean":"2023-09-23T20:52:15.717046+0000","last_became_active":"2023-09-23T20:50:08.083061+0000","last_became_peered":"2023-09-23T20:50:08.083061+0000","last_unstale":"2023-09-23T20:52:15.717046+0000","last_undegraded":"2023-09-23T20:52:15.717046+0000","last_fullsized":"2023-09-23T20:52:15.717046+0000","mapping_epoch":164,"log_start":"168'200","ondisk_log_start":"168'200","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7724,"log_dups_size":0,"ondisk_log_size":7724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:41.569707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322043904,"num_objects":4914,"num_object_clones":0,"num_object_copies":14742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7924,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":29,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.787449+0000","last_change":"2023-09-23T20:51:05.957996+0000","last_active":"2023-09-23T20:52:10.787449+0000","last_peered":"2023-09-23T20:52:10.787449+0000","last_clean":"2023-09-23T20:52:10.787449+0000","last_became_active":"2023-09-23T20:51:05.957651+0000","last_became_peered":"2023-09-23T20:51:05.957651+0000","last_unstale":"2023-09-23T20:52:10.787449+0000","last_undegraded":"2023-09-23T20:52:10.787449+0000","last_fullsized":"2023-09-23T20:52:10.787449+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:05.872498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":150,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779352+0000","last_change":"2023-09-23T20:45:00.005077+0000","last_active":"2023-09-23T20:52:10.779352+0000","last_peered":"2023-09-23T20:52:10.779352+0000","last_clean":"2023-09-23T20:52:10.779352+0000","last_became_active":"2023-09-23T20:45:00.004383+0000","last_became_peered":"2023-09-23T20:45:00.004383+0000","last_unstale":"2023-09-23T20:52:10.779352+0000","last_undegraded":"2023-09-23T20:52:10.779352+0000","last_fullsized":"2023-09-23T20:52:10.779352+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:45.178382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788166+0000","last_change":"2023-09-23T20:51:05.959327+0000","last_active":"2023-09-23T20:52:10.788166+0000","last_peered":"2023-09-23T20:52:10.788166+0000","last_clean":"2023-09-23T20:52:10.788166+0000","last_became_active":"2023-09-23T20:51:05.958816+0000","last_became_peered":"2023-09-23T20:51:05.958816+0000","last_unstale":"2023-09-23T20:52:10.788166+0000","last_undegraded":"2023-09-23T20:52:10.788166+0000","last_fullsized":"2023-09-23T20:52:10.788166+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:52.674764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":134,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779421+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:52:10.779421+0000","last_peered":"2023-09-23T20:52:10.779421+0000","last_clean":"2023-09-23T20:52:10.779421+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:52:10.779421+0000","last_undegraded":"2023-09-23T20:52:10.779421+0000","last_fullsized":"2023-09-23T20:52:10.779421+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":171,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.777165+0000","last_change":"2023-09-23T20:48:07.539824+0000","last_active":"2023-09-23T20:52:10.777165+0000","last_peered":"2023-09-23T20:52:10.777165+0000","last_clean":"2023-09-23T20:52:10.777165+0000","last_became_active":"2023-09-23T20:48:07.539530+0000","last_became_peered":"2023-09-23T20:48:07.539530+0000","last_unstale":"2023-09-23T20:52:10.777165+0000","last_undegraded":"2023-09-23T20:52:10.777165+0000","last_fullsized":"2023-09-23T20:52:10.777165+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:21.997036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.774594+0000","last_change":"2023-09-23T20:45:00.008268+0000","last_active":"2023-09-23T20:52:10.774594+0000","last_peered":"2023-09-23T20:52:10.774594+0000","last_clean":"2023-09-23T20:52:10.774594+0000","last_became_active":"2023-09-23T20:45:00.006393+0000","last_became_peered":"2023-09-23T20:45:00.006393+0000","last_unstale":"2023-09-23T20:52:10.774594+0000","last_undegraded":"2023-09-23T20:52:10.774594+0000","last_fullsized":"2023-09-23T20:52:10.774594+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:41.255405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":173,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788510+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:52:10.788510+0000","last_peered":"2023-09-23T20:52:10.788510+0000","last_clean":"2023-09-23T20:52:10.788510+0000","last_became_active":"2023-09-23T20:48:07.544066+0000","last_became_peered":"2023-09-23T20:48:07.544066+0000","last_unstale":"2023-09-23T20:52:10.788510+0000","last_undegraded":"2023-09-23T20:52:10.788510+0000","last_fullsized":"2023-09-23T20:52:10.788510+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:28.905486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":170,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.775029+0000","last_change":"2023-09-23T20:51:05.969594+0000","last_active":"2023-09-23T20:52:10.775029+0000","last_peered":"2023-09-23T20:52:10.775029+0000","last_clean":"2023-09-23T20:52:10.775029+0000","last_became_active":"2023-09-23T20:51:05.959688+0000","last_became_peered":"2023-09-23T20:51:05.959688+0000","last_unstale":"2023-09-23T20:52:10.775029+0000","last_undegraded":"2023-09-23T20:52:10.775029+0000","last_fullsized":"2023-09-23T20:52:10.775029+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:59.980249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":173,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788947+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:52:10.788947+0000","last_peered":"2023-09-23T20:52:10.788947+0000","last_clean":"2023-09-23T20:52:10.788947+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:52:10.788947+0000","last_undegraded":"2023-09-23T20:52:10.788947+0000","last_fullsized":"2023-09-23T20:52:10.788947+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":172,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766642+0000","last_change":"2023-09-23T20:51:05.999322+0000","last_active":"2023-09-23T20:52:10.766642+0000","last_peered":"2023-09-23T20:52:10.766642+0000","last_clean":"2023-09-23T20:52:10.766642+0000","last_became_active":"2023-09-23T20:51:05.998993+0000","last_became_peered":"2023-09-23T20:51:05.998993+0000","last_unstale":"2023-09-23T20:52:10.766642+0000","last_undegraded":"2023-09-23T20:52:10.766642+0000","last_fullsized":"2023-09-23T20:52:10.766642+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:47.495994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":168,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.767602+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:52:10.767602+0000","last_peered":"2023-09-23T20:52:10.767602+0000","last_clean":"2023-09-23T20:52:10.767602+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:52:10.767602+0000","last_undegraded":"2023-09-23T20:52:10.767602+0000","last_fullsized":"2023-09-23T20:52:10.767602+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":150,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.779791+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:52:10.779791+0000","last_peered":"2023-09-23T20:52:10.779791+0000","last_clean":"2023-09-23T20:52:10.779791+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:52:10.779791+0000","last_undegraded":"2023-09-23T20:52:10.779791+0000","last_fullsized":"2023-09-23T20:52:10.779791+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":176,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.788064+0000","last_change":"2023-09-23T20:51:06.011816+0000","last_active":"2023-09-23T20:52:10.788064+0000","last_peered":"2023-09-23T20:52:10.788064+0000","last_clean":"2023-09-23T20:52:10.788064+0000","last_became_active":"2023-09-23T20:51:06.011506+0000","last_became_peered":"2023-09-23T20:51:06.011506+0000","last_unstale":"2023-09-23T20:52:10.788064+0000","last_undegraded":"2023-09-23T20:52:10.788064+0000","last_fullsized":"2023-09-23T20:52:10.788064+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:40.603970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5305991192,"num_objects":80964,"num_object_clones":0,"num_object_copies":242892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80964,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124808,"num_write_kb":6584641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3638,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15935746048,"data_stored":15935733832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123308,"ondisk_log_size":123308,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201562,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3275808,"kb_used_data":2914264,"kb_used_omap":0,"kb_used_meta":361536,"kb_avail":90469344,"statfs":{"total":95995035648,"available":92640608256,"internally_reserved":0,"allocated":2984206336,"data_stored":2983626136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":370212864},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":3.879,"15min":2.4289999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.20499999999999999},"max":{"1min":6.0890000000000004,"5min":225.56999999999999,"15min":225.56999999999999},"last":7.0679999999999996},{"interface":"front","average":{"1min":1.5389999999999999,"5min":4.0659999999999998,"15min":2.738},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.21299999999999999},"max":{"1min":5.3330000000000002,"5min":225.61099999999999,"15min":225.61099999999999},"last":7.0789999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":3.2290000000000001,"15min":1.8540000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.6219999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.23400000000000001},{"interface":"front","average":{"1min":1.5489999999999999,"5min":6.2569999999999997,"15min":2.8730000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3810000000000002,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.316}]},{"osd":2,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":3.286,"15min":3.1880000000000002},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.6150000000000002,"5min":142.321,"15min":142.321},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.7189999999999999,"15min":2.464},"min":{"1min":0.35999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.6020000000000003,"5min":142.29300000000001,"15min":142.29300000000001},"last":3.1150000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":6.7389999999999999,"15min":9.0519999999999996},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":31.631,"5min":138.345,"15min":138.345},"last":0.26500000000000001},{"interface":"front","average":{"1min":2.7909999999999999,"5min":6.3150000000000004,"15min":8.3870000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":31.617000000000001,"5min":133.76400000000001,"15min":133.76400000000001},"last":9.1769999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.411,"15min":1.139},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.216},"max":{"1min":4.5,"5min":9.1400000000000006,"15min":11.834},"last":1.9950000000000001},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.3069999999999999,"15min":1.1819999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.6420000000000003,"5min":8.4649999999999999,"15min":10.077999999999999},"last":2.6459999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.3580000000000001,"15min":1.2889999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.0789999999999997,"5min":9.4440000000000008,"15min":14.58},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.7350000000000001,"15min":1.2949999999999999},"min":{"1min":0.32500000000000001,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":4.5620000000000003,"5min":13.156000000000001,"15min":13.156000000000001},"last":2.7269999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.6779999999999999,"15min":1.2969999999999999},"min":{"1min":0.39100000000000001,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":4.4740000000000002,"5min":10.509,"15min":10.509},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.1160000000000001,"15min":1.0980000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.419,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":2.8079999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463199,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275660,"kb_used_data":987588,"kb_used_omap":0,"kb_used_meta":288064,"kb_avail":92469492,"statfs":{"total":95995035648,"available":94688759808,"internally_reserved":0,"allocated":1011290112,"data_stored":1010730015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":294977536},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":3.0710000000000002,"15min":2.3039999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":3.6459999999999999,"5min":92.164000000000001,"15min":92.164000000000001},"last":2.9470000000000001},{"interface":"front","average":{"1min":0.70399999999999996,"5min":2.5830000000000002,"15min":2.2360000000000002},"min":{"1min":0.32100000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":2.8100000000000001,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.77500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:51:15 2023","interfaces":[{"interface":"back","average":{"1min":3.641,"5min":3.081,"15min":2.1309999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25800000000000001},"max":{"1min":19.994,"5min":96.623999999999995,"15min":96.623999999999995},"last":6.5519999999999996},{"interface":"front","average":{"1min":4.1509999999999998,"5min":3.206,"15min":1.972},"min":{"1min":0.29299999999999998,"5min":0.223,"15min":0.223},"max":{"1min":19.916,"5min":96.254000000000005,"15min":96.254000000000005},"last":2.1579999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:51:15 2023","interfaces":[{"interface":"back","average":{"1min":2.8919999999999999,"5min":2.1000000000000001,"15min":2.226},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":24.440999999999999,"5min":24.440999999999999,"15min":24.440999999999999},"last":3.1579999999999999},{"interface":"front","average":{"1min":2.5539999999999998,"5min":2.206,"15min":2.4670000000000001},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":23.916,"5min":23.916,"15min":23.916},"last":3.0760000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":4.0090000000000003,"15min":5.4189999999999996},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.996,"5min":79.668000000000006,"15min":79.668000000000006},"last":1.5249999999999999},{"interface":"front","average":{"1min":1.1439999999999999,"5min":3.9500000000000002,"15min":5.3150000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.673,"5min":79.700000000000003,"15min":79.700000000000003},"last":0.874}]},{"osd":4,"last update":"Sat Sep 23 20:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.2210000000000001,"15min":1.0289999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":6.2569999999999997,"5min":9.0709999999999997,"15min":9.0709999999999997},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.5900000000000001,"15min":1.292},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27000000000000002},"max":{"1min":5.2220000000000004,"5min":9.8650000000000002,"15min":10.058999999999999},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:51:15 2023","interfaces":[{"interface":"back","average":{"1min":2.548,"5min":1.7609999999999999,"15min":1.329},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.26000000000000001},"max":{"1min":23.071999999999999,"5min":23.071999999999999,"15min":23.071999999999999},"last":1.391},{"interface":"front","average":{"1min":2.597,"5min":1.851,"15min":1.4350000000000001},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.29399999999999998},"max":{"1min":23.294,"5min":23.294,"15min":23.294},"last":0.42799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":1.6080000000000001,"15min":1.25},"min":{"1min":0.33700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.5640000000000001,"5min":9.3249999999999993,"15min":19.451000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.27,"5min":1.528,"15min":1.3500000000000001},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.32100000000000001},"max":{"1min":6.4109999999999996,"5min":9.7959999999999994,"15min":16.295999999999999},"last":0.40400000000000003}]}]},{"osd":1,"up_from":114,"seq":489626271833,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3294156,"kb_used_data":2940996,"kb_used_omap":4,"kb_used_meta":353147,"kb_avail":90450996,"statfs":{"total":95995035648,"available":92621819904,"internally_reserved":0,"allocated":3011579904,"data_stored":3010786383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":361623438},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":5.649,"5min":3.3239999999999998,"15min":2.399},"min":{"1min":0.21199999999999999,"5min":0.189,"15min":0.189},"max":{"1min":61.957000000000001,"5min":64.622,"15min":64.622},"last":0.246},{"interface":"front","average":{"1min":5.3330000000000002,"5min":3.3340000000000001,"15min":2.359},"min":{"1min":0.26100000000000001,"5min":0.222,"15min":0.222},"max":{"1min":62.005000000000003,"5min":63.808,"15min":63.808},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":1.587,"15min":2.0299999999999998},"min":{"1min":0.23499999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.8049999999999997,"5min":11.593999999999999,"15min":11.593999999999999},"last":0.498},{"interface":"front","average":{"1min":1.409,"5min":1.7909999999999999,"15min":2.581},"min":{"1min":0.26800000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.8689999999999998,"5min":14.153,"15min":14.153},"last":0.30099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.1469999999999998,"15min":2.5350000000000001},"min":{"1min":0.30499999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.4060000000000006,"5min":8.4969999999999999,"15min":8.4969999999999999},"last":2.891},{"interface":"front","average":{"1min":2.5710000000000002,"5min":1.839,"15min":1.9730000000000001},"min":{"1min":0.307,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.047000000000001,"5min":11.047000000000001,"15min":11.047000000000001},"last":5.9370000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.4570000000000001,"15min":1.2989999999999999},"min":{"1min":0.27500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.0629999999999997,"5min":10.439,"15min":10.439},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.2730000000000001,"5min":1.51,"15min":1.24},"min":{"1min":0.35799999999999998,"5min":0.316,"15min":0.307},"max":{"1min":9.8889999999999993,"5min":10.34,"15min":10.34},"last":0.52900000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":1.962,"15min":1.788},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":5.117,"5min":10.177,"15min":10.177},"last":0.432},{"interface":"front","average":{"1min":3.242,"5min":1.8109999999999999,"15min":1.3859999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.6519999999999999,"15min":1.6060000000000001},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.4829999999999997,"5min":10.25,"15min":10.25},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.9950000000000001,"5min":1.6990000000000001,"15min":1.4059999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":9.8369999999999997,"5min":9.8369999999999997,"15min":9.8369999999999997},"last":0.58999999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":4.8090000000000002,"5min":2.3860000000000001,"15min":1.6930000000000001},"min":{"1min":0.38400000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":25.32,"5min":25.32,"15min":25.32},"last":0.64400000000000002},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.1099999999999999,"15min":1.913},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":25.344000000000001,"5min":25.344000000000001,"15min":25.344000000000001},"last":0.51900000000000002}]}]},{"osd":0,"up_from":92,"seq":395136991370,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1796176,"kb_used_data":1327688,"kb_used_omap":5,"kb_used_meta":468474,"kb_avail":91948976,"statfs":{"total":95995035648,"available":94155751424,"internally_reserved":0,"allocated":1359552512,"data_stored":1358744672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":479718010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":8.516,"5min":4.2370000000000001,"15min":2.7719999999999998},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":134.322,"5min":134.322,"15min":134.322},"last":0.248},{"interface":"front","average":{"1min":1.6399999999999999,"5min":2.9529999999999998,"15min":2.3889999999999998},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":7.5750000000000002,"5min":59.570999999999998,"15min":59.570999999999998},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":2.113,"15min":1.988},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":12.582000000000001,"5min":49.652000000000001,"15min":49.652000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.5249999999999999,"5min":2.6400000000000001,"15min":3.8740000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8200000000000003,"5min":49.595999999999997,"15min":49.595999999999997},"last":0.34399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":10.429,"5min":4.2069999999999999,"15min":3.6749999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":94.947999999999993,"5min":94.947999999999993,"15min":94.947999999999993},"last":0.375},{"interface":"front","average":{"1min":9.7249999999999996,"5min":4.2530000000000001,"15min":3.851},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":94.382999999999996,"5min":94.382999999999996,"15min":94.382999999999996},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":1.7410000000000001,"15min":1.7769999999999999},"min":{"1min":0.36599999999999999,"5min":0.27400000000000002,"15min":0.26800000000000002},"max":{"1min":10.525,"5min":12.973000000000001,"15min":12.973000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.907,"5min":1.3360000000000001,"15min":1.9890000000000001},"min":{"1min":0.32600000000000001,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":12.461,"5min":12.461,"15min":12.461},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":2.2770000000000001,"15min":2.1549999999999998},"min":{"1min":0.40200000000000002,"5min":0.30199999999999999,"15min":0.28799999999999998},"max":{"1min":10.470000000000001,"5min":14.051,"15min":14.051},"last":124.40600000000001},{"interface":"front","average":{"1min":1.909,"5min":1.5549999999999999,"15min":2.1299999999999999},"min":{"1min":0.36799999999999999,"5min":0.307,"15min":0.28899999999999998},"max":{"1min":12.488,"5min":12.488,"15min":12.488},"last":124.363}]},{"osd":6,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":1.8320000000000001,"15min":2.1949999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.307},"max":{"1min":12.510999999999999,"5min":14.037000000000001,"15min":14.037000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.095,"5min":1.847,"15min":1.6519999999999999},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.6289999999999996,"5min":20.399999999999999,"15min":20.399999999999999},"last":0.40200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":1.5549999999999999,"15min":2.274},"min":{"1min":0.42299999999999999,"5min":0.39800000000000002,"15min":0.35099999999999998},"max":{"1min":12.534000000000001,"5min":12.534000000000001,"15min":12.534000000000001},"last":60.037999999999997},{"interface":"front","average":{"1min":1.665,"5min":1.885,"15min":1.76},"min":{"1min":0.40999999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.5739999999999998,"5min":20.422999999999998,"15min":20.422999999999998},"last":60.012}]}]},{"osd":2,"up_from":129,"seq":554050781260,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292000,"kb_used_data":980888,"kb_used_omap":7,"kb_used_meta":311096,"kb_avail":92453152,"statfs":{"total":95995035648,"available":94672027648,"internally_reserved":0,"allocated":1004429312,"data_stored":1003673117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":318562385},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.7120000000000002,"5min":2.2450000000000001,"15min":2.911},"min":{"1min":0.20599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":15.539999999999999,"5min":89.644999999999996,"15min":89.644999999999996},"last":0.20599999999999999},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.1309999999999998,"15min":2.6949999999999998},"min":{"1min":0.26700000000000002,"5min":0.186,"15min":0.186},"max":{"1min":15.593999999999999,"5min":88.599999999999994,"15min":88.599999999999994},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.8439999999999999,"15min":2.9750000000000001},"min":{"1min":0.22500000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5879999999999992,"5min":91.325000000000003,"15min":91.325000000000003},"last":0.22500000000000001},{"interface":"front","average":{"1min":1.119,"5min":2.3599999999999999,"15min":2.996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2489999999999997,"5min":91.799999999999997,"15min":91.799999999999997},"last":0.28299999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":8.6180000000000003,"5min":2.7970000000000002,"15min":2.0470000000000002},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":88.859999999999999,"5min":88.859999999999999,"15min":88.859999999999999},"last":151.43799999999999},{"interface":"front","average":{"1min":8.5410000000000004,"5min":2.391,"15min":1.4750000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":89.704999999999998,"5min":89.704999999999998,"15min":89.704999999999998},"last":152.20699999999999}]},{"osd":4,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":1.369,"15min":1.7869999999999999},"min":{"1min":0.30499999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.7230000000000008,"5min":8.7230000000000008,"15min":8.7230000000000008},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.294,"5min":1.1679999999999999,"15min":2.3180000000000001},"min":{"1min":0.42399999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.5519999999999996,"5min":6.5519999999999996,"15min":8.1199999999999992},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":11.076000000000001,"5min":3.1110000000000002,"15min":3.2370000000000001},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":163.352,"5min":163.352,"15min":163.352},"last":0.38900000000000001},{"interface":"front","average":{"1min":10.865,"5min":3.2839999999999998,"15min":3.121},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":163.089,"5min":163.089,"15min":163.089},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.2090000000000001,"15min":2.786},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.4500000000000002,"5min":6.3449999999999998,"15min":7.476},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.3359999999999999,"5min":1.6679999999999999,"15min":2.7000000000000002},"min":{"1min":0.42499999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.6809999999999992,"5min":9.5220000000000002,"15min":12.595000000000001},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":11.727,"5min":3.327,"15min":2.8999999999999999},"min":{"1min":0.42299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":160.08099999999999,"5min":160.08099999999999,"15min":160.08099999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":11.019,"5min":3.1800000000000002,"15min":3.1019999999999999},"min":{"1min":0.44900000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":160.13800000000001,"5min":160.13800000000001,"15min":160.13800000000001},"last":0.502}]}]},{"osd":3,"up_from":148,"seq":635655159860,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2773604,"kb_used_data":2559516,"kb_used_omap":2,"kb_used_meta":214013,"kb_avail":90971548,"statfs":{"total":95995035648,"available":93154865152,"internally_reserved":0,"allocated":2620944384,"data_stored":2620183733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":219149365},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9810000000000001,"5min":6.6669999999999998,"15min":11.776999999999999},"min":{"1min":0.18099999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":10.470000000000001,"5min":253.68100000000001,"15min":253.68100000000001},"last":2.0590000000000002},{"interface":"front","average":{"1min":2.3370000000000002,"5min":6.2830000000000004,"15min":10.898999999999999},"min":{"1min":0.20999999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.3380000000000001,"5min":253.46199999999999,"15min":253.46199999999999},"last":3.9199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":5.0099999999999998,"15min":8.6150000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.0140000000000002,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.218},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.7919999999999998,"15min":7.8689999999999998},"min":{"1min":0.25,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.692,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.66400000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.3069999999999999,"15min":1.127},"min":{"1min":0.27300000000000002,"5min":0.215,"15min":0.215},"max":{"1min":11.317,"5min":11.317,"15min":11.317},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.835,"5min":1.296,"15min":1.0600000000000001},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.0720000000000001,"5min":8.8819999999999997,"15min":8.8819999999999997},"last":3.8370000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.552,"5min":1.556,"15min":1.6399999999999999},"min":{"1min":0.32600000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.9000000000000004,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.516,"5min":1.76,"15min":2.0640000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.276,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":1.675,"15min":1.341},"min":{"1min":0.39400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.057,"5min":12.057,"15min":12.057},"last":4.4610000000000003},{"interface":"front","average":{"1min":3.1379999999999999,"5min":1.978,"15min":1.964},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":14.054,"5min":14.054,"15min":14.054},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.8620000000000001,"15min":1.9350000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":11.632,"5min":11.632,"15min":11.632},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.9950000000000001,"15min":2.4260000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.404999999999999,"5min":16.451000000000001,"15min":16.451000000000001},"last":1.2110000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0139999999999998,"5min":1.8029999999999999,"15min":1.643},"min":{"1min":0.41399999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":12.25,"5min":12.25,"15min":12.25},"last":3.8029999999999999},{"interface":"front","average":{"1min":2.7440000000000002,"5min":1.8740000000000001,"15min":2.1920000000000002},"min":{"1min":0.45200000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.9690000000000003,"5min":11.561999999999999,"15min":11.561999999999999},"last":1.3560000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986472,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1317720,"kb_used_data":984720,"kb_used_omap":0,"kb_used_meta":332992,"kb_avail":92427432,"statfs":{"total":95995035648,"available":94645690368,"internally_reserved":0,"allocated":1008353280,"data_stored":1007780957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340983808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.8999999999999999,"15min":2.1360000000000001},"min":{"1min":0.25,"5min":0.216,"15min":0.214},"max":{"1min":45.552,"5min":45.552,"15min":45.552},"last":7.1909999999999998},{"interface":"front","average":{"1min":3.964,"5min":2.6040000000000001,"15min":1.782},"min":{"1min":0.245,"5min":0.245,"15min":0.22600000000000001},"max":{"1min":45.529000000000003,"5min":45.529000000000003,"15min":45.529000000000003},"last":0.25}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":4.2939999999999996,"5min":7.4859999999999998,"15min":3.589},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":45.712000000000003,"5min":286.31099999999998,"15min":286.31099999999998},"last":6.3899999999999997},{"interface":"front","average":{"1min":4.6509999999999998,"5min":7.2949999999999999,"15min":4.1310000000000002},"min":{"1min":0.27900000000000003,"5min":0.25700000000000001,"15min":0.24299999999999999},"max":{"1min":45.460000000000001,"5min":285.97000000000003,"15min":285.97000000000003},"last":2.0830000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9059999999999999,"5min":1.339,"15min":1.3280000000000001},"min":{"1min":0.30199999999999999,"5min":0.28499999999999998,"15min":0.25},"max":{"1min":8.6639999999999997,"5min":8.6639999999999997,"15min":8.6639999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.7969999999999999,"5min":1.617,"15min":2.1190000000000002},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0730000000000004,"5min":12.773999999999999,"15min":12.773999999999999},"last":7.2279999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":5.2039999999999997,"15min":6.7999999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":28.183,"5min":107.285,"15min":107.285},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.7200000000000002,"5min":4.9669999999999996,"15min":6.4500000000000002},"min":{"1min":0.37,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":31.763999999999999,"5min":107.303,"15min":107.303},"last":6.4219999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:51:49 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":1.101,"15min":0.92900000000000005},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.0960000000000001,"5min":7.1740000000000004,"15min":7.1740000000000004},"last":1.9159999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":1.1319999999999999,"15min":1.0469999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":2.6190000000000002,"5min":9.0660000000000007,"15min":12.209},"last":7.2960000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.103,"15min":1.1259999999999999},"min":{"1min":0.313,"5min":0.28499999999999998,"15min":0.28000000000000003},"max":{"1min":4.1989999999999998,"5min":7.4109999999999996,"15min":16.271000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.109,"5min":1.0649999999999999,"15min":1.03},"min":{"1min":0.34200000000000003,"5min":0.33800000000000002,"15min":0.30399999999999999},"max":{"1min":4.7329999999999997,"5min":7.617,"15min":11.718},"last":0.86599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:51:49 2023","interfaces":[{"interface":"back","average":{"1min":4.2949999999999999,"5min":1.962,"15min":1.4430000000000001},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":73.268000000000001,"5min":73.268000000000001,"15min":73.268000000000001},"last":7.2619999999999996},{"interface":"front","average":{"1min":4.5830000000000002,"5min":1.853,"15min":1.357},"min":{"1min":0.35099999999999998,"5min":0.33700000000000002,"15min":0.317},"max":{"1min":69.760000000000005,"5min":69.760000000000005,"15min":69.760000000000005},"last":0.44400000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724836,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3367116,"kb_used_data":2932420,"kb_used_omap":0,"kb_used_meta":434688,"kb_avail":90378036,"statfs":{"total":95995035648,"available":92547108864,"internally_reserved":0,"allocated":3002798080,"data_stored":3002233813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":445120512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":8.5879999999999992,"5min":2.649,"15min":2.1230000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":124.873,"5min":124.873,"15min":124.873},"last":0.61199999999999999},{"interface":"front","average":{"1min":8.6940000000000008,"5min":2.8540000000000001,"15min":1.849},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":123.554,"5min":123.554,"15min":123.554},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":11.34,"5min":4.9880000000000004,"15min":3.407},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":184.31200000000001,"5min":184.31200000000001,"15min":323.541},"last":0.69199999999999995},{"interface":"front","average":{"1min":11.313000000000001,"5min":5.0149999999999997,"15min":3.6469999999999998},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":184.67400000000001,"5min":184.67400000000001,"15min":324.072},"last":4.0830000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.8759999999999999,"15min":2.2429999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.3760000000000003,"5min":13.394,"15min":13.394},"last":0.64300000000000002},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.5129999999999999,"15min":1.5580000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.3810000000000002,"5min":14.263,"15min":14.263},"last":4.2039999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":19.783000000000001,"5min":5.1799999999999997,"15min":2.968},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":198.33099999999999,"5min":198.33099999999999,"15min":198.33099999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":20.48,"5min":5.1040000000000001,"15min":2.6869999999999998},"min":{"1min":0.32300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":198.47300000000001,"5min":198.47300000000001,"15min":198.47300000000001},"last":1.198}]},{"osd":4,"last update":"Sat Sep 23 20:51:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.012,"15min":1.0009999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.216},"max":{"1min":4.625,"5min":5.8529999999999998,"15min":9.8800000000000008},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.21,"15min":1.1839999999999999},"min":{"1min":0.29999999999999999,"5min":0.28699999999999998,"15min":0.23999999999999999},"max":{"1min":4.3470000000000004,"5min":8.3170000000000002,"15min":8.798},"last":4.2350000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:51:13 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.3149999999999999,"15min":1.133},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.27000000000000002},"max":{"1min":4.4160000000000004,"5min":8.3010000000000002,"15min":8.8239999999999998},"last":4.0030000000000001},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.2290000000000001,"15min":0.97599999999999998},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1890000000000001,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":0.72899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:44 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.716,"15min":1.3100000000000001},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":5.0839999999999996,"5min":14.217000000000001,"15min":14.217000000000001},"last":4.0499999999999998},{"interface":"front","average":{"1min":1.012,"5min":0.996,"15min":1.004},"min":{"1min":0.36799999999999999,"5min":0.29899999999999999,"15min":0.28699999999999998},"max":{"1min":4.4950000000000001,"5min":4.9669999999999996,"15min":7.6130000000000004},"last":0.66700000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1345716224,"data_stored":1345716224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3009286144,"data_stored":3009282072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1001652224,"data_stored":1001652224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2614165504,"data_stored":2614165504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1001259008,"data_stored":1001259008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2986086400,"data_stored":2986082328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1001127936,"data_stored":1001127936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2976452608,"data_stored":2976448536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:52:19.726 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:52:19.726 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:19.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:19 smithi121 ceph-mon[211526]: from='client.45260 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:19.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:19 smithi121 ceph-mon[211526]: pgmap v620: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T20:52:19.805 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:19 smithi121 ceph-mon[219709]: from='client.45260 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:19.806 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:19 smithi121 ceph-mon[219709]: pgmap v620: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T20:52:19.825 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:19.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:19.934 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:19.938 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:19 smithi123 ceph-mon[193903]: from='client.45260 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:19 smithi123 ceph-mon[193903]: pgmap v620: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T20:52:20.049 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:20.276 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:20.277 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:20.323 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:20.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:20.428 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:20.459 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:20.503 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:20.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:20.607 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:20.678 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:20.818 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:52:20.898 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:20.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:20.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1646443172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1646443172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:21.001 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:21.057 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:21.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1646443172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:21.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:21.301 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:21.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:21.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:21.317 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:52:21.406 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:21.500 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: pgmap v621: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[219709]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.065 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: pgmap v621: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:52:22.065 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:22.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:22.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:52:22.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:21 smithi121 ceph-mon[211526]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.109 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:22.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:22.155 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:22.220 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: pgmap v621: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T20:52:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:52:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:21 smithi123 ceph-mon[193903]: failed to encode map e198 with expected crc 2023-09-23T20:52:22.307 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:22.684 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:22.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:22.760 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:22.790 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:22.889 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.057 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:52:23.105 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:52:23.146 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:23.146 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:52:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:23.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:23.179 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:52:23.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:52:23.255 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:23.271 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:52:23.291 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.292 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:52:23.312 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:52:23.334 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:52:23.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:23.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:23.437 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:52:23.464 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.504 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:52:23.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:23.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:52:23.608 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:23.672 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:23 smithi121 ceph-mon[219709]: pgmap v623: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T20:52:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/112822168' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:23.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:23 smithi121 ceph-mon[211526]: pgmap v623: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T20:52:23.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/112822168' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:24.083 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:52:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:23 smithi123 ceph-mon[193903]: pgmap v623: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T20:52:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/112822168' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:24.762 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:24.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:24.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:24.873 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:24.911 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:52:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:52:24] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-09-23T20:52:24.984 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:25.493 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:25.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:25.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:25.599 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:25.657 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:25.814 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:25.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:25.926 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:26.007 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:26.152 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:25 smithi121 ceph-mon[219709]: pgmap v624: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:52:26.170 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:25 smithi121 ceph-mon[211526]: pgmap v624: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:52:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:25 smithi123 ceph-mon[193903]: pgmap v624: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:52:26.300 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:26.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:26.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:26.415 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:26.532 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:26.588 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:52:26.617 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:52:26.677 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:52:26.677 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:52:26.677 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T20:52:26.677 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:52:26.709 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:26.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:26.815 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:26.857 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:26.935 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:52:26.972 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:26.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:26.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:27.083 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:27.175 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:27 smithi121 ceph-mon[211526]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:27 smithi121 ceph-mon[211526]: pgmap v625: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:52:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:27 smithi121 ceph-mon[219709]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:27 smithi121 ceph-mon[219709]: pgmap v625: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:52:27.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:27 smithi123 ceph-mon[193903]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:27.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:27 smithi123 ceph-mon[193903]: pgmap v625: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-09-23T20:52:28.183 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:28.184 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:28.330 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-09-23T20:52:24.704716+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4914878392,"num_objects":75011,"num_object_clones":0,"num_object_copies":225033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75011,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":131085,"num_write_kb":6587086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3668,"num_bytes_recovered":241381416,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126401,"ondisk_log_size":126401,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17235260,"kb_used_data":14428860,"kb_used_omap":20,"kb_used_meta":2806315,"kb_avail":732725956,"statfs":{"total":767960285184,"available":750311378944,"internally_reserved":0,"allocated":14775152640,"data_stored":14769745275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2873667214},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,5,1,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-446169088,"num_objects":-6808,"num_object_clones":0,"num_object_copies":-20424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6808,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002481"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":176,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159561+0000","last_change":"2023-09-23T20:46:05.100324+0000","last_active":"2023-09-23T20:52:12.159561+0000","last_peered":"2023-09-23T20:52:12.159561+0000","last_clean":"2023-09-23T20:52:12.159561+0000","last_became_active":"2023-09-23T20:46:05.099981+0000","last_became_peered":"2023-09-23T20:46:05.099981+0000","last_unstale":"2023-09-23T20:52:12.159561+0000","last_undegraded":"2023-09-23T20:52:12.159561+0000","last_fullsized":"2023-09-23T20:52:12.159561+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:19.876971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766367+0000","last_change":"2023-09-23T20:48:07.517973+0000","last_active":"2023-09-23T20:52:10.766367+0000","last_peered":"2023-09-23T20:52:10.766367+0000","last_clean":"2023-09-23T20:52:10.766367+0000","last_became_active":"2023-09-23T20:48:07.517681+0000","last_became_peered":"2023-09-23T20:48:07.517681+0000","last_unstale":"2023-09-23T20:52:10.766367+0000","last_undegraded":"2023-09-23T20:52:10.766367+0000","last_fullsized":"2023-09-23T20:52:10.766367+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:47.789594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":123,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:13.374618+0000","last_change":"2023-09-23T20:48:07.541333+0000","last_active":"2023-09-23T20:52:13.374618+0000","last_peered":"2023-09-23T20:52:13.374618+0000","last_clean":"2023-09-23T20:52:13.374618+0000","last_became_active":"2023-09-23T20:48:07.541014+0000","last_became_peered":"2023-09-23T20:48:07.541014+0000","last_unstale":"2023-09-23T20:52:13.374618+0000","last_undegraded":"2023-09-23T20:52:13.374618+0000","last_fullsized":"2023-09-23T20:52:13.374618+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:48.678651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":148,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.145464+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:52:12.145464+0000","last_peered":"2023-09-23T20:52:12.145464+0000","last_clean":"2023-09-23T20:52:12.145464+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:52:12.145464+0000","last_undegraded":"2023-09-23T20:52:12.145464+0000","last_fullsized":"2023-09-23T20:52:12.145464+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":154,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766311+0000","last_change":"2023-09-23T20:48:07.540435+0000","last_active":"2023-09-23T20:52:10.766311+0000","last_peered":"2023-09-23T20:52:10.766311+0000","last_clean":"2023-09-23T20:52:10.766311+0000","last_became_active":"2023-09-23T20:48:07.540062+0000","last_became_peered":"2023-09-23T20:48:07.540062+0000","last_unstale":"2023-09-23T20:52:10.766311+0000","last_undegraded":"2023-09-23T20:52:10.766311+0000","last_fullsized":"2023-09-23T20:52:10.766311+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:19.449218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":30,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156554+0000","last_change":"2023-09-23T20:51:05.998368+0000","last_active":"2023-09-23T20:52:12.156554+0000","last_peered":"2023-09-23T20:52:12.156554+0000","last_clean":"2023-09-23T20:52:12.156554+0000","last_became_active":"2023-09-23T20:51:05.997978+0000","last_became_peered":"2023-09-23T20:51:05.997978+0000","last_unstale":"2023-09-23T20:52:12.156554+0000","last_undegraded":"2023-09-23T20:52:12.156554+0000","last_fullsized":"2023-09-23T20:52:12.156554+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:02.281720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.148786+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:52:12.148786+0000","last_peered":"2023-09-23T20:52:12.148786+0000","last_clean":"2023-09-23T20:52:12.148786+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:52:12.148786+0000","last_undegraded":"2023-09-23T20:52:12.148786+0000","last_fullsized":"2023-09-23T20:52:12.148786+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":154,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766224+0000","last_change":"2023-09-23T20:51:06.005347+0000","last_active":"2023-09-23T20:52:10.766224+0000","last_peered":"2023-09-23T20:52:10.766224+0000","last_clean":"2023-09-23T20:52:10.766224+0000","last_became_active":"2023-09-23T20:51:06.005036+0000","last_became_peered":"2023-09-23T20:51:06.005036+0000","last_unstale":"2023-09-23T20:52:10.766224+0000","last_undegraded":"2023-09-23T20:52:10.766224+0000","last_fullsized":"2023-09-23T20:52:10.766224+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:19.359325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":158,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.155380+0000","last_change":"2023-09-23T20:45:00.007125+0000","last_active":"2023-09-23T20:52:12.155380+0000","last_peered":"2023-09-23T20:52:12.155380+0000","last_clean":"2023-09-23T20:52:12.155380+0000","last_became_active":"2023-09-23T20:45:00.006403+0000","last_became_peered":"2023-09-23T20:45:00.006403+0000","last_unstale":"2023-09-23T20:52:12.155380+0000","last_undegraded":"2023-09-23T20:52:12.155380+0000","last_fullsized":"2023-09-23T20:52:12.155380+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:27.906490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":132,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.767459+0000","last_change":"2023-09-23T20:51:06.024444+0000","last_active":"2023-09-23T20:52:10.767459+0000","last_peered":"2023-09-23T20:52:10.767459+0000","last_clean":"2023-09-23T20:52:10.767459+0000","last_became_active":"2023-09-23T20:51:06.023727+0000","last_became_peered":"2023-09-23T20:51:06.023727+0000","last_unstale":"2023-09-23T20:52:10.767459+0000","last_undegraded":"2023-09-23T20:52:10.767459+0000","last_fullsized":"2023-09-23T20:52:10.767459+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:24.860774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":161,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.154498+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:52:12.154498+0000","last_peered":"2023-09-23T20:52:12.154498+0000","last_clean":"2023-09-23T20:52:12.154498+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:52:12.154498+0000","last_undegraded":"2023-09-23T20:52:12.154498+0000","last_fullsized":"2023-09-23T20:52:12.154498+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156285+0000","last_change":"2023-09-23T20:51:06.002973+0000","last_active":"2023-09-23T20:52:12.156285+0000","last_peered":"2023-09-23T20:52:12.156285+0000","last_clean":"2023-09-23T20:52:12.156285+0000","last_became_active":"2023-09-23T20:51:06.001732+0000","last_became_peered":"2023-09-23T20:51:06.001732+0000","last_unstale":"2023-09-23T20:52:12.156285+0000","last_undegraded":"2023-09-23T20:52:12.156285+0000","last_fullsized":"2023-09-23T20:52:12.156285+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:44.025395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":203,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.160015+0000","last_change":"2023-09-23T20:51:05.966238+0000","last_active":"2023-09-23T20:52:12.160015+0000","last_peered":"2023-09-23T20:52:12.160015+0000","last_clean":"2023-09-23T20:52:12.160015+0000","last_became_active":"2023-09-23T20:51:05.959313+0000","last_became_peered":"2023-09-23T20:51:05.959313+0000","last_unstale":"2023-09-23T20:52:12.160015+0000","last_undegraded":"2023-09-23T20:52:12.160015+0000","last_fullsized":"2023-09-23T20:52:12.160015+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:02.228179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159977+0000","last_change":"2023-09-23T20:51:05.998598+0000","last_active":"2023-09-23T20:52:12.159977+0000","last_peered":"2023-09-23T20:52:12.159977+0000","last_clean":"2023-09-23T20:52:12.159977+0000","last_became_active":"2023-09-23T20:51:05.998180+0000","last_became_peered":"2023-09-23T20:51:05.998180+0000","last_unstale":"2023-09-23T20:52:12.159977+0000","last_undegraded":"2023-09-23T20:52:12.159977+0000","last_fullsized":"2023-09-23T20:52:12.159977+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:56.185992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":160,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156757+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:52:12.156757+0000","last_peered":"2023-09-23T20:52:12.156757+0000","last_clean":"2023-09-23T20:52:12.156757+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:52:12.156757+0000","last_undegraded":"2023-09-23T20:52:12.156757+0000","last_fullsized":"2023-09-23T20:52:12.156757+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":136,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.767994+0000","last_change":"2023-09-23T20:48:07.540014+0000","last_active":"2023-09-23T20:52:10.767994+0000","last_peered":"2023-09-23T20:52:10.767994+0000","last_clean":"2023-09-23T20:52:10.767994+0000","last_became_active":"2023-09-23T20:48:07.539722+0000","last_became_peered":"2023-09-23T20:48:07.539722+0000","last_unstale":"2023-09-23T20:52:10.767994+0000","last_undegraded":"2023-09-23T20:52:10.767994+0000","last_fullsized":"2023-09-23T20:52:10.767994+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.937232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":146,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.148331+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:52:12.148331+0000","last_peered":"2023-09-23T20:52:12.148331+0000","last_clean":"2023-09-23T20:52:12.148331+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:52:12.148331+0000","last_undegraded":"2023-09-23T20:52:12.148331+0000","last_fullsized":"2023-09-23T20:52:12.148331+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":157,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.153477+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:52:12.153477+0000","last_peered":"2023-09-23T20:52:12.153477+0000","last_clean":"2023-09-23T20:52:12.153477+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:52:12.153477+0000","last_undegraded":"2023-09-23T20:52:12.153477+0000","last_fullsized":"2023-09-23T20:52:12.153477+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":161,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.155100+0000","last_change":"2023-09-23T20:51:05.958217+0000","last_active":"2023-09-23T20:52:12.155100+0000","last_peered":"2023-09-23T20:52:12.155100+0000","last_clean":"2023-09-23T20:52:12.155100+0000","last_became_active":"2023-09-23T20:51:05.957874+0000","last_became_peered":"2023-09-23T20:51:05.957874+0000","last_unstale":"2023-09-23T20:52:12.155100+0000","last_undegraded":"2023-09-23T20:52:12.155100+0000","last_fullsized":"2023-09-23T20:52:12.155100+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:13.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.145593+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:52:12.145593+0000","last_peered":"2023-09-23T20:52:12.145593+0000","last_clean":"2023-09-23T20:52:12.145593+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:52:12.145593+0000","last_undegraded":"2023-09-23T20:52:12.145593+0000","last_fullsized":"2023-09-23T20:52:12.145593+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":40,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.155863+0000","last_change":"2023-09-23T20:51:06.038448+0000","last_active":"2023-09-23T20:52:12.155863+0000","last_peered":"2023-09-23T20:52:12.155863+0000","last_clean":"2023-09-23T20:52:12.155863+0000","last_became_active":"2023-09-23T20:51:06.005965+0000","last_became_peered":"2023-09-23T20:51:06.005965+0000","last_unstale":"2023-09-23T20:52:12.155863+0000","last_undegraded":"2023-09-23T20:52:12.155863+0000","last_fullsized":"2023-09-23T20:52:12.155863+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:13.397771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":136,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766878+0000","last_change":"2023-09-23T20:48:07.557905+0000","last_active":"2023-09-23T20:52:10.766878+0000","last_peered":"2023-09-23T20:52:10.766878+0000","last_clean":"2023-09-23T20:52:10.766878+0000","last_became_active":"2023-09-23T20:48:07.557568+0000","last_became_peered":"2023-09-23T20:48:07.557568+0000","last_unstale":"2023-09-23T20:52:10.766878+0000","last_undegraded":"2023-09-23T20:52:10.766878+0000","last_fullsized":"2023-09-23T20:52:10.766878+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:14.229704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"198'7978","reported_seq":8022,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:22.789346+0000","last_change":"2023-09-23T20:50:08.077384+0000","last_active":"2023-09-23T20:52:22.789346+0000","last_peered":"2023-09-23T20:52:22.789346+0000","last_clean":"2023-09-23T20:52:22.789346+0000","last_became_active":"2023-09-23T20:50:08.077095+0000","last_became_peered":"2023-09-23T20:50:08.077095+0000","last_unstale":"2023-09-23T20:52:22.789346+0000","last_undegraded":"2023-09-23T20:52:22.789346+0000","last_fullsized":"2023-09-23T20:52:22.789346+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7978,"log_dups_size":0,"ondisk_log_size":7978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:34.442024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303824896,"num_objects":4636,"num_object_clones":0,"num_object_copies":13908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7978,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":159,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.155053+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:52:12.155053+0000","last_peered":"2023-09-23T20:52:12.155053+0000","last_clean":"2023-09-23T20:52:12.155053+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:52:12.155053+0000","last_undegraded":"2023-09-23T20:52:12.155053+0000","last_fullsized":"2023-09-23T20:52:12.155053+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":160,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.152277+0000","last_change":"2023-09-23T20:51:05.962659+0000","last_active":"2023-09-23T20:52:12.152277+0000","last_peered":"2023-09-23T20:52:12.152277+0000","last_clean":"2023-09-23T20:52:12.152277+0000","last_became_active":"2023-09-23T20:51:05.962048+0000","last_became_peered":"2023-09-23T20:51:05.962048+0000","last_unstale":"2023-09-23T20:52:12.152277+0000","last_undegraded":"2023-09-23T20:52:12.152277+0000","last_fullsized":"2023-09-23T20:52:12.152277+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:03.530084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"198'8245","reported_seq":8289,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.241934+0000","last_change":"2023-09-23T20:50:08.083329+0000","last_active":"2023-09-23T20:52:23.241934+0000","last_peered":"2023-09-23T20:52:23.241934+0000","last_clean":"2023-09-23T20:52:23.241934+0000","last_became_active":"2023-09-23T20:50:08.082984+0000","last_became_peered":"2023-09-23T20:50:08.082984+0000","last_unstale":"2023-09-23T20:52:23.241934+0000","last_undegraded":"2023-09-23T20:52:23.241934+0000","last_fullsized":"2023-09-23T20:52:23.241934+0000","mapping_epoch":164,"log_start":"168'900","ondisk_log_start":"168'900","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.393867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312016896,"num_objects":4761,"num_object_clones":0,"num_object_copies":14283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8245,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":160,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.155196+0000","last_change":"2023-09-23T20:45:00.003948+0000","last_active":"2023-09-23T20:52:12.155196+0000","last_peered":"2023-09-23T20:52:12.155196+0000","last_clean":"2023-09-23T20:52:12.155196+0000","last_became_active":"2023-09-23T20:45:00.003639+0000","last_became_peered":"2023-09-23T20:45:00.003639+0000","last_unstale":"2023-09-23T20:52:12.155196+0000","last_undegraded":"2023-09-23T20:52:12.155196+0000","last_fullsized":"2023-09-23T20:52:12.155196+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:59.619696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":174,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159319+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:52:12.159319+0000","last_peered":"2023-09-23T20:52:12.159319+0000","last_clean":"2023-09-23T20:52:12.159319+0000","last_became_active":"2023-09-23T20:48:07.544029+0000","last_became_peered":"2023-09-23T20:48:07.544029+0000","last_unstale":"2023-09-23T20:52:12.159319+0000","last_undegraded":"2023-09-23T20:52:12.159319+0000","last_fullsized":"2023-09-23T20:52:12.159319+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:52.671448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"198'8146","reported_seq":8188,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.819637+0000","last_change":"2023-09-23T20:50:08.078801+0000","last_active":"2023-09-23T20:52:23.819637+0000","last_peered":"2023-09-23T20:52:23.819637+0000","last_clean":"2023-09-23T20:52:23.819637+0000","last_became_active":"2023-09-23T20:50:08.078386+0000","last_became_peered":"2023-09-23T20:50:08.078386+0000","last_unstale":"2023-09-23T20:52:23.819637+0000","last_undegraded":"2023-09-23T20:52:23.819637+0000","last_fullsized":"2023-09-23T20:52:23.819637+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":8146,"log_dups_size":0,"ondisk_log_size":8146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:47.134724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301334528,"num_objects":4598,"num_object_clones":0,"num_object_copies":13794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8146,"num_write_kb":407808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":170,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.154351+0000","last_change":"2023-09-23T20:51:05.967804+0000","last_active":"2023-09-23T20:52:12.154351+0000","last_peered":"2023-09-23T20:52:12.154351+0000","last_clean":"2023-09-23T20:52:12.154351+0000","last_became_active":"2023-09-23T20:51:05.959914+0000","last_became_peered":"2023-09-23T20:51:05.959914+0000","last_unstale":"2023-09-23T20:52:12.154351+0000","last_undegraded":"2023-09-23T20:52:12.154351+0000","last_fullsized":"2023-09-23T20:52:12.154351+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:03.210361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":30,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156094+0000","last_change":"2023-09-23T20:51:05.957335+0000","last_active":"2023-09-23T20:52:12.156094+0000","last_peered":"2023-09-23T20:52:12.156094+0000","last_clean":"2023-09-23T20:52:12.156094+0000","last_became_active":"2023-09-23T20:51:05.956858+0000","last_became_peered":"2023-09-23T20:51:05.956858+0000","last_unstale":"2023-09-23T20:52:12.156094+0000","last_undegraded":"2023-09-23T20:52:12.156094+0000","last_fullsized":"2023-09-23T20:52:12.156094+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:31.085919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"198'8083","reported_seq":8125,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.824621+0000","last_change":"2023-09-23T20:50:08.648424+0000","last_active":"2023-09-23T20:52:23.824621+0000","last_peered":"2023-09-23T20:52:23.824621+0000","last_clean":"2023-09-23T20:52:23.824621+0000","last_became_active":"2023-09-23T20:50:08.648031+0000","last_became_peered":"2023-09-23T20:50:08.648031+0000","last_unstale":"2023-09-23T20:52:23.824621+0000","last_undegraded":"2023-09-23T20:52:23.824621+0000","last_fullsized":"2023-09-23T20:52:23.824621+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":8083,"log_dups_size":0,"ondisk_log_size":8083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:22.291407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299958272,"num_objects":4577,"num_object_clones":0,"num_object_copies":13731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8083,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":178,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.152496+0000","last_change":"2023-09-23T20:51:05.995979+0000","last_active":"2023-09-23T20:52:12.152496+0000","last_peered":"2023-09-23T20:52:12.152496+0000","last_clean":"2023-09-23T20:52:12.152496+0000","last_became_active":"2023-09-23T20:51:05.995591+0000","last_became_peered":"2023-09-23T20:51:05.995591+0000","last_unstale":"2023-09-23T20:52:12.152496+0000","last_undegraded":"2023-09-23T20:52:12.152496+0000","last_fullsized":"2023-09-23T20:52:12.152496+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:35.414294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":148,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.146496+0000","last_change":"2023-09-23T20:45:00.002738+0000","last_active":"2023-09-23T20:52:12.146496+0000","last_peered":"2023-09-23T20:52:12.146496+0000","last_clean":"2023-09-23T20:52:12.146496+0000","last_became_active":"2023-09-23T20:45:00.002403+0000","last_became_peered":"2023-09-23T20:45:00.002403+0000","last_unstale":"2023-09-23T20:52:12.146496+0000","last_undegraded":"2023-09-23T20:52:12.146496+0000","last_fullsized":"2023-09-23T20:52:12.146496+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:54.551205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"197'8031","reported_seq":10938,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:20.753672+0000","last_change":"2023-09-23T20:51:15.811325+0000","last_active":"2023-09-23T20:52:20.753672+0000","last_peered":"2023-09-23T20:52:20.753672+0000","last_clean":"2023-09-23T20:52:20.753672+0000","last_became_active":"2023-09-23T20:51:15.805454+0000","last_became_peered":"2023-09-23T20:51:15.805454+0000","last_unstale":"2023-09-23T20:52:20.753672+0000","last_undegraded":"2023-09-23T20:52:20.753672+0000","last_fullsized":"2023-09-23T20:52:20.753672+0000","mapping_epoch":180,"log_start":"168'300","ondisk_log_start":"168'300","created":164,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7731,"log_dups_size":0,"ondisk_log_size":7731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:13.254228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306380800,"num_objects":4675,"num_object_clones":0,"num_object_copies":14025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8031,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1418,"num_bytes_recovered":92930048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766162+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:52:10.766162+0000","last_peered":"2023-09-23T20:52:10.766162+0000","last_clean":"2023-09-23T20:52:10.766162+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:52:10.766162+0000","last_undegraded":"2023-09-23T20:52:10.766162+0000","last_fullsized":"2023-09-23T20:52:10.766162+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159940+0000","last_change":"2023-09-23T20:51:06.001664+0000","last_active":"2023-09-23T20:52:12.159940+0000","last_peered":"2023-09-23T20:52:12.159940+0000","last_clean":"2023-09-23T20:52:12.159940+0000","last_became_active":"2023-09-23T20:51:06.001352+0000","last_became_peered":"2023-09-23T20:51:06.001352+0000","last_unstale":"2023-09-23T20:52:12.159940+0000","last_undegraded":"2023-09-23T20:52:12.159940+0000","last_fullsized":"2023-09-23T20:52:12.159940+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.688374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"198'8243","reported_seq":9973,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.246452+0000","last_change":"2023-09-23T20:51:30.833121+0000","last_active":"2023-09-23T20:52:23.246452+0000","last_peered":"2023-09-23T20:52:23.246452+0000","last_clean":"2023-09-23T20:52:23.246452+0000","last_became_active":"2023-09-23T20:51:30.828670+0000","last_became_peered":"2023-09-23T20:51:30.828670+0000","last_unstale":"2023-09-23T20:52:23.246452+0000","last_undegraded":"2023-09-23T20:52:23.246452+0000","last_fullsized":"2023-09-23T20:52:23.246452+0000","mapping_epoch":186,"log_start":"168'900","ondisk_log_start":"168'900","created":164,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:07.018027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304414720,"num_objects":4645,"num_object_clones":0,"num_object_copies":13935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8243,"num_write_kb":412416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":829,"num_bytes_recovered":54329344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":30,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156142+0000","last_change":"2023-09-23T20:51:06.005884+0000","last_active":"2023-09-23T20:52:12.156142+0000","last_peered":"2023-09-23T20:52:12.156142+0000","last_clean":"2023-09-23T20:52:12.156142+0000","last_became_active":"2023-09-23T20:51:06.005399+0000","last_became_peered":"2023-09-23T20:51:06.005399+0000","last_unstale":"2023-09-23T20:52:12.156142+0000","last_undegraded":"2023-09-23T20:52:12.156142+0000","last_fullsized":"2023-09-23T20:52:12.156142+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:56.910456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":174,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.161427+0000","last_change":"2023-09-23T20:46:05.106502+0000","last_active":"2023-09-23T20:52:12.161427+0000","last_peered":"2023-09-23T20:52:12.161427+0000","last_clean":"2023-09-23T20:52:12.161427+0000","last_became_active":"2023-09-23T20:46:05.106134+0000","last_became_peered":"2023-09-23T20:46:05.106134+0000","last_unstale":"2023-09-23T20:52:12.161427+0000","last_undegraded":"2023-09-23T20:52:12.161427+0000","last_fullsized":"2023-09-23T20:52:12.161427+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:39.257351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"198'8208","reported_seq":8250,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.825460+0000","last_change":"2023-09-23T20:50:08.077812+0000","last_active":"2023-09-23T20:52:23.825460+0000","last_peered":"2023-09-23T20:52:23.825460+0000","last_clean":"2023-09-23T20:52:23.825460+0000","last_became_active":"2023-09-23T20:50:08.077458+0000","last_became_peered":"2023-09-23T20:50:08.077458+0000","last_unstale":"2023-09-23T20:52:23.825460+0000","last_undegraded":"2023-09-23T20:52:23.825460+0000","last_fullsized":"2023-09-23T20:52:23.825460+0000","mapping_epoch":164,"log_start":"168'500","ondisk_log_start":"168'500","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7708,"log_dups_size":0,"ondisk_log_size":7708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.729804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304087040,"num_objects":4640,"num_object_clones":0,"num_object_copies":13920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8208,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":158,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159094+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:52:12.159094+0000","last_peered":"2023-09-23T20:52:12.159094+0000","last_clean":"2023-09-23T20:52:12.159094+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:52:12.159094+0000","last_undegraded":"2023-09-23T20:52:12.159094+0000","last_fullsized":"2023-09-23T20:52:12.159094+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.160192+0000","last_change":"2023-09-23T20:51:06.002223+0000","last_active":"2023-09-23T20:52:12.160192+0000","last_peered":"2023-09-23T20:52:12.160192+0000","last_clean":"2023-09-23T20:52:12.160192+0000","last_became_active":"2023-09-23T20:51:06.001904+0000","last_became_peered":"2023-09-23T20:51:06.001904+0000","last_unstale":"2023-09-23T20:52:12.160192+0000","last_undegraded":"2023-09-23T20:52:12.160192+0000","last_fullsized":"2023-09-23T20:52:12.160192+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:32.282239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"197'8136","reported_seq":8179,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:20.758622+0000","last_change":"2023-09-23T20:50:08.647934+0000","last_active":"2023-09-23T20:52:20.758622+0000","last_peered":"2023-09-23T20:52:20.758622+0000","last_clean":"2023-09-23T20:52:20.758622+0000","last_became_active":"2023-09-23T20:50:08.647621+0000","last_became_peered":"2023-09-23T20:50:08.647621+0000","last_unstale":"2023-09-23T20:52:20.758622+0000","last_undegraded":"2023-09-23T20:52:20.758622+0000","last_fullsized":"2023-09-23T20:52:20.758622+0000","mapping_epoch":164,"log_start":"168'400","ondisk_log_start":"168'400","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7736,"log_dups_size":0,"ondisk_log_size":7736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.046520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8136,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":137,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766157+0000","last_change":"2023-09-23T20:48:07.540484+0000","last_active":"2023-09-23T20:52:10.766157+0000","last_peered":"2023-09-23T20:52:10.766157+0000","last_clean":"2023-09-23T20:52:10.766157+0000","last_became_active":"2023-09-23T20:48:07.540219+0000","last_became_peered":"2023-09-23T20:48:07.540219+0000","last_unstale":"2023-09-23T20:52:10.766157+0000","last_undegraded":"2023-09-23T20:52:10.766157+0000","last_fullsized":"2023-09-23T20:52:10.766157+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:54.792744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.160646+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:52:12.160646+0000","last_peered":"2023-09-23T20:52:12.160646+0000","last_clean":"2023-09-23T20:52:12.160646+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:52:12.160646+0000","last_undegraded":"2023-09-23T20:52:12.160646+0000","last_fullsized":"2023-09-23T20:52:12.160646+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"198'8310","reported_seq":8352,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.797452+0000","last_change":"2023-09-23T20:50:08.078863+0000","last_active":"2023-09-23T20:52:23.797452+0000","last_peered":"2023-09-23T20:52:23.797452+0000","last_clean":"2023-09-23T20:52:23.797452+0000","last_became_active":"2023-09-23T20:50:08.078560+0000","last_became_peered":"2023-09-23T20:50:08.078560+0000","last_unstale":"2023-09-23T20:52:23.797452+0000","last_undegraded":"2023-09-23T20:52:23.797452+0000","last_fullsized":"2023-09-23T20:52:23.797452+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":8310,"log_dups_size":0,"ondisk_log_size":8310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:41.488151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310640640,"num_objects":4740,"num_object_clones":0,"num_object_copies":14220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8310,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":156,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.154044+0000","last_change":"2023-09-23T20:45:00.001830+0000","last_active":"2023-09-23T20:52:12.154044+0000","last_peered":"2023-09-23T20:52:12.154044+0000","last_clean":"2023-09-23T20:52:12.154044+0000","last_became_active":"2023-09-23T20:45:00.001532+0000","last_became_peered":"2023-09-23T20:45:00.001532+0000","last_unstale":"2023-09-23T20:52:12.154044+0000","last_undegraded":"2023-09-23T20:52:12.154044+0000","last_fullsized":"2023-09-23T20:52:12.154044+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:17.499694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":175,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.153555+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:52:12.153555+0000","last_peered":"2023-09-23T20:52:12.153555+0000","last_clean":"2023-09-23T20:52:12.153555+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:52:12.153555+0000","last_undegraded":"2023-09-23T20:52:12.153555+0000","last_fullsized":"2023-09-23T20:52:12.153555+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":109,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156247+0000","last_change":"2023-09-23T20:45:00.004588+0000","last_active":"2023-09-23T20:52:12.156247+0000","last_peered":"2023-09-23T20:52:12.156247+0000","last_clean":"2023-09-23T20:52:12.156247+0000","last_became_active":"2023-09-23T20:45:00.004185+0000","last_became_peered":"2023-09-23T20:45:00.004185+0000","last_unstale":"2023-09-23T20:52:12.156247+0000","last_undegraded":"2023-09-23T20:52:12.156247+0000","last_fullsized":"2023-09-23T20:52:12.156247+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:47.406920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"198'8369","reported_seq":8413,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:22.784603+0000","last_change":"2023-09-23T20:50:08.076706+0000","last_active":"2023-09-23T20:52:22.784603+0000","last_peered":"2023-09-23T20:52:22.784603+0000","last_clean":"2023-09-23T20:52:22.784603+0000","last_became_active":"2023-09-23T20:50:08.076360+0000","last_became_peered":"2023-09-23T20:50:08.076360+0000","last_unstale":"2023-09-23T20:52:22.784603+0000","last_undegraded":"2023-09-23T20:52:22.784603+0000","last_fullsized":"2023-09-23T20:52:22.784603+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":8369,"log_dups_size":0,"ondisk_log_size":8369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.596097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8369,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":182,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.154139+0000","last_change":"2023-09-23T20:51:06.003661+0000","last_active":"2023-09-23T20:52:12.154139+0000","last_peered":"2023-09-23T20:52:12.154139+0000","last_clean":"2023-09-23T20:52:12.154139+0000","last_became_active":"2023-09-23T20:51:06.003277+0000","last_became_peered":"2023-09-23T20:51:06.003277+0000","last_unstale":"2023-09-23T20:52:12.154139+0000","last_undegraded":"2023-09-23T20:52:12.154139+0000","last_fullsized":"2023-09-23T20:52:12.154139+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:32.933372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":178,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159776+0000","last_change":"2023-09-23T20:51:06.033002+0000","last_active":"2023-09-23T20:52:12.159776+0000","last_peered":"2023-09-23T20:52:12.159776+0000","last_clean":"2023-09-23T20:52:12.159776+0000","last_became_active":"2023-09-23T20:51:06.032657+0000","last_became_peered":"2023-09-23T20:51:06.032657+0000","last_unstale":"2023-09-23T20:52:12.159776+0000","last_undegraded":"2023-09-23T20:52:12.159776+0000","last_fullsized":"2023-09-23T20:52:12.159776+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:07.293307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":167,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.152364+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:52:12.152364+0000","last_peered":"2023-09-23T20:52:12.152364+0000","last_clean":"2023-09-23T20:52:12.152364+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:52:12.152364+0000","last_undegraded":"2023-09-23T20:52:12.152364+0000","last_fullsized":"2023-09-23T20:52:12.152364+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"198'8222","reported_seq":8264,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.794500+0000","last_change":"2023-09-23T20:50:08.080121+0000","last_active":"2023-09-23T20:52:23.794500+0000","last_peered":"2023-09-23T20:52:23.794500+0000","last_clean":"2023-09-23T20:52:23.794500+0000","last_became_active":"2023-09-23T20:50:08.079818+0000","last_became_peered":"2023-09-23T20:50:08.079818+0000","last_unstale":"2023-09-23T20:52:23.794500+0000","last_undegraded":"2023-09-23T20:52:23.794500+0000","last_fullsized":"2023-09-23T20:52:23.794500+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":8222,"log_dups_size":0,"ondisk_log_size":8222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:03.610431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300679168,"num_objects":4588,"num_object_clones":0,"num_object_copies":13764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8222,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":156,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:18.270358+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:52:18.270358+0000","last_peered":"2023-09-23T20:52:18.270358+0000","last_clean":"2023-09-23T20:52:18.270358+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:52:18.270358+0000","last_undegraded":"2023-09-23T20:52:18.270358+0000","last_fullsized":"2023-09-23T20:52:18.270358+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.152907+0000","last_change":"2023-09-23T20:51:06.002709+0000","last_active":"2023-09-23T20:52:12.152907+0000","last_peered":"2023-09-23T20:52:12.152907+0000","last_clean":"2023-09-23T20:52:12.152907+0000","last_became_active":"2023-09-23T20:51:06.002425+0000","last_became_peered":"2023-09-23T20:51:06.002425+0000","last_unstale":"2023-09-23T20:52:12.152907+0000","last_undegraded":"2023-09-23T20:52:12.152907+0000","last_fullsized":"2023-09-23T20:52:12.152907+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:45.344651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.153049+0000","last_change":"2023-09-23T20:45:00.002611+0000","last_active":"2023-09-23T20:52:12.153049+0000","last_peered":"2023-09-23T20:52:12.153049+0000","last_clean":"2023-09-23T20:52:12.153049+0000","last_became_active":"2023-09-23T20:45:00.002331+0000","last_became_peered":"2023-09-23T20:45:00.002331+0000","last_unstale":"2023-09-23T20:52:12.153049+0000","last_undegraded":"2023-09-23T20:52:12.153049+0000","last_fullsized":"2023-09-23T20:52:12.153049+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:19.968130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"197'7940","reported_seq":7982,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:20.759463+0000","last_change":"2023-09-23T20:50:08.078559+0000","last_active":"2023-09-23T20:52:20.759463+0000","last_peered":"2023-09-23T20:52:20.759463+0000","last_clean":"2023-09-23T20:52:20.759463+0000","last_became_active":"2023-09-23T20:50:08.078294+0000","last_became_peered":"2023-09-23T20:50:08.078294+0000","last_unstale":"2023-09-23T20:52:20.759463+0000","last_undegraded":"2023-09-23T20:52:20.759463+0000","last_fullsized":"2023-09-23T20:52:20.759463+0000","mapping_epoch":164,"log_start":"168'200","ondisk_log_start":"168'200","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7740,"log_dups_size":0,"ondisk_log_size":7740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:11.281483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305922048,"num_objects":4668,"num_object_clones":0,"num_object_copies":14004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7940,"num_write_kb":403456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":162,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.152789+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:52:12.152789+0000","last_peered":"2023-09-23T20:52:12.152789+0000","last_clean":"2023-09-23T20:52:12.152789+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:52:12.152789+0000","last_undegraded":"2023-09-23T20:52:12.152789+0000","last_fullsized":"2023-09-23T20:52:12.152789+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":590,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.154309+0000","last_change":"2023-09-23T20:51:30.823817+0000","last_active":"2023-09-23T20:52:12.154309+0000","last_peered":"2023-09-23T20:52:12.154309+0000","last_clean":"2023-09-23T20:52:12.154309+0000","last_became_active":"2023-09-23T20:51:30.823372+0000","last_became_peered":"2023-09-23T20:51:30.823372+0000","last_unstale":"2023-09-23T20:52:12.154309+0000","last_undegraded":"2023-09-23T20:52:12.154309+0000","last_fullsized":"2023-09-23T20:52:12.154309+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:09.432277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":132,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.150730+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:52:12.150730+0000","last_peered":"2023-09-23T20:52:12.150730+0000","last_clean":"2023-09-23T20:52:12.150730+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:52:12.150730+0000","last_undegraded":"2023-09-23T20:52:12.150730+0000","last_fullsized":"2023-09-23T20:52:12.150730+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":30,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156403+0000","last_change":"2023-09-23T20:51:05.957381+0000","last_active":"2023-09-23T20:52:12.156403+0000","last_peered":"2023-09-23T20:52:12.156403+0000","last_clean":"2023-09-23T20:52:12.156403+0000","last_became_active":"2023-09-23T20:51:05.956991+0000","last_became_peered":"2023-09-23T20:51:05.956991+0000","last_unstale":"2023-09-23T20:52:12.156403+0000","last_undegraded":"2023-09-23T20:52:12.156403+0000","last_fullsized":"2023-09-23T20:52:12.156403+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:20.613174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"198'8319","reported_seq":11172,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:22.766186+0000","last_change":"2023-09-23T20:51:23.778886+0000","last_active":"2023-09-23T20:52:22.766186+0000","last_peered":"2023-09-23T20:52:22.766186+0000","last_clean":"2023-09-23T20:52:22.766186+0000","last_became_active":"2023-09-23T20:51:23.771838+0000","last_became_peered":"2023-09-23T20:51:23.771838+0000","last_unstale":"2023-09-23T20:52:22.766186+0000","last_undegraded":"2023-09-23T20:52:22.766186+0000","last_fullsized":"2023-09-23T20:52:22.766186+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":8319,"log_dups_size":0,"ondisk_log_size":8319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:54.532463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316473344,"num_objects":4829,"num_object_clones":0,"num_object_copies":14487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8319,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1391,"num_bytes_recovered":91160576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":179,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.154181+0000","last_change":"2023-09-23T20:45:00.001911+0000","last_active":"2023-09-23T20:52:12.154181+0000","last_peered":"2023-09-23T20:52:12.154181+0000","last_clean":"2023-09-23T20:52:12.154181+0000","last_became_active":"2023-09-23T20:45:00.001531+0000","last_became_peered":"2023-09-23T20:45:00.001531+0000","last_unstale":"2023-09-23T20:52:12.154181+0000","last_undegraded":"2023-09-23T20:52:12.154181+0000","last_fullsized":"2023-09-23T20:52:12.154181+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:17.927950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.147236+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:52:12.147236+0000","last_peered":"2023-09-23T20:52:12.147236+0000","last_clean":"2023-09-23T20:52:12.147236+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:52:12.147236+0000","last_undegraded":"2023-09-23T20:52:12.147236+0000","last_fullsized":"2023-09-23T20:52:12.147236+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":163,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.145529+0000","last_change":"2023-09-23T20:46:05.077316+0000","last_active":"2023-09-23T20:52:12.145529+0000","last_peered":"2023-09-23T20:52:12.145529+0000","last_clean":"2023-09-23T20:52:12.145529+0000","last_became_active":"2023-09-23T20:46:05.077044+0000","last_became_peered":"2023-09-23T20:46:05.077044+0000","last_unstale":"2023-09-23T20:52:12.145529+0000","last_undegraded":"2023-09-23T20:52:12.145529+0000","last_fullsized":"2023-09-23T20:52:12.145529+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:56.517854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"198'8295","reported_seq":8336,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.831315+0000","last_change":"2023-09-23T20:50:08.083097+0000","last_active":"2023-09-23T20:52:23.831315+0000","last_peered":"2023-09-23T20:52:23.831315+0000","last_clean":"2023-09-23T20:52:23.831315+0000","last_became_active":"2023-09-23T20:50:08.082761+0000","last_became_peered":"2023-09-23T20:50:08.082761+0000","last_unstale":"2023-09-23T20:52:23.831315+0000","last_undegraded":"2023-09-23T20:52:23.831315+0000","last_fullsized":"2023-09-23T20:52:23.831315+0000","mapping_epoch":164,"log_start":"168'600","ondisk_log_start":"168'600","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7695,"log_dups_size":0,"ondisk_log_size":7695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:29.125314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305463296,"num_objects":4661,"num_object_clones":0,"num_object_copies":13983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8295,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":162,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.154423+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:52:12.154423+0000","last_peered":"2023-09-23T20:52:12.154423+0000","last_clean":"2023-09-23T20:52:12.154423+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:52:12.154423+0000","last_undegraded":"2023-09-23T20:52:12.154423+0000","last_fullsized":"2023-09-23T20:52:12.154423+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.153978+0000","last_change":"2023-09-23T20:46:05.079434+0000","last_active":"2023-09-23T20:52:12.153978+0000","last_peered":"2023-09-23T20:52:12.153978+0000","last_clean":"2023-09-23T20:52:12.153978+0000","last_became_active":"2023-09-23T20:46:05.079111+0000","last_became_peered":"2023-09-23T20:46:05.079111+0000","last_unstale":"2023-09-23T20:52:12.153978+0000","last_undegraded":"2023-09-23T20:52:12.153978+0000","last_fullsized":"2023-09-23T20:52:12.153978+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:11.869457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":133,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.146364+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:52:12.146364+0000","last_peered":"2023-09-23T20:52:12.146364+0000","last_clean":"2023-09-23T20:52:12.146364+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:52:12.146364+0000","last_undegraded":"2023-09-23T20:52:12.146364+0000","last_fullsized":"2023-09-23T20:52:12.146364+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"198'8148","reported_seq":8189,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T20:52:23.854011+0000","last_change":"2023-09-23T20:50:08.078428+0000","last_active":"2023-09-23T20:52:23.854011+0000","last_peered":"2023-09-23T20:52:23.854011+0000","last_clean":"2023-09-23T20:52:23.854011+0000","last_became_active":"2023-09-23T20:50:08.078116+0000","last_became_peered":"2023-09-23T20:50:08.078116+0000","last_unstale":"2023-09-23T20:52:23.854011+0000","last_undegraded":"2023-09-23T20:52:23.854011+0000","last_fullsized":"2023-09-23T20:52:23.854011+0000","mapping_epoch":164,"log_start":"168'400","ondisk_log_start":"168'400","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7748,"log_dups_size":0,"ondisk_log_size":7748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:06.919475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298647576,"num_objects":4558,"num_object_clones":0,"num_object_copies":13674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4558,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8148,"num_write_kb":406529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":159,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766954+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:52:10.766954+0000","last_peered":"2023-09-23T20:52:10.766954+0000","last_clean":"2023-09-23T20:52:10.766954+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:52:10.766954+0000","last_undegraded":"2023-09-23T20:52:10.766954+0000","last_fullsized":"2023-09-23T20:52:10.766954+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":168,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.153521+0000","last_change":"2023-09-23T20:51:05.997935+0000","last_active":"2023-09-23T20:52:12.153521+0000","last_peered":"2023-09-23T20:52:12.153521+0000","last_clean":"2023-09-23T20:52:12.153521+0000","last_became_active":"2023-09-23T20:51:05.997637+0000","last_became_peered":"2023-09-23T20:51:05.997637+0000","last_unstale":"2023-09-23T20:52:12.153521+0000","last_undegraded":"2023-09-23T20:52:12.153521+0000","last_fullsized":"2023-09-23T20:52:12.153521+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:27.645262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":157,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766080+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:52:10.766080+0000","last_peered":"2023-09-23T20:52:10.766080+0000","last_clean":"2023-09-23T20:52:10.766080+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:52:10.766080+0000","last_undegraded":"2023-09-23T20:52:10.766080+0000","last_fullsized":"2023-09-23T20:52:10.766080+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"197'8112","reported_seq":8154,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:20.730341+0000","last_change":"2023-09-23T20:50:08.083439+0000","last_active":"2023-09-23T20:52:20.730341+0000","last_peered":"2023-09-23T20:52:20.730341+0000","last_clean":"2023-09-23T20:52:20.730341+0000","last_became_active":"2023-09-23T20:50:08.083061+0000","last_became_peered":"2023-09-23T20:50:08.083061+0000","last_unstale":"2023-09-23T20:52:20.730341+0000","last_undegraded":"2023-09-23T20:52:20.730341+0000","last_fullsized":"2023-09-23T20:52:20.730341+0000","mapping_epoch":164,"log_start":"168'400","ondisk_log_start":"168'400","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7712,"log_dups_size":0,"ondisk_log_size":7712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:41.569707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309723136,"num_objects":4726,"num_object_clones":0,"num_object_copies":14178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8112,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":30,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.156483+0000","last_change":"2023-09-23T20:51:05.957996+0000","last_active":"2023-09-23T20:52:12.156483+0000","last_peered":"2023-09-23T20:52:12.156483+0000","last_clean":"2023-09-23T20:52:12.156483+0000","last_became_active":"2023-09-23T20:51:05.957651+0000","last_became_peered":"2023-09-23T20:51:05.957651+0000","last_unstale":"2023-09-23T20:52:12.156483+0000","last_undegraded":"2023-09-23T20:52:12.156483+0000","last_fullsized":"2023-09-23T20:52:12.156483+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:05.872498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":151,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.145339+0000","last_change":"2023-09-23T20:45:00.005077+0000","last_active":"2023-09-23T20:52:12.145339+0000","last_peered":"2023-09-23T20:52:12.145339+0000","last_clean":"2023-09-23T20:52:12.145339+0000","last_became_active":"2023-09-23T20:45:00.004383+0000","last_became_peered":"2023-09-23T20:45:00.004383+0000","last_unstale":"2023-09-23T20:52:12.145339+0000","last_undegraded":"2023-09-23T20:52:12.145339+0000","last_fullsized":"2023-09-23T20:52:12.145339+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:45.178382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":160,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159176+0000","last_change":"2023-09-23T20:51:05.959327+0000","last_active":"2023-09-23T20:52:12.159176+0000","last_peered":"2023-09-23T20:52:12.159176+0000","last_clean":"2023-09-23T20:52:12.159176+0000","last_became_active":"2023-09-23T20:51:05.958816+0000","last_became_peered":"2023-09-23T20:51:05.958816+0000","last_unstale":"2023-09-23T20:52:12.159176+0000","last_undegraded":"2023-09-23T20:52:12.159176+0000","last_fullsized":"2023-09-23T20:52:12.159176+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:52.674764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":135,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.146607+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:52:12.146607+0000","last_peered":"2023-09-23T20:52:12.146607+0000","last_clean":"2023-09-23T20:52:12.146607+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:52:12.146607+0000","last_undegraded":"2023-09-23T20:52:12.146607+0000","last_fullsized":"2023-09-23T20:52:12.146607+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":172,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.158842+0000","last_change":"2023-09-23T20:48:07.539824+0000","last_active":"2023-09-23T20:52:12.158842+0000","last_peered":"2023-09-23T20:52:12.158842+0000","last_clean":"2023-09-23T20:52:12.158842+0000","last_became_active":"2023-09-23T20:48:07.539530+0000","last_became_peered":"2023-09-23T20:48:07.539530+0000","last_unstale":"2023-09-23T20:52:12.158842+0000","last_undegraded":"2023-09-23T20:52:12.158842+0000","last_fullsized":"2023-09-23T20:52:12.158842+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:21.997036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":160,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.152577+0000","last_change":"2023-09-23T20:45:00.008268+0000","last_active":"2023-09-23T20:52:12.152577+0000","last_peered":"2023-09-23T20:52:12.152577+0000","last_clean":"2023-09-23T20:52:12.152577+0000","last_became_active":"2023-09-23T20:45:00.006393+0000","last_became_peered":"2023-09-23T20:45:00.006393+0000","last_unstale":"2023-09-23T20:52:12.152577+0000","last_undegraded":"2023-09-23T20:52:12.152577+0000","last_fullsized":"2023-09-23T20:52:12.152577+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:41.255405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159850+0000","last_change":"2023-09-23T20:48:07.544433+0000","last_active":"2023-09-23T20:52:12.159850+0000","last_peered":"2023-09-23T20:52:12.159850+0000","last_clean":"2023-09-23T20:52:12.159850+0000","last_became_active":"2023-09-23T20:48:07.544066+0000","last_became_peered":"2023-09-23T20:48:07.544066+0000","last_unstale":"2023-09-23T20:52:12.159850+0000","last_undegraded":"2023-09-23T20:52:12.159850+0000","last_fullsized":"2023-09-23T20:52:12.159850+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:28.905486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":171,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.153642+0000","last_change":"2023-09-23T20:51:05.969594+0000","last_active":"2023-09-23T20:52:12.153642+0000","last_peered":"2023-09-23T20:52:12.153642+0000","last_clean":"2023-09-23T20:52:12.153642+0000","last_became_active":"2023-09-23T20:51:05.959688+0000","last_became_peered":"2023-09-23T20:51:05.959688+0000","last_unstale":"2023-09-23T20:52:12.153642+0000","last_undegraded":"2023-09-23T20:52:12.153642+0000","last_fullsized":"2023-09-23T20:52:12.153642+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:59.980249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":174,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159684+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:52:12.159684+0000","last_peered":"2023-09-23T20:52:12.159684+0000","last_clean":"2023-09-23T20:52:12.159684+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:52:12.159684+0000","last_undegraded":"2023-09-23T20:52:12.159684+0000","last_fullsized":"2023-09-23T20:52:12.159684+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":172,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.766642+0000","last_change":"2023-09-23T20:51:05.999322+0000","last_active":"2023-09-23T20:52:10.766642+0000","last_peered":"2023-09-23T20:52:10.766642+0000","last_clean":"2023-09-23T20:52:10.766642+0000","last_became_active":"2023-09-23T20:51:05.998993+0000","last_became_peered":"2023-09-23T20:51:05.998993+0000","last_unstale":"2023-09-23T20:52:10.766642+0000","last_undegraded":"2023-09-23T20:52:10.766642+0000","last_fullsized":"2023-09-23T20:52:10.766642+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:47.495994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":168,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T20:52:10.767602+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:52:10.767602+0000","last_peered":"2023-09-23T20:52:10.767602+0000","last_clean":"2023-09-23T20:52:10.767602+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:52:10.767602+0000","last_undegraded":"2023-09-23T20:52:10.767602+0000","last_fullsized":"2023-09-23T20:52:10.767602+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":151,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.148636+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:52:12.148636+0000","last_peered":"2023-09-23T20:52:12.148636+0000","last_clean":"2023-09-23T20:52:12.148636+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:52:12.148636+0000","last_undegraded":"2023-09-23T20:52:12.148636+0000","last_fullsized":"2023-09-23T20:52:12.148636+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":177,"reported_epoch":197,"state":"active+clean","last_fresh":"2023-09-23T20:52:12.159033+0000","last_change":"2023-09-23T20:51:06.011816+0000","last_active":"2023-09-23T20:52:12.159033+0000","last_peered":"2023-09-23T20:52:12.159033+0000","last_clean":"2023-09-23T20:52:12.159033+0000","last_became_active":"2023-09-23T20:51:06.011506+0000","last_became_peered":"2023-09-23T20:51:06.011506+0000","last_unstale":"2023-09-23T20:52:12.159033+0000","last_undegraded":"2023-09-23T20:52:12.159033+0000","last_fullsized":"2023-09-23T20:52:12.159033+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:40.603970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4914282520,"num_objects":74987,"num_object_clones":0,"num_object_copies":224961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74987,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130785,"num_write_kb":6584641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3638,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14702751744,"data_stored":14702739528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126185,"ondisk_log_size":126185,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201564,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3031784,"kb_used_data":2672032,"kb_used_omap":0,"kb_used_meta":359744,"kb_avail":90713368,"statfs":{"total":95995035648,"available":92890488832,"internally_reserved":0,"allocated":2736160768,"data_stored":2735580446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":368377856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,0,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":6.2670000000000003,"5min":4.4699999999999998,"15min":2.7509999999999999},"min":{"1min":0.245,"5min":0.221,"15min":0.20499999999999999},"max":{"1min":70.781999999999996,"5min":225.56999999999999,"15min":225.56999999999999},"last":4.242},{"interface":"front","average":{"1min":6.415,"5min":4.649,"15min":3.036},"min":{"1min":0.41099999999999998,"5min":0.26200000000000001,"15min":0.21299999999999999},"max":{"1min":70.876000000000005,"5min":225.61099999999999,"15min":225.61099999999999},"last":3.629}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7439999999999998,"5min":3.8559999999999999,"15min":2.0920000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":24.016999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":7.9749999999999996},{"interface":"front","average":{"1min":4.827,"5min":6.7590000000000003,"15min":3.1160000000000001},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.991,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.249}]},{"osd":2,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":3.1539999999999999,"15min":3.1440000000000001},"min":{"1min":0.29399999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.044,"5min":142.321,"15min":142.321},"last":8.9610000000000003},{"interface":"front","average":{"1min":2.2549999999999999,"5min":2.7029999999999998,"15min":2.4590000000000001},"min":{"1min":0.45300000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.5540000000000003,"5min":142.29300000000001,"15min":142.29300000000001},"last":8.9800000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":3.5830000000000002,"5min":5.4139999999999997,"15min":8.6099999999999994},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":26.234000000000002,"5min":138.345,"15min":138.345},"last":4.3109999999999999},{"interface":"front","average":{"1min":4.4880000000000004,"5min":5.3280000000000003,"15min":8.0579999999999998},"min":{"1min":0.42599999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":26.099,"5min":133.76400000000001,"15min":133.76400000000001},"last":5.2569999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.411,"15min":1.139},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.216},"max":{"1min":4.5,"5min":9.1400000000000006,"15min":11.834},"last":3.6030000000000002},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.3069999999999999,"15min":1.1819999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.6420000000000003,"5min":8.4649999999999999,"15min":10.077999999999999},"last":4.4130000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.3580000000000001,"15min":1.2889999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.0789999999999997,"5min":9.4440000000000008,"15min":14.58},"last":4.46},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.7350000000000001,"15min":1.2949999999999999},"min":{"1min":0.32500000000000001,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":4.5620000000000003,"5min":13.156000000000001,"15min":13.156000000000001},"last":4.2880000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.6779999999999999,"15min":1.2969999999999999},"min":{"1min":0.39100000000000001,"5min":0.34399999999999997,"15min":0.28899999999999998},"max":{"1min":4.4740000000000002,"5min":10.509,"15min":10.509},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.1160000000000001,"15min":1.0980000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.419,"5min":8.4350000000000005,"15min":8.4350000000000005},"last":4.335}]}]},{"osd":6,"up_from":47,"seq":201863463201,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196820,"kb_used_data":903820,"kb_used_omap":0,"kb_used_meta":292992,"kb_avail":92548332,"statfs":{"total":95995035648,"available":94769491968,"internally_reserved":0,"allocated":925511680,"data_stored":924951461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":300023808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":4.5860000000000003,"5min":3.6070000000000002,"15min":2.4860000000000002},"min":{"1min":0.22500000000000001,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":31.263999999999999,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":3.7650000000000001,"5min":2.8700000000000001,"15min":2.3479999999999999},"min":{"1min":0.27000000000000002,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":31.244,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.47999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":5.1440000000000001,"5min":3.6960000000000002,"15min":2.3639999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":35.262,"5min":96.623999999999995,"15min":96.623999999999995},"last":2.9100000000000001},{"interface":"front","average":{"1min":4.6790000000000003,"5min":3.7440000000000002,"15min":2.1930000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.223},"max":{"1min":35.350000000000001,"5min":96.254000000000005,"15min":96.254000000000005},"last":0.58799999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.9399999999999999,"15min":2.1720000000000002},"min":{"1min":0.29799999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.9589999999999996,"5min":24.440999999999999,"15min":24.440999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.49,"5min":1.984,"15min":2.3929999999999998},"min":{"1min":0.34899999999999998,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":4.4539999999999997,"5min":23.916,"15min":23.916},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":7.2830000000000004,"5min":4.2409999999999997,"15min":5.4969999999999999},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":74.683999999999997,"5min":79.668000000000006,"15min":79.668000000000006},"last":250.804},{"interface":"front","average":{"1min":7.8760000000000003,"5min":4.3259999999999996,"15min":5.4400000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":74.531999999999996,"5min":79.700000000000003,"15min":79.700000000000003},"last":250.446}]},{"osd":4,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.1279999999999999,"15min":1.012},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.248},"max":{"1min":3.9079999999999999,"5min":9.0709999999999997,"15min":9.0709999999999997},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.516,"15min":1.304},"min":{"1min":0.313,"5min":0.313,"15min":0.27000000000000002},"max":{"1min":6.4340000000000002,"5min":9.8650000000000002,"15min":10.058999999999999},"last":2.952}]},{"osd":5,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":7.976,"5min":3.0819999999999999,"15min":1.7889999999999999},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.26000000000000001},"max":{"1min":105.121,"5min":105.121,"15min":105.121},"last":0.52900000000000003},{"interface":"front","average":{"1min":6.9219999999999997,"5min":2.96,"15min":1.841},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.29399999999999998},"max":{"1min":106.30500000000001,"5min":106.30500000000001,"15min":106.30500000000001},"last":9.4469999999999992}]},{"osd":7,"last update":"Sat Sep 23 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":1.6080000000000001,"15min":1.25},"min":{"1min":0.33700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.5640000000000001,"5min":9.3249999999999993,"15min":19.451000000000001},"last":7.5369999999999999},{"interface":"front","average":{"1min":1.27,"5min":1.528,"15min":1.3500000000000001},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.32100000000000001},"max":{"1min":6.4109999999999996,"5min":9.7959999999999994,"15min":16.295999999999999},"last":2.9369999999999998}]}]},{"osd":1,"up_from":114,"seq":489626271835,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3039064,"kb_used_data":2690000,"kb_used_omap":4,"kb_used_meta":349051,"kb_avail":90706088,"statfs":{"total":95995035648,"available":92883034112,"internally_reserved":0,"allocated":2754560000,"data_stored":2753762592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":357429134},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":5.649,"5min":3.3239999999999998,"15min":2.399},"min":{"1min":0.21199999999999999,"5min":0.189,"15min":0.189},"max":{"1min":61.957000000000001,"5min":64.622,"15min":64.622},"last":0.35099999999999998},{"interface":"front","average":{"1min":5.3330000000000002,"5min":3.3340000000000001,"15min":2.359},"min":{"1min":0.26100000000000001,"5min":0.222,"15min":0.222},"max":{"1min":62.005000000000003,"5min":63.808,"15min":63.808},"last":4.8550000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:52:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5539999999999998,"5min":1.647,"15min":2.0499999999999998},"min":{"1min":0.221,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.7840000000000007,"5min":9.8049999999999997,"15min":11.593999999999999},"last":4.2560000000000002},{"interface":"front","average":{"1min":1.619,"5min":1.5189999999999999,"15min":2.4900000000000002},"min":{"1min":0.27700000000000002,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":5.4589999999999996,"5min":10.465999999999999,"15min":14.153},"last":0.60899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:52:18 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":2.238,"15min":2.5659999999999998},"min":{"1min":0.36699999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.0359999999999996,"5min":9.0359999999999996,"15min":9.0359999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.909,"15min":1.996},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":8.0980000000000008,"5min":11.047000000000001,"15min":11.047000000000001},"last":0.63700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.4570000000000001,"15min":1.2989999999999999},"min":{"1min":0.27500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.0629999999999997,"5min":10.439,"15min":10.439},"last":0.75700000000000001},{"interface":"front","average":{"1min":2.2730000000000001,"5min":1.51,"15min":1.24},"min":{"1min":0.35799999999999998,"5min":0.316,"15min":0.307},"max":{"1min":9.8889999999999993,"5min":10.34,"15min":10.34},"last":4.3070000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":1.962,"15min":1.788},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.28899999999999998},"max":{"1min":5.117,"5min":10.177,"15min":10.177},"last":0.77900000000000003},{"interface":"front","average":{"1min":3.242,"5min":1.8109999999999999,"15min":1.3859999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.7590000000000003,"5min":9.7590000000000003,"15min":9.7590000000000003},"last":4.3390000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.6519999999999999,"15min":1.6060000000000001},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.4829999999999997,"5min":10.25,"15min":10.25},"last":0.73499999999999999},{"interface":"front","average":{"1min":2.9950000000000001,"5min":1.6990000000000001,"15min":1.4059999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":9.8369999999999997,"5min":9.8369999999999997,"15min":9.8369999999999997},"last":4.3650000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":4.8090000000000002,"5min":2.3860000000000001,"15min":1.6930000000000001},"min":{"1min":0.38400000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":25.32,"5min":25.32,"15min":25.32},"last":4.3899999999999997},{"interface":"front","average":{"1min":3.7109999999999999,"5min":2.1099999999999999,"15min":1.913},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":25.344000000000001,"5min":25.344000000000001,"15min":25.344000000000001},"last":0.70299999999999996}]}]},{"osd":0,"up_from":92,"seq":395136991372,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1713700,"kb_used_data":1223068,"kb_used_omap":5,"kb_used_meta":490618,"kb_avail":92031452,"statfs":{"total":95995035648,"available":94240206848,"internally_reserved":0,"allocated":1252421632,"data_stored":1251609783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":502393466},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0919999999999996,"5min":4.4889999999999999,"15min":2.9089999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":31.515000000000001,"5min":134.322,"15min":134.322},"last":0.20200000000000001},{"interface":"front","average":{"1min":4.4530000000000003,"5min":3.1400000000000001,"15min":2.5449999999999999},"min":{"1min":0.21199999999999999,"5min":0.19,"15min":0.19},"max":{"1min":31.48,"5min":59.570999999999998,"15min":59.570999999999998},"last":4.5199999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":2.0979999999999999,"15min":1.9830000000000001},"min":{"1min":0.23599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.7300000000000004,"5min":49.652000000000001,"15min":49.652000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.468,"5min":2.2349999999999999,"15min":3.7389999999999999},"min":{"1min":0.26400000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.06,"5min":49.595999999999997,"15min":49.595999999999997},"last":0.25800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":6.4039999999999999,"5min":4.806,"15min":3.8740000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":72.131,"5min":94.947999999999993,"15min":94.947999999999993},"last":4.4720000000000004},{"interface":"front","average":{"1min":6.8079999999999998,"5min":4.8840000000000003,"15min":4.0620000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":69.988,"5min":94.382999999999996,"15min":94.382999999999996},"last":4.5519999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.546,"15min":1.79},"min":{"1min":0.32200000000000001,"5min":0.29899999999999999,"15min":0.26800000000000002},"max":{"1min":5.9989999999999997,"5min":10.525,"15min":12.973000000000001},"last":4.0640000000000001},{"interface":"front","average":{"1min":1.546,"5min":1.395,"15min":1.9339999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7939999999999996,"5min":12.461,"15min":12.461},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":9.2420000000000009,"5min":3.2189999999999999,"15min":2.6520000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":124.40600000000001,"5min":124.40600000000001,"15min":124.40600000000001},"last":4.032},{"interface":"front","average":{"1min":7.931,"5min":2.71,"15min":2.4900000000000002},"min":{"1min":0.31,"5min":0.307,"15min":0.28899999999999998},"max":{"1min":124.363,"5min":124.363,"15min":124.363},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4139999999999999,"5min":1.472,"15min":2.1190000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.307},"max":{"1min":6.4160000000000004,"5min":12.510999999999999,"15min":14.037000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.819,"5min":1.6180000000000001,"15min":1.6839999999999999},"min":{"1min":0.40200000000000002,"5min":0.34999999999999998,"15min":0.34799999999999998},"max":{"1min":5.6900000000000004,"5min":9.1189999999999998,"15min":20.399999999999999},"last":4.4349999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":5.4219999999999997,"5min":2.206,"15min":2.4500000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35099999999999998},"max":{"1min":60.037999999999997,"5min":60.037999999999997,"15min":60.037999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":5.8419999999999996,"5min":2.3380000000000001,"15min":2.056},"min":{"1min":0.46400000000000002,"5min":0.38200000000000001,"15min":0.38100000000000001},"max":{"1min":60.012,"5min":60.012,"15min":60.012},"last":4.4870000000000001}]}]},{"osd":2,"up_from":129,"seq":554050781262,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1214828,"kb_used_data":895524,"kb_used_omap":7,"kb_used_meta":319288,"kb_avail":92530324,"statfs":{"total":95995035648,"available":94751051776,"internally_reserved":0,"allocated":917016576,"data_stored":916256494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":326950993},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.7120000000000002,"5min":2.2450000000000001,"15min":2.911},"min":{"1min":0.20599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":15.539999999999999,"5min":89.644999999999996,"15min":89.644999999999996},"last":1.0680000000000001},{"interface":"front","average":{"1min":2.3130000000000002,"5min":2.1309999999999998,"15min":2.6949999999999998},"min":{"1min":0.26700000000000002,"5min":0.186,"15min":0.186},"max":{"1min":15.593999999999999,"5min":88.599999999999994,"15min":88.599999999999994},"last":8.4700000000000006}]},{"osd":1,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.8439999999999999,"15min":2.9750000000000001},"min":{"1min":0.22500000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5879999999999992,"5min":91.325000000000003,"15min":91.325000000000003},"last":8.5809999999999995},{"interface":"front","average":{"1min":1.119,"5min":2.3599999999999999,"15min":2.996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2489999999999997,"5min":91.799999999999997,"15min":91.799999999999997},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":11.372,"5min":4.7370000000000001,"15min":2.6930000000000001},"min":{"1min":0.26600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":151.43799999999999,"5min":151.43799999999999,"15min":151.43799999999999},"last":5.5},{"interface":"front","average":{"1min":12.388999999999999,"5min":4.6660000000000004,"15min":2.2330000000000001},"min":{"1min":0.317,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":152.20699999999999,"5min":152.20699999999999,"15min":152.20699999999999},"last":0.77900000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":1.369,"15min":1.7869999999999999},"min":{"1min":0.30499999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.7230000000000008,"5min":8.7230000000000008,"15min":8.7230000000000008},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.294,"5min":1.1679999999999999,"15min":2.3180000000000001},"min":{"1min":0.42399999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.5519999999999996,"5min":6.5519999999999996,"15min":8.1199999999999992},"last":8.5350000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":11.076000000000001,"5min":3.1110000000000002,"15min":3.2370000000000001},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":163.352,"5min":163.352,"15min":163.352},"last":8.5090000000000003},{"interface":"front","average":{"1min":10.865,"5min":3.2839999999999998,"15min":3.121},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":163.089,"5min":163.089,"15min":163.089},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.2090000000000001,"15min":2.786},"min":{"1min":0.34599999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.4500000000000002,"5min":6.3449999999999998,"15min":7.476},"last":4.8920000000000003},{"interface":"front","average":{"1min":2.3359999999999999,"5min":1.6679999999999999,"15min":2.7000000000000002},"min":{"1min":0.42499999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.6809999999999992,"5min":9.5220000000000002,"15min":12.595000000000001},"last":6.6059999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":11.727,"5min":3.327,"15min":2.8999999999999999},"min":{"1min":0.42299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":160.08099999999999,"5min":160.08099999999999,"15min":160.08099999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":11.019,"5min":3.1800000000000002,"15min":3.1019999999999999},"min":{"1min":0.44900000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":160.13800000000001,"5min":160.13800000000001,"15min":160.13800000000001},"last":8.5559999999999992}]}]},{"osd":3,"up_from":148,"seq":635655159861,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2677924,"kb_used_data":2454044,"kb_used_omap":2,"kb_used_meta":223869,"kb_avail":91067228,"statfs":{"total":95995035648,"available":93252841472,"internally_reserved":0,"allocated":2512941056,"data_stored":2512180405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":229241909},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9810000000000001,"5min":6.6669999999999998,"15min":11.776999999999999},"min":{"1min":0.18099999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":10.470000000000001,"5min":253.68100000000001,"15min":253.68100000000001},"last":2.0590000000000002},{"interface":"front","average":{"1min":2.3370000000000002,"5min":6.2830000000000004,"15min":10.898999999999999},"min":{"1min":0.20999999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.3380000000000001,"5min":253.46199999999999,"15min":253.46199999999999},"last":3.9199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":5.0099999999999998,"15min":8.6150000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.0140000000000002,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.218},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.7919999999999998,"15min":7.8689999999999998},"min":{"1min":0.25,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.692,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.66400000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":1.3069999999999999,"15min":1.127},"min":{"1min":0.27300000000000002,"5min":0.215,"15min":0.215},"max":{"1min":11.317,"5min":11.317,"15min":11.317},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.835,"5min":1.296,"15min":1.0600000000000001},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.0720000000000001,"5min":8.8819999999999997,"15min":8.8819999999999997},"last":3.8370000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.552,"5min":1.556,"15min":1.6399999999999999},"min":{"1min":0.32600000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.9000000000000004,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.516,"5min":1.76,"15min":2.0640000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.276,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":1.675,"15min":1.341},"min":{"1min":0.39400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.057,"5min":12.057,"15min":12.057},"last":4.4610000000000003},{"interface":"front","average":{"1min":3.1379999999999999,"5min":1.978,"15min":1.964},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":14.054,"5min":14.054,"15min":14.054},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.8620000000000001,"15min":1.9350000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":11.632,"5min":11.632,"15min":11.632},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.9950000000000001,"15min":2.4260000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.404999999999999,"5min":16.451000000000001,"15min":16.451000000000001},"last":1.2110000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0139999999999998,"5min":1.8029999999999999,"15min":1.643},"min":{"1min":0.41399999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":12.25,"5min":12.25,"15min":12.25},"last":3.8029999999999999},{"interface":"front","average":{"1min":2.7440000000000002,"5min":1.8740000000000001,"15min":2.1920000000000002},"min":{"1min":0.45200000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.9690000000000003,"5min":11.561999999999999,"15min":11.561999999999999},"last":1.3560000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986474,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1242400,"kb_used_data":904472,"kb_used_omap":0,"kb_used_meta":337920,"kb_avail":92502752,"statfs":{"total":95995035648,"available":94722818048,"internally_reserved":0,"allocated":926179328,"data_stored":925606883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":346030080},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.8999999999999999,"15min":2.1360000000000001},"min":{"1min":0.25,"5min":0.216,"15min":0.214},"max":{"1min":45.552,"5min":45.552,"15min":45.552},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.964,"5min":2.6040000000000001,"15min":1.782},"min":{"1min":0.245,"5min":0.245,"15min":0.22600000000000001},"max":{"1min":45.529000000000003,"5min":45.529000000000003,"15min":45.529000000000003},"last":3.1120000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":4.2939999999999996,"5min":7.4859999999999998,"15min":3.589},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":45.712000000000003,"5min":286.31099999999998,"15min":286.31099999999998},"last":0.318},{"interface":"front","average":{"1min":4.6509999999999998,"5min":7.2949999999999999,"15min":4.1310000000000002},"min":{"1min":0.27900000000000003,"5min":0.25700000000000001,"15min":0.24299999999999999},"max":{"1min":45.460000000000001,"5min":285.97000000000003,"15min":285.97000000000003},"last":9.2210000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9059999999999999,"5min":1.339,"15min":1.3280000000000001},"min":{"1min":0.30199999999999999,"5min":0.28499999999999998,"15min":0.25},"max":{"1min":8.6639999999999997,"5min":8.6639999999999997,"15min":8.6639999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.7969999999999999,"5min":1.617,"15min":2.1190000000000002},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.0730000000000004,"5min":12.773999999999999,"15min":12.773999999999999},"last":0.65100000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:52:20 2023","interfaces":[{"interface":"back","average":{"1min":3.484,"5min":4.3810000000000002,"15min":6.5259999999999998},"min":{"1min":0.313,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":45.491,"5min":107.285,"15min":107.285},"last":0.495},{"interface":"front","average":{"1min":4.3099999999999996,"5min":4.391,"15min":6.258},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":46.109999999999999,"5min":107.303,"15min":107.303},"last":2.3149999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:51:49 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":1.101,"15min":0.92900000000000005},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.0960000000000001,"5min":7.1740000000000004,"15min":7.1740000000000004},"last":2.9020000000000001},{"interface":"front","average":{"1min":0.83299999999999996,"5min":1.1319999999999999,"15min":1.0469999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":2.6190000000000002,"5min":9.0660000000000007,"15min":12.209},"last":0.34799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.103,"15min":1.1259999999999999},"min":{"1min":0.313,"5min":0.28499999999999998,"15min":0.28000000000000003},"max":{"1min":4.1989999999999998,"5min":7.4109999999999996,"15min":16.271000000000001},"last":3.0499999999999998},{"interface":"front","average":{"1min":1.109,"5min":1.0649999999999999,"15min":1.03},"min":{"1min":0.34200000000000003,"5min":0.33800000000000002,"15min":0.30399999999999999},"max":{"1min":4.7329999999999997,"5min":7.617,"15min":11.718},"last":2.8199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:49 2023","interfaces":[{"interface":"back","average":{"1min":4.2949999999999999,"5min":1.962,"15min":1.4430000000000001},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":73.268000000000001,"5min":73.268000000000001,"15min":73.268000000000001},"last":1.5},{"interface":"front","average":{"1min":4.5830000000000002,"5min":1.853,"15min":1.357},"min":{"1min":0.35099999999999998,"5min":0.33700000000000002,"15min":0.317},"max":{"1min":69.760000000000005,"5min":69.760000000000005,"15min":69.760000000000005},"last":3.0859999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724838,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3118740,"kb_used_data":2685900,"kb_used_omap":0,"kb_used_meta":432832,"kb_avail":90626412,"statfs":{"total":95995035648,"available":92801445888,"internally_reserved":0,"allocated":2750361600,"data_stored":2749797211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":443219968},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":2.7559999999999998,"15min":2.1429999999999998},"min":{"1min":0.224,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":10.763,"5min":124.873,"15min":124.873},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.7270000000000001,"5min":2.8130000000000002,"15min":1.887},"min":{"1min":0.23100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":8.5960000000000001,"5min":123.554,"15min":123.554},"last":2.3849999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":11.34,"5min":4.9880000000000004,"15min":3.407},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":184.31200000000001,"5min":184.31200000000001,"15min":323.541},"last":2.6970000000000001},{"interface":"front","average":{"1min":11.313000000000001,"5min":5.0149999999999997,"15min":3.6469999999999998},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":184.67400000000001,"5min":184.67400000000001,"15min":324.072},"last":2.4329999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":1.8580000000000001,"15min":2.2370000000000001},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.859,"5min":12.585000000000001,"15min":13.394},"last":2.923},{"interface":"front","average":{"1min":2.214,"5min":1.639,"15min":1.6000000000000001},"min":{"1min":0.34399999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.6699999999999999,"5min":14.263,"15min":14.263},"last":2.3530000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3410000000000002,"5min":5.2750000000000004,"15min":3},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":9.8879999999999999,"5min":198.33099999999999,"15min":198.33099999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.6699999999999999,"5min":5.3419999999999996,"15min":2.766},"min":{"1min":0.313,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.882999999999999,"5min":198.47300000000001,"15min":198.47300000000001},"last":3.1589999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:51:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.012,"15min":1.0009999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.216},"max":{"1min":4.625,"5min":5.8529999999999998,"15min":9.8800000000000008},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.21,"15min":1.1839999999999999},"min":{"1min":0.29999999999999999,"5min":0.28699999999999998,"15min":0.23999999999999999},"max":{"1min":4.3470000000000004,"5min":8.3170000000000002,"15min":8.798},"last":0.73499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.3340000000000001,"15min":1.1779999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":6.2119999999999997,"5min":8.3010000000000002,"15min":8.8239999999999998},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.607,"5min":1.294,"15min":1.034},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.8180000000000001,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":2.1899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:51:44 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.716,"15min":1.3100000000000001},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":5.0839999999999996,"5min":14.217000000000001,"15min":14.217000000000001},"last":3.4889999999999999},{"interface":"front","average":{"1min":1.012,"5min":0.996,"15min":1.004},"min":{"1min":0.36799999999999999,"5min":0.29899999999999999,"15min":0.28699999999999998},"max":{"1min":4.4950000000000001,"5min":4.9669999999999996,"15min":7.6130000000000004},"last":2.448}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1236795392,"data_stored":1236795392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2752385024,"data_stored":2752380952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":914882560,"data_stored":914882560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2501705728,"data_stored":2501705728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":919273472,"data_stored":919273472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2737836032,"data_stored":2737831960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":915996672,"data_stored":915996672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2723876864,"data_stored":2723872792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:52:28.336 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:52:28.336 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:52:28.336 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:52:28.337 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T20:52:28.337 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:52:28.460 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:28.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:28.468 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:28 smithi121 systemd[1]: Stopping Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:28.568 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:28.622 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:28.709 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:28.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:28.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:28.825 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:28.853 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:29.009 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:52:29.176 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:29.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:29.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:29.285 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:29.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:29 smithi121 ceph-mon[219709]: pgmap v626: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T20:52:29.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:29 smithi121 ceph-mon[211526]: pgmap v626: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T20:52:29.431 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:29 smithi123 ceph-mon[193903]: pgmap v626: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-09-23T20:52:29.509 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:29.548 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:29.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:29.664 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:29.698 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:29 smithi121 conmon[251098]: 2023-09-23T20:52:29.690+0000 7fdffdf7f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:52:29.698 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:29 smithi121 conmon[251098]: 2023-09-23T20:52:29.690+0000 7fdffdf7f700 -1 osd.1 198 *** Got signal Terminated *** 2023-09-23T20:52:29.698 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:29 smithi121 conmon[251098]: 2023-09-23T20:52:29.690+0000 7fdffdf7f700 -1 osd.1 198 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:52:29.726 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:29.809 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:29.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:29.919 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:29.998 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:30.214 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:30.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:30.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:30.291 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:30 smithi121 ceph-mon[211526]: osd.1 marked itself down and dead 2023-09-23T20:52:30.291 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:30 smithi121 ceph-mon[219709]: osd.1 marked itself down and dead 2023-09-23T20:52:30.331 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:30.356 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:30.397 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:30.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:30.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:30 smithi123 ceph-mon[193903]: osd.1 marked itself down and dead 2023-09-23T20:52:30.502 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:30.528 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:30.581 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:30.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:30.686 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:30.716 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:30.791 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:30.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:30.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:30.894 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:30.924 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:31.121 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:31.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:31.225 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:31.232 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:52:31.265 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:31.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[219709]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:52:31.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[219709]: failed to encode map e199 with expected crc 2023-09-23T20:52:31.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[219709]: osdmap e199: 8 total, 7 up, 8 in 2023-09-23T20:52:31.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3246760055' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:31.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:31.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[219709]: pgmap v628: 89 pgs: 10 stale+active+clean, 79 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T20:52:31.414 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:31 smithi121 bash[323855]: 015c027d535f9bed98928477a1e628a4be5b5b93dd854c9916a2a5cc9fa5d398 2023-09-23T20:52:31.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[211526]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:52:31.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[211526]: failed to encode map e199 with expected crc 2023-09-23T20:52:31.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[211526]: osdmap e199: 8 total, 7 up, 8 in 2023-09-23T20:52:31.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3246760055' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:31.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:31.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:31 smithi121 ceph-mon[211526]: pgmap v628: 89 pgs: 10 stale+active+clean, 79 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T20:52:31.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:31 smithi123 ceph-mon[193903]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:52:31.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:31 smithi123 ceph-mon[193903]: failed to encode map e199 with expected crc 2023-09-23T20:52:31.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:31 smithi123 ceph-mon[193903]: osdmap e199: 8 total, 7 up, 8 in 2023-09-23T20:52:31.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3246760055' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:31.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:31 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:52:31.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:31 smithi123 ceph-mon[193903]: pgmap v628: 89 pgs: 10 stale+active+clean, 79 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-09-23T20:52:32.068 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:32.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:32.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:32.173 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:32.215 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:32.448 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:32.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: osdmap e200: 8 total, 7 up, 8 in 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:32 smithi123 ceph-mon[193903]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: osdmap e200: 8 total, 7 up, 8 in 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[219709]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: osdmap e200: 8 total, 7 up, 8 in 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:32 smithi121 ceph-mon[211526]: failed to encode map e200 with expected crc 2023-09-23T20:52:32.501 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:32.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:32.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:32.609 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:32.674 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:32.802 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:32.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:32.911 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:32.983 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:32.988 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:52:33.018 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:52:33.048 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:52:33.078 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:52:33.102 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:33.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:33.115 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:52:33.145 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:52:33.181 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:52:33.182 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:52:33.209 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:33.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:33 smithi123 ceph-mon[193903]: pgmap v630: 89 pgs: 8 peering, 9 stale+active+clean, 72 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T20:52:33.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1768300943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:33.266 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:33.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:33 smithi121 ceph-mon[219709]: pgmap v630: 89 pgs: 8 peering, 9 stale+active+clean, 72 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T20:52:33.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1768300943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:33.372 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:33 smithi121 ceph-mon[211526]: pgmap v630: 89 pgs: 8 peering, 9 stale+active+clean, 72 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T20:52:33.372 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1768300943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:33.375 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:33.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:33.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:33.484 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:33.565 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:33.664 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:33.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:33.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:33.769 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:33.807 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:33.932 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:33.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:34.047 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:34.091 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:34 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:52:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:34 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:52:34.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:34 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:52:34.484 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:52:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:52:34.749 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:52:34.780 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:34.787 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:52:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:52:34] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-09-23T20:52:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:34.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:34.890 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:34.977 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:35.121 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:35.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:35.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:35.226 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:35.298 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:35.386 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:35.386 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:35 smithi121 ceph-mon[211526]: pgmap v631: 89 pgs: 11 active+undersized+degraded, 22 active+undersized, 8 peering, 48 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 33570/203739 objects degraded (16.477%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:35.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:35 smithi121 ceph-mon[219709]: pgmap v631: 89 pgs: 11 active+undersized+degraded, 22 active+undersized, 8 peering, 48 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 33570/203739 objects degraded (16.477%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:35.430 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:35.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:35 smithi123 ceph-mon[193903]: pgmap v631: 89 pgs: 11 active+undersized+degraded, 22 active+undersized, 8 peering, 48 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 33570/203739 objects degraded (16.477%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:35.544 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:35.630 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:35.740 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:35.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:35.858 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:35.883 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:35.973 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:35.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:35.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:36.092 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:36.139 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:36.318 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:36.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:36.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:36 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 33570/203739 objects degraded (16.477%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:36.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:36 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 33570/203739 objects degraded (16.477%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:36.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:36.472 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:36.624 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:52:36.639 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:36.648 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T20:52:36.706 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:36.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:36.721 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:36 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T20:52:36.721 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:36 smithi121 systemd[1]: Stopped Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:52:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:36 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 33570/203739 objects degraded (16.477%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:36.751 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:52:36.798 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:52:36.798 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T20:52:36.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:36.872 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:36.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:36.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:37.096 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:37.167 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:37.248 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:37 smithi121 ceph-mon[211526]: pgmap v632: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 37705/200691 objects degraded (18.788%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:37.248 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:37 smithi121 ceph-mon[219709]: pgmap v632: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 37705/200691 objects degraded (18.788%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:37.278 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:37.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:37.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:37.410 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:37.446 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:37.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:37 smithi123 ceph-mon[193903]: pgmap v632: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 37705/200691 objects degraded (18.788%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:37.980 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:37.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:37.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:38.049 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:38.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:38.155 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:38.242 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:38.249 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:38 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:52:38.255 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:38.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:38.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:38.362 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:38.417 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:38.533 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:38 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:52:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:38 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:52:39.260 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:52:39.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:39 smithi121 ceph-mon[211526]: pgmap v633: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 37208/196695 objects degraded (18.917%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:39.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2166228435' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:39.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:39 smithi121 ceph-mon[219709]: pgmap v633: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 37208/196695 objects degraded (18.917%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2166228435' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:39 smithi123 ceph-mon[193903]: pgmap v633: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 37208/196695 objects degraded (18.917%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2166228435' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:39.742 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:39.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:39.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:39.850 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:39.889 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:39.960 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:39.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:40.077 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:40.168 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:40.251 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:40.267 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: osdmap e201: 8 total, 7 up, 8 in 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[211526]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:40.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: osdmap e201: 8 total, 7 up, 8 in 2023-09-23T20:52:40.269 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.269 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.269 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.269 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.269 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:40 smithi121 ceph-mon[219709]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.358 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:40.425 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:52:40.425 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:40.426 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: osdmap e201: 8 total, 7 up, 8 in 2023-09-23T20:52:40.426 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.426 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.426 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.426 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.426 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:40 smithi123 ceph-mon[193903]: failed to encode map e201 with expected crc 2023-09-23T20:52:40.429 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:40.560 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:40.672 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:40.770 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:40.914 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:40.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:41.037 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:41.102 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:41.207 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:41.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:41.286 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:41 smithi121 ceph-mon[211526]: pgmap v635: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 35446/190617 objects degraded (18.595%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:41.286 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 37208/196695 objects degraded (18.917%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:41.286 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:41 smithi121 ceph-mon[219709]: pgmap v635: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 35446/190617 objects degraded (18.595%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:41.286 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 37208/196695 objects degraded (18.917%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:41.296 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:41.296 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:41.323 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:41.366 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:41.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:41 smithi123 ceph-mon[193903]: pgmap v635: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 35446/190617 objects degraded (18.595%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:41.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 37208/196695 objects degraded (18.917%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:41.798 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:52:41.799 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:52:41.799 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T20:52:41.799 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:52:41.799 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:52:41.799 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T20:52:42.098 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:42.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:42.103 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:52:42.131 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:52:42.154 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:52:42.196 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:52:42.204 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:42.232 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:52:42.270 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:52:42.293 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:42.312 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:52:42.312 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:52:42.313 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:52:42.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:52:42.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:52:42.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:52:42.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:52:42.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:52:42.313 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:52:42.314 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:52:42.340 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:52:42.396 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:42.400 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3784548024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:42.400 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3784548024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:42.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:42.508 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:42.569 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:42.600 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:52:42.634 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:42.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:42.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:42.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3784548024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:42.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:42.814 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:52:43.094 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:52:43.185 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:43.244 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:43.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:43.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:43.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:43 smithi121 ceph-mon[211526]: pgmap v636: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 35190/187578 objects degraded (18.760%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:43.604 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:43.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:43.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:43 smithi121 ceph-mon[219709]: pgmap v636: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 35190/187578 objects degraded (18.760%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:43.717 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:43 smithi123 ceph-mon[193903]: pgmap v636: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 35190/187578 objects degraded (18.760%); 0 B/s, 0 objects/s recovering 2023-09-23T20:52:43.821 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:44.315 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:44.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:44.419 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:44.505 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:44.761 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:44.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:44.869 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:44.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:52:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:52:44] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T20:52:44.930 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:45.063 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:45.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:45.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:45.095 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:52:45.173 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:45.232 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:45.352 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:45.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:45.464 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:45.493 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:45.665 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:52:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:52:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:148443 9.2 e201/199 9:4abb3380:::benchmark_data_smithi121_293770_object45555:head v 201'9345, mlcod=201'9342)", 2023-09-23T20:52:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:52:45.654448+0000", 2023-09-23T20:52:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0082757070000000002, 2023-09-23T20:52:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0083063749999999995, 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.654448+0000", 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.654448+0000", 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.654447+0000", 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.654455+0000", 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7840000000000004e-06 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.654470+0000", 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5251e-05 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.654477+0000", 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2749999999999998e-06 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.655369+0000", 2023-09-23T20:52:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089202400000000005 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.655391+0000", 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1135e-05 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:148446 9.2 e201/199 9:45beefdb:::benchmark_data_smithi121_293770_object45558:head v 201'9346, mlcod=201'9342)", 2023-09-23T20:52:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:52:45.658807+0000", 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0039165880000000004, 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004014933, 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658807+0000", 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:45.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658807+0000", 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658806+0000", 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:45.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658814+0000", 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6120000000000001e-06 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658824+0000", 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0498e-05 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658831+0000", 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1359999999999999e-06 2023-09-23T20:52:45.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658963+0000", 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013200199999999999 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.658984+0000", 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0786999999999999e-05 2023-09-23T20:52:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:148449 9.2 e201/199 9:4ec8df05:::benchmark_data_smithi121_293770_object45561:head v 201'9347, mlcod=201'9342)", 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:52:45.661166+0000", 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0015578390000000001, 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001720615, 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:45.674 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.661166+0000", 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.661166+0000", 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.661165+0000", 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.661173+0000", 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9440000000000005e-06 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.661185+0000", 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2284e-05 2023-09-23T20:52:45.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.661193+0000", 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8599999999999993e-06 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.662296+0000", 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001103473 2023-09-23T20:52:45.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:45.662317+0000", 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0080000000000001e-05 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 3 2023-09-23T20:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:52:45.707 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:45.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:45.811 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:45.888 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:46.023 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:46.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:46.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:46.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:45 smithi123 ceph-mon[193903]: pgmap v637: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 33173/178392 objects degraded (18.596%) 2023-09-23T20:52:46.135 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:46.162 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:46.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:45 smithi121 ceph-mon[211526]: pgmap v637: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 33173/178392 objects degraded (18.596%) 2023-09-23T20:52:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:45 smithi121 ceph-mon[219709]: pgmap v637: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 33173/178392 objects degraded (18.596%) 2023-09-23T20:52:46.327 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:46.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:46.433 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:46.480 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:46.586 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:46.598 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:46.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:46.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:46.614 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:52:46.632 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:52:46.633 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:52:46.633 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:52:46.706 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:46.752 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:46.891 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:46.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:46.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:46.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 33173/178392 objects degraded (18.596%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:46.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/482160864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:46.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:46 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:47.001 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:47.038 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:47.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 33173/178392 objects degraded (18.596%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:47.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/482160864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:47.063 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:46 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:47.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 33173/178392 objects degraded (18.596%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:47.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/482160864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:47.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:46 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:52:47.080 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:47.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:47.145 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:52:47.184 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:47.213 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:47.274 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:47.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:47.378 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:47.415 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:47.828 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:47.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:47.933 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:47.966 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:47.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:47 smithi123 ceph-mon[193903]: pgmap v638: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 32932/175539 objects degraded (18.761%) 2023-09-23T20:52:47.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:47 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:47.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:47 smithi123 ceph-mon[193903]: failed to encode map e202 with expected crc 2023-09-23T20:52:47.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:47 smithi123 ceph-mon[193903]: failed to encode map e202 with expected crc 2023-09-23T20:52:47.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:47.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:47 smithi123 ceph-mon[193903]: osdmap e202: 8 total, 7 up, 8 in 2023-09-23T20:52:48.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[219709]: pgmap v638: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 32932/175539 objects degraded (18.761%) 2023-09-23T20:52:48.173 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:48.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[219709]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[219709]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:48.174 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[219709]: osdmap e202: 8 total, 7 up, 8 in 2023-09-23T20:52:48.174 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[211526]: pgmap v638: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 32932/175539 objects degraded (18.761%) 2023-09-23T20:52:48.174 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:48.174 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[211526]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.175 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[211526]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.175 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:52:48.175 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:47 smithi121 ceph-mon[211526]: osdmap e202: 8 total, 7 up, 8 in 2023-09-23T20:52:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:48.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:48.471 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:48.507 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:48.547 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:48.793 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:48.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:48.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:48.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:48.950 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:48 smithi121 ceph-mon[219709]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.950 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:48 smithi121 ceph-mon[219709]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.950 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:48 smithi121 ceph-mon[219709]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.952 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:48 smithi121 ceph-mon[211526]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:48 smithi121 ceph-mon[211526]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:48 smithi121 ceph-mon[211526]: failed to encode map e202 with expected crc 2023-09-23T20:52:48.973 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:52:48.974 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:49.012 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:52:49.049 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:52:49.052 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:49.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:49.085 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:52:49.115 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:52:49.153 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:52:49.157 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:49.183 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:52:49.184 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:52:49.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:48 smithi123 ceph-mon[193903]: failed to encode map e202 with expected crc 2023-09-23T20:52:49.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:48 smithi123 ceph-mon[193903]: failed to encode map e202 with expected crc 2023-09-23T20:52:49.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:48 smithi123 ceph-mon[193903]: failed to encode map e202 with expected crc 2023-09-23T20:52:49.224 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:49.357 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:49.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:49.474 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:49.506 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:49.867 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:49.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:49.974 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:50.020 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:49 smithi121 ceph-mon[219709]: pgmap v640: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 32391/171759 objects degraded (18.858%) 2023-09-23T20:52:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/90596302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:49 smithi121 ceph-mon[211526]: pgmap v640: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 32391/171759 objects degraded (18.858%) 2023-09-23T20:52:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/90596302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:50.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:49 smithi123 ceph-mon[193903]: pgmap v640: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 32391/171759 objects degraded (18.858%) 2023-09-23T20:52:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/90596302' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:50.397 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:52:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:52:50.581 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:52:50.597 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:50.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:50.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:50.709 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:50.786 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:50.892 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:50.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:51.013 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:51.076 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:51.085 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:51.114 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:52:51.115 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:52:51.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:51.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:51.348 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:52:51.408 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:51.527 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:52:51.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:51.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:52:51.644 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:51.679 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.018 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:52.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:52.034 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:51 smithi121 ceph-mon[211526]: pgmap v641: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 30988/166812 objects degraded (18.577%) 2023-09-23T20:52:52.034 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30988/166812 objects degraded (18.577%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:52.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:51 smithi121 ceph-mon[219709]: pgmap v641: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 30988/166812 objects degraded (18.577%) 2023-09-23T20:52:52.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30988/166812 objects degraded (18.577%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:52.098 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:52:52.130 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:52.194 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:51 smithi123 ceph-mon[193903]: pgmap v641: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 30988/166812 objects degraded (18.577%) 2023-09-23T20:52:52.194 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30988/166812 objects degraded (18.577%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:52.203 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.329 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:52.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:52.444 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:52.547 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.582 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:52:52.657 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:52:52.657 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:52:52.657 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T20:52:52.658 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T20:52:52.658 DEBUG:teuthology.orchestra.run.smithi121:osd.1> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T20:52:52.660 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl start ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:52:52.679 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:52.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:52.700 INFO:journalctl@ceph.osd.1.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:52:52.800 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:52.868 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:52.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:52 smithi123 ceph-mon[193903]: pgmap v642: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 30730/163884 objects degraded (18.751%) 2023-09-23T20:52:52.964 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:52 smithi121 systemd[1]: Starting Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:52.964 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:52 smithi121 ceph-mon[211526]: pgmap v642: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 30730/163884 objects degraded (18.751%) 2023-09-23T20:52:52.965 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:52 smithi121 ceph-mon[219709]: pgmap v642: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 30730/163884 objects degraded (18.751%) 2023-09-23T20:52:52.995 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:53.101 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:53.119 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:53.182 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.289 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:53.405 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:53.432 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:53.480 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:53.683 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:53.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:53.791 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:53.838 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:54.165 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:52:54.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3689745908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:54.492 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3689745908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:54.492 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3689745908' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:52:55.039 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:52:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:52:54] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T20:52:55.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:55.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3689745908' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: osdmap e203: 8 total, 7 up, 8 in 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[211526]: pgmap v644: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 28661/154389 objects degraded (18.564%) 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3689745908' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:55.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: osdmap e203: 8 total, 7 up, 8 in 2023-09-23T20:52:55.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:55 smithi121 ceph-mon[219709]: pgmap v644: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 28661/154389 objects degraded (18.564%) 2023-09-23T20:52:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3689745908' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: osdmap e203: 8 total, 7 up, 8 in 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: failed to encode map e203 with expected crc 2023-09-23T20:52:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:55 smithi123 ceph-mon[193903]: pgmap v644: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 28661/154389 objects degraded (18.564%) 2023-09-23T20:52:55.906 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:55.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:56.027 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:56.120 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:56.174 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 28661/154389 objects degraded (18.564%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:56.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 28661/154389 objects degraded (18.564%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:56.203 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:56.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:56.318 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:52:56.372 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 28661/154389 objects degraded (18.564%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:52:56.377 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:56.471 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:52:56.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:56.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:52:56.587 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:52:56.616 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:52:56.640 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:56.649 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:52:56.677 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:52:56.723 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:52:56.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:52:56.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:56.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:52:56.757 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:52:56.797 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:52:56.835 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:52:56.835 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:52:56.836 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:52:56.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:52:56.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:52:56.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:52:56.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:52:56.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:52:56.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:52:56.837 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:56.906 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:57.033 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:52:57.034 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:52:57.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:52:57.034 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:52:57.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.034 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b7ed88ab:::benchmark_data_smithi121_252781_object87596:head, version: 125'5492, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b7ed88ab:::benchmark_data_smithi121_252781_object87596:head@125'5492, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:52:57.034 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:52:57.034 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 316.42568413499998, 2023-09-23T20:52:57.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17604561799999999, 2023-09-23T20:52:57.035 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.035 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:57.035 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605345+0000", 2023-09-23T20:52:57.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6996000000000001e-05 2023-09-23T20:52:57.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605357+0000", 2023-09-23T20:52:57.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2374e-05 2023-09-23T20:52:57.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778323+0000", 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.172965539 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778421+0000", 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8530999999999996e-05 2023-09-23T20:52:57.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.040 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.040 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778437+0000", 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5136999999999999e-05 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.781334+0000", 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028975680000000001 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.042 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b02b4366:::benchmark_data_smithi121_252781_object90395:head, version: 125'5662, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b02b4366:::benchmark_data_smithi121_252781_object90395:head@125'5662, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 314.10496919899998, 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22766398199999999, 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.925997+0000", 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999938 2023-09-23T20:52:57.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926050+0000", 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2899999999999998e-05 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150250+0000", 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22419977199999999 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150290+0000", 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9712000000000002e-05 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150341+0000", 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1696000000000001e-05 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150349+0000", 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7409999999999992e-06 2023-09-23T20:52:57.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.153667+0000", 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033184040000000001 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b6dc2210:::benchmark_data_smithi121_252781_object93072:head, version: 127'5840, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b6dc2210:::benchmark_data_smithi121_252781_object93072:head@127'5840, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 311.65886059899998, 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16357189899999999, 2023-09-23T20:52:57.047 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372112+0000", 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.372135+0000", 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3283000000000001e-05 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.529427+0000", 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.157291876 2023-09-23T20:52:57.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.529452+0000", 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5023000000000001e-05 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.531548+0000", 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020965010000000002 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.531560+0000", 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1425e-05 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:45.535684+0000", 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041242800000000001 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172419 8.f 8:f3e775cc:::benchmark_data_smithi121_252781_object63244:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 309.91391316099998, 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0270669539999999, 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172419 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.052 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117058+0000", 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117065+0000", 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9940000000000003e-06 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117075+0000", 2023-09-23T20:52:57.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8649999999999996e-06 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117083+0000", 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7399999999999993e-06 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117315+0000", 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00023133999999999999 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117325+0000", 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0008e-05 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144044+0000", 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.026719535 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144126+0000", 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2231000000000006e-05 2023-09-23T20:52:57.056 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172440 8.f 8:fcfd2d26:::benchmark_data_smithi121_252781_object63265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 309.91387557000002, 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047514855, 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:52:57.057 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172440 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:52:57.058 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117099+0000", 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117106+0000", 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2210000000000002e-06 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117111+0000", 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2360000000000004e-06 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.060 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117343+0000", 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000232597 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117349+0000", 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1930000000000001e-06 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144377+0000", 2023-09-23T20:52:57.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.027027844 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144494+0000", 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000117108 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144624+0000", 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000130677 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.155446+0000", 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010821377 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164470+0000", 2023-09-23T20:52:57.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090247639999999994 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164526+0000", 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5665999999999999e-05 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164612+0000", 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.577e-05 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172441 8.f 8:f67e9cac:::benchmark_data_smithi121_252781_object63266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 309.91385748699997, 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047671113, 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.065 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172441 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117118+0000", 2023-09-23T20:52:57.068 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8109999999999999e-06 2023-09-23T20:52:57.068 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.068 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.068 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.068 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117122+0000", 2023-09-23T20:52:57.068 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7160000000000002e-06 2023-09-23T20:52:57.068 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117127+0000", 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4100000000000001e-06 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.069 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117369+0000", 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024246900000000001 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117375+0000", 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3000000000000001e-06 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148014+0000", 2023-09-23T20:52:57.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.030639455 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148124+0000", 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110089 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148227+0000", 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010324599999999999 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:52:57.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.155452+0000", 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072244780000000003 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164664+0000", 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092118040000000005 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164708+0000", 2023-09-23T20:52:57.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4780999999999997e-05 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164786+0000", 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7847000000000002e-05 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.a 143/130 [PushOp(8:5b7183a5:::benchmark_data_smithi121_252781_object76859:head, version: 125'4857, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:5b7183a5:::benchmark_data_smithi121_252781_object76859:head@125'4857, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:48.626718+0000", 2023-09-23T20:52:57.073 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 308.40425503900002, 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10125018, 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626718+0000", 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626718+0000", 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626717+0000", 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626738+0000", 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1288999999999999e-05 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626749+0000", 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0528e-05 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626758+0000", 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8429999999999999e-06 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.626810+0000", 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2621999999999999e-05 2023-09-23T20:52:57.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.724953+0000", 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.098142939999999998 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.727968+0000", 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003014507 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:174677 8.d e144/143 8:b7ddb6cb:::benchmark_data_smithi121_252781_object65502:head v 144'10966, mlcod=144'10963)", 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 305.62156131900002, 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.043301345999999998, 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.078 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409411+0000", 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409416+0000", 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2820000000000004e-06 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409424+0000", 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4670000000000004e-06 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409430+0000", 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3650000000000003e-06 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409472+0000", 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2267e-05 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.409491+0000", 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9117999999999999e-05 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.413890+0000", 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004399053 2023-09-23T20:52:57.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452713+0000", 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038822397000000002 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:174680 8.f 8:fb6dc5c3:::benchmark_data_smithi121_252781_object65505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:52:57.082 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.412110+0000", 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 305.61886292200001, 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044728539999999997, 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 174680 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.083 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412110+0000", 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412110+0000", 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412108+0000", 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412116+0000", 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7719999999999994e-06 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412130+0000", 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3902e-05 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412139+0000", 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1519999999999998e-06 2023-09-23T20:52:57.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412206+0000", 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7294999999999999e-05 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412337+0000", 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013082600000000001 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412457+0000", 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011976000000000001 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456665+0000", 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044208285999999999 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456694+0000", 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9335999999999998e-05 2023-09-23T20:52:57.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456747+0000", 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2333999999999997e-05 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.456838+0000", 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1529999999999999e-05 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.088 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:174681 8.f 8:fb2398b1:::benchmark_data_smithi121_252781_object65506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 305.61881832500001, 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.055719985999999999, 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 174681 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412154+0000", 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412157+0000", 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.095e-06 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412163+0000", 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6460000000000004e-06 2023-09-23T20:52:57.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.412169+0000", 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1399999999999999e-06 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452744+0000", 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.040575356999999999 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452875+0000", 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013148599999999999 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452994+0000", 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000118662 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:52:57.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.463838+0000", 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010843983 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.467748+0000", 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0039096160000000003 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.467792+0000", 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4431999999999997e-05 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.467874+0000", 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2099000000000001e-05 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:174679 8.d e144/143 8:be774cb5:::benchmark_data_smithi121_252781_object65504:head v 144'10967, mlcod=144'10964)", 2023-09-23T20:52:57.095 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:51.414661+0000", 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 305.61631154899999, 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050115271000000003, 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414661+0000", 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414661+0000", 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414660+0000", 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.414668+0000", 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1589999999999993e-06 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452693+0000", 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038024924000000002 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452706+0000", 2023-09-23T20:52:57.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2772000000000001e-05 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452744+0000", 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8118000000000003e-05 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.452765+0000", 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1257999999999999e-05 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.464754+0000", 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011988421000000001 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:51.464776+0000", 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2609e-05 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:177178 8.f 8:f0621e84:::benchmark_data_smithi121_252781_object68003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:52:57.100 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 302.677162915, 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36433712000000001, 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 177178 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:52:57.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353808+0000", 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353817+0000", 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6180000000000001e-06 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353835+0000", 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7601e-05 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353845+0000", 2023-09-23T20:52:57.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0268000000000001e-05 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353898+0000", 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3173000000000001e-05 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.354023+0000", 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000124456 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:52:57.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.354142+0000", 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000118789 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.357799+0000", 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036574469999999999 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718014+0000", 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36021456600000001 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718061+0000", 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7530000000000001e-05 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718147+0000", 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5612999999999995e-05 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:177186 8.f 8:f634efb7:::benchmark_data_smithi121_252781_object68011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 302.66423578899997, 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.361305761, 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 177186 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366735+0000", 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:52:57.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366742+0000", 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8569999999999997e-06 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366756+0000", 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3405e-05 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366765+0000", 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7969999999999995e-06 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366817+0000", 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2021000000000002e-05 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366923+0000", 2023-09-23T20:52:57.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106098 2023-09-23T20:52:57.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:52:57.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366996+0000", 2023-09-23T20:52:57.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2354999999999994e-05 2023-09-23T20:52:57.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:52:57.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.370015+0000", 2023-09-23T20:52:57.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030194229999999998 2023-09-23T20:52:57.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:52:57.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.727915+0000", 2023-09-23T20:52:57.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35789987899999998 2023-09-23T20:52:57.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.727963+0000", 2023-09-23T20:52:57.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7817e-05 2023-09-23T20:52:57.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.728043+0000", 2023-09-23T20:52:57.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9654999999999995e-05 2023-09-23T20:52:57.115 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.115 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.115 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185557 8.4 e146/134 8:23b68925:::benchmark_data_smithi121_252781_object76382:head v 146'11606, mlcod=146'11603)", 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.53852410899998, 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33023901100000003, 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:52:57.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492455+0000", 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5970000000000003e-06 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492464+0000", 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9279999999999999e-06 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492471+0000", 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4699999999999999e-06 2023-09-23T20:52:57.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492508+0000", 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7024999999999999e-05 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492525+0000", 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6776e-05 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822566+0000", 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33004159199999999 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822688+0000", 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121403 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185566 8.d e146/143 8:bb91d518:::benchmark_data_smithi121_252781_object76391:head v 146'11646, mlcod=146'11643)", 2023-09-23T20:52:57.120 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.53595250000001, 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32767350499999998, 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:52:57.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495025+0000", 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6550000000000003e-06 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495035+0000", 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.057e-05 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495043+0000", 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9699999999999999e-06 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495079+0000", 2023-09-23T20:52:57.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6300999999999997e-05 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495094+0000", 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4430000000000001e-05 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822604+0000", 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32751017900000001 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822694+0000", 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9649999999999997e-05 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185568 8.4 e146/134 8:2156ec6b:::benchmark_data_smithi121_252781_object76393:head v 146'11607, mlcod=146'11604)", 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.530331725, 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33877399600000002, 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.125 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500640+0000", 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500649+0000", 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0450000000000006e-06 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822380+0000", 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32173094000000002 2023-09-23T20:52:57.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822388+0000", 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1999999999999994e-06 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.824928+0000", 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025395029999999998 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.824942+0000", 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4331999999999999e-05 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.839373+0000", 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01443056 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.839415+0000", 2023-09-23T20:52:57.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2320000000000001e-05 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185570 8.a e147/145 8:550a95bc:::benchmark_data_smithi121_252781_object76395:head v 147'11761, mlcod=146'11759)", 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.52922042599999, 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32342548900000001, 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.130 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501756+0000", 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9689999999999996e-06 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822281+0000", 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32052530800000001 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822294+0000", 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2822e-05 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822388+0000", 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4215999999999994e-05 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822411+0000", 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3161999999999999e-05 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.825145+0000", 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027331389999999999 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.825178+0000", 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3142999999999998e-05 2023-09-23T20:52:57.134 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:57315 9.0 e171/164 9:0328d5f3:::benchmark_data_smithi121_293770_object57314:head v 171'3588, mlcod=171'3586)", 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.087617+0000", 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 129.943355196, 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057652203999999999, 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087617+0000", 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087617+0000", 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087616+0000", 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:57.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087640+0000", 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4063e-05 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087653+0000", 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2913e-05 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087660+0000", 2023-09-23T20:52:57.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6959999999999999e-06 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087753+0000", 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3505999999999995e-05 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.087776+0000", 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2350000000000001e-05 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.145270+0000", 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057493778000000002 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:57324 9.2 e171/164 9:4fe6fb3c:::benchmark_data_smithi121_293770_object57323:head v 171'3588, mlcod=171'3586)", 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.096479+0000", 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 129.934493874, 2023-09-23T20:52:57.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.059996267999999998, 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096479+0000", 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096479+0000", 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096478+0000", 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096507+0000", 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9575000000000001e-05 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142872+0000", 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.046364378999999997 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142882+0000", 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0198e-05 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143205+0000", 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00032271200000000001 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143220+0000", 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5903e-05 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156416+0000", 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013195231 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156475+0000", 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9321999999999999e-05 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.144 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:57334 9.6 e171/164 9:6e49c932:::benchmark_data_smithi121_293770_object57333:head v 171'3517, mlcod=171'3515)", 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 129.92493239999999, 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050547171000000002, 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:52:57.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106040+0000", 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:52:57.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.106071+0000", 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1495999999999998e-05 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142607+0000", 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036535521000000001 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:52:57.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.142621+0000", 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4562000000000001e-05 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143462+0000", 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00084066100000000003 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.143487+0000", 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4536e-05 2023-09-23T20:52:57.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156547+0000", 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013060824 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.156587+0000", 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9947000000000002e-05 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.149 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:52:57.150 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:52:57.150 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:52:57.150 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:52:57.150 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:52:57.150 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T20:52:57.150 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:52:57.150 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:52:57.151 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T20:52:57.151 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:52:57.151 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:56 smithi121 bash[330240]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T20:52:57.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3356221433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:57.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:57 smithi121 ceph-mon[211526]: pgmap v645: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 28419/151374 objects degraded (18.774%) 2023-09-23T20:52:57.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3356221433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:57.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:57 smithi121 ceph-mon[219709]: pgmap v645: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 28419/151374 objects degraded (18.774%) 2023-09-23T20:52:57.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3356221433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:52:57.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:57 smithi123 ceph-mon[193903]: pgmap v645: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 28419/151374 objects degraded (18.774%) 2023-09-23T20:52:57.974 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:52:58.076 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:58.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:52:58.185 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:52:58.235 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:52:58.290 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:52:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:52:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:58.869 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:52:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:58.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:52:58.952 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:52:58.974 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:52:59.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:59.101 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:59.177 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:52:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:59.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:52:59.290 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:52:59.345 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:59.474 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:52:59.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:59.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:52:59.568 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:52:59 smithi121 ceph-mon[211526]: pgmap v646: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 27928/147591 objects degraded (18.923%) 2023-09-23T20:52:59.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:52:59 smithi121 ceph-mon[219709]: pgmap v646: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 27928/147591 objects degraded (18.923%) 2023-09-23T20:52:59.597 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:52:59.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:52:59.681 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:52:59.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:52:59 smithi123 ceph-mon[193903]: pgmap v646: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 27928/147591 objects degraded (18.923%) 2023-09-23T20:52:59.913 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:59 smithi121 bash[331459]: 47e41b5525ab93df0b6493030d551d69c9082934ed6df0977d43df6d053eb377 2023-09-23T20:52:59.914 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:59 smithi121 ceph-osd[331640]: -- 172.21.15.121:0/40368605 <== mon.2 v2:172.21.15.121:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556e908e04e0 con 0x556e908b3000 2023-09-23T20:52:59.914 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:52:59 smithi121 systemd[1]: Started Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:52:59.974 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:52:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:52:59.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:00.085 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:00.156 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:00.198 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:00.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:00.291 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:00.304 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:00.315 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:53:00 smithi121 conmon[331554]: 2023-09-23T20:53:00.196+0000 7fced673d640 -1 Falling back to public interface 2023-09-23T20:53:00.329 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:00.400 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:00.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:00.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:00.514 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:00.571 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:00.703 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:00.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:00.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:00.735 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:00.819 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:00.880 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:00.890 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:00.989 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:01.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:01.112 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:01.162 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:01.292 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:01.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:01.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:01.408 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:01.480 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:01.597 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:01.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:01.712 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:01.796 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:01.945 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:01.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:01.964 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:01 smithi123 ceph-mon[193903]: pgmap v647: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 26466/142500 objects degraded (18.573%) 2023-09-23T20:53:01.964 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 26466/142500 objects degraded (18.573%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:02.064 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:02.098 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:01 smithi121 ceph-mon[219709]: pgmap v647: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 26466/142500 objects degraded (18.573%) 2023-09-23T20:53:02.098 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 26466/142500 objects degraded (18.573%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:02.098 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:01 smithi121 ceph-mon[211526]: pgmap v647: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 26466/142500 objects degraded (18.573%) 2023-09-23T20:53:02.099 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 26466/142500 objects degraded (18.573%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:02.105 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:02.199 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:02.384 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:02.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:02.489 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:02.565 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:02.676 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:02.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:02.783 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:02.804 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:53:02.834 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:02.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/234789805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:02.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:02 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:03.093 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/234789805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:03.094 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:02 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:03.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/234789805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:03.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:02 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:03.135 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:03.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:03.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:03.242 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:03.305 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:03.429 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:03.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:03.541 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:03.579 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:03.751 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:03.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:03.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:03.863 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: pgmap v648: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 26217/139698 objects degraded (18.767%) 2023-09-23T20:53:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:03.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: osdmap e204: 8 total, 7 up, 8 in 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[211526]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: pgmap v648: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 26217/139698 objects degraded (18.767%) 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: osdmap e204: 8 total, 7 up, 8 in 2023-09-23T20:53:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:03 smithi121 ceph-mon[219709]: failed to encode map e204 with expected crc 2023-09-23T20:53:03.937 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:04.038 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:04.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:04.155 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:04.209 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:04.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: pgmap v648: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 26217/139698 objects degraded (18.767%) 2023-09-23T20:53:04.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:04.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: failed to encode map e204 with expected crc 2023-09-23T20:53:04.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:04.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: osdmap e204: 8 total, 7 up, 8 in 2023-09-23T20:53:04.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: failed to encode map e204 with expected crc 2023-09-23T20:53:04.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: failed to encode map e204 with expected crc 2023-09-23T20:53:04.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: failed to encode map e204 with expected crc 2023-09-23T20:53:04.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: failed to encode map e204 with expected crc 2023-09-23T20:53:04.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:03 smithi123 ceph-mon[193903]: failed to encode map e204 with expected crc 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:no valid command found; 10 closest matches: 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:0 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:1 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:2 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:abort 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:assert 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:bluefs debug_inject_read_zeros 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:bluefs files list 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:bluefs stats 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:bluestore allocator dump block 2023-09-23T20:53:04.801 INFO:teuthology.orchestra.run.smithi121.stderr:bluestore allocator fragmentation block 2023-09-23T20:53:04.802 INFO:teuthology.orchestra.run.smithi121.stderr:admin_socket: invalid command 2023-09-23T20:53:05.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:53:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:53:04] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.43.0" 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:164210 9.7 e204/164 9:e42ff152:::benchmark_data_smithi121_293770_object61322:head v 204'10236, mlcod=204'10231)", 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:05.238333+0000", 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.011482119000000001, 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011511484000000001, 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:05.252 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.238333+0000", 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.238333+0000", 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.253 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.238332+0000", 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.238339+0000", 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4320000000000002e-06 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:05.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.238351+0000", 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1270000000000001e-05 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.238364+0000", 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3424000000000001e-05 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.244510+0000", 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061457660000000004 2023-09-23T20:53:05.255 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.244528+0000", 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7901999999999999e-05 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:164217 9.f e204/199 9:f2c81add:::benchmark_data_smithi121_293770_object61329:head v 204'10404, mlcod=204'10402)", 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:05.245829+0000", 2023-09-23T20:53:05.256 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0039854740000000001, 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040808930000000004, 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245829+0000", 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245829+0000", 2023-09-23T20:53:05.257 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245829+0000", 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245837+0000", 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1170000000000004e-06 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.258 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245852+0000", 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.571e-05 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245860+0000", 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8620000000000003e-06 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.249334+0000", 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034734589999999999 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.249349+0000", 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5285999999999999e-05 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.260 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:164216 9.4 e204/199 9:2992eb03:::benchmark_data_smithi121_293770_object61328:head v 204'10368, mlcod=204'10366)", 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:05.245919+0000", 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0038957879999999999, 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040519459999999998, 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245919+0000", 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.261 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245919+0000", 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245919+0000", 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.262 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245924+0000", 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7640000000000002e-06 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245935+0000", 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029999999999999e-05 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245941+0000", 2023-09-23T20:53:05.263 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3140000000000003e-06 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245990+0000", 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9211e-05 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246012+0000", 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2425e-05 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.264 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:164218 9.c 9:3b74b1f7:::benchmark_data_smithi121_293770_object61330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:05.246000+0000", 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0038151090000000001, 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040332639999999999, 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:05.265 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 164218 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246000+0000", 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246000+0000", 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.266 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.245998+0000", 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246005+0000", 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8709999999999999e-06 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.267 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246018+0000", 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2921e-05 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246028+0000", 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5559999999999996e-06 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246231+0000", 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020350300000000001 2023-09-23T20:53:05.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246381+0000", 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015034299999999999 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.246511+0000", 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012944399999999999 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:05.269 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:164220 9.3 e204/199 9:cb6f00c0:::benchmark_data_smithi121_293770_object61332:head v 204'10207, mlcod=204'10205)", 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:05.247183+0000", 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0026319440000000002, 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002924842, 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.270 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247183+0000", 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247183+0000", 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247182+0000", 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:05.271 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247192+0000", 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6639999999999993e-06 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247202+0000", 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0081000000000001e-05 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:05.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247209+0000", 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7750000000000004e-06 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247281+0000", 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2114999999999996e-05 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:05.247302+0000", 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1287000000000001e-05 2023-09-23T20:53:05.273 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.274 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:05.274 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.274 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:05.274 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:53:05.274 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 5 2023-09-23T20:53:05.274 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:53:05.345 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:05.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:05.469 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:05.545 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:05.594 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:05.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:05.679 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:53:05.701 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:05.841 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:05.959 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:05.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:06.063 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:06.108 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:05 smithi121 ceph-mon[211526]: pgmap v650: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 24258/130767 objects degraded (18.551%) 2023-09-23T20:53:06.112 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:06.117 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:05 smithi121 ceph-mon[219709]: pgmap v650: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 24258/130767 objects degraded (18.551%) 2023-09-23T20:53:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:05 smithi123 ceph-mon[193903]: pgmap v650: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 24258/130767 objects degraded (18.551%) 2023-09-23T20:53:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:53:06.564 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-09-23T20:53:06.678 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:06.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:06.709 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:53:06.780 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:53:06 smithi121 conmon[331554]: 2023-09-23T20:53:06.619+0000 7fced673d640 -1 osd.1 198 log_to_monitors true 2023-09-23T20:53:06.783 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:06.836 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:06.927 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:06.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:06.975 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:53:06.987 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:07.017 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:53:07.039 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:07.075 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:53:07.117 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:53:07.120 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:07.158 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:53:07.168 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:07 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 24258/130767 objects degraded (18.551%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:07.168 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:07 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:53:07.168 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:07 smithi123 ceph-mon[193903]: pgmap v651: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 23992/127623 objects degraded (18.799%) 2023-09-23T20:53:07.169 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3994721294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:07.212 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:53:07.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:53:07.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:53:07.362 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:53:07 smithi121 conmon[331554]: 2023-09-23T20:53:07.217+0000 7fcec4ba4700 -1 osd.1 198 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:53:07.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 24258/130767 objects degraded (18.551%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:07.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:53:07.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[219709]: pgmap v651: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 23992/127623 objects degraded (18.799%) 2023-09-23T20:53:07.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3994721294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:07.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 24258/130767 objects degraded (18.551%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:07.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:53:07.364 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[211526]: pgmap v651: 89 pgs: 12 active+undersized+degraded, 29 active+undersized, 48 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 23992/127623 objects degraded (18.799%) 2023-09-23T20:53:07.364 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3994721294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:07.880 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:07.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:07.989 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:08.038 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:08.307 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:08.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[219709]: failed to encode map e205 with expected crc 2023-09-23T20:53:08.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:53:08.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[219709]: osdmap e205: 8 total, 7 up, 8 in 2023-09-23T20:53:08.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:53:08.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[219709]: failed to encode map e205 with expected crc 2023-09-23T20:53:08.309 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[211526]: failed to encode map e205 with expected crc 2023-09-23T20:53:08.309 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:53:08.310 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[211526]: osdmap e205: 8 total, 7 up, 8 in 2023-09-23T20:53:08.310 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:53:08.310 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:08 smithi121 ceph-mon[211526]: failed to encode map e205 with expected crc 2023-09-23T20:53:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:08.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:08.411 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:08.437 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:08.478 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:08 smithi123 ceph-mon[193903]: failed to encode map e205 with expected crc 2023-09-23T20:53:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:08 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:53:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:08 smithi123 ceph-mon[193903]: osdmap e205: 8 total, 7 up, 8 in 2023-09-23T20:53:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:08 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:53:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:08 smithi123 ceph-mon[193903]: failed to encode map e205 with expected crc 2023-09-23T20:53:08.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:08.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:08.583 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:08.610 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:53:08.754 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:53:08.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:08.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:09.046 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:09.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:09.078 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:09.122 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:09.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:09.201 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:53:09.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: failed to encode map e206 with expected crc 2023-09-23T20:53:09.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148] boot 2023-09-23T20:53:09.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:53:09.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: failed to encode map e206 with expected crc 2023-09-23T20:53:09.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:53:09.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: pgmap v654: 89 pgs: 10 peering, 12 active+undersized+degraded, 19 active+undersized, 48 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 23992/125289 objects degraded (19.149%) 2023-09-23T20:53:09.202 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[211526]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:53:09.207 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:53:09.207 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: failed to encode map e206 with expected crc 2023-09-23T20:53:09.207 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148] boot 2023-09-23T20:53:09.208 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:53:09.208 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: failed to encode map e206 with expected crc 2023-09-23T20:53:09.208 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:53:09.208 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: pgmap v654: 89 pgs: 10 peering, 12 active+undersized+degraded, 19 active+undersized, 48 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 23992/125289 objects degraded (19.149%) 2023-09-23T20:53:09.208 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:09 smithi121 ceph-mon[219709]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:53:09.228 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:09.257 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:09.301 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:09.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:09.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:09.406 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:09.429 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:53:09.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: failed to encode map e206 with expected crc 2023-09-23T20:53:09.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: osd.1 [v2:172.21.15.121:6810/211524148,v1:172.21.15.121:6811/211524148] boot 2023-09-23T20:53:09.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:53:09.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: failed to encode map e206 with expected crc 2023-09-23T20:53:09.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:53:09.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: pgmap v654: 89 pgs: 10 peering, 12 active+undersized+degraded, 19 active+undersized, 48 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 23992/125289 objects degraded (19.149%) 2023-09-23T20:53:09.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:09 smithi123 ceph-mon[193903]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:53:09.432 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:09.473 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:09.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:09.579 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:09.604 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:09.645 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:09.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:09.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:09.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:09.776 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:09.818 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:09.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:09.923 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:09.952 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:09.996 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:09.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:09.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:10.099 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:10.125 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:10.167 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:10.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:10.179 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:53:10.275 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:10.300 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:10.349 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:10.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:10.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:10.469 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2441731916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:10 smithi123 ceph-mon[193903]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:53:10.509 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:10.563 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:10.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:10.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:10.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2441731916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[211526]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: failed to encode map e207 with expected crc 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2441731916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:10.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:10 smithi121 ceph-mon[219709]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:53:10.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:10.695 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:10.755 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:53:10.771 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:10.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:10.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:10.884 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:10.918 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:11.006 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:11.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:11.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:11.219 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:11.399 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:11.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: pgmap v657: 89 pgs: 21 peering, 4 active+undersized+degraded, 16 active+undersized, 48 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 8027/120873 objects degraded (6.641%) 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 23992/125289 objects degraded (19.149%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: pgmap v657: 89 pgs: 21 peering, 4 active+undersized+degraded, 16 active+undersized, 48 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 8027/120873 objects degraded (6.641%) 2023-09-23T20:53:11.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 23992/125289 objects degraded (19.149%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:11.514 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:11.553 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: failed to encode map e208 with expected crc 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: pgmap v657: 89 pgs: 21 peering, 4 active+undersized+degraded, 16 active+undersized, 48 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 8027/120873 objects degraded (6.641%) 2023-09-23T20:53:11.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 23992/125289 objects degraded (19.149%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:11.556 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:11.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:53:11.640 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:11.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:11.670 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:11.670 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:11.678 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:11.748 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:11.784 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:11.867 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:11.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:11.982 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:12.067 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:12.703 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:12.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:12.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:12.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:12.866 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:53:12.909 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:13.030 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:13.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:13.123 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:53:13.139 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:13.268 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:13.486 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:13.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:13.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:13.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:13 smithi121 ceph-mon[211526]: pgmap v658: 89 pgs: 21 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized, 62 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6532/118590 objects degraded (5.508%); 0 B/s, 0 objects/s recovering 2023-09-23T20:53:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:13 smithi121 ceph-mon[219709]: pgmap v658: 89 pgs: 21 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized, 62 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6532/118590 objects degraded (5.508%); 0 B/s, 0 objects/s recovering 2023-09-23T20:53:13.687 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:13.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:13.767 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:13.767 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:13 smithi123 ceph-mon[193903]: pgmap v658: 89 pgs: 21 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized, 62 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6532/118590 objects degraded (5.508%); 0 B/s, 0 objects/s recovering 2023-09-23T20:53:14.216 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:14.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:14.328 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:14.428 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:14.454 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:14.531 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:14.583 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:14.606 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:14.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:14.707 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:14.747 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:14.812 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:14.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:14.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:14.864 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:53:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:53:14] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-09-23T20:53:14.925 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:14.971 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:15.059 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:15.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:15.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:15.170 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:15.214 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:15.242 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:53:15.269 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:15.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:15.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:15.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:53:15.336 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:53:15.372 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:53:15.376 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:15.438 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:53:15.472 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:53:15.493 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:15.499 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:53:15.549 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:53:15.552 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:53:15.553 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:53:15.553 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:53:15.553 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:53:15.553 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:53:15.553 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:53:15.553 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:53:15.553 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:53:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:15 smithi121 ceph-mon[211526]: pgmap v659: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 15114/114165 objects degraded (13.239%); 0 B/s, 18 objects/s recovering 2023-09-23T20:53:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3796538142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:15 smithi121 ceph-mon[219709]: pgmap v659: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 15114/114165 objects degraded (13.239%); 0 B/s, 18 objects/s recovering 2023-09-23T20:53:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3796538142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:16.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:15 smithi123 ceph-mon[193903]: pgmap v659: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 15114/114165 objects degraded (13.239%); 0 B/s, 18 objects/s recovering 2023-09-23T20:53:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3796538142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:17.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15114/114165 objects degraded (13.239%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:17.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15114/114165 objects degraded (13.239%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15114/114165 objects degraded (13.239%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:17.330 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:17.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:17.436 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:17.513 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:17.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:17 smithi121 ceph-mon[219709]: pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 15114/111345 objects degraded (13.574%); 0 B/s, 15 objects/s recovering 2023-09-23T20:53:17.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:53:17.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:17 smithi121 ceph-mon[211526]: pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 15114/111345 objects degraded (13.574%); 0 B/s, 15 objects/s recovering 2023-09-23T20:53:17.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:53:18.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:17 smithi123 ceph-mon[193903]: pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 15114/111345 objects degraded (13.574%); 0 B/s, 15 objects/s recovering 2023-09-23T20:53:18.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:53:18.279 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:18.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:18.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:18.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:18.547 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:18.836 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:18.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:18.957 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:18.999 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:19.067 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:19.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:19.173 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:19.234 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:19.306 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:19.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:19.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:19.412 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:19.460 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:19.543 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:19.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:19.649 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:19.684 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:19.762 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:53:19.762 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:53:19.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.762 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:172256 9.f e208/207 9:f1adcc5c:::benchmark_data_smithi121_293770_object69368:head v 208'10932, mlcod=198'8616)", 2023-09-23T20:53:19.763 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:19.741305+0000", 2023-09-23T20:53:19.763 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.014482893, 2023-09-23T20:53:19.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014512647, 2023-09-23T20:53:19.763 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.763 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:19.763 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741305+0000", 2023-09-23T20:53:19.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741305+0000", 2023-09-23T20:53:19.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741305+0000", 2023-09-23T20:53:19.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:19.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741311+0000", 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7860000000000002e-06 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741320+0000", 2023-09-23T20:53:19.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5960000000000004e-06 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741324+0000", 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6210000000000003e-06 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743657+0000", 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002333075 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743681+0000", 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4051999999999999e-05 2023-09-23T20:53:19.768 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:172258 9.f e208/207 9:fea1b994:::benchmark_data_smithi121_293770_object69370:head v 208'10933, mlcod=198'8616)", 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:19.741335+0000", 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.014453268, 2023-09-23T20:53:19.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01455186, 2023-09-23T20:53:19.770 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.770 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:19.770 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741335+0000", 2023-09-23T20:53:19.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741335+0000", 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741335+0000", 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741338+0000", 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3100000000000001e-06 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741343+0000", 2023-09-23T20:53:19.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8189999999999999e-06 2023-09-23T20:53:19.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.741347+0000", 2023-09-23T20:53:19.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7519999999999998e-06 2023-09-23T20:53:19.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.752032+0000", 2023-09-23T20:53:19.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010685617999999999 2023-09-23T20:53:19.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.752062+0000", 2023-09-23T20:53:19.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9832000000000002e-05 2023-09-23T20:53:19.775 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.775 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.775 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.776 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:172264 9.f e208/207)", 2023-09-23T20:53:19.776 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:19.743506+0000", 2023-09-23T20:53:19.776 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.012282533, 2023-09-23T20:53:19.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012438631, 2023-09-23T20:53:19.776 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743506+0000", 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743506+0000", 2023-09-23T20:53:19.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743505+0000", 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743512+0000", 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8120000000000002e-06 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743523+0000", 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0900000000000001e-05 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.743531+0000", 2023-09-23T20:53:19.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0860000000000002e-06 2023-09-23T20:53:19.781 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.781 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.781 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.781 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:172266 9.6 e208/207 9:6d5e9868:::benchmark_data_smithi121_293770_object69378:head v 208'10550, mlcod=198'8378)", 2023-09-23T20:53:19.781 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:19.745994+0000", 2023-09-23T20:53:19.782 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.009793988, 2023-09-23T20:53:19.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0099984659999999993, 2023-09-23T20:53:19.782 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.782 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:19.782 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.745994+0000", 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.745994+0000", 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.745993+0000", 2023-09-23T20:53:19.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:53:19.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.746002+0000", 2023-09-23T20:53:19.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0100000000000001e-06 2023-09-23T20:53:19.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.746019+0000", 2023-09-23T20:53:19.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6739999999999999e-05 2023-09-23T20:53:19.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.746027+0000", 2023-09-23T20:53:19.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9279999999999993e-06 2023-09-23T20:53:19.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.746114+0000", 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7465000000000004e-05 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.746141+0000", 2023-09-23T20:53:19.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6418999999999999e-05 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:172254 9.4 e208/207 9:2b9a1d87:::benchmark_data_smithi121_293770_object69366:head v 208'10885, mlcod=198'8557)", 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:19.754054+0000", 2023-09-23T20:53:19.788 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0017341769999999999, 2023-09-23T20:53:19.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019946569999999999, 2023-09-23T20:53:19.789 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.789 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:53:19.789 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754054+0000", 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754054+0000", 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754053+0000", 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754062+0000", 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2369999999999992e-06 2023-09-23T20:53:19.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754078+0000", 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6132000000000001e-05 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754086+0000", 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.977e-06 2023-09-23T20:53:19.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754152+0000", 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6322000000000006e-05 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:19.754176+0000", 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3898999999999999e-05 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.793 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.794 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:53:19.794 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 5 2023-09-23T20:53:19.794 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:53:19.794 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:19.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:19.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:19.871 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194759 8.b 8:d1bf2064:::benchmark_data_smithi121_252781_object85584:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 299.69438299900003, 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0533581320000001, 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.872 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194759 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170899+0000", 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170910+0000", 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1206000000000001e-05 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170925+0000", 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4596999999999999e-05 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170936+0000", 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1445e-05 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171104+0000", 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016754899999999999 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171116+0000", 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1989e-05 2023-09-23T20:53:19.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224175+0000", 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0530593850000001 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224259+0000", 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3837000000000004e-05 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.877 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194764 8.b 8:d2640955:::benchmark_data_smithi121_252781_object85589:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 299.69432821999999, 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.053600919, 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.878 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194764 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:53:19.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170955+0000", 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170961+0000", 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2619999999999998e-06 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170968+0000", 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.345e-06 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170973+0000", 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3410000000000002e-06 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171165+0000", 2023-09-23T20:53:19.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019139 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171173+0000", 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6260000000000001e-06 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224498+0000", 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0533245570000001 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224557+0000", 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8866000000000002e-05 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194773 8.b 8:dae7dbf1:::benchmark_data_smithi121_252781_object85598:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 299.69430518899998, 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067740498, 2023-09-23T20:53:19.883 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194773 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170978+0000", 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170981+0000", 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.249e-06 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170986+0000", 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6590000000000004e-06 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170991+0000", 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7709999999999997e-06 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171203+0000", 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021228600000000001 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171210+0000", 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0199999999999997e-06 2023-09-23T20:53:19.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224626+0000", 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0534161630000001 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224718+0000", 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1296000000000001e-05 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238636+0000", 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013917802 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238719+0000", 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3721999999999996e-05 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194787 8.b 8:dd8b32fd:::benchmark_data_smithi121_252781_object85612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 299.694288529, 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067236472, 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194787 2023-09-23T20:53:19.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170997+0000", 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2220000000000001e-06 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171003+0000", 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4620000000000004e-06 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171008+0000", 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9250000000000003e-06 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171302+0000", 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000294331 2023-09-23T20:53:19.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171311+0000", 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7250000000000003e-06 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224751+0000", 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.053440331 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224842+0000", 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1292000000000006e-05 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224927+0000", 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5044999999999996e-05 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237978+0000", 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013050277000000001 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237987+0000", 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3090000000000001e-06 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238081+0000", 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4129999999999995e-05 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238135+0000", 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4212999999999997e-05 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238232+0000", 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6571999999999996e-05 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194805 8.b 8:de52f621:::benchmark_data_smithi121_252781_object85630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 299.67719051099999, 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0503357229999999, 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194805 2023-09-23T20:53:19.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188092+0000", 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188100+0000", 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.729e-06 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188113+0000", 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3016999999999999e-05 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188122+0000", 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3230000000000004e-06 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188182+0000", 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9775000000000003e-05 2023-09-23T20:53:19.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188194+0000", 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2452e-05 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227071+0000", 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.038876744 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227205+0000", 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013409199999999999 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227299+0000", 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3601e-05 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237991+0000", 2023-09-23T20:53:19.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010692421000000001 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238032+0000", 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0800000000000002e-05 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238293+0000", 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000260938 2023-09-23T20:53:19.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238331+0000", 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8381999999999999e-05 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238429+0000", 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7999999999999997e-05 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.905 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194818 8.b 8:db0a1c54:::benchmark_data_smithi121_252781_object85643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 299.65933167999998, 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0410233019999999, 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.906 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194818 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:53:19.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205951+0000", 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205957+0000", 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205975+0000", 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.766e-05 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205984+0000", 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4099999999999997e-06 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.206051+0000", 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6910999999999997e-05 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.206063+0000", 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2242000000000001e-05 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230211+0000", 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.024147707 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230377+0000", 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000165635 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230528+0000", 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015100899999999999 2023-09-23T20:53:19.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238042+0000", 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075139159999999998 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238773+0000", 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00073170099999999997 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246813+0000", 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080395229999999998 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246870+0000", 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6894000000000001e-05 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246976+0000", 2023-09-23T20:53:19.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000105787 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194827 8.b 8:d1592b5b:::benchmark_data_smithi121_252781_object85652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 299.642376576, 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0286707319999999, 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.914 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194827 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222906+0000", 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222913+0000", 2023-09-23T20:53:19.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3719999999999999e-06 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222926+0000", 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3294e-05 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222935+0000", 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4789999999999996e-06 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222996+0000", 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1631000000000007e-05 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.223009+0000", 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2109e-05 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.234721+0000", 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0117129 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.234865+0000", 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014378500000000001 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.235011+0000", 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014604 2023-09-23T20:53:19.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238786+0000", 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003774576 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238821+0000", 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5635e-05 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251421+0000", 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012599552 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251476+0000", 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5387999999999999e-05 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251578+0000", 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000101727 2023-09-23T20:53:19.921 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216597 8.0 8:06510814:::benchmark_data_smithi121_252781_object107422:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.15334528300002, 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.054665387, 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:53:19.922 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216597 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711937+0000", 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711944+0000", 2023-09-23T20:53:19.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.25e-06 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711957+0000", 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2999e-05 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711966+0000", 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9719999999999998e-06 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.712019+0000", 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2799000000000001e-05 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.712028+0000", 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.003e-06 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766511+0000", 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.054482731 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766604+0000", 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3139000000000004e-05 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.927 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216620 8.0 8:04259452:::benchmark_data_smithi121_252781_object107445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.13004106300002, 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0569858000000001, 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216620 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735241+0000", 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735249+0000", 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6429999999999995e-06 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735279+0000", 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9805e-05 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735289+0000", 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9820000000000003e-06 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735397+0000", 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107877 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735410+0000", 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2962000000000001e-05 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766912+0000", 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.031502782 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.767011+0000", 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8770999999999993e-05 2023-09-23T20:53:19.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.767472+0000", 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000460922 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785103+0000", 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017630777 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787562+0000", 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024589770000000002 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792167+0000", 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046052189999999998 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792202+0000", 2023-09-23T20:53:19.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5182000000000003e-05 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792229+0000", 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6404999999999999e-05 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216632 8.0 8:09792f9f:::benchmark_data_smithi121_252781_object107457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.935 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.12025404000002, 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0473433670000001, 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216632 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745028+0000", 2023-09-23T20:53:19.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745036+0000", 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1319999999999994e-06 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745053+0000", 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6854000000000002e-05 2023-09-23T20:53:19.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745062+0000", 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4940000000000003e-06 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745125+0000", 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3314999999999999e-05 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745136+0000", 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0546e-05 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782661+0000", 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.037525241 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782813+0000", 2023-09-23T20:53:19.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015243099999999999 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782964+0000", 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000150925 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792128+0000", 2023-09-23T20:53:19.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091633930000000006 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792279+0000", 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015110899999999999 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792323+0000", 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3788000000000002e-05 2023-09-23T20:53:19.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792350+0000", 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7915e-05 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792373+0000", 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2816e-05 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.943 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216637 8.0 8:0d71f1da:::benchmark_data_smithi121_252781_object107462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.11085663099999, 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0470203279999999, 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.944 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216637 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754426+0000", 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754433+0000", 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5839999999999997e-06 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754450+0000", 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7129000000000002e-05 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754458+0000", 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1689999999999992e-06 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754544+0000", 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5140000000000001e-05 2023-09-23T20:53:19.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754557+0000", 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3107e-05 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785135+0000", 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0305782050000001 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785220+0000", 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4860999999999999e-05 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785315+0000", 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5088999999999997e-05 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792290+0000", 2023-09-23T20:53:19.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069754370000000001 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792437+0000", 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014619000000000001 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801364+0000", 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089271820000000005 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801415+0000", 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1214999999999997e-05 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801448+0000", 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.277e-05 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.951 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216642 8.0 8:0cd41a68:::benchmark_data_smithi121_252781_object107467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.11081030499997, 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047113078, 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.952 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216642 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754473+0000", 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754476+0000", 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8870000000000001e-06 2023-09-23T20:53:19.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754483+0000", 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.985e-06 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754488+0000", 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.977e-06 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754584+0000", 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6200999999999996e-05 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754593+0000", 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5310000000000001e-06 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787595+0000", 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.033002559 2023-09-23T20:53:19.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787703+0000", 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107431 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787777+0000", 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4598000000000006e-05 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792296+0000", 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045180970000000004 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792476+0000", 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000180669 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801501+0000", 2023-09-23T20:53:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090245069999999993 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801549+0000", 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8742000000000002e-05 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801587+0000", 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7317000000000002e-05 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.959 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216645 8.0 8:049490ab:::benchmark_data_smithi121_252781_object107470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.10197511299998, 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.038903957, 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.960 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216645 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763307+0000", 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763315+0000", 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0779999999999998e-06 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763331+0000", 2023-09-23T20:53:19.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6860000000000001e-05 2023-09-23T20:53:19.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763341+0000", 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4250000000000004e-06 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.765814+0000", 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024728929999999999 2023-09-23T20:53:19.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.765825+0000", 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1243000000000001e-05 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.789907+0000", 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.024082245 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.790014+0000", 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106702 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.790102+0000", 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7682999999999994e-05 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792526+0000", 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024248820000000002 2023-09-23T20:53:19.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801599+0000", 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090724640000000006 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802118+0000", 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00051864700000000001 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802173+0000", 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5547000000000002e-05 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802213+0000", 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9702e-05 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.969 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216663 8.0 8:0ce516a0:::benchmark_data_smithi121_252781_object107488:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.08010590100002, 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.035871486, 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216663 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.970 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785177+0000", 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785184+0000", 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8469999999999998e-06 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785199+0000", 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5784999999999999e-05 2023-09-23T20:53:19.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785209+0000", 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7880000000000001e-06 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785281+0000", 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2002000000000006e-05 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785295+0000", 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3675e-05 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792572+0000", 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0072767949999999 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792688+0000", 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116554 2023-09-23T20:53:19.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792802+0000", 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011395099999999999 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801610+0000", 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088076600000000001 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.814161+0000", 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012550716 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.820977+0000", 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068159379999999997 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821010+0000", 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3442000000000003e-05 2023-09-23T20:53:19.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821050+0000", 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9595999999999998e-05 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216676 8.0 8:0d0e61e5:::benchmark_data_smithi121_252781_object107501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 269.06178163599998, 2023-09-23T20:53:19.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.017644091, 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216676 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:53:19.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803507+0000", 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8500000000000002e-06 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803521+0000", 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4942e-05 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803529+0000", 2023-09-23T20:53:19.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1300000000000003e-06 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803579+0000", 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0317e-05 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803589+0000", 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8409999999999995e-06 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795218+0000", 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99162894599999996 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795317+0000", 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9674999999999997e-05 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:53:19.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795433+0000", 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011584 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801615+0000", 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061815459999999996 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.820910+0000", 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01929556 2023-09-23T20:53:19.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821082+0000", 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017190599999999999 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821115+0000", 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2373000000000003e-05 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821146+0000", 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1846e-05 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57331 9.1 9:8abe57f8:::benchmark_data_smithi121_293770_object57330:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.769093957, 2023-09-23T20:53:19.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1261466759999998, 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57331 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:53:19.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096188+0000", 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096226+0000", 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7861999999999998e-05 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096239+0000", 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2727e-05 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096289+0000", 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0037999999999998e-05 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.100438+0000", 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041489939999999996 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.100451+0000", 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2942000000000001e-05 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093896+0000", 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99344541600000003 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093910+0000", 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3175e-05 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212636+0000", 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.118726068 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212786+0000", 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015021899999999999 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212927+0000", 2023-09-23T20:53:19.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014154 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219297+0000", 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063694299999999997 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219311+0000", 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3786e-05 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219344+0000", 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3769000000000001e-05 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219408+0000", 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3341000000000002e-05 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:19.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222337+0000", 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029289189999999999 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57332 9.1 9:830fc147:::benchmark_data_smithi121_293770_object57331:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.76392392400001, 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1239288190000001, 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:19.994 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57332 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101359+0000", 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:19.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101380+0000", 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1217e-05 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101396+0000", 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5953000000000001e-05 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101405+0000", 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7679999999999997e-06 2023-09-23T20:53:19.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101456+0000", 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1297e-05 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101470+0000", 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4127e-05 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093968+0000", 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.992497713 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093975+0000", 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5719999999999997e-06 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217098+0000", 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1231222729999999 2023-09-23T20:53:19.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217225+0000", 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000127253 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217358+0000", 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013342700000000001 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222449+0000", 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050906379999999998 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222491+0000", 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1764999999999997e-05 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222529+0000", 2023-09-23T20:53:20.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8318999999999997e-05 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222585+0000", 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6302999999999999e-05 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225289+0000", 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027034759999999998 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.002 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57335 9.1 9:8de699c5:::benchmark_data_smithi121_293770_object57334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.763765635, 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1379105439999999, 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:53:20.003 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57335 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:53:20.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:20.006 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.006 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.006 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:20.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:53:20.006 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:53:20.006 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101541+0000", 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3251000000000002e-05 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101553+0000", 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2714999999999999e-05 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:20.007 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101561+0000", 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8099999999999998e-06 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101611+0000", 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9898000000000002e-05 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101618+0000", 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3380000000000001e-06 2023-09-23T20:53:20.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094001+0000", 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99238338000000004 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094008+0000", 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7530000000000004e-06 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225332+0000", 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1313241270000001 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225481+0000", 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000149269 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225643+0000", 2023-09-23T20:53:20.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016234999999999999 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.230166+0000", 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045225370000000001 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231616+0000", 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0014502429999999999 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236427+0000", 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048111459999999997 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236496+0000", 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8417999999999994e-05 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:20.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239429+0000", 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002933125 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57345 9.1 9:89ab31a2:::benchmark_data_smithi121_293770_object57344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.73748399300001, 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1243519640000001, 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:20.013 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57345 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.014 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127798+0000", 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:20.015 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127828+0000", 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9895000000000001e-05 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127846+0000", 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8281e-05 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127857+0000", 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.081e-05 2023-09-23T20:53:20.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127899+0000", 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2271999999999998e-05 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127908+0000", 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5469999999999997e-06 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094072+0000", 2023-09-23T20:53:20.018 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96616466300000003 2023-09-23T20:53:20.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.018 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.018 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:20.019 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:19 smithi123 ceph-mon[193903]: pgmap v661: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 15114/108762 objects degraded (13.896%); 0 B/s, 39 objects/s recovering 2023-09-23T20:53:20.019 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094083+0000", 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.098e-05 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227666+0000", 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.133582614 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:20.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227790+0000", 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012393300000000001 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227940+0000", 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000149989 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231704+0000", 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037640120000000002 2023-09-23T20:53:20.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239483+0000", 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0077796499999999999 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249105+0000", 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096215469999999994 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249162+0000", 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7080999999999998e-05 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252152+0000", 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002989916 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:136306 9.2 9:41ee0b2f:::benchmark_data_smithi121_293770_object33418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:52:30.175927+0000", 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 49.689356668000002, 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067013811, 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 136306 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.024 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175927+0000", 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175927+0000", 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175926+0000", 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175932+0000", 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6309999999999997e-06 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:53:20.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175943+0000", 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0485e-05 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175959+0000", 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5865000000000001e-05 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.180074+0000", 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004115563 2023-09-23T20:53:20.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.180213+0000", 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013848699999999999 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.180353+0000", 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139915 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.191511+0000", 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01115794 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.195631+0000", 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041203580000000002 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.229020+0000", 2023-09-23T20:53:20.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.033389094000000001 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.229031+0000", 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0849999999999999e-05 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:31.242859+0000", 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.013828344 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:53:20.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:31.242941+0000", 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1901000000000001e-05 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:53:20.031 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:53:20.053 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:20.142 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:20.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:19 smithi121 ceph-mon[211526]: pgmap v661: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 15114/108762 objects degraded (13.896%); 0 B/s, 39 objects/s recovering 2023-09-23T20:53:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:19 smithi121 ceph-mon[219709]: pgmap v661: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 15114/108762 objects degraded (13.896%); 0 B/s, 39 objects/s recovering 2023-09-23T20:53:20.256 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:20.319 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:20.448 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:20.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:20.572 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:20.634 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:20.716 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:20.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:20.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:20.739 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:20.823 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:20.850 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:20.853 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:20 smithi121 ceph-mon[211526]: pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 15114/104562 objects degraded (14.455%); 0 B/s, 35 objects/s recovering 2023-09-23T20:53:20.929 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:20.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:20 smithi123 ceph-mon[193903]: pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 15114/104562 objects degraded (14.455%); 0 B/s, 35 objects/s recovering 2023-09-23T20:53:20.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:20.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:21.035 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:21.108 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:21.170 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:20 smithi121 ceph-mon[219709]: pgmap v662: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 15114/104562 objects degraded (14.455%); 0 B/s, 35 objects/s recovering 2023-09-23T20:53:21.207 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:21.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:21.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:21.320 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:21.395 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:21.552 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:21.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:21.669 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:21.751 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:21.880 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:53:21.998 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:53:22.001 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2534062972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.001 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.001 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15114/104562 objects degraded (14.455%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:22.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2534062972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15114/104562 objects degraded (14.455%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:22.143 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:22.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:22.173 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2534062972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.173 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:22.173 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15114/104562 objects degraded (14.455%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:22.258 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:22.347 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:22.714 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: failed to encode map e209 with expected crc 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: failed to encode map e209 with expected crc 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:23.168 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[211526]: pgmap v664: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.1 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15114/101610 objects degraded (14.875%); 0 B/s, 37 objects/s recovering 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: failed to encode map e209 with expected crc 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: failed to encode map e209 with expected crc 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:23.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:22 smithi121 ceph-mon[219709]: pgmap v664: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.1 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15114/101610 objects degraded (14.875%); 0 B/s, 37 objects/s recovering 2023-09-23T20:53:23.176 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:23.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: failed to encode map e209 with expected crc 2023-09-23T20:53:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: failed to encode map e209 with expected crc 2023-09-23T20:53:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:53:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:22 smithi123 ceph-mon[193903]: pgmap v664: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.1 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 15114/101610 objects degraded (14.875%); 0 B/s, 37 objects/s recovering 2023-09-23T20:53:23.293 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:23.366 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:23.468 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:23.486 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:23.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:23.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:23.597 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:23.674 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:23.852 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:23.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:23.965 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:24.050 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:24.346 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:24.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:24.450 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:24.478 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:24.782 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:53:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:53:24] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-09-23T20:53:25.171 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:25.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:25.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:25.280 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:25.292 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:53:25.334 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:25.466 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:25.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:25.570 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:25.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:25.700 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:53:25.700 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:53:25.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.700 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:175786 9.4 e209/207 9:24ef5f97:::benchmark_data_smithi121_293770_object72898:head v 209'11092, mlcod=198'8557)", 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:25.698516+0000", 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0012145490000000001, 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012427670000000001, 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698516+0000", 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698516+0000", 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698515+0000", 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698523+0000", 2023-09-23T20:53:25.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1480000000000006e-06 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698536+0000", 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2391999999999999e-05 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698540+0000", 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7099999999999998e-06 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698585+0000", 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4610000000000001e-05 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:25.698607+0000", 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1586e-05 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T20:53:25.704 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 1 2023-09-23T20:53:25.705 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:53:25.762 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:25.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:25.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:25.874 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:25.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:25 smithi121 ceph-mon[211526]: pgmap v665: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 15114/95061 objects degraded (15.899%); 0 B/s, 54 objects/s recovering 2023-09-23T20:53:25.925 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:25 smithi121 ceph-mon[219709]: pgmap v665: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 15114/95061 objects degraded (15.899%); 0 B/s, 54 objects/s recovering 2023-09-23T20:53:25.929 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:25 smithi123 ceph-mon[193903]: pgmap v665: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 15114/95061 objects degraded (15.899%); 0 B/s, 54 objects/s recovering 2023-09-23T20:53:26.018 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:26.067 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:53:26.089 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:53:26.113 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:53:26.128 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:26.138 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:53:26.154 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:26.159 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:53:26.196 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:26.300 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:26.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:53:26.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:53:26.376 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:26.402 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:53:26.407 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:26.451 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.452 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.452 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.452 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.452 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.452 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.452 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.452 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:53:26.591 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:53:26.657 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:26.764 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:26.837 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1674528077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:26.837 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15114/95061 objects degraded (15.899%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:26.840 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1674528077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15114/95061 objects degraded (15.899%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1674528077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15114/95061 objects degraded (15.899%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:26.925 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:53:26.926 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:53:26.926 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T20:53:26.927 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:53:26.949 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:26.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:26.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:27.061 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:27.123 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:27.188 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:27.204 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:27.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:27.314 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:27.391 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:27.977 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:27.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:27.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:28.085 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:28.122 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:27 smithi121 ceph-mon[211526]: pgmap v666: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 15114/92211 objects degraded (16.391%); 0 B/s, 54 objects/s recovering 2023-09-23T20:53:28.122 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:28.122 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:28.123 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:27 smithi121 ceph-mon[219709]: pgmap v666: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 15114/92211 objects degraded (16.391%); 0 B/s, 54 objects/s recovering 2023-09-23T20:53:28.123 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:28.123 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:28.126 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:27 smithi123 ceph-mon[193903]: pgmap v666: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 15114/92211 objects degraded (16.391%); 0 B/s, 54 objects/s recovering 2023-09-23T20:53:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:29.136 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:29.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:29.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:29.251 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:29.301 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:29.332 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:29.332 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:29.387 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:29.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:29.493 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:29.501 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:29.590 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:29.842 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:29 smithi121 ceph-mon[211526]: pgmap v667: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 15114/90138 objects degraded (16.768%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:29.842 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:29 smithi121 ceph-mon[219709]: pgmap v667: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 15114/90138 objects degraded (16.768%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:29.921 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:53:29.940 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:29.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:29.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:30.059 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:30.181 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:30.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:29 smithi123 ceph-mon[193903]: pgmap v667: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 15114/90138 objects degraded (16.768%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:30.335 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:53:30.849 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:31.378 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:31.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:31.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:31.485 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:31.564 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:31.684 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:31.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:31.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:31.786 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:31.802 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:31.833 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:31.844 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:53:32.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[211526]: pgmap v668: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15114/86640 objects degraded (17.445%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:32.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3905013680' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:32.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15114/86640 objects degraded (17.445%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:32.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:32.045 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:32.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[219709]: pgmap v668: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15114/86640 objects degraded (17.445%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:32.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3905013680' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:32.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15114/86640 objects degraded (17.445%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:32.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:32.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:31 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:32.202 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:32.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:32.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:32.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:31 smithi123 ceph-mon[193903]: pgmap v668: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 15114/86640 objects degraded (17.445%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:32.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3905013680' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:32.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15114/86640 objects degraded (17.445%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:32.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:31 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:32.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:31 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:32.314 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:32.403 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:32.615 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:32.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:32.722 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:32.820 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:32.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:32 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:32.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3905013680' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:32.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:32 smithi123 ceph-mon[193903]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:53:32.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:32 smithi123 ceph-mon[193903]: failed to encode map e210 with expected crc 2023-09-23T20:53:32.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:32 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:53:32.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:32 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:53:32.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:32 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:32.920 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:32.920 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:33.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:33.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3905013680' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:33.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[219709]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[219709]: failed to encode map e210 with expected crc 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3905013680' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[211526]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:53:33.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[211526]: failed to encode map e210 with expected crc 2023-09-23T20:53:33.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:53:33.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:53:33.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:32 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:33.253 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:33.271 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:53:33.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:33.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:33.306 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:53:33.373 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:33.435 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:33.878 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:53:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[211526]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[211526]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[211526]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[211526]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[211526]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[211526]: pgmap v670: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 15114/83937 objects degraded (18.006%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[211526]: from='client.55384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[219709]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[219709]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[219709]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[219709]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[219709]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[219709]: pgmap v670: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 15114/83937 objects degraded (18.006%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:34.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:33 smithi121 ceph-mon[219709]: from='client.55384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:34.238 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:33 smithi123 ceph-mon[193903]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.238 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:33 smithi123 ceph-mon[193903]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.238 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:33 smithi123 ceph-mon[193903]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.238 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:33 smithi123 ceph-mon[193903]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.238 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:33 smithi123 ceph-mon[193903]: failed to encode map e210 with expected crc 2023-09-23T20:53:34.238 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:33 smithi123 ceph-mon[193903]: pgmap v670: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 15114/83937 objects degraded (18.006%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:34.238 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:33 smithi123 ceph-mon[193903]: from='client.55384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:34.335 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:34.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:34.440 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:34.489 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:34.502 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:34.536 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:34.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:34.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:34.647 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:34.674 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:34.838 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":670,"stamp":"2023-09-23T20:53:32.735532+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1832589240,"num_objects":27979,"num_object_clones":0,"num_object_copies":83937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15114,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27979,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":178117,"num_write_kb":6587086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4764,"num_bytes_recovered":241381416,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142308,"ondisk_log_size":142308,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9109764,"kb_used_data":6373124,"kb_used_omap":21,"kb_used_meta":2736554,"kb_avail":740851452,"statfs":{"total":767960285184,"available":758631886848,"internally_reserved":0,"allocated":6526078976,"data_stored":6520435120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":2802231779},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,6,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-386072576,"num_objects":-5891,"num_object_clones":0,"num_object_copies":-17673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5891,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":713,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011335"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":204,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197399+0000","last_change":"2023-09-23T20:53:09.184160+0000","last_active":"2023-09-23T20:53:10.197399+0000","last_peered":"2023-09-23T20:53:10.197399+0000","last_clean":"2023-09-23T20:53:10.197399+0000","last_became_active":"2023-09-23T20:53:09.183837+0000","last_became_peered":"2023-09-23T20:53:09.183837+0000","last_unstale":"2023-09-23T20:53:10.197399+0000","last_undegraded":"2023-09-23T20:53:10.197399+0000","last_fullsized":"2023-09-23T20:53:10.197399+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:03.396066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":164,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.176483+0000","last_change":"2023-09-23T20:53:09.205867+0000","last_active":"2023-09-23T20:53:10.176483+0000","last_peered":"2023-09-23T20:53:10.176483+0000","last_clean":"2023-09-23T20:53:10.176483+0000","last_became_active":"2023-09-23T20:53:09.205534+0000","last_became_peered":"2023-09-23T20:53:09.205534+0000","last_unstale":"2023-09-23T20:53:10.176483+0000","last_undegraded":"2023-09-23T20:53:10.176483+0000","last_fullsized":"2023-09-23T20:53:10.176483+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:23.682188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.869712+0000","last_change":"2023-09-23T20:53:09.200700+0000","last_active":"2023-09-23T20:53:21.869712+0000","last_peered":"2023-09-23T20:53:21.869712+0000","last_clean":"2023-09-23T20:53:21.869712+0000","last_became_active":"2023-09-23T20:53:09.200262+0000","last_became_peered":"2023-09-23T20:53:09.200262+0000","last_unstale":"2023-09-23T20:53:21.869712+0000","last_undegraded":"2023-09-23T20:53:21.869712+0000","last_fullsized":"2023-09-23T20:53:21.869712+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:04.656116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":160,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.286098+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:53:10.286098+0000","last_peered":"2023-09-23T20:53:10.286098+0000","last_clean":"2023-09-23T20:53:10.286098+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:53:10.286098+0000","last_undegraded":"2023-09-23T20:53:10.286098+0000","last_fullsized":"2023-09-23T20:53:10.286098+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":182,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.176364+0000","last_change":"2023-09-23T20:53:09.190070+0000","last_active":"2023-09-23T20:53:10.176364+0000","last_peered":"2023-09-23T20:53:10.176364+0000","last_clean":"2023-09-23T20:53:10.176364+0000","last_became_active":"2023-09-23T20:53:09.189785+0000","last_became_peered":"2023-09-23T20:53:09.189785+0000","last_unstale":"2023-09-23T20:53:10.176364+0000","last_undegraded":"2023-09-23T20:53:10.176364+0000","last_fullsized":"2023-09-23T20:53:10.176364+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.370133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":58,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.187972+0000","last_change":"2023-09-23T20:53:09.187404+0000","last_active":"2023-09-23T20:53:10.187972+0000","last_peered":"2023-09-23T20:53:10.187972+0000","last_clean":"2023-09-23T20:53:10.187972+0000","last_became_active":"2023-09-23T20:53:09.187078+0000","last_became_peered":"2023-09-23T20:53:09.187078+0000","last_unstale":"2023-09-23T20:53:10.187972+0000","last_undegraded":"2023-09-23T20:53:10.187972+0000","last_fullsized":"2023-09-23T20:53:10.187972+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:44.084084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":158,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.285917+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:53:10.285917+0000","last_peered":"2023-09-23T20:53:10.285917+0000","last_clean":"2023-09-23T20:53:10.285917+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:53:10.285917+0000","last_undegraded":"2023-09-23T20:53:10.285917+0000","last_fullsized":"2023-09-23T20:53:10.285917+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":182,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.176251+0000","last_change":"2023-09-23T20:53:09.194688+0000","last_active":"2023-09-23T20:53:10.176251+0000","last_peered":"2023-09-23T20:53:10.176251+0000","last_clean":"2023-09-23T20:53:10.176251+0000","last_became_active":"2023-09-23T20:53:09.194399+0000","last_became_peered":"2023-09-23T20:53:09.194399+0000","last_unstale":"2023-09-23T20:53:10.176251+0000","last_undegraded":"2023-09-23T20:53:10.176251+0000","last_fullsized":"2023-09-23T20:53:10.176251+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:05.137531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":186,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.196475+0000","last_change":"2023-09-23T20:53:09.187702+0000","last_active":"2023-09-23T20:53:10.196475+0000","last_peered":"2023-09-23T20:53:10.196475+0000","last_clean":"2023-09-23T20:53:10.196475+0000","last_became_active":"2023-09-23T20:53:09.187353+0000","last_became_peered":"2023-09-23T20:53:09.187353+0000","last_unstale":"2023-09-23T20:53:10.196475+0000","last_undegraded":"2023-09-23T20:53:10.196475+0000","last_fullsized":"2023-09-23T20:53:10.196475+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:12.645048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":145,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.176410+0000","last_change":"2023-09-23T20:51:06.024444+0000","last_active":"2023-09-23T20:53:10.176410+0000","last_peered":"2023-09-23T20:53:10.176410+0000","last_clean":"2023-09-23T20:53:10.176410+0000","last_became_active":"2023-09-23T20:51:06.023727+0000","last_became_peered":"2023-09-23T20:51:06.023727+0000","last_unstale":"2023-09-23T20:53:10.176410+0000","last_undegraded":"2023-09-23T20:53:10.176410+0000","last_fullsized":"2023-09-23T20:53:10.176410+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:24.860774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":172,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.193108+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:53:10.193108+0000","last_peered":"2023-09-23T20:53:10.193108+0000","last_clean":"2023-09-23T20:53:10.193108+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:53:10.193108+0000","last_undegraded":"2023-09-23T20:53:10.193108+0000","last_fullsized":"2023-09-23T20:53:10.193108+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.869939+0000","last_change":"2023-09-23T20:53:09.188835+0000","last_active":"2023-09-23T20:53:21.869939+0000","last_peered":"2023-09-23T20:53:21.869939+0000","last_clean":"2023-09-23T20:53:21.869939+0000","last_became_active":"2023-09-23T20:53:09.188246+0000","last_became_peered":"2023-09-23T20:53:09.188246+0000","last_unstale":"2023-09-23T20:53:21.869939+0000","last_undegraded":"2023-09-23T20:53:21.869939+0000","last_fullsized":"2023-09-23T20:53:21.869939+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:27.239470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":214,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197617+0000","last_change":"2023-09-23T20:51:05.966238+0000","last_active":"2023-09-23T20:53:10.197617+0000","last_peered":"2023-09-23T20:53:10.197617+0000","last_clean":"2023-09-23T20:53:10.197617+0000","last_became_active":"2023-09-23T20:51:05.959313+0000","last_became_peered":"2023-09-23T20:51:05.959313+0000","last_unstale":"2023-09-23T20:53:10.197617+0000","last_undegraded":"2023-09-23T20:53:10.197617+0000","last_fullsized":"2023-09-23T20:53:10.197617+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:02.228179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":205,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197928+0000","last_change":"2023-09-23T20:53:09.191934+0000","last_active":"2023-09-23T20:53:10.197928+0000","last_peered":"2023-09-23T20:53:10.197928+0000","last_clean":"2023-09-23T20:53:10.197928+0000","last_became_active":"2023-09-23T20:53:09.191583+0000","last_became_peered":"2023-09-23T20:53:09.191583+0000","last_unstale":"2023-09-23T20:53:10.197928+0000","last_undegraded":"2023-09-23T20:53:10.197928+0000","last_fullsized":"2023-09-23T20:53:10.197928+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:45.729672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":171,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.193663+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:53:10.193663+0000","last_peered":"2023-09-23T20:53:10.193663+0000","last_clean":"2023-09-23T20:53:10.193663+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:53:10.193663+0000","last_undegraded":"2023-09-23T20:53:10.193663+0000","last_fullsized":"2023-09-23T20:53:10.193663+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":164,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.194780+0000","last_change":"2023-09-23T20:53:09.219437+0000","last_active":"2023-09-23T20:53:10.194780+0000","last_peered":"2023-09-23T20:53:10.194780+0000","last_clean":"2023-09-23T20:53:10.194780+0000","last_became_active":"2023-09-23T20:53:09.219177+0000","last_became_peered":"2023-09-23T20:53:09.219177+0000","last_unstale":"2023-09-23T20:53:10.194780+0000","last_undegraded":"2023-09-23T20:53:10.194780+0000","last_fullsized":"2023-09-23T20:53:10.194780+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:22.008304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":158,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.285435+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:53:10.285435+0000","last_peered":"2023-09-23T20:53:10.285435+0000","last_clean":"2023-09-23T20:53:10.285435+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:53:10.285435+0000","last_undegraded":"2023-09-23T20:53:10.285435+0000","last_fullsized":"2023-09-23T20:53:10.285435+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":168,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179490+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:53:10.179490+0000","last_peered":"2023-09-23T20:53:10.179490+0000","last_clean":"2023-09-23T20:53:10.179490+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:53:10.179490+0000","last_undegraded":"2023-09-23T20:53:10.179490+0000","last_fullsized":"2023-09-23T20:53:10.179490+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":172,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.180672+0000","last_change":"2023-09-23T20:51:05.958217+0000","last_active":"2023-09-23T20:53:10.180672+0000","last_peered":"2023-09-23T20:53:10.180672+0000","last_clean":"2023-09-23T20:53:10.180672+0000","last_became_active":"2023-09-23T20:51:05.957874+0000","last_became_peered":"2023-09-23T20:51:05.957874+0000","last_unstale":"2023-09-23T20:53:10.180672+0000","last_undegraded":"2023-09-23T20:53:10.180672+0000","last_fullsized":"2023-09-23T20:53:10.180672+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:13.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":159,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.286245+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:53:10.286245+0000","last_peered":"2023-09-23T20:53:10.286245+0000","last_clean":"2023-09-23T20:53:10.286245+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:53:10.286245+0000","last_undegraded":"2023-09-23T20:53:10.286245+0000","last_fullsized":"2023-09-23T20:53:10.286245+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":51,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.187319+0000","last_change":"2023-09-23T20:51:06.038448+0000","last_active":"2023-09-23T20:53:10.187319+0000","last_peered":"2023-09-23T20:53:10.187319+0000","last_clean":"2023-09-23T20:53:10.187319+0000","last_became_active":"2023-09-23T20:51:06.005965+0000","last_became_peered":"2023-09-23T20:51:06.005965+0000","last_unstale":"2023-09-23T20:53:10.187319+0000","last_undegraded":"2023-09-23T20:53:10.187319+0000","last_fullsized":"2023-09-23T20:53:10.187319+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:13.397771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":164,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179668+0000","last_change":"2023-09-23T20:53:09.232052+0000","last_active":"2023-09-23T20:53:10.179668+0000","last_peered":"2023-09-23T20:53:10.179668+0000","last_clean":"2023-09-23T20:53:10.179668+0000","last_became_active":"2023-09-23T20:53:09.231748+0000","last_became_peered":"2023-09-23T20:53:09.231748+0000","last_unstale":"2023-09-23T20:53:10.179668+0000","last_undegraded":"2023-09-23T20:53:10.179668+0000","last_fullsized":"2023-09-23T20:53:10.179668+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:04.683102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"209'10826","reported_seq":10881,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:27.747597+0000","last_change":"2023-09-23T20:50:08.077384+0000","last_active":"2023-09-23T20:53:27.747597+0000","last_peered":"2023-09-23T20:53:27.747597+0000","last_clean":"2023-09-23T20:53:27.747597+0000","last_became_active":"2023-09-23T20:50:08.077095+0000","last_became_peered":"2023-09-23T20:50:08.077095+0000","last_unstale":"2023-09-23T20:53:27.747597+0000","last_undegraded":"2023-09-23T20:53:27.747597+0000","last_fullsized":"2023-09-23T20:53:27.747597+0000","mapping_epoch":164,"log_start":"168'800","ondisk_log_start":"168'800","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:34.442024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117178368,"num_objects":1788,"num_object_clones":0,"num_object_copies":5364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10826,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":170,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.178133+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:53:10.178133+0000","last_peered":"2023-09-23T20:53:10.178133+0000","last_clean":"2023-09-23T20:53:10.178133+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:53:10.178133+0000","last_undegraded":"2023-09-23T20:53:10.178133+0000","last_fullsized":"2023-09-23T20:53:10.178133+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":171,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179073+0000","last_change":"2023-09-23T20:51:05.962659+0000","last_active":"2023-09-23T20:53:10.179073+0000","last_peered":"2023-09-23T20:53:10.179073+0000","last_clean":"2023-09-23T20:53:10.179073+0000","last_became_active":"2023-09-23T20:51:05.962048+0000","last_became_peered":"2023-09-23T20:51:05.962048+0000","last_unstale":"2023-09-23T20:53:10.179073+0000","last_undegraded":"2023-09-23T20:53:10.179073+0000","last_fullsized":"2023-09-23T20:53:10.179073+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:03.530084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"209'11284","reported_seq":11351,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:30.706907+0000","last_change":"2023-09-23T20:53:10.348437+0000","last_active":"2023-09-23T20:53:30.706907+0000","last_peered":"2023-09-23T20:53:30.706907+0000","last_clean":"2023-09-23T20:52:30.157076+0000","last_became_active":"2023-09-23T20:53:10.306549+0000","last_became_peered":"2023-09-23T20:53:10.306549+0000","last_unstale":"2023-09-23T20:53:30.706907+0000","last_undegraded":"2023-09-23T20:53:10.193751+0000","last_fullsized":"2023-09-23T20:53:10.176070+0000","mapping_epoch":207,"log_start":"171'3525","ondisk_log_start":"171'3525","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7759,"log_dups_size":0,"ondisk_log_size":7759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:52.651633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112852992,"num_objects":1722,"num_object_clones":0,"num_object_copies":5166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11284,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1722}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":188,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.177952+0000","last_change":"2023-09-23T20:53:09.185607+0000","last_active":"2023-09-23T20:53:10.177952+0000","last_peered":"2023-09-23T20:53:10.177952+0000","last_clean":"2023-09-23T20:53:10.177952+0000","last_became_active":"2023-09-23T20:53:09.185271+0000","last_became_peered":"2023-09-23T20:53:09.185271+0000","last_unstale":"2023-09-23T20:53:10.177952+0000","last_undegraded":"2023-09-23T20:53:10.177952+0000","last_fullsized":"2023-09-23T20:53:10.177952+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:57.917928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":202,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197716+0000","last_change":"2023-09-23T20:53:09.191763+0000","last_active":"2023-09-23T20:53:10.197716+0000","last_peered":"2023-09-23T20:53:10.197716+0000","last_clean":"2023-09-23T20:53:10.197716+0000","last_became_active":"2023-09-23T20:53:09.191442+0000","last_became_peered":"2023-09-23T20:53:09.191442+0000","last_unstale":"2023-09-23T20:53:10.197716+0000","last_undegraded":"2023-09-23T20:53:10.197716+0000","last_fullsized":"2023-09-23T20:53:10.197716+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:02.859784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"209'10997","reported_seq":11050,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:28.823689+0000","last_change":"2023-09-23T20:50:08.078801+0000","last_active":"2023-09-23T20:53:28.823689+0000","last_peered":"2023-09-23T20:53:28.823689+0000","last_clean":"2023-09-23T20:53:28.823689+0000","last_became_active":"2023-09-23T20:50:08.078386+0000","last_became_peered":"2023-09-23T20:50:08.078386+0000","last_unstale":"2023-09-23T20:53:28.823689+0000","last_undegraded":"2023-09-23T20:53:28.823689+0000","last_fullsized":"2023-09-23T20:53:28.823689+0000","mapping_epoch":164,"log_start":"168'900","ondisk_log_start":"168'900","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10097,"log_dups_size":0,"ondisk_log_size":10097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:47.134724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114491392,"num_objects":1747,"num_object_clones":0,"num_object_copies":5241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10997,"num_write_kb":407808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":181,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.192852+0000","last_change":"2023-09-23T20:51:05.967804+0000","last_active":"2023-09-23T20:53:10.192852+0000","last_peered":"2023-09-23T20:53:10.192852+0000","last_clean":"2023-09-23T20:53:10.192852+0000","last_became_active":"2023-09-23T20:51:05.959914+0000","last_became_peered":"2023-09-23T20:51:05.959914+0000","last_unstale":"2023-09-23T20:53:10.192852+0000","last_undegraded":"2023-09-23T20:53:10.192852+0000","last_fullsized":"2023-09-23T20:53:10.192852+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:03.210361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":41,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.187456+0000","last_change":"2023-09-23T20:51:05.957335+0000","last_active":"2023-09-23T20:53:10.187456+0000","last_peered":"2023-09-23T20:53:10.187456+0000","last_clean":"2023-09-23T20:53:10.187456+0000","last_became_active":"2023-09-23T20:51:05.956858+0000","last_became_peered":"2023-09-23T20:51:05.956858+0000","last_unstale":"2023-09-23T20:53:10.187456+0000","last_undegraded":"2023-09-23T20:53:10.187456+0000","last_fullsized":"2023-09-23T20:53:10.187456+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:31.085919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"209'10906","reported_seq":10981,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:28.784019+0000","last_change":"2023-09-23T20:53:10.403698+0000","last_active":"2023-09-23T20:53:28.784019+0000","last_peered":"2023-09-23T20:53:28.784019+0000","last_clean":"2023-09-23T20:52:30.196736+0000","last_became_active":"2023-09-23T20:53:10.355510+0000","last_became_peered":"2023-09-23T20:53:10.355510+0000","last_unstale":"2023-09-23T20:53:28.784019+0000","last_undegraded":"2023-09-23T20:53:10.196118+0000","last_fullsized":"2023-09-23T20:53:10.178982+0000","mapping_epoch":207,"log_start":"168'900","ondisk_log_start":"168'900","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:55.710095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1832,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10906,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":1754}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179417+0000","last_change":"2023-09-23T20:51:05.995979+0000","last_active":"2023-09-23T20:53:10.179417+0000","last_peered":"2023-09-23T20:53:10.179417+0000","last_clean":"2023-09-23T20:53:10.179417+0000","last_became_active":"2023-09-23T20:51:05.995591+0000","last_became_peered":"2023-09-23T20:51:05.995591+0000","last_unstale":"2023-09-23T20:53:10.179417+0000","last_undegraded":"2023-09-23T20:53:10.179417+0000","last_fullsized":"2023-09-23T20:53:10.179417+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:35.414294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":174,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.286620+0000","last_change":"2023-09-23T20:53:09.185147+0000","last_active":"2023-09-23T20:53:10.286620+0000","last_peered":"2023-09-23T20:53:10.286620+0000","last_clean":"2023-09-23T20:53:10.286620+0000","last_became_active":"2023-09-23T20:53:09.184855+0000","last_became_peered":"2023-09-23T20:53:09.184855+0000","last_unstale":"2023-09-23T20:53:10.286620+0000","last_undegraded":"2023-09-23T20:53:10.286620+0000","last_fullsized":"2023-09-23T20:53:10.286620+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:45.699896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"209'11025","reported_seq":13944,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:30.757055+0000","last_change":"2023-09-23T20:51:15.811325+0000","last_active":"2023-09-23T20:53:30.757055+0000","last_peered":"2023-09-23T20:53:30.757055+0000","last_clean":"2023-09-23T20:53:30.757055+0000","last_became_active":"2023-09-23T20:51:15.805454+0000","last_became_peered":"2023-09-23T20:51:15.805454+0000","last_unstale":"2023-09-23T20:53:30.757055+0000","last_undegraded":"2023-09-23T20:53:30.757055+0000","last_fullsized":"2023-09-23T20:53:30.757055+0000","mapping_epoch":180,"log_start":"171'3300","ondisk_log_start":"171'3300","created":164,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7725,"log_dups_size":0,"ondisk_log_size":7725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:13.254228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110166016,"num_objects":1681,"num_object_clones":0,"num_object_copies":5043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11025,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1418,"num_bytes_recovered":92930048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":172,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.176426+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:53:10.176426+0000","last_peered":"2023-09-23T20:53:10.176426+0000","last_clean":"2023-09-23T20:53:10.176426+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:53:10.176426+0000","last_undegraded":"2023-09-23T20:53:10.176426+0000","last_fullsized":"2023-09-23T20:53:10.176426+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197568+0000","last_change":"2023-09-23T20:51:06.001664+0000","last_active":"2023-09-23T20:53:10.197568+0000","last_peered":"2023-09-23T20:53:10.197568+0000","last_clean":"2023-09-23T20:53:10.197568+0000","last_became_active":"2023-09-23T20:51:06.001352+0000","last_became_peered":"2023-09-23T20:51:06.001352+0000","last_unstale":"2023-09-23T20:53:10.197568+0000","last_undegraded":"2023-09-23T20:53:10.197568+0000","last_fullsized":"2023-09-23T20:53:10.197568+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.688374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"209'11097","reported_seq":15022,"reported_epoch":209,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:53:28.286524+0000","last_change":"2023-09-23T20:53:10.343981+0000","last_active":"2023-09-23T20:53:28.286524+0000","last_peered":"2023-09-23T20:53:28.286524+0000","last_clean":"2023-09-23T20:52:30.193025+0000","last_became_active":"2023-09-23T20:53:10.276010+0000","last_became_peered":"2023-09-23T20:53:10.276010+0000","last_unstale":"2023-09-23T20:53:28.286524+0000","last_undegraded":"2023-09-23T20:53:28.286524+0000","last_fullsized":"2023-09-23T20:53:10.174804+0000","mapping_epoch":207,"log_start":"168'1200","ondisk_log_start":"168'1200","created":164,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":9897,"log_dups_size":0,"ondisk_log_size":9897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:28.272259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117374976,"num_objects":1791,"num_object_clones":0,"num_object_copies":5373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11097,"num_write_kb":412416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1917,"num_bytes_recovered":54329344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":1791}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":58,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.187538+0000","last_change":"2023-09-23T20:53:09.236307+0000","last_active":"2023-09-23T20:53:10.187538+0000","last_peered":"2023-09-23T20:53:10.187538+0000","last_clean":"2023-09-23T20:53:10.187538+0000","last_became_active":"2023-09-23T20:53:09.235961+0000","last_became_peered":"2023-09-23T20:53:09.235961+0000","last_unstale":"2023-09-23T20:53:10.187538+0000","last_undegraded":"2023-09-23T20:53:10.187538+0000","last_fullsized":"2023-09-23T20:53:10.187538+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:54.763339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":202,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197339+0000","last_change":"2023-09-23T20:53:09.184166+0000","last_active":"2023-09-23T20:53:10.197339+0000","last_peered":"2023-09-23T20:53:10.197339+0000","last_clean":"2023-09-23T20:53:10.197339+0000","last_became_active":"2023-09-23T20:53:09.183862+0000","last_became_peered":"2023-09-23T20:53:09.183862+0000","last_unstale":"2023-09-23T20:53:10.197339+0000","last_undegraded":"2023-09-23T20:53:10.197339+0000","last_fullsized":"2023-09-23T20:53:10.197339+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:40.482320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"209'11026","reported_seq":11103,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:28.821492+0000","last_change":"2023-09-23T20:53:10.383476+0000","last_active":"2023-09-23T20:53:28.821492+0000","last_peered":"2023-09-23T20:53:28.821492+0000","last_clean":"2023-09-23T20:52:30.214684+0000","last_became_active":"2023-09-23T20:53:10.343781+0000","last_became_peered":"2023-09-23T20:53:10.343781+0000","last_unstale":"2023-09-23T20:53:28.821492+0000","last_undegraded":"2023-09-23T20:53:10.193078+0000","last_fullsized":"2023-09-23T20:53:10.177310+0000","mapping_epoch":207,"log_start":"171'3300","ondisk_log_start":"171'3300","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7726,"log_dups_size":0,"ondisk_log_size":7726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:11.933025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119406592,"num_objects":1822,"num_object_clones":0,"num_object_copies":5466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11026,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1822}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":169,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197668+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:53:10.197668+0000","last_peered":"2023-09-23T20:53:10.197668+0000","last_clean":"2023-09-23T20:53:10.197668+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:53:10.197668+0000","last_undegraded":"2023-09-23T20:53:10.197668+0000","last_fullsized":"2023-09-23T20:53:10.197668+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":205,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197921+0000","last_change":"2023-09-23T20:53:09.192221+0000","last_active":"2023-09-23T20:53:10.197921+0000","last_peered":"2023-09-23T20:53:10.197921+0000","last_clean":"2023-09-23T20:53:10.197921+0000","last_became_active":"2023-09-23T20:53:09.191599+0000","last_became_peered":"2023-09-23T20:53:09.191599+0000","last_unstale":"2023-09-23T20:53:10.197921+0000","last_undegraded":"2023-09-23T20:53:10.197921+0000","last_fullsized":"2023-09-23T20:53:10.197921+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:54.725504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"209'11266","reported_seq":11335,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:30.996433+0000","last_change":"2023-09-23T20:53:10.344228+0000","last_active":"2023-09-23T20:53:30.996433+0000","last_peered":"2023-09-23T20:53:30.996433+0000","last_clean":"2023-09-23T20:52:30.196344+0000","last_became_active":"2023-09-23T20:53:10.314804+0000","last_became_peered":"2023-09-23T20:53:10.314804+0000","last_unstale":"2023-09-23T20:53:30.996433+0000","last_undegraded":"2023-09-23T20:53:10.193272+0000","last_fullsized":"2023-09-23T20:53:10.176077+0000","mapping_epoch":207,"log_start":"171'3500","ondisk_log_start":"171'3500","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7766,"log_dups_size":0,"ondisk_log_size":7766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:48.249155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113246208,"num_objects":1728,"num_object_clones":0,"num_object_copies":5184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11266,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1728}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":165,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.176298+0000","last_change":"2023-09-23T20:53:09.188782+0000","last_active":"2023-09-23T20:53:10.176298+0000","last_peered":"2023-09-23T20:53:10.176298+0000","last_clean":"2023-09-23T20:53:10.176298+0000","last_became_active":"2023-09-23T20:53:09.188563+0000","last_became_peered":"2023-09-23T20:53:09.188563+0000","last_unstale":"2023-09-23T20:53:10.176298+0000","last_undegraded":"2023-09-23T20:53:10.176298+0000","last_fullsized":"2023-09-23T20:53:10.176298+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:33.898108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":162,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.175552+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:53:10.175552+0000","last_peered":"2023-09-23T20:53:10.175552+0000","last_clean":"2023-09-23T20:53:10.175552+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:53:10.175552+0000","last_undegraded":"2023-09-23T20:53:10.175552+0000","last_fullsized":"2023-09-23T20:53:10.175552+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"209'11308","reported_seq":11379,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:28.824066+0000","last_change":"2023-09-23T20:53:10.403424+0000","last_active":"2023-09-23T20:53:28.824066+0000","last_peered":"2023-09-23T20:53:28.824066+0000","last_clean":"2023-09-23T20:52:30.157599+0000","last_became_active":"2023-09-23T20:53:10.359278+0000","last_became_peered":"2023-09-23T20:53:10.359278+0000","last_unstale":"2023-09-23T20:53:28.824066+0000","last_undegraded":"2023-09-23T20:53:10.196743+0000","last_fullsized":"2023-09-23T20:53:10.179018+0000","mapping_epoch":207,"log_start":"168'1300","ondisk_log_start":"168'1300","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:52.816379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114163712,"num_objects":1742,"num_object_clones":0,"num_object_copies":5226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1922,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11308,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1742}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":172,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.870196+0000","last_change":"2023-09-23T20:53:09.191673+0000","last_active":"2023-09-23T20:53:21.870196+0000","last_peered":"2023-09-23T20:53:21.870196+0000","last_clean":"2023-09-23T20:53:21.870196+0000","last_became_active":"2023-09-23T20:53:09.191277+0000","last_became_peered":"2023-09-23T20:53:09.191277+0000","last_unstale":"2023-09-23T20:53:21.870196+0000","last_undegraded":"2023-09-23T20:53:21.870196+0000","last_fullsized":"2023-09-23T20:53:21.870196+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:21.050943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":186,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179576+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:53:10.179576+0000","last_peered":"2023-09-23T20:53:10.179576+0000","last_clean":"2023-09-23T20:53:10.179576+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:53:10.179576+0000","last_undegraded":"2023-09-23T20:53:10.179576+0000","last_fullsized":"2023-09-23T20:53:10.179576+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.870007+0000","last_change":"2023-09-23T20:53:09.196041+0000","last_active":"2023-09-23T20:53:21.870007+0000","last_peered":"2023-09-23T20:53:21.870007+0000","last_clean":"2023-09-23T20:53:21.870007+0000","last_became_active":"2023-09-23T20:53:09.191615+0000","last_became_peered":"2023-09-23T20:53:09.191615+0000","last_unstale":"2023-09-23T20:53:21.870007+0000","last_undegraded":"2023-09-23T20:53:21.870007+0000","last_fullsized":"2023-09-23T20:53:21.870007+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:25.505589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"209'11408","reported_seq":11463,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:27.796805+0000","last_change":"2023-09-23T20:50:08.076706+0000","last_active":"2023-09-23T20:53:27.796805+0000","last_peered":"2023-09-23T20:53:27.796805+0000","last_clean":"2023-09-23T20:53:27.796805+0000","last_became_active":"2023-09-23T20:50:08.076360+0000","last_became_peered":"2023-09-23T20:50:08.076360+0000","last_unstale":"2023-09-23T20:53:27.796805+0000","last_undegraded":"2023-09-23T20:53:27.796805+0000","last_fullsized":"2023-09-23T20:53:27.796805+0000","mapping_epoch":164,"log_start":"168'1400","ondisk_log_start":"168'1400","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.596097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117178368,"num_objects":1788,"num_object_clones":0,"num_object_copies":5364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11408,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":210,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.194812+0000","last_change":"2023-09-23T20:53:09.184802+0000","last_active":"2023-09-23T20:53:10.194812+0000","last_peered":"2023-09-23T20:53:10.194812+0000","last_clean":"2023-09-23T20:53:10.194812+0000","last_became_active":"2023-09-23T20:53:09.184501+0000","last_became_peered":"2023-09-23T20:53:09.184501+0000","last_unstale":"2023-09-23T20:53:10.194812+0000","last_undegraded":"2023-09-23T20:53:10.194812+0000","last_fullsized":"2023-09-23T20:53:10.194812+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:56.145028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197748+0000","last_change":"2023-09-23T20:51:06.033002+0000","last_active":"2023-09-23T20:53:10.197748+0000","last_peered":"2023-09-23T20:53:10.197748+0000","last_clean":"2023-09-23T20:53:10.197748+0000","last_became_active":"2023-09-23T20:51:06.032657+0000","last_became_peered":"2023-09-23T20:51:06.032657+0000","last_unstale":"2023-09-23T20:53:10.197748+0000","last_undegraded":"2023-09-23T20:53:10.197748+0000","last_fullsized":"2023-09-23T20:53:10.197748+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:07.293307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":178,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179208+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:53:10.179208+0000","last_peered":"2023-09-23T20:53:10.179208+0000","last_clean":"2023-09-23T20:53:10.179208+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:53:10.179208+0000","last_undegraded":"2023-09-23T20:53:10.179208+0000","last_fullsized":"2023-09-23T20:53:10.179208+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"209'11062","reported_seq":11132,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:28.805204+0000","last_change":"2023-09-23T20:53:10.375283+0000","last_active":"2023-09-23T20:53:28.805204+0000","last_peered":"2023-09-23T20:53:28.805204+0000","last_clean":"2023-09-23T20:52:30.211455+0000","last_became_active":"2023-09-23T20:53:10.333927+0000","last_became_peered":"2023-09-23T20:53:10.333927+0000","last_unstale":"2023-09-23T20:53:28.805204+0000","last_undegraded":"2023-09-23T20:53:10.194479+0000","last_fullsized":"2023-09-23T20:53:10.178986+0000","mapping_epoch":207,"log_start":"168'1000","ondisk_log_start":"168'1000","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:47:13.773667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114556928,"num_objects":1748,"num_object_clones":0,"num_object_copies":5244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1872,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11062,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1748}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":167,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.174945+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:53:10.174945+0000","last_peered":"2023-09-23T20:53:10.174945+0000","last_clean":"2023-09-23T20:53:10.174945+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:53:10.174945+0000","last_undegraded":"2023-09-23T20:53:10.174945+0000","last_fullsized":"2023-09-23T20:53:10.174945+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":137,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.876552+0000","last_change":"2023-09-23T20:53:09.187057+0000","last_active":"2023-09-23T20:53:21.876552+0000","last_peered":"2023-09-23T20:53:21.876552+0000","last_clean":"2023-09-23T20:53:21.876552+0000","last_became_active":"2023-09-23T20:53:09.186597+0000","last_became_peered":"2023-09-23T20:53:09.186597+0000","last_unstale":"2023-09-23T20:53:21.876552+0000","last_undegraded":"2023-09-23T20:53:21.876552+0000","last_fullsized":"2023-09-23T20:53:21.876552+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.376692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":167,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.870981+0000","last_change":"2023-09-23T20:53:09.193262+0000","last_active":"2023-09-23T20:53:21.870981+0000","last_peered":"2023-09-23T20:53:21.870981+0000","last_clean":"2023-09-23T20:53:21.870981+0000","last_became_active":"2023-09-23T20:53:09.192941+0000","last_became_peered":"2023-09-23T20:53:09.192941+0000","last_unstale":"2023-09-23T20:53:21.870981+0000","last_undegraded":"2023-09-23T20:53:21.870981+0000","last_fullsized":"2023-09-23T20:53:21.870981+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:06.470020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"209'10994","reported_seq":11049,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:30.690439+0000","last_change":"2023-09-23T20:50:08.078559+0000","last_active":"2023-09-23T20:53:30.690439+0000","last_peered":"2023-09-23T20:53:30.690439+0000","last_clean":"2023-09-23T20:53:30.690439+0000","last_became_active":"2023-09-23T20:50:08.078294+0000","last_became_peered":"2023-09-23T20:50:08.078294+0000","last_unstale":"2023-09-23T20:53:30.690439+0000","last_undegraded":"2023-09-23T20:53:30.690439+0000","last_fullsized":"2023-09-23T20:53:30.690439+0000","mapping_epoch":164,"log_start":"171'3300","ondisk_log_start":"171'3300","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7694,"log_dups_size":0,"ondisk_log_size":7694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:11.281483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105775104,"num_objects":1614,"num_object_clones":0,"num_object_copies":4842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10994,"num_write_kb":403456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":173,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179512+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:53:10.179512+0000","last_peered":"2023-09-23T20:53:10.179512+0000","last_clean":"2023-09-23T20:53:10.179512+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:53:10.179512+0000","last_undegraded":"2023-09-23T20:53:10.179512+0000","last_fullsized":"2023-09-23T20:53:10.179512+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":601,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.177368+0000","last_change":"2023-09-23T20:51:30.823817+0000","last_active":"2023-09-23T20:53:10.177368+0000","last_peered":"2023-09-23T20:53:10.177368+0000","last_clean":"2023-09-23T20:53:10.177368+0000","last_became_active":"2023-09-23T20:51:30.823372+0000","last_became_peered":"2023-09-23T20:51:30.823372+0000","last_unstale":"2023-09-23T20:53:10.177368+0000","last_undegraded":"2023-09-23T20:53:10.177368+0000","last_fullsized":"2023-09-23T20:53:10.177368+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:09.432277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":144,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.193681+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:53:10.193681+0000","last_peered":"2023-09-23T20:53:10.193681+0000","last_clean":"2023-09-23T20:53:10.193681+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:53:10.193681+0000","last_undegraded":"2023-09-23T20:53:10.193681+0000","last_fullsized":"2023-09-23T20:53:10.193681+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.187808+0000","last_change":"2023-09-23T20:51:05.957381+0000","last_active":"2023-09-23T20:53:10.187808+0000","last_peered":"2023-09-23T20:53:10.187808+0000","last_clean":"2023-09-23T20:53:10.187808+0000","last_became_active":"2023-09-23T20:51:05.956991+0000","last_became_peered":"2023-09-23T20:51:05.956991+0000","last_unstale":"2023-09-23T20:53:10.187808+0000","last_undegraded":"2023-09-23T20:53:10.187808+0000","last_fullsized":"2023-09-23T20:53:10.187808+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:20.613174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"209'11244","reported_seq":14108,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:27.808810+0000","last_change":"2023-09-23T20:51:23.778886+0000","last_active":"2023-09-23T20:53:27.808810+0000","last_peered":"2023-09-23T20:53:27.808810+0000","last_clean":"2023-09-23T20:53:27.808810+0000","last_became_active":"2023-09-23T20:51:23.771838+0000","last_became_peered":"2023-09-23T20:51:23.771838+0000","last_unstale":"2023-09-23T20:53:27.808810+0000","last_undegraded":"2023-09-23T20:53:27.808810+0000","last_fullsized":"2023-09-23T20:53:27.808810+0000","mapping_epoch":183,"log_start":"168'1200","ondisk_log_start":"168'1200","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:54.532463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124780544,"num_objects":1904,"num_object_clones":0,"num_object_copies":5712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11244,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1391,"num_bytes_recovered":91160576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":207,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.177346+0000","last_change":"2023-09-23T20:53:09.183474+0000","last_active":"2023-09-23T20:53:10.177346+0000","last_peered":"2023-09-23T20:53:10.177346+0000","last_clean":"2023-09-23T20:53:10.177346+0000","last_became_active":"2023-09-23T20:53:09.183165+0000","last_became_peered":"2023-09-23T20:53:09.183165+0000","last_unstale":"2023-09-23T20:53:10.177346+0000","last_undegraded":"2023-09-23T20:53:10.177346+0000","last_fullsized":"2023-09-23T20:53:10.177346+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:11.886478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":159,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.863614+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:53:21.863614+0000","last_peered":"2023-09-23T20:53:21.863614+0000","last_clean":"2023-09-23T20:53:21.863614+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:53:21.863614+0000","last_undegraded":"2023-09-23T20:53:21.863614+0000","last_fullsized":"2023-09-23T20:53:21.863614+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.286171+0000","last_change":"2023-09-23T20:53:09.181989+0000","last_active":"2023-09-23T20:53:10.286171+0000","last_peered":"2023-09-23T20:53:10.286171+0000","last_clean":"2023-09-23T20:53:10.286171+0000","last_became_active":"2023-09-23T20:53:09.181672+0000","last_became_peered":"2023-09-23T20:53:09.181672+0000","last_unstale":"2023-09-23T20:53:10.286171+0000","last_undegraded":"2023-09-23T20:53:10.286171+0000","last_fullsized":"2023-09-23T20:53:10.286171+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:39.497624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"209'11278","reported_seq":11353,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:28.793019+0000","last_change":"2023-09-23T20:53:10.407911+0000","last_active":"2023-09-23T20:53:28.793019+0000","last_peered":"2023-09-23T20:53:28.793019+0000","last_clean":"2023-09-23T20:52:30.212933+0000","last_became_active":"2023-09-23T20:53:10.358990+0000","last_became_peered":"2023-09-23T20:53:10.358990+0000","last_unstale":"2023-09-23T20:53:28.793019+0000","last_undegraded":"2023-09-23T20:53:10.195827+0000","last_fullsized":"2023-09-23T20:53:10.177309+0000","mapping_epoch":207,"log_start":"171'3500","ondisk_log_start":"171'3500","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7778,"log_dups_size":0,"ondisk_log_size":7778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:09.500397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1955,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11278,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1678}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":173,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.192836+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:53:10.192836+0000","last_peered":"2023-09-23T20:53:10.192836+0000","last_clean":"2023-09-23T20:53:10.192836+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:53:10.192836+0000","last_undegraded":"2023-09-23T20:53:10.192836+0000","last_fullsized":"2023-09-23T20:53:10.192836+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":112,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.872808+0000","last_change":"2023-09-23T20:53:09.199776+0000","last_active":"2023-09-23T20:53:21.872808+0000","last_peered":"2023-09-23T20:53:21.872808+0000","last_clean":"2023-09-23T20:53:21.872808+0000","last_became_active":"2023-09-23T20:53:09.199309+0000","last_became_peered":"2023-09-23T20:53:09.199309+0000","last_unstale":"2023-09-23T20:53:21.872808+0000","last_undegraded":"2023-09-23T20:53:21.872808+0000","last_fullsized":"2023-09-23T20:53:21.872808+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:32.904643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":145,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.863939+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:53:21.863939+0000","last_peered":"2023-09-23T20:53:21.863939+0000","last_clean":"2023-09-23T20:53:21.863939+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:53:21.863939+0000","last_undegraded":"2023-09-23T20:53:21.863939+0000","last_fullsized":"2023-09-23T20:53:21.863939+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"209'10922","reported_seq":10999,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:28.840581+0000","last_change":"2023-09-23T20:53:10.346064+0000","last_active":"2023-09-23T20:53:28.840581+0000","last_peered":"2023-09-23T20:53:28.840581+0000","last_clean":"2023-09-23T20:52:30.141642+0000","last_became_active":"2023-09-23T20:53:10.309106+0000","last_became_peered":"2023-09-23T20:53:10.309106+0000","last_unstale":"2023-09-23T20:53:28.840581+0000","last_undegraded":"2023-09-23T20:53:10.192526+0000","last_fullsized":"2023-09-23T20:53:10.177303+0000","mapping_epoch":207,"log_start":"171'3200","ondisk_log_start":"171'3200","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:03.627658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116850712,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1843,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10922,"num_write_kb":406529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1784}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":172,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179791+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:53:10.179791+0000","last_peered":"2023-09-23T20:53:10.179791+0000","last_clean":"2023-09-23T20:53:10.179791+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:53:10.179791+0000","last_undegraded":"2023-09-23T20:53:10.179791+0000","last_fullsized":"2023-09-23T20:53:10.179791+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":163,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.870359+0000","last_change":"2023-09-23T20:53:09.214852+0000","last_active":"2023-09-23T20:53:21.870359+0000","last_peered":"2023-09-23T20:53:21.870359+0000","last_clean":"2023-09-23T20:53:21.870359+0000","last_became_active":"2023-09-23T20:53:09.195778+0000","last_became_peered":"2023-09-23T20:53:09.195778+0000","last_unstale":"2023-09-23T20:53:21.870359+0000","last_undegraded":"2023-09-23T20:53:21.870359+0000","last_fullsized":"2023-09-23T20:53:21.870359+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.010784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.176136+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:53:10.176136+0000","last_peered":"2023-09-23T20:53:10.176136+0000","last_clean":"2023-09-23T20:53:10.176136+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:53:10.176136+0000","last_undegraded":"2023-09-23T20:53:10.176136+0000","last_fullsized":"2023-09-23T20:53:10.176136+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"209'11174","reported_seq":11229,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:30.765579+0000","last_change":"2023-09-23T20:50:08.083439+0000","last_active":"2023-09-23T20:53:30.765579+0000","last_peered":"2023-09-23T20:53:30.765579+0000","last_clean":"2023-09-23T20:53:30.765579+0000","last_became_active":"2023-09-23T20:50:08.083061+0000","last_became_peered":"2023-09-23T20:50:08.083061+0000","last_unstale":"2023-09-23T20:53:30.765579+0000","last_undegraded":"2023-09-23T20:53:30.765579+0000","last_fullsized":"2023-09-23T20:53:30.765579+0000","mapping_epoch":164,"log_start":"171'3400","ondisk_log_start":"171'3400","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7774,"log_dups_size":0,"ondisk_log_size":7774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:41.569707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109051904,"num_objects":1664,"num_object_clones":0,"num_object_copies":4992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11174,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":41,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.187901+0000","last_change":"2023-09-23T20:51:05.957996+0000","last_active":"2023-09-23T20:53:10.187901+0000","last_peered":"2023-09-23T20:53:10.187901+0000","last_clean":"2023-09-23T20:53:10.187901+0000","last_became_active":"2023-09-23T20:51:05.957651+0000","last_became_peered":"2023-09-23T20:51:05.957651+0000","last_unstale":"2023-09-23T20:53:10.187901+0000","last_undegraded":"2023-09-23T20:53:10.187901+0000","last_fullsized":"2023-09-23T20:53:10.187901+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:05.872498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":177,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.286020+0000","last_change":"2023-09-23T20:53:09.184461+0000","last_active":"2023-09-23T20:53:10.286020+0000","last_peered":"2023-09-23T20:53:10.286020+0000","last_clean":"2023-09-23T20:53:10.286020+0000","last_became_active":"2023-09-23T20:53:09.184067+0000","last_became_peered":"2023-09-23T20:53:09.184067+0000","last_unstale":"2023-09-23T20:53:10.286020+0000","last_undegraded":"2023-09-23T20:53:10.286020+0000","last_fullsized":"2023-09-23T20:53:10.286020+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:08.928487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197498+0000","last_change":"2023-09-23T20:51:05.959327+0000","last_active":"2023-09-23T20:53:10.197498+0000","last_peered":"2023-09-23T20:53:10.197498+0000","last_clean":"2023-09-23T20:53:10.197498+0000","last_became_active":"2023-09-23T20:51:05.958816+0000","last_became_peered":"2023-09-23T20:51:05.958816+0000","last_unstale":"2023-09-23T20:53:10.197498+0000","last_undegraded":"2023-09-23T20:53:10.197498+0000","last_fullsized":"2023-09-23T20:53:10.197498+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:52.674764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":147,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.864132+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:53:21.864132+0000","last_peered":"2023-09-23T20:53:21.864132+0000","last_clean":"2023-09-23T20:53:21.864132+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:53:21.864132+0000","last_undegraded":"2023-09-23T20:53:21.864132+0000","last_fullsized":"2023-09-23T20:53:21.864132+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":198,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.177378+0000","last_change":"2023-09-23T20:53:09.190345+0000","last_active":"2023-09-23T20:53:10.177378+0000","last_peered":"2023-09-23T20:53:10.177378+0000","last_clean":"2023-09-23T20:53:10.177378+0000","last_became_active":"2023-09-23T20:53:09.190072+0000","last_became_peered":"2023-09-23T20:53:09.190072+0000","last_unstale":"2023-09-23T20:53:10.177378+0000","last_undegraded":"2023-09-23T20:53:10.177378+0000","last_fullsized":"2023-09-23T20:53:10.177378+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:42.575345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":188,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179523+0000","last_change":"2023-09-23T20:53:09.184633+0000","last_active":"2023-09-23T20:53:10.179523+0000","last_peered":"2023-09-23T20:53:10.179523+0000","last_clean":"2023-09-23T20:53:10.179523+0000","last_became_active":"2023-09-23T20:53:09.184285+0000","last_became_peered":"2023-09-23T20:53:09.184285+0000","last_unstale":"2023-09-23T20:53:10.179523+0000","last_undegraded":"2023-09-23T20:53:10.179523+0000","last_fullsized":"2023-09-23T20:53:10.179523+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:38.473632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":202,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197823+0000","last_change":"2023-09-23T20:53:09.191904+0000","last_active":"2023-09-23T20:53:10.197823+0000","last_peered":"2023-09-23T20:53:10.197823+0000","last_clean":"2023-09-23T20:53:10.197823+0000","last_became_active":"2023-09-23T20:53:09.191558+0000","last_became_peered":"2023-09-23T20:53:09.191558+0000","last_unstale":"2023-09-23T20:53:10.197823+0000","last_undegraded":"2023-09-23T20:53:10.197823+0000","last_fullsized":"2023-09-23T20:53:10.197823+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:10.469531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":182,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179657+0000","last_change":"2023-09-23T20:51:05.969594+0000","last_active":"2023-09-23T20:53:10.179657+0000","last_peered":"2023-09-23T20:53:10.179657+0000","last_clean":"2023-09-23T20:53:10.179657+0000","last_became_active":"2023-09-23T20:51:05.959688+0000","last_became_peered":"2023-09-23T20:51:05.959688+0000","last_unstale":"2023-09-23T20:53:10.179657+0000","last_undegraded":"2023-09-23T20:53:10.179657+0000","last_fullsized":"2023-09-23T20:53:10.179657+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:59.980249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":185,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197531+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:53:10.197531+0000","last_peered":"2023-09-23T20:53:10.197531+0000","last_clean":"2023-09-23T20:53:10.197531+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:53:10.197531+0000","last_undegraded":"2023-09-23T20:53:10.197531+0000","last_fullsized":"2023-09-23T20:53:10.197531+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":184,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179402+0000","last_change":"2023-09-23T20:51:05.999322+0000","last_active":"2023-09-23T20:53:10.179402+0000","last_peered":"2023-09-23T20:53:10.179402+0000","last_clean":"2023-09-23T20:53:10.179402+0000","last_became_active":"2023-09-23T20:51:05.998993+0000","last_became_peered":"2023-09-23T20:51:05.998993+0000","last_unstale":"2023-09-23T20:53:10.179402+0000","last_undegraded":"2023-09-23T20:53:10.179402+0000","last_fullsized":"2023-09-23T20:53:10.179402+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:47.495994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":181,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.179603+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:53:10.179603+0000","last_peered":"2023-09-23T20:53:10.179603+0000","last_clean":"2023-09-23T20:53:10.179603+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:53:10.179603+0000","last_undegraded":"2023-09-23T20:53:10.179603+0000","last_fullsized":"2023-09-23T20:53:10.179603+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":163,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:53:21.862310+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:53:21.862310+0000","last_peered":"2023-09-23T20:53:21.862310+0000","last_clean":"2023-09-23T20:53:21.862310+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:53:21.862310+0000","last_undegraded":"2023-09-23T20:53:21.862310+0000","last_fullsized":"2023-09-23T20:53:21.862310+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":205,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:53:10.197342+0000","last_change":"2023-09-23T20:53:09.192021+0000","last_active":"2023-09-23T20:53:10.197342+0000","last_peered":"2023-09-23T20:53:10.197342+0000","last_clean":"2023-09-23T20:53:10.197342+0000","last_became_active":"2023-09-23T20:53:09.191668+0000","last_became_peered":"2023-09-23T20:53:09.191668+0000","last_unstale":"2023-09-23T20:53:10.197342+0000","last_undegraded":"2023-09-23T20:53:10.197342+0000","last_fullsized":"2023-09-23T20:53:10.197342+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:10.164608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1831993368,"num_objects":27955,"num_object_clones":0,"num_object_copies":83865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15114,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27955,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":177817,"num_write_kb":6584641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4734,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6465990656,"data_stored":6465978440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142092,"ondisk_log_size":142092,"up":48,"acting":39,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201577,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1425164,"kb_used_data":1019268,"kb_used_omap":0,"kb_used_meta":405888,"kb_avail":92319988,"statfs":{"total":95995035648,"available":94535667712,"internally_reserved":0,"allocated":1043730432,"data_stored":1043137263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":415629312},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.3149999999999999,"15min":2.7890000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20499999999999999},"max":{"1min":4.9820000000000002,"5min":70.781999999999996,"15min":225.56999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":3.327,"5min":2.661,"15min":3.129},"min":{"1min":0.307,"5min":0.29299999999999998,"15min":0.21299999999999999},"max":{"1min":14.035,"5min":70.876000000000005,"15min":225.61099999999999},"last":0.35099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7439999999999998,"5min":3.8559999999999999,"15min":2.0920000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":24.016999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":4.827,"5min":6.7590000000000003,"15min":3.1160000000000001},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.991,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.27400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":11.965,"5min":3.5590000000000002,"15min":3.7320000000000002},"min":{"1min":0.46100000000000002,"5min":0.29399999999999998,"15min":0.22900000000000001},"max":{"1min":147.309,"5min":147.309,"15min":147.309},"last":1.0309999999999999},{"interface":"front","average":{"1min":11.414999999999999,"5min":3.262,"15min":3.0640000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":147.32400000000001,"5min":147.32400000000001,"15min":147.32400000000001},"last":1.0449999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":9.1080000000000005,"5min":5.194,"15min":8.5370000000000008},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":121.90900000000001,"5min":138.345,"15min":138.345},"last":0.59899999999999998},{"interface":"front","average":{"1min":10.446999999999999,"5min":5.5330000000000004,"15min":8.1259999999999994},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":121.748,"5min":133.76400000000001,"15min":133.76400000000001},"last":4.4180000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:52:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0790000000000002,"5min":1.665,"15min":1.292},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.216},"max":{"1min":16.648,"5min":16.648,"15min":16.648},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.7490000000000001,"5min":1.5249999999999999,"15min":1.284},"min":{"1min":0.307,"5min":0.247,"15min":0.247},"max":{"1min":10.172000000000001,"5min":10.172000000000001,"15min":10.172000000000001},"last":4.5789999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:52:46 2023","interfaces":[{"interface":"back","average":{"1min":8.8670000000000009,"5min":2.8130000000000002,"15min":1.7789999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":126.28,"5min":126.28,"15min":126.28},"last":0.627},{"interface":"front","average":{"1min":11.44,"5min":3.5760000000000001,"15min":2.0009999999999999},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":126.465,"5min":126.465,"15min":126.465},"last":3.8799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:52:46 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":1.7370000000000001,"15min":1.401},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.28899999999999998},"max":{"1min":16.678000000000001,"5min":16.678000000000001,"15min":16.678000000000001},"last":0.443},{"interface":"front","average":{"1min":2.7229999999999999,"5min":1.4390000000000001,"15min":1.2110000000000001},"min":{"1min":0.42399999999999999,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":10.154,"5min":10.154,"15min":10.154},"last":4.4509999999999996}]}]},{"osd":6,"up_from":47,"seq":201863463214,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":653240,"kb_used_data":353392,"kb_used_omap":0,"kb_used_meta":299840,"kb_avail":93091912,"statfs":{"total":95995035648,"available":95326117888,"internally_reserved":0,"allocated":361873408,"data_stored":361300342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":307036160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":5.9589999999999996,"5min":3.9089999999999998,"15min":2.7599999999999998},"min":{"1min":0.224,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":79.016000000000005,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.94599999999999995},{"interface":"front","average":{"1min":5.8209999999999997,"5min":3.6200000000000001,"15min":2.5979999999999999},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.20000000000000001},"max":{"1min":78.843000000000004,"5min":92.138000000000005,"15min":92.138000000000005},"last":2.9649999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":5.1440000000000001,"5min":3.6960000000000002,"15min":2.3639999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":35.262,"5min":96.623999999999995,"15min":96.623999999999995},"last":2.7229999999999999},{"interface":"front","average":{"1min":4.6790000000000003,"5min":3.7440000000000002,"15min":2.1930000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.223},"max":{"1min":35.350000000000001,"5min":96.254000000000005,"15min":96.254000000000005},"last":2.746}]},{"osd":2,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.8080000000000001,"15min":2.1589999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":8.2370000000000001,"5min":24.440999999999999,"15min":24.440999999999999},"last":1.208},{"interface":"front","average":{"1min":2.6880000000000002,"5min":1.9570000000000001,"15min":2.399},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.1109999999999998,"5min":23.916,"15min":23.916},"last":0.51000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":22.062000000000001,"5min":7.4279999999999999,"15min":6.5590000000000002},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":250.804,"5min":250.804,"15min":250.804},"last":1.325},{"interface":"front","average":{"1min":22.256,"5min":7.577,"15min":6.524},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":250.446,"5min":250.446,"15min":250.446},"last":0.39200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.2410000000000001,"15min":1.111},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":9.1590000000000007,"5min":9.1590000000000007,"15min":9.1590000000000007},"last":0.433},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.4359999999999999,"15min":1.3819999999999999},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.27000000000000002},"max":{"1min":7.5890000000000004,"5min":9.8650000000000002,"15min":10.058999999999999},"last":0.36599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.2599999999999998,"15min":1.9370000000000001},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.26000000000000001},"max":{"1min":18.962,"5min":105.121,"15min":105.121},"last":0.47899999999999998},{"interface":"front","average":{"1min":3.2770000000000001,"5min":3.117,"15min":1.994},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":9.4469999999999992,"5min":106.30500000000001,"15min":106.30500000000001},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":6.907,"5min":2.7570000000000001,"15min":1.6200000000000001},"min":{"1min":0.371,"5min":0.30199999999999999,"15min":0.22600000000000001},"max":{"1min":101.04600000000001,"5min":101.04600000000001,"15min":101.04600000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":6.8200000000000003,"5min":2.5070000000000001,"15min":1.726},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.32100000000000001},"max":{"1min":101.081,"5min":101.081,"15min":101.081},"last":0.52600000000000002}]}]},{"osd":1,"up_from":206,"seq":884763262982,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2101304,"kb_used_data":1953840,"kb_used_omap":5,"kb_used_meta":147450,"kb_avail":91643848,"statfs":{"total":95995035648,"available":93843300352,"internally_reserved":0,"allocated":2000732160,"data_stored":1999876277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":150989539},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.1280000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.9980000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.2949999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.1120000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44500000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.0380000000000003}]}]},{"osd":0,"up_from":92,"seq":395136991385,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":949340,"kb_used_data":472212,"kb_used_omap":5,"kb_used_meta":477114,"kb_avail":92795812,"statfs":{"total":95995035648,"available":95022911488,"internally_reserved":0,"allocated":483545088,"data_stored":482702867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":488565370},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0919999999999996,"5min":4.4889999999999999,"15min":2.9089999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":31.515000000000001,"5min":134.322,"15min":134.322},"last":6.3609999999999998},{"interface":"front","average":{"1min":4.4530000000000003,"5min":3.1400000000000001,"15min":2.5449999999999999},"min":{"1min":0.21199999999999999,"5min":0.19,"15min":0.19},"max":{"1min":31.48,"5min":59.570999999999998,"15min":59.570999999999998},"last":4.0149999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":7.5499999999999998,"5min":2.8170000000000002,"15min":2.3580000000000001},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":114.002,"5min":114.002,"15min":114.002},"last":8.1140000000000008},{"interface":"front","average":{"1min":7.5949999999999998,"5min":2.7829999999999999,"15min":3.9460000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":113.69499999999999,"5min":113.69499999999999,"15min":113.69499999999999},"last":10.329000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":12.102,"5min":6.5439999999999996,"15min":4.4539999999999997},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":89.665999999999997,"5min":94.947999999999993,"15min":94.947999999999993},"last":5.7000000000000002},{"interface":"front","average":{"1min":12.561,"5min":6.6660000000000004,"15min":4.6559999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":89.533000000000001,"5min":94.382999999999996,"15min":94.382999999999996},"last":3.9670000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.7669999999999999,"15min":1.839},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":5.4210000000000003,"5min":10.525,"15min":12.973000000000001},"last":5.5380000000000003},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.389,"15min":1.847},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5279999999999996,"5min":12.461,"15min":12.461},"last":5.7489999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":3.617,"15min":2.8039999999999998},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":18.146000000000001,"5min":124.40600000000001,"15min":124.40600000000001},"last":5.5750000000000002},{"interface":"front","average":{"1min":1.2450000000000001,"5min":2.8050000000000002,"15min":2.4049999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.5990000000000002,"5min":124.363,"15min":124.363},"last":8.5449999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":1.5349999999999999,"15min":2.0510000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.307},"max":{"1min":5.5519999999999996,"5min":12.510999999999999,"15min":14.037000000000001},"last":8.5749999999999993},{"interface":"front","average":{"1min":2.9630000000000001,"5min":1.7549999999999999,"15min":1.7929999999999999},"min":{"1min":0.36599999999999999,"5min":0.35599999999999998,"15min":0.34799999999999998},"max":{"1min":10.026999999999999,"5min":10.026999999999999,"15min":20.399999999999999},"last":3.9329999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":2.3799999999999999,"15min":2.363},"min":{"1min":0.39600000000000002,"5min":0.35699999999999998,"15min":0.35099999999999998},"max":{"1min":6.46,"5min":60.037999999999997,"15min":60.037999999999997},"last":8.0700000000000003},{"interface":"front","average":{"1min":1.7170000000000001,"5min":2.383,"15min":2.0760000000000001},"min":{"1min":0.46300000000000002,"5min":0.39800000000000002,"15min":0.38100000000000001},"max":{"1min":5.3399999999999999,"5min":60.012,"15min":60.012},"last":3.992}]}]},{"osd":2,"up_from":129,"seq":554050781275,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":691248,"kb_used_data":353896,"kb_used_omap":7,"kb_used_meta":337336,"kb_avail":93053904,"statfs":{"total":95995035648,"available":95287197696,"internally_reserved":0,"allocated":362389504,"data_stored":361587806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":345432145},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":2.4220000000000002,"15min":2.8530000000000002},"min":{"1min":0.218,"5min":0.182,"15min":0.17599999999999999},"max":{"1min":19.902000000000001,"5min":89.644999999999996,"15min":89.644999999999996},"last":2.6720000000000002},{"interface":"front","average":{"1min":3.3260000000000001,"5min":2.464,"15min":2.718},"min":{"1min":0.253,"5min":0.217,"15min":0.186},"max":{"1min":21.472000000000001,"5min":88.599999999999994,"15min":88.599999999999994},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.8439999999999999,"15min":2.9750000000000001},"min":{"1min":0.22500000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5879999999999992,"5min":91.325000000000003,"15min":91.325000000000003},"last":2.169},{"interface":"front","average":{"1min":1.119,"5min":2.3599999999999999,"15min":2.996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2489999999999997,"5min":91.799999999999997,"15min":91.799999999999997},"last":0.28199999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:53:22 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":4.9960000000000004,"15min":2.7799999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.753,"5min":151.43799999999999,"15min":151.43799999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.8900000000000001,"5min":5.04,"15min":2.3580000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":23.725000000000001,"5min":152.20699999999999,"15min":152.20699999999999},"last":2.8130000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.415,"15min":1.7629999999999999},"min":{"1min":0.29999999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7960000000000003,"5min":8.7230000000000008,"15min":8.7230000000000008},"last":2.895},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.3100000000000001,"15min":2.282},"min":{"1min":0.36599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.5350000000000001,"5min":8.5350000000000001,"15min":8.5350000000000001},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":3.2170000000000001,"15min":3.1869999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.5090000000000003,"5min":163.352,"15min":163.352},"last":173.34},{"interface":"front","average":{"1min":1.629,"5min":3.2810000000000001,"15min":3.0270000000000001},"min":{"1min":0.36699999999999999,"5min":0.316,"15min":0.30299999999999999},"max":{"1min":5.1909999999999998,"5min":163.089,"15min":163.089},"last":173.15000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.367,"15min":2.6779999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.8920000000000003,"5min":6.3449999999999998,"15min":7.476},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.744,"15min":2.6600000000000001},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.6580000000000004,"5min":9.5220000000000002,"15min":12.595000000000001},"last":2.9239999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":3.4860000000000002,"15min":2.8820000000000001},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.6630000000000003,"5min":160.08099999999999,"15min":160.08099999999999},"last":177.30799999999999},{"interface":"front","average":{"1min":3.008,"5min":3.427,"15min":3.0979999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.5559999999999992,"5min":160.13800000000001,"15min":160.13800000000001},"last":177.40899999999999}]}]},{"osd":3,"up_from":148,"seq":635655159875,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129856,"kb_used_data":865784,"kb_used_omap":2,"kb_used_meta":264061,"kb_avail":92615296,"statfs":{"total":95995035648,"available":94838063104,"internally_reserved":0,"allocated":886562816,"data_stored":885747525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":270398517},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":7.391,"5min":5.2789999999999999,"15min":11.314},"min":{"1min":0.215,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":95.507999999999996,"5min":253.68100000000001,"15min":253.68100000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":7.1139999999999999,"5min":5.0640000000000001,"15min":10.493},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":95.259,"5min":253.46199999999999,"15min":253.46199999999999},"last":3.5489999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":5.0099999999999998,"15min":8.6150000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.0140000000000002,"5min":172.77099999999999,"15min":172.77099999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.7919999999999998,"15min":7.8689999999999998},"min":{"1min":0.25,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.692,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.55400000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.3100000000000001,"15min":1.1279999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":3.3159999999999998,"5min":11.317,"15min":11.317},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.409,"15min":1.097},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.3049999999999997,"5min":8.8819999999999997,"15min":8.8819999999999997},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.5609999999999999,"15min":1.641},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.9749999999999996,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":0.626},{"interface":"front","average":{"1min":1.27,"5min":1.571,"15min":2.0009999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.7930000000000001,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.432}]},{"osd":5,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.847,"15min":1.399},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.2949999999999999,"5min":12.057,"15min":12.057},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.1099999999999999,"5min":2.008,"15min":1.974},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":5.274,"5min":14.054,"15min":14.054},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.8859999999999999,"15min":1.9430000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6909999999999998,"5min":11.632,"15min":11.632},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.0649999999999999,"5min":1.879,"15min":2.387},"min":{"1min":0.40300000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5309999999999997,"5min":16.451000000000001,"15min":16.451000000000001},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.024,"15min":1.7170000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7110000000000003,"5min":12.25,"15min":12.25},"last":2.2370000000000001},{"interface":"front","average":{"1min":2.3940000000000001,"5min":1.883,"15min":2.1949999999999998},"min":{"1min":0.42399999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.3639999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":2.5489999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986487,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":686788,"kb_used_data":340220,"kb_used_omap":0,"kb_used_meta":346560,"kb_avail":93058364,"statfs":{"total":95995035648,"available":95291764736,"internally_reserved":0,"allocated":348385280,"data_stored":347799988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":354877440},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":8.3989999999999991,"5min":3.831,"15min":2.6000000000000001},"min":{"1min":0.22600000000000001,"5min":0.216,"15min":0.214},"max":{"1min":146.25299999999999,"5min":146.25299999999999,"15min":146.25299999999999},"last":1.6659999999999999},{"interface":"front","average":{"1min":7.9160000000000004,"5min":3.7010000000000001,"15min":2.262},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22600000000000001},"max":{"1min":144.77000000000001,"5min":144.77000000000001,"15min":144.77000000000001},"last":1.5640000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":4.2939999999999996,"5min":7.4859999999999998,"15min":3.589},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":45.712000000000003,"5min":286.31099999999998,"15min":286.31099999999998},"last":1.4870000000000001},{"interface":"front","average":{"1min":4.6509999999999998,"5min":7.2949999999999999,"15min":4.1310000000000002},"min":{"1min":0.27900000000000003,"5min":0.25700000000000001,"15min":0.24299999999999999},"max":{"1min":45.460000000000001,"5min":285.97000000000003,"15min":285.97000000000003},"last":0.72599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.1819999999999999,"15min":1.3009999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.25},"max":{"1min":8.2400000000000002,"5min":8.6639999999999997,"15min":8.6639999999999997},"last":1.474},{"interface":"front","average":{"1min":1.944,"5min":1.478,"15min":2.0899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.4509999999999996,"5min":7.4509999999999996,"15min":12.773999999999999},"last":1.635}]},{"osd":3,"last update":"Sat Sep 23 20:53:23 2023","interfaces":[{"interface":"back","average":{"1min":11.016,"5min":5.0640000000000001,"15min":6.7539999999999996},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":142.28399999999999,"5min":142.28399999999999,"15min":142.28399999999999},"last":8.2959999999999994},{"interface":"front","average":{"1min":11.807,"5min":5.3140000000000001,"15min":6.5659999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":142.34700000000001,"5min":142.34700000000001,"15min":142.34700000000001},"last":2.4849999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:52:50 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":1.0509999999999999,"15min":0.97999999999999998},"min":{"1min":0.28799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":9.2270000000000003},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.105,"5min":1.236,"15min":1.1319999999999999},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":7.2960000000000003,"5min":7.2960000000000003,"15min":12.209},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":1.1100000000000001,"15min":1.135},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28000000000000003},"max":{"1min":5.2220000000000004,"5min":7.4109999999999996,"15min":16.271000000000001},"last":0.308},{"interface":"front","average":{"1min":1.337,"5min":0.99299999999999999,"15min":1.0700000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.30399999999999999},"max":{"1min":8.1609999999999996,"5min":8.1609999999999996,"15min":11.718},"last":0.32500000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:52:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":1.9410000000000001,"15min":1.458},"min":{"1min":0.35999999999999999,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":7.2619999999999996,"5min":73.268000000000001,"15min":73.268000000000001},"last":0.373},{"interface":"front","average":{"1min":2.129,"5min":1.9199999999999999,"15min":1.383},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":9.1679999999999993,"5min":69.760000000000005,"15min":69.760000000000005},"last":0.34000000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724851,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1472824,"kb_used_data":1014512,"kb_used_omap":0,"kb_used_meta":458304,"kb_avail":92272328,"statfs":{"total":95995035648,"available":94486863872,"internally_reserved":0,"allocated":1038860288,"data_stored":1038283052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":469303296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.7890000000000001,"15min":2.1299999999999999},"min":{"1min":0.25700000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":4.8680000000000003,"5min":124.873,"15min":124.873},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.829,"5min":2.8119999999999998,"15min":1.9330000000000001},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":9.2789999999999999,"5min":123.554,"15min":123.554},"last":0.30599999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":11.34,"5min":4.9880000000000004,"15min":3.407},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":184.31200000000001,"5min":184.31200000000001,"15min":323.541},"last":0.372},{"interface":"front","average":{"1min":11.313000000000001,"5min":5.0149999999999997,"15min":3.6469999999999998},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":184.67400000000001,"5min":184.67400000000001,"15min":324.072},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.9279999999999999,"15min":2.2090000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.9059999999999997,"5min":12.585000000000001,"15min":13.394},"last":2.02},{"interface":"front","average":{"1min":1.869,"5min":1.391,"15min":1.619},"min":{"1min":0.313,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.8929999999999998,"5min":7.8929999999999998,"15min":14.263},"last":0.44500000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6240000000000001,"5min":5.4279999999999999,"15min":3.0510000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.5839999999999996,"5min":198.33099999999999,"15min":198.33099999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.2309999999999999,"5min":5.4930000000000003,"15min":2.8170000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.202,"5min":198.47300000000001,"15min":198.47300000000001},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:52:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.149,"15min":1.0449999999999999},"min":{"1min":0.34000000000000002,"5min":0.25900000000000001,"15min":0.216},"max":{"1min":10.906000000000001,"5min":10.906000000000001,"15min":10.906000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.5739999999999998,"5min":1.2410000000000001,"15min":1.175},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":8.3940000000000001,"5min":8.3940000000000001,"15min":8.3940000000000001},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.238,"15min":1.1259999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":6.327,"5min":6.327,"15min":8.3010000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.7429999999999999,"5min":1.407,"15min":1.1279999999999999},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":8.6219999999999999,"5min":8.9870000000000001,"15min":9.6370000000000005},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:52:45 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":1.569,"15min":1.349},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.28199999999999997},"max":{"1min":7.9370000000000003,"5min":7.9370000000000003,"15min":14.217000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.613,"5min":1.236,"15min":1.0980000000000001},"min":{"1min":0.35499999999999998,"5min":0.29899999999999999,"15min":0.28699999999999998},"max":{"1min":10.836,"5min":10.836,"15min":10.836},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":470679552,"data_stored":470679552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1990397952,"data_stored":1990393880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":353566720,"data_stored":353566720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":883621888,"data_stored":883621888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":342622208,"data_stored":342622208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1036652544,"data_stored":1036648472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":355074048,"data_stored":355074048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1033375744,"data_stored":1033371672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:53:34.844 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T20:53:34.946 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:34 smithi121 ceph-mon[211526]: pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 15114/77988 objects degraded (19.380%); 0 B/s, 77 objects/s recovering 2023-09-23T20:53:34.947 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:53:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:53:34] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.43.0" 2023-09-23T20:53:34.947 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:34 smithi121 ceph-mon[219709]: pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 15114/77988 objects degraded (19.380%); 0 B/s, 77 objects/s recovering 2023-09-23T20:53:34.983 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:34.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:34.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:35.096 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:35.138 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:34 smithi123 ceph-mon[193903]: pgmap v671: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 15114/77988 objects degraded (19.380%); 0 B/s, 77 objects/s recovering 2023-09-23T20:53:35.147 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:35.259 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:35.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:35.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:35.274 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:53:35.362 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:35.483 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:35.631 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:35.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:35.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:35.742 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:35.863 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:35.958 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:35.976 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:53:35.992 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:36.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:36.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:36.023 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:53:36.078 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:36.084 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:53:36.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:36.131 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:53:36.181 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:36.182 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:53:36.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1574839179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:36.240 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:53:36.286 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:53:36.325 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1574839179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:36.326 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1574839179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:36.338 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:53:36.347 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:36.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:36.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:36.369 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.369 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.369 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.369 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.370 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.370 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.370 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.370 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:53:36.457 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:36.522 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:36.536 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:53:36.536 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:53:36.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.536 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:182091 9.b e210/183 9:da0c081e:::benchmark_data_smithi121_293770_object79203:head v 210'11584, mlcod=210'11582)", 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:36.519921+0000", 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.00930484, 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0093521160000000006, 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.519921+0000", 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.519921+0000", 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.519920+0000", 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.519928+0000", 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1829999999999994e-06 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.519944+0000", 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5418e-05 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:36.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.519950+0000", 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8089999999999998e-06 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.520005+0000", 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5309000000000002e-05 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.520023+0000", 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8219999999999998e-05 2023-09-23T20:53:36.540 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:182097 9.b e210/183 9:d057db34:::benchmark_data_smithi121_293770_object79209:head v 210'11585, mlcod=210'11582)", 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:36.525378+0000", 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.003848256, 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003963261, 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525378+0000", 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525378+0000", 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525376+0000", 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:36.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525385+0000", 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2119999999999992e-06 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525401+0000", 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5814000000000001e-05 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525408+0000", 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7190000000000004e-06 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525457+0000", 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8568e-05 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525480+0000", 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3206000000000001e-05 2023-09-23T20:53:36.544 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:182095 9.f 9:f77766de:::benchmark_data_smithi121_293770_object79207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:36.525942+0000", 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0032841210000000001, 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003463537, 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 182095 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525942+0000", 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525942+0000", 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525941+0000", 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525944+0000", 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9609999999999999e-06 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525952+0000", 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9580000000000007e-06 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.525957+0000", 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2010000000000001e-06 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.526596+0000", 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00063971600000000003 2023-09-23T20:53:36.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.526724+0000", 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012715599999999999 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.526849+0000", 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012597300000000001 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:182100 9.d e210/207)", 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:36.529367+0000", 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 4294967295.9998584, 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107796, 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:36.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.529367+0000", 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.529367+0000", 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.529366+0000", 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.529376+0000", 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0026e-05 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.529389+0000", 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2612999999999999e-05 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.552 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.529396+0000", 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9140000000000002e-06 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:36.529470+0000", 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3905000000000005e-05 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T20:53:36.553 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 4 2023-09-23T20:53:36.554 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:53:36.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15114/77988 objects degraded (19.380%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:36.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:36 smithi123 ceph-mon[193903]: pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 15114/75150 objects degraded (20.112%); 0 B/s, 77 objects/s recovering 2023-09-23T20:53:37.140 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:37.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:37.244 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:37.270 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15114/77988 objects degraded (19.380%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:37.270 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:36 smithi121 ceph-mon[211526]: pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 15114/75150 objects degraded (20.112%); 0 B/s, 77 objects/s recovering 2023-09-23T20:53:37.270 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15114/77988 objects degraded (19.380%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:37.270 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:36 smithi121 ceph-mon[219709]: pgmap v672: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 15114/75150 objects degraded (20.112%); 0 B/s, 77 objects/s recovering 2023-09-23T20:53:37.272 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:37.550 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:53:38.161 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:38.397 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:38.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:38.506 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:38.570 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:38.685 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:38.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:38.789 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:38.827 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.106 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:39.209 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:39.237 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.282 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:39.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:39.385 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:39.415 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.510 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:39.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:39.614 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:39.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[211526]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[211526]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:53:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[211526]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[211526]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[211526]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[211526]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[211526]: pgmap v674: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 15114/72831 objects degraded (20.752%); 0 B/s, 93 objects/s recovering 2023-09-23T20:53:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[219709]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[219709]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:53:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[219709]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[219709]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[219709]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[219709]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:39 smithi121 ceph-mon[219709]: pgmap v674: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 15114/72831 objects degraded (20.752%); 0 B/s, 93 objects/s recovering 2023-09-23T20:53:39.686 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.803 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:39.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:39.917 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:39.980 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:39.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:39 smithi123 ceph-mon[193903]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:39 smithi123 ceph-mon[193903]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:53:39.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:39 smithi123 ceph-mon[193903]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:39 smithi123 ceph-mon[193903]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:39 smithi123 ceph-mon[193903]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:39 smithi123 ceph-mon[193903]: failed to encode map e211 with expected crc 2023-09-23T20:53:39.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:39 smithi123 ceph-mon[193903]: pgmap v674: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 15114/72831 objects degraded (20.752%); 0 B/s, 93 objects/s recovering 2023-09-23T20:53:40.110 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:40.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:40.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:40.217 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:40.256 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:40.304 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:40.368 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:40.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:40.483 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:40.523 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:53:40.524 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:53:40.524 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T20:53:40.524 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T20:53:40.524 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3936932506' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1595081542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:40.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:40.559 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 9 pgp_num to 16 2023-09-23T20:53:40.568 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:53:40.574 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:40.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:53:40.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3936932506' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:40.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:40.796 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1595081542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:40.797 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:53:40.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: failed to encode map e212 with expected crc 2023-09-23T20:53:40.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3936932506' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:40.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:40.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1595081542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:40.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:53:41.646 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:41.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:41.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:41.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:41.824 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:53:41.824 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:41.824 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:53:41.824 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: failed to encode map e213 with expected crc 2023-09-23T20:53:41.824 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: failed to encode map e213 with expected crc 2023-09-23T20:53:41.824 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: pgmap v677: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 13271/68598 objects degraded (19.346%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:41.824 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15114/72831 objects degraded (20.752%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: failed to encode map e213 with expected crc 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: failed to encode map e213 with expected crc 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: pgmap v677: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 13271/68598 objects degraded (19.346%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:41.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15114/72831 objects degraded (20.752%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:41.827 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: failed to encode map e213 with expected crc 2023-09-23T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: failed to encode map e213 with expected crc 2023-09-23T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: pgmap v677: 89 pgs: 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 13271/68598 objects degraded (19.346%); 0 B/s, 71 objects/s recovering 2023-09-23T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15114/72831 objects degraded (20.752%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:41.977 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:53:42.260 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:42.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:42.366 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:42.375 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:42.446 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:42.529 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:42.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:42.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:42.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:42.582 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:42.647 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:42.696 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:42.822 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:42.827 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:42.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:42.938 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:42.993 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:43.249 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:43.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:43.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:43.358 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:43.387 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:43.430 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:43.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:43.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:43.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:43.600 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:43 smithi123 ceph-mon[193903]: pgmap v678: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 13271/65550 objects degraded (20.246%); 0 B/s, 73 objects/s recovering 2023-09-23T20:53:43.847 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:43.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:43.859 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:43 smithi121 ceph-mon[211526]: pgmap v678: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 13271/65550 objects degraded (20.246%); 0 B/s, 73 objects/s recovering 2023-09-23T20:53:43.859 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:43 smithi121 ceph-mon[219709]: pgmap v678: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 13271/65550 objects degraded (20.246%); 0 B/s, 73 objects/s recovering 2023-09-23T20:53:43.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:43.961 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:44.067 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:44.282 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:53:44.346 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:44.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:44.451 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:44.502 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:44.692 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:44.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:44.723 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:53:44.764 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:53:44.796 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:44.809 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:53:44.859 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:53:44.873 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:44.914 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:53:44.959 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:19254 8.3 8:c32fabf0:::benchmark_data_smithi121_252781_object19253:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:44:59.997455+0000", 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 524.95805871699997, 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2634655770000001, 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:53:44.960 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 19254 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997455+0000", 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997455+0000", 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997453+0000", 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997700+0000", 2023-09-23T20:53:44.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000246405 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997714+0000", 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.485e-05 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997725+0000", 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0322999999999999e-05 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998826+0000", 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011006989999999999 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998835+0000", 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5009999999999993e-06 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.260835+0000", 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.261999836 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.260920+0000", 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5592000000000003e-05 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:19291 8.3 8:c64c6b63:::benchmark_data_smithi121_252781_object19290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:53:44.965 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:44:59.997734+0000", 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 524.95777963900002, 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2702351030000001, 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 19291 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997734+0000", 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997734+0000", 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997733+0000", 2023-09-23T20:53:44.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997799+0000", 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5835999999999998e-05 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997810+0000", 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1031e-05 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997816+0000", 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9630000000000003e-06 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998860+0000", 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0010436289999999999 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998866+0000", 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2369999999999997e-06 2023-09-23T20:53:44.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.261444+0000", 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.262578027 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.261578+0000", 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000133793 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.261706+0000", 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012807199999999999 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.263821+0000", 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021155929999999998 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.263872+0000", 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0754999999999998e-05 2023-09-23T20:53:44.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.266921+0000", 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0030492079999999999 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.266985+0000", 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3360000000000003e-05 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:44.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.267969+0000", 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00098445600000000009 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:19300 8.3 8:cd31d4d8:::benchmark_data_smithi121_252781_object19299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 524.95769109499997, 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2736549619999999, 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:44.973 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 19300 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997822+0000", 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997875+0000", 2023-09-23T20:53:44.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3315e-05 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997884+0000", 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1230000000000006e-06 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.997891+0000", 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7789999999999998e-06 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998892+0000", 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0010008300000000001 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:44:59.998898+0000", 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3430000000000001e-06 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.262620+0000", 2023-09-23T20:53:44.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2637219449999999 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.262722+0000", 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010137900000000001 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.262886+0000", 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016391999999999999 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.264035+0000", 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001149033 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.264274+0000", 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00023902299999999999 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.270500+0000", 2023-09-23T20:53:44.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062266190000000001 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.270562+0000", 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1509999999999999e-05 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:45:01.271477+0000", 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000915715 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.980 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95048 8.4 8:2454d6a3:::benchmark_data_smithi121_252781_object95047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 460.89138206000001, 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2752492069999999, 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:44.981 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95048 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064130+0000", 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064169+0000", 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8658999999999997e-05 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064181+0000", 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2089999999999999e-05 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064188+0000", 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2400000000000001e-06 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064346+0000", 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015842800000000001 2023-09-23T20:53:44.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064356+0000", 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.01e-05 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072299+0000", 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0079425179999999 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072311+0000", 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1701e-05 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.324944+0000", 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.252633109 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.325108+0000", 2023-09-23T20:53:44.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016413699999999999 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.325270+0000", 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000162201 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333069+0000", 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077994290000000001 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:44.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333084+0000", 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4813e-05 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.338377+0000", 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0052931949999999997 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.338442+0000", 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4246999999999997e-05 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.339381+0000", 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00093899200000000004 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95052 8.d 8:bca20ec2:::benchmark_data_smithi121_252781_object95051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 460.89131362799998, 2023-09-23T20:53:44.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2831711729999999, 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95052 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:53:44.990 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064199+0000", 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064234+0000", 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4929999999999999e-05 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064242+0000", 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8450000000000003e-06 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064248+0000", 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8989999999999998e-06 2023-09-23T20:53:44.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064373+0000", 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012531700000000001 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064383+0000", 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4509999999999998e-06 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072655+0000", 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.008272584 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072666+0000", 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0275000000000001e-05 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336059+0000", 2023-09-23T20:53:44.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2633933260000001 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336211+0000", 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000152299 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336390+0000", 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017826800000000001 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.340844+0000", 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044546439999999998 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.340887+0000", 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2695999999999999e-05 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346536+0000", 2023-09-23T20:53:44.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056488220000000004 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346600+0000", 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3899000000000006e-05 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.347371+0000", 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00077147299999999995 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.997 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95057 8.4 8:2a7799b6:::benchmark_data_smithi121_252781_object95056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 460.88485835, 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2767475730000002, 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:44.998 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95057 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:44.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070654+0000", 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070684+0000", 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0334000000000001e-05 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070699+0000", 2023-09-23T20:53:45.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4211e-05 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070706+0000", 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7249999999999997e-06 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070765+0000", 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8112999999999997e-05 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070775+0000", 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0766e-05 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072335+0000", 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0015602290000001 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072343+0000", 2023-09-23T20:53:45.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0609999999999998e-06 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329624+0000", 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2572813840000001 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329748+0000", 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000123678 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329878+0000", 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013038500000000001 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333185+0000", 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033074290000000002 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333622+0000", 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043623000000000002 2023-09-23T20:53:45.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346585+0000", 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012963585 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346641+0000", 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6227999999999998e-05 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.347403+0000", 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00076131300000000001 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:172444 8.8 8:1dfce91d:::benchmark_data_smithi121_252781_object63269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:47:47.118875+0000", 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 357.83663904000002, 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.058494244, 2023-09-23T20:53:45.006 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 172444 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118875+0000", 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118875+0000", 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118874+0000", 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118877+0000", 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5569999999999998e-06 2023-09-23T20:53:45.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118883+0000", 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1650000000000003e-06 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.118887+0000", 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1359999999999999e-06 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.121980+0000", 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0030927149999999998 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:47.121985+0000", 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4410000000000001e-06 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.151693+0000", 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.029708158 2023-09-23T20:53:45.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.151806+0000", 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000112716 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 7", 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.151920+0000", 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113928 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.165278+0000", 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013357707 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177275+0000", 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011997173 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177303+0000", 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8028999999999999e-05 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:47:48.177369+0000", 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5949000000000002e-05 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186212 8.8 8:10804386:::benchmark_data_smithi121_252781_object77037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 338.46002363500003, 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1681942240000001, 2023-09-23T20:53:45.013 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186212 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495489+0000", 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495493+0000", 2023-09-23T20:53:45.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1779999999999999e-06 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495500+0000", 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7600000000000002e-06 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495504+0000", 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9720000000000003e-06 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500662+0000", 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051571189999999999 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500891+0000", 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022905000000000001 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 7", 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.501010+0000", 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119268 2023-09-23T20:53:45.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509849+0000", 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088391219999999996 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509859+0000", 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0032e-05 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513758+0000", 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0038989819999999 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513773+0000", 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5116999999999999e-05 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602445+0000", 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0886714449999999 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602553+0000", 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010843199999999999 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663605+0000", 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.061052150999999999 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663684+0000", 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9040000000000002e-05 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186226 8.8 8:109cf42b:::benchmark_data_smithi121_252781_object77051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 338.42589118500001, 2023-09-23T20:53:45.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0872477360000001, 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186226 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529620+0000", 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529629+0000", 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8680000000000005e-06 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529649+0000", 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9874e-05 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529657+0000", 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6680000000000001e-06 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529724+0000", 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7138000000000007e-05 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529737+0000", 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3083e-05 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513799+0000", 2023-09-23T20:53:45.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98406165899999998 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513808+0000", 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8489999999999995e-06 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602617+0000", 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0888096490000001 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602726+0000", 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010896 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602846+0000", 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011963099999999999 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616554+0000", 2023-09-23T20:53:45.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013708016 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616622+0000", 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7794999999999997e-05 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616656+0000", 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4397999999999998e-05 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616718+0000", 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1920000000000003e-05 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616870+0000", 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015218 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.029 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186232 8.8 8:18ce4fcc:::benchmark_data_smithi121_252781_object77057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 338.42259466500002, 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1309886809999998, 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186232 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532918+0000", 2023-09-23T20:53:45.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532923+0000", 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7319999999999998e-06 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532936+0000", 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.293e-05 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532942+0000", 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6980000000000003e-06 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532993+0000", 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1447999999999999e-05 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532999+0000", 2023-09-23T20:53:45.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3389999999999998e-06 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513832+0000", 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98083224099999999 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513840+0000", 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1070000000000005e-06 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607582+0000", 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0937428 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607747+0000", 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016451500000000001 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.608823+0000", 2023-09-23T20:53:45.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001075559 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616564+0000", 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077410530000000003 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616924+0000", 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00036078199999999998 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663714+0000", 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.046789848000000002 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663776+0000", 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2114999999999997e-05 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663908+0000", 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013124799999999999 2023-09-23T20:53:45.037 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186215 8.0 8:00f6bf9b:::benchmark_data_smithi121_252781_object77040:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 337.44123676800001, 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.162298345, 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.038 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186215 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514275+0000", 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514284+0000", 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9570000000000008e-06 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514297+0000", 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2663000000000001e-05 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514305+0000", 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2970000000000004e-06 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514439+0000", 2023-09-23T20:53:45.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013414199999999999 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514449+0000", 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3139999999999993e-06 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649698+0000", 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1352495979999999 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.042 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649809+0000", 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110911 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676465+0000", 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026655729999999999 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676575+0000", 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110305 2023-09-23T20:53:45.043 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186222 8.0 8:0eef9874:::benchmark_data_smithi121_252781_object77047:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 337.44117917199998, 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1614129209999999, 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.044 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186222 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514339+0000", 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1680000000000003e-06 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514348+0000", 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0140000000000004e-06 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514353+0000", 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514474+0000", 2023-09-23T20:53:45.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121105 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514482+0000", 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4039999999999998e-06 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649859+0000", 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.135377012 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649954+0000", 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4914999999999995e-05 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.650033+0000", 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9120000000000001e-05 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663245+0000", 2023-09-23T20:53:45.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013211926000000001 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674215+0000", 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010970126 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675506+0000", 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001290914 2023-09-23T20:53:45.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675559+0000", 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2643999999999999e-05 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675747+0000", 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000188815 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186230 8.0 8:041ac878:::benchmark_data_smithi121_252781_object77055:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 337.44115379499999, 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1616748569999999, 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186230 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.052 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514359+0000", 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514363+0000", 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8360000000000002e-06 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514370+0000", 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4899999999999997e-06 2023-09-23T20:53:45.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514374+0000", 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6750000000000001e-06 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514504+0000", 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012982599999999999 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514512+0000", 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5959999999999998e-06 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654104+0000", 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.139592663 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654250+0000", 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014595400000000001 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654394+0000", 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143442 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663683+0000", 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092891689999999999 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674249+0000", 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010566002 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675814+0000", 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015649539999999999 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675875+0000", 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1224999999999997e-05 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676035+0000", 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015948000000000001 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186231 8.0 8:0bcd4c07:::benchmark_data_smithi121_252781_object77056:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 337.44113315999999, 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.168650363, 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.059 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186231 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514383+0000", 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:53:45.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514388+0000", 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5969999999999999e-06 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514393+0000", 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.16e-06 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514533+0000", 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140841 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514540+0000", 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9890000000000003e-06 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659037+0000", 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.144496156 2023-09-23T20:53:45.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659183+0000", 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014606600000000001 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659334+0000", 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015159700000000001 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663874+0000", 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045394600000000004 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674301+0000", 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010427439 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682815+0000", 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085137149999999998 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682865+0000", 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0510999999999998e-05 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.683031+0000", 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016547400000000001 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209245 8.8 8:1443b4f6:::benchmark_data_smithi121_252781_object100070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.22713935500002, 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.08464988, 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209245 2023-09-23T20:53:45.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728373+0000", 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728379+0000", 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5089999999999999e-06 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728395+0000", 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5710999999999999e-05 2023-09-23T20:53:45.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728402+0000", 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7190000000000004e-06 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728795+0000", 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039319799999999999 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728805+0000", 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6690000000000001e-06 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757335+0000", 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0285302249999999 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757427+0000", 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1818999999999996e-05 2023-09-23T20:53:45.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757509+0000", 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1922000000000006e-05 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768732+0000", 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011223713 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812940+0000", 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.044207659000000003 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812972+0000", 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1633e-05 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813001+0000", 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9300000000000001e-05 2023-09-23T20:53:45.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813024+0000", 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3244999999999999e-05 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209248 8.8 8:136e166f:::benchmark_data_smithi121_252781_object100073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.227102892, 2023-09-23T20:53:45.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.084736245, 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209248 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728410+0000", 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728413+0000", 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.756e-06 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728419+0000", 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3940000000000001e-06 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728423+0000", 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9400000000000004e-06 2023-09-23T20:53:45.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728828+0000", 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00040448300000000002 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728835+0000", 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8469999999999998e-06 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761382+0000", 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0325472419999999 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761496+0000", 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113714 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:53:45.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761583+0000", 2023-09-23T20:53:45.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7459000000000004e-05 2023-09-23T20:53:45.081 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.772112+0000", 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010529221 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812957+0000", 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040844889000000002 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813061+0000", 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000104036 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813109+0000", 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7704999999999998e-05 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813147+0000", 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8046000000000001e-05 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209254 8.8 8:15bdd1a9:::benchmark_data_smithi121_252781_object100079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:53:45.084 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.740455+0000", 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.21505885699997, 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0728612630000001, 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209254 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740455+0000", 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740455+0000", 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740454+0000", 2023-09-23T20:53:45.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740459+0000", 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9130000000000002e-06 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740471+0000", 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2014999999999999e-05 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740476+0000", 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9389999999999996e-06 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740521+0000", 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5689000000000003e-05 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.740535+0000", 2023-09-23T20:53:45.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.381e-05 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.765266+0000", 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247304669999999 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.765359+0000", 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3289999999999996e-05 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.765433+0000", 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4147999999999995e-05 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.775265+0000", 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0098324220000000004 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813215+0000", 2023-09-23T20:53:45.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.037949825999999999 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813268+0000", 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3152999999999997e-05 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813294+0000", 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5103e-05 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813316+0000", 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2408e-05 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209264 8.8 8:1f8be86d:::benchmark_data_smithi121_252781_object100089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.752342+0000", 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.203171736, 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.061116331, 2023-09-23T20:53:45.092 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209264 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752342+0000", 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752342+0000", 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752341+0000", 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752345+0000", 2023-09-23T20:53:45.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9480000000000001e-06 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752355+0000", 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0171e-05 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752360+0000", 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3150000000000002e-06 2023-09-23T20:53:45.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752402+0000", 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2116999999999997e-05 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.752408+0000", 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.981e-06 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768763+0000", 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.016355417 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768864+0000", 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010038400000000001 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768936+0000", 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2478999999999995e-05 2023-09-23T20:53:45.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.781881+0000", 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012944496 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813390+0000", 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031509523999999997 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813413+0000", 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2639000000000001e-05 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813438+0000", 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5664999999999999e-05 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813458+0000", 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9808000000000001e-05 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136323 9.6 9:6c612c1e:::benchmark_data_smithi121_293770_object33435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 74.740847670999997, 2023-09-23T20:53:45.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0778323409999999, 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136323 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214665+0000", 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214669+0000", 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2310000000000001e-06 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214675+0000", 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6250000000000001e-06 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214680+0000", 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.301e-06 2023-09-23T20:53:45.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214759+0000", 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9574999999999996e-05 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214869+0000", 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010993399999999999 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:53:45.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214978+0000", 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000108773 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224530+0000", 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095521080000000001 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224538+0000", 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0970000000000006e-06 2023-09-23T20:53:45.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.283176+0000", 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0586380129999999 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.283513+0000", 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000336769 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292448+0000", 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0089352670000000002 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292498+0000", 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0102000000000002e-05 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.107 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136328 9.6 9:660f4d25:::benchmark_data_smithi121_293770_object33440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214690+0000", 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 74.740823207999995, 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.077717885, 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136328 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:45.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214690+0000", 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214690+0000", 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214690+0000", 2023-09-23T20:53:45.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214693+0000", 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5289999999999998e-06 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214698+0000", 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2079999999999999e-06 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214701+0000", 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3780000000000001e-06 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224558+0000", 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0098567189999999999 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:45.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224562+0000", 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4830000000000001e-06 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.284447+0000", 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0598843870000001 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.284597+0000", 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000150507 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.284745+0000", 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000147625 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.289594+0000", 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048490169999999997 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292306+0000", 2023-09-23T20:53:45.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027123260000000001 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292341+0000", 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.519e-05 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292408+0000", 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6869e-05 2023-09-23T20:53:45.114 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.115 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.115 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.115 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:45.115 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:45.115 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:53:45.115 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2739318211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:45.116 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:53:45.116 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:53:45.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.117 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.117 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.117 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.117 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:53:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:45.117 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2739318211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:45.117 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2739318211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:45.118 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:53:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:53:44] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T20:53:45.118 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:45.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:45.219 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:45.282 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:45.374 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:45.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:45.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:45.478 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:45.513 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:45.554 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:45.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:45.658 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:45.687 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:45 smithi123 ceph-mon[193903]: pgmap v679: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 13271/60162 objects degraded (22.059%); 0 B/s, 62 objects/s recovering 2023-09-23T20:53:46.132 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:53:46.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:45 smithi121 ceph-mon[219709]: pgmap v679: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 13271/60162 objects degraded (22.059%); 0 B/s, 62 objects/s recovering 2023-09-23T20:53:46.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:45 smithi121 ceph-mon[211526]: pgmap v679: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 13271/60162 objects degraded (22.059%); 0 B/s, 62 objects/s recovering 2023-09-23T20:53:46.461 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:46.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:46.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:46.576 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:46.603 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:46.665 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:46.920 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:46.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:47.008 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:53:47.008 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:53:47.008 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T20:53:47.008 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:53:47.012 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13271/60162 objects degraded (22.059%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:47.026 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:47.027 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13271/60162 objects degraded (22.059%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:47.028 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13271/60162 objects degraded (22.059%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:47.129 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:47.249 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:47.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:47.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:47.354 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:47.394 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:47.469 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:47.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:47.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:47.578 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:53:47.632 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:47.739 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:53:47.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:47.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:53:47.770 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:47 smithi121 ceph-mon[211526]: pgmap v680: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 13271/57387 objects degraded (23.125%); 0 B/s, 48 objects/s recovering 2023-09-23T20:53:47.835 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:47.843 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:47.976 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:47 smithi121 ceph-mon[219709]: pgmap v680: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 13271/57387 objects degraded (23.125%); 0 B/s, 48 objects/s recovering 2023-09-23T20:53:48.184 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:48.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:48.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:47 smithi123 ceph-mon[193903]: pgmap v680: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 13271/57387 objects degraded (23.125%); 0 B/s, 48 objects/s recovering 2023-09-23T20:53:48.295 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:48.371 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.448 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:53:48.469 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:48.576 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:48.669 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:48.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2139763786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:48.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:48.789 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2139763786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:48.789 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:48.815 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:53:48.910 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:48.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:48.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:48.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2139763786' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:48.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:53:49.012 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:49.012 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:49.014 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:53:49.090 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:49.625 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:53:49.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:49.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:53:49.732 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:49.771 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:49.838 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:49.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:49.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[211526]: pgmap v681: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 13271/54993 objects degraded (24.132%); 0 B/s, 38 objects/s recovering 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[211526]: failed to encode map e214 with expected crc 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[211526]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[211526]: failed to encode map e214 with expected crc 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[219709]: pgmap v681: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 13271/54993 objects degraded (24.132%); 0 B/s, 38 objects/s recovering 2023-09-23T20:53:49.886 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:49.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[219709]: failed to encode map e214 with expected crc 2023-09-23T20:53:49.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:49.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[219709]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:53:49.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:49 smithi121 ceph-mon[219709]: failed to encode map e214 with expected crc 2023-09-23T20:53:49.948 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:50.008 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:50.158 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:50.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:50.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:50.162 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:49 smithi123 ceph-mon[193903]: pgmap v681: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 13271/54993 objects degraded (24.132%); 0 B/s, 38 objects/s recovering 2023-09-23T20:53:50.162 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:49 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:53:50.162 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:49 smithi123 ceph-mon[193903]: failed to encode map e214 with expected crc 2023-09-23T20:53:50.162 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:49 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:53:50.162 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:49 smithi123 ceph-mon[193903]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:53:50.163 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:49 smithi123 ceph-mon[193903]: failed to encode map e214 with expected crc 2023-09-23T20:53:50.231 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:53:50.263 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:50.373 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:50.537 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:50.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:50.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:50.706 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:50.757 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:50.820 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:50.853 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:50.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:50.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:50.961 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:51.047 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:53:51.060 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:51.071 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:53:51.313 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:51.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:51.420 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:53:51.471 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:51.690 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:53:51.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:51.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:53:51.809 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:53:51.837 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:51.879 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:53:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:51.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:53:51.979 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:51 smithi121 ceph-mon[211526]: pgmap v683: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13271/50835 objects degraded (26.106%); 0 B/s, 60 objects/s recovering 2023-09-23T20:53:51.979 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13271/50835 objects degraded (26.106%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:51.979 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:51 smithi121 ceph-mon[219709]: pgmap v683: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13271/50835 objects degraded (26.106%); 0 B/s, 60 objects/s recovering 2023-09-23T20:53:51.979 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13271/50835 objects degraded (26.106%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:51.982 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:52.007 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:52.132 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:53:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:51 smithi123 ceph-mon[193903]: pgmap v683: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13271/50835 objects degraded (26.106%); 0 B/s, 60 objects/s recovering 2023-09-23T20:53:52.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13271/50835 objects degraded (26.106%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:52.479 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:52.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:52.503 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":683,"stamp":"2023-09-23T20:53:50.742178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1109465016,"num_objects":16945,"num_object_clones":0,"num_object_copies":50835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13271,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16945,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":189151,"num_write_kb":6587086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6157,"num_bytes_recovered":241381416,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139783,"ondisk_log_size":139783,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6933484,"kb_used_data":4205228,"kb_used_omap":21,"kb_used_meta":2727978,"kb_avail":743027732,"statfs":{"total":767960285184,"available":760860397568,"internally_reserved":0,"allocated":4306153472,"data_stored":4300345047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":2793449955},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-388038656,"num_objects":-5921,"num_object_clones":0,"num_object_copies":-17763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5921,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":606,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004370"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":208,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606525+0000","last_change":"2023-09-23T20:53:09.184160+0000","last_active":"2023-09-23T20:53:39.606525+0000","last_peered":"2023-09-23T20:53:39.606525+0000","last_clean":"2023-09-23T20:53:39.606525+0000","last_became_active":"2023-09-23T20:53:09.183837+0000","last_became_peered":"2023-09-23T20:53:09.183837+0000","last_unstale":"2023-09-23T20:53:39.606525+0000","last_undegraded":"2023-09-23T20:53:39.606525+0000","last_fullsized":"2023-09-23T20:53:39.606525+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:03.396066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":168,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526820+0000","last_change":"2023-09-23T20:53:09.205867+0000","last_active":"2023-09-23T20:53:39.526820+0000","last_peered":"2023-09-23T20:53:39.526820+0000","last_clean":"2023-09-23T20:53:39.526820+0000","last_became_active":"2023-09-23T20:53:09.205534+0000","last_became_peered":"2023-09-23T20:53:09.205534+0000","last_unstale":"2023-09-23T20:53:39.526820+0000","last_undegraded":"2023-09-23T20:53:39.526820+0000","last_fullsized":"2023-09-23T20:53:39.526820+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:23.682188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":134,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529457+0000","last_change":"2023-09-23T20:53:09.200700+0000","last_active":"2023-09-23T20:53:39.529457+0000","last_peered":"2023-09-23T20:53:39.529457+0000","last_clean":"2023-09-23T20:53:39.529457+0000","last_became_active":"2023-09-23T20:53:09.200262+0000","last_became_peered":"2023-09-23T20:53:09.200262+0000","last_unstale":"2023-09-23T20:53:39.529457+0000","last_undegraded":"2023-09-23T20:53:39.529457+0000","last_fullsized":"2023-09-23T20:53:39.529457+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:04.656116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":164,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618681+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:53:39.618681+0000","last_peered":"2023-09-23T20:53:39.618681+0000","last_clean":"2023-09-23T20:53:39.618681+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:53:39.618681+0000","last_undegraded":"2023-09-23T20:53:39.618681+0000","last_fullsized":"2023-09-23T20:53:39.618681+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":186,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526465+0000","last_change":"2023-09-23T20:53:09.190070+0000","last_active":"2023-09-23T20:53:39.526465+0000","last_peered":"2023-09-23T20:53:39.526465+0000","last_clean":"2023-09-23T20:53:39.526465+0000","last_became_active":"2023-09-23T20:53:09.189785+0000","last_became_peered":"2023-09-23T20:53:09.189785+0000","last_unstale":"2023-09-23T20:53:39.526465+0000","last_undegraded":"2023-09-23T20:53:39.526465+0000","last_fullsized":"2023-09-23T20:53:39.526465+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.370133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.534436+0000","last_change":"2023-09-23T20:53:09.187404+0000","last_active":"2023-09-23T20:53:39.534436+0000","last_peered":"2023-09-23T20:53:39.534436+0000","last_clean":"2023-09-23T20:53:39.534436+0000","last_became_active":"2023-09-23T20:53:09.187078+0000","last_became_peered":"2023-09-23T20:53:09.187078+0000","last_unstale":"2023-09-23T20:53:39.534436+0000","last_undegraded":"2023-09-23T20:53:39.534436+0000","last_fullsized":"2023-09-23T20:53:39.534436+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:44.084084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":162,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.619194+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:53:39.619194+0000","last_peered":"2023-09-23T20:53:39.619194+0000","last_clean":"2023-09-23T20:53:39.619194+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:53:39.619194+0000","last_undegraded":"2023-09-23T20:53:39.619194+0000","last_fullsized":"2023-09-23T20:53:39.619194+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":186,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526375+0000","last_change":"2023-09-23T20:53:09.194688+0000","last_active":"2023-09-23T20:53:39.526375+0000","last_peered":"2023-09-23T20:53:39.526375+0000","last_clean":"2023-09-23T20:53:39.526375+0000","last_became_active":"2023-09-23T20:53:09.194399+0000","last_became_peered":"2023-09-23T20:53:09.194399+0000","last_unstale":"2023-09-23T20:53:39.526375+0000","last_undegraded":"2023-09-23T20:53:39.526375+0000","last_fullsized":"2023-09-23T20:53:39.526375+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:05.137531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":191,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.545698+0000","last_change":"2023-09-23T20:53:09.187702+0000","last_active":"2023-09-23T20:53:40.545698+0000","last_peered":"2023-09-23T20:53:40.545698+0000","last_clean":"2023-09-23T20:53:40.545698+0000","last_became_active":"2023-09-23T20:53:09.187353+0000","last_became_peered":"2023-09-23T20:53:09.187353+0000","last_unstale":"2023-09-23T20:53:40.545698+0000","last_undegraded":"2023-09-23T20:53:40.545698+0000","last_fullsized":"2023-09-23T20:53:40.545698+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:12.645048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":149,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526546+0000","last_change":"2023-09-23T20:51:06.024444+0000","last_active":"2023-09-23T20:53:39.526546+0000","last_peered":"2023-09-23T20:53:39.526546+0000","last_clean":"2023-09-23T20:53:39.526546+0000","last_became_active":"2023-09-23T20:51:06.023727+0000","last_became_peered":"2023-09-23T20:51:06.023727+0000","last_unstale":"2023-09-23T20:53:39.526546+0000","last_undegraded":"2023-09-23T20:53:39.526546+0000","last_fullsized":"2023-09-23T20:53:39.526546+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:24.860774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.536195+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:53:40.536195+0000","last_peered":"2023-09-23T20:53:40.536195+0000","last_clean":"2023-09-23T20:53:40.536195+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:53:40.536195+0000","last_undegraded":"2023-09-23T20:53:40.536195+0000","last_fullsized":"2023-09-23T20:53:40.536195+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":140,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529749+0000","last_change":"2023-09-23T20:53:09.188835+0000","last_active":"2023-09-23T20:53:39.529749+0000","last_peered":"2023-09-23T20:53:39.529749+0000","last_clean":"2023-09-23T20:53:39.529749+0000","last_became_active":"2023-09-23T20:53:09.188246+0000","last_became_peered":"2023-09-23T20:53:09.188246+0000","last_unstale":"2023-09-23T20:53:39.529749+0000","last_undegraded":"2023-09-23T20:53:39.529749+0000","last_fullsized":"2023-09-23T20:53:39.529749+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:27.239470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":218,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.607484+0000","last_change":"2023-09-23T20:51:05.966238+0000","last_active":"2023-09-23T20:53:39.607484+0000","last_peered":"2023-09-23T20:53:39.607484+0000","last_clean":"2023-09-23T20:53:39.607484+0000","last_became_active":"2023-09-23T20:51:05.959313+0000","last_became_peered":"2023-09-23T20:51:05.959313+0000","last_unstale":"2023-09-23T20:53:39.607484+0000","last_undegraded":"2023-09-23T20:53:39.607484+0000","last_fullsized":"2023-09-23T20:53:39.607484+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:02.228179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":209,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606909+0000","last_change":"2023-09-23T20:53:09.191934+0000","last_active":"2023-09-23T20:53:39.606909+0000","last_peered":"2023-09-23T20:53:39.606909+0000","last_clean":"2023-09-23T20:53:39.606909+0000","last_became_active":"2023-09-23T20:53:09.191583+0000","last_became_peered":"2023-09-23T20:53:09.191583+0000","last_unstale":"2023-09-23T20:53:39.606909+0000","last_undegraded":"2023-09-23T20:53:39.606909+0000","last_fullsized":"2023-09-23T20:53:39.606909+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:45.729672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":176,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.535749+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:53:40.535749+0000","last_peered":"2023-09-23T20:53:40.535749+0000","last_clean":"2023-09-23T20:53:40.535749+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:53:40.535749+0000","last_undegraded":"2023-09-23T20:53:40.535749+0000","last_fullsized":"2023-09-23T20:53:40.535749+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":168,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.527168+0000","last_change":"2023-09-23T20:53:09.219437+0000","last_active":"2023-09-23T20:53:39.527168+0000","last_peered":"2023-09-23T20:53:39.527168+0000","last_clean":"2023-09-23T20:53:39.527168+0000","last_became_active":"2023-09-23T20:53:09.219177+0000","last_became_peered":"2023-09-23T20:53:09.219177+0000","last_unstale":"2023-09-23T20:53:39.527168+0000","last_undegraded":"2023-09-23T20:53:39.527168+0000","last_fullsized":"2023-09-23T20:53:39.527168+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:22.008304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":162,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618443+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:53:39.618443+0000","last_peered":"2023-09-23T20:53:39.618443+0000","last_clean":"2023-09-23T20:53:39.618443+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:53:39.618443+0000","last_undegraded":"2023-09-23T20:53:39.618443+0000","last_fullsized":"2023-09-23T20:53:39.618443+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.545437+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:53:40.545437+0000","last_peered":"2023-09-23T20:53:40.545437+0000","last_clean":"2023-09-23T20:53:40.545437+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:53:40.545437+0000","last_undegraded":"2023-09-23T20:53:40.545437+0000","last_fullsized":"2023-09-23T20:53:40.545437+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.535052+0000","last_change":"2023-09-23T20:51:05.958217+0000","last_active":"2023-09-23T20:53:40.535052+0000","last_peered":"2023-09-23T20:53:40.535052+0000","last_clean":"2023-09-23T20:53:40.535052+0000","last_became_active":"2023-09-23T20:51:05.957874+0000","last_became_peered":"2023-09-23T20:51:05.957874+0000","last_unstale":"2023-09-23T20:53:40.535052+0000","last_undegraded":"2023-09-23T20:53:40.535052+0000","last_fullsized":"2023-09-23T20:53:40.535052+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:13.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618815+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:53:39.618815+0000","last_peered":"2023-09-23T20:53:39.618815+0000","last_clean":"2023-09-23T20:53:39.618815+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:53:39.618815+0000","last_undegraded":"2023-09-23T20:53:39.618815+0000","last_fullsized":"2023-09-23T20:53:39.618815+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":55,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.534430+0000","last_change":"2023-09-23T20:51:06.038448+0000","last_active":"2023-09-23T20:53:39.534430+0000","last_peered":"2023-09-23T20:53:39.534430+0000","last_clean":"2023-09-23T20:53:39.534430+0000","last_became_active":"2023-09-23T20:51:06.005965+0000","last_became_peered":"2023-09-23T20:51:06.005965+0000","last_unstale":"2023-09-23T20:53:39.534430+0000","last_undegraded":"2023-09-23T20:53:39.534430+0000","last_fullsized":"2023-09-23T20:53:39.534430+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:13.397771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":168,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526652+0000","last_change":"2023-09-23T20:53:09.232052+0000","last_active":"2023-09-23T20:53:39.526652+0000","last_peered":"2023-09-23T20:53:39.526652+0000","last_clean":"2023-09-23T20:53:39.526652+0000","last_became_active":"2023-09-23T20:53:09.231748+0000","last_became_peered":"2023-09-23T20:53:09.231748+0000","last_unstale":"2023-09-23T20:53:39.526652+0000","last_undegraded":"2023-09-23T20:53:39.526652+0000","last_fullsized":"2023-09-23T20:53:39.526652+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:04.683102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"213'11572","reported_seq":11631,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:47.700220+0000","last_change":"2023-09-23T20:50:08.077384+0000","last_active":"2023-09-23T20:53:47.700220+0000","last_peered":"2023-09-23T20:53:47.700220+0000","last_clean":"2023-09-23T20:53:47.700220+0000","last_became_active":"2023-09-23T20:50:08.077095+0000","last_became_peered":"2023-09-23T20:50:08.077095+0000","last_unstale":"2023-09-23T20:53:47.700220+0000","last_undegraded":"2023-09-23T20:53:47.700220+0000","last_fullsized":"2023-09-23T20:53:47.700220+0000","mapping_epoch":164,"log_start":"168'1500","ondisk_log_start":"168'1500","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10072,"log_dups_size":0,"ondisk_log_size":10072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:34.442024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68288512,"num_objects":1042,"num_object_clones":0,"num_object_copies":3126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11572,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":175,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.539298+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:53:40.539298+0000","last_peered":"2023-09-23T20:53:40.539298+0000","last_clean":"2023-09-23T20:53:40.539298+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:53:40.539298+0000","last_undegraded":"2023-09-23T20:53:40.539298+0000","last_fullsized":"2023-09-23T20:53:40.539298+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":176,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.539568+0000","last_change":"2023-09-23T20:51:05.962659+0000","last_active":"2023-09-23T20:53:40.539568+0000","last_peered":"2023-09-23T20:53:40.539568+0000","last_clean":"2023-09-23T20:53:40.539568+0000","last_became_active":"2023-09-23T20:51:05.962048+0000","last_became_peered":"2023-09-23T20:51:05.962048+0000","last_unstale":"2023-09-23T20:53:40.539568+0000","last_undegraded":"2023-09-23T20:53:40.539568+0000","last_fullsized":"2023-09-23T20:53:40.539568+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:03.530084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"213'11875","reported_seq":11946,"reported_epoch":213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:46.009399+0000","last_change":"2023-09-23T20:53:10.348437+0000","last_active":"2023-09-23T20:53:46.009399+0000","last_peered":"2023-09-23T20:53:46.009399+0000","last_clean":"2023-09-23T20:52:30.157076+0000","last_became_active":"2023-09-23T20:53:10.306549+0000","last_became_peered":"2023-09-23T20:53:10.306549+0000","last_unstale":"2023-09-23T20:53:46.009399+0000","last_undegraded":"2023-09-23T20:53:10.193751+0000","last_fullsized":"2023-09-23T20:53:10.176070+0000","mapping_epoch":207,"log_start":"176'4125","ondisk_log_start":"176'4125","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7750,"log_dups_size":0,"ondisk_log_size":7750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:52.651633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74121216,"num_objects":1131,"num_object_clones":0,"num_object_copies":3393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11875,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1131}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":193,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.535888+0000","last_change":"2023-09-23T20:53:09.185607+0000","last_active":"2023-09-23T20:53:40.535888+0000","last_peered":"2023-09-23T20:53:40.535888+0000","last_clean":"2023-09-23T20:53:40.535888+0000","last_became_active":"2023-09-23T20:53:09.185271+0000","last_became_peered":"2023-09-23T20:53:09.185271+0000","last_unstale":"2023-09-23T20:53:40.535888+0000","last_undegraded":"2023-09-23T20:53:40.535888+0000","last_fullsized":"2023-09-23T20:53:40.535888+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:57.917928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":206,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606382+0000","last_change":"2023-09-23T20:53:09.191763+0000","last_active":"2023-09-23T20:53:39.606382+0000","last_peered":"2023-09-23T20:53:39.606382+0000","last_clean":"2023-09-23T20:53:39.606382+0000","last_became_active":"2023-09-23T20:53:09.191442+0000","last_became_peered":"2023-09-23T20:53:09.191442+0000","last_unstale":"2023-09-23T20:53:39.606382+0000","last_undegraded":"2023-09-23T20:53:39.606382+0000","last_fullsized":"2023-09-23T20:53:39.606382+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:02.859784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"214'11739","reported_seq":11797,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.895562+0000","last_change":"2023-09-23T20:50:08.078801+0000","last_active":"2023-09-23T20:53:48.895562+0000","last_peered":"2023-09-23T20:53:48.895562+0000","last_clean":"2023-09-23T20:53:48.895562+0000","last_became_active":"2023-09-23T20:50:08.078386+0000","last_became_peered":"2023-09-23T20:50:08.078386+0000","last_unstale":"2023-09-23T20:53:48.895562+0000","last_undegraded":"2023-09-23T20:53:48.895562+0000","last_fullsized":"2023-09-23T20:53:48.895562+0000","mapping_epoch":164,"log_start":"168'1700","ondisk_log_start":"168'1700","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:47.134724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863680,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11739,"num_write_kb":407808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":186,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.536035+0000","last_change":"2023-09-23T20:51:05.967804+0000","last_active":"2023-09-23T20:53:40.536035+0000","last_peered":"2023-09-23T20:53:40.536035+0000","last_clean":"2023-09-23T20:53:40.536035+0000","last_became_active":"2023-09-23T20:51:05.959914+0000","last_became_peered":"2023-09-23T20:51:05.959914+0000","last_unstale":"2023-09-23T20:53:40.536035+0000","last_undegraded":"2023-09-23T20:53:40.536035+0000","last_fullsized":"2023-09-23T20:53:40.536035+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:03.210361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":45,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.534368+0000","last_change":"2023-09-23T20:51:05.957335+0000","last_active":"2023-09-23T20:53:39.534368+0000","last_peered":"2023-09-23T20:53:39.534368+0000","last_clean":"2023-09-23T20:53:39.534368+0000","last_became_active":"2023-09-23T20:51:05.956858+0000","last_became_peered":"2023-09-23T20:51:05.956858+0000","last_unstale":"2023-09-23T20:53:39.534368+0000","last_undegraded":"2023-09-23T20:53:39.534368+0000","last_fullsized":"2023-09-23T20:53:39.534368+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:31.085919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"214'11648","reported_seq":11728,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:48.843249+0000","last_change":"2023-09-23T20:53:10.403698+0000","last_active":"2023-09-23T20:53:48.843249+0000","last_peered":"2023-09-23T20:53:48.843249+0000","last_clean":"2023-09-23T20:52:30.196736+0000","last_became_active":"2023-09-23T20:53:10.355510+0000","last_became_peered":"2023-09-23T20:53:10.355510+0000","last_unstale":"2023-09-23T20:53:48.843249+0000","last_undegraded":"2023-09-23T20:53:10.196118+0000","last_fullsized":"2023-09-23T20:53:10.178982+0000","mapping_epoch":207,"log_start":"168'1600","ondisk_log_start":"168'1600","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10048,"log_dups_size":0,"ondisk_log_size":10048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:55.710095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66322432,"num_objects":1012,"num_object_clones":0,"num_object_copies":3036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1832,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11648,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":1012}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":194,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.542457+0000","last_change":"2023-09-23T20:51:05.995979+0000","last_active":"2023-09-23T20:53:40.542457+0000","last_peered":"2023-09-23T20:53:40.542457+0000","last_clean":"2023-09-23T20:53:40.542457+0000","last_became_active":"2023-09-23T20:51:05.995591+0000","last_became_peered":"2023-09-23T20:51:05.995591+0000","last_unstale":"2023-09-23T20:53:40.542457+0000","last_undegraded":"2023-09-23T20:53:40.542457+0000","last_fullsized":"2023-09-23T20:53:40.542457+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:35.414294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":178,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618653+0000","last_change":"2023-09-23T20:53:09.185147+0000","last_active":"2023-09-23T20:53:39.618653+0000","last_peered":"2023-09-23T20:53:39.618653+0000","last_clean":"2023-09-23T20:53:39.618653+0000","last_became_active":"2023-09-23T20:53:09.184855+0000","last_became_peered":"2023-09-23T20:53:09.184855+0000","last_unstale":"2023-09-23T20:53:39.618653+0000","last_undegraded":"2023-09-23T20:53:39.618653+0000","last_fullsized":"2023-09-23T20:53:39.618653+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:45.699896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"213'11554","reported_seq":14477,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:45.907328+0000","last_change":"2023-09-23T20:51:15.811325+0000","last_active":"2023-09-23T20:53:45.907328+0000","last_peered":"2023-09-23T20:53:45.907328+0000","last_clean":"2023-09-23T20:53:45.907328+0000","last_became_active":"2023-09-23T20:51:15.805454+0000","last_became_peered":"2023-09-23T20:51:15.805454+0000","last_unstale":"2023-09-23T20:53:45.907328+0000","last_undegraded":"2023-09-23T20:53:45.907328+0000","last_fullsized":"2023-09-23T20:53:45.907328+0000","mapping_epoch":180,"log_start":"176'3800","ondisk_log_start":"176'3800","created":164,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:13.254228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75497472,"num_objects":1152,"num_object_clones":0,"num_object_copies":3456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11554,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1418,"num_bytes_recovered":92930048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":176,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526430+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:53:39.526430+0000","last_peered":"2023-09-23T20:53:39.526430+0000","last_clean":"2023-09-23T20:53:39.526430+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:53:39.526430+0000","last_undegraded":"2023-09-23T20:53:39.526430+0000","last_fullsized":"2023-09-23T20:53:39.526430+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.607436+0000","last_change":"2023-09-23T20:51:06.001664+0000","last_active":"2023-09-23T20:53:39.607436+0000","last_peered":"2023-09-23T20:53:39.607436+0000","last_clean":"2023-09-23T20:53:39.607436+0000","last_became_active":"2023-09-23T20:51:06.001352+0000","last_became_peered":"2023-09-23T20:51:06.001352+0000","last_unstale":"2023-09-23T20:53:39.607436+0000","last_undegraded":"2023-09-23T20:53:39.607436+0000","last_fullsized":"2023-09-23T20:53:39.607436+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.688374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"213'11777","reported_seq":17251,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:47.320533+0000","last_change":"2023-09-23T20:53:39.589168+0000","last_active":"2023-09-23T20:53:47.320533+0000","last_peered":"2023-09-23T20:53:47.320533+0000","last_clean":"2023-09-23T20:53:47.320533+0000","last_became_active":"2023-09-23T20:53:39.535053+0000","last_became_peered":"2023-09-23T20:53:39.535053+0000","last_unstale":"2023-09-23T20:53:47.320533+0000","last_undegraded":"2023-09-23T20:53:47.320533+0000","last_fullsized":"2023-09-23T20:53:47.320533+0000","mapping_epoch":211,"log_start":"178'4459","ondisk_log_start":"178'4459","created":164,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7318,"log_dups_size":0,"ondisk_log_size":7318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.122744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72810496,"num_objects":1111,"num_object_clones":0,"num_object_copies":3333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11777,"num_write_kb":412416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2684,"num_bytes_recovered":54329344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":62,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.534448+0000","last_change":"2023-09-23T20:53:09.236307+0000","last_active":"2023-09-23T20:53:39.534448+0000","last_peered":"2023-09-23T20:53:39.534448+0000","last_clean":"2023-09-23T20:53:39.534448+0000","last_became_active":"2023-09-23T20:53:09.235961+0000","last_became_peered":"2023-09-23T20:53:09.235961+0000","last_unstale":"2023-09-23T20:53:39.534448+0000","last_undegraded":"2023-09-23T20:53:39.534448+0000","last_fullsized":"2023-09-23T20:53:39.534448+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:54.763339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":206,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606149+0000","last_change":"2023-09-23T20:53:09.184166+0000","last_active":"2023-09-23T20:53:39.606149+0000","last_peered":"2023-09-23T20:53:39.606149+0000","last_clean":"2023-09-23T20:53:39.606149+0000","last_became_active":"2023-09-23T20:53:09.183862+0000","last_became_peered":"2023-09-23T20:53:09.183862+0000","last_unstale":"2023-09-23T20:53:39.606149+0000","last_undegraded":"2023-09-23T20:53:39.606149+0000","last_fullsized":"2023-09-23T20:53:39.606149+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:40.482320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"214'11844","reported_seq":11926,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:48.855697+0000","last_change":"2023-09-23T20:53:10.383476+0000","last_active":"2023-09-23T20:53:48.855697+0000","last_peered":"2023-09-23T20:53:48.855697+0000","last_clean":"2023-09-23T20:52:30.214684+0000","last_became_active":"2023-09-23T20:53:10.343781+0000","last_became_peered":"2023-09-23T20:53:10.343781+0000","last_unstale":"2023-09-23T20:53:48.855697+0000","last_undegraded":"2023-09-23T20:53:10.193078+0000","last_fullsized":"2023-09-23T20:53:10.177310+0000","mapping_epoch":207,"log_start":"176'4100","ondisk_log_start":"176'4100","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:11.933025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65798144,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11844,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1004}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606169+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:53:39.606169+0000","last_peered":"2023-09-23T20:53:39.606169+0000","last_clean":"2023-09-23T20:53:39.606169+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:53:39.606169+0000","last_undegraded":"2023-09-23T20:53:39.606169+0000","last_fullsized":"2023-09-23T20:53:39.606169+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":209,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.609185+0000","last_change":"2023-09-23T20:53:09.192221+0000","last_active":"2023-09-23T20:53:39.609185+0000","last_peered":"2023-09-23T20:53:39.609185+0000","last_clean":"2023-09-23T20:53:39.609185+0000","last_became_active":"2023-09-23T20:53:09.191599+0000","last_became_peered":"2023-09-23T20:53:09.191599+0000","last_unstale":"2023-09-23T20:53:39.609185+0000","last_undegraded":"2023-09-23T20:53:39.609185+0000","last_fullsized":"2023-09-23T20:53:39.609185+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:54.725504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"213'11818","reported_seq":11892,"reported_epoch":213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:45.990651+0000","last_change":"2023-09-23T20:53:10.344228+0000","last_active":"2023-09-23T20:53:45.990651+0000","last_peered":"2023-09-23T20:53:45.990651+0000","last_clean":"2023-09-23T20:52:30.196344+0000","last_became_active":"2023-09-23T20:53:10.314804+0000","last_became_peered":"2023-09-23T20:53:10.314804+0000","last_unstale":"2023-09-23T20:53:45.990651+0000","last_undegraded":"2023-09-23T20:53:10.193272+0000","last_fullsized":"2023-09-23T20:53:10.176077+0000","mapping_epoch":207,"log_start":"176'4100","ondisk_log_start":"176'4100","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7718,"log_dups_size":0,"ondisk_log_size":7718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:48.249155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77070336,"num_objects":1176,"num_object_clones":0,"num_object_copies":3528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11818,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1176}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":169,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526605+0000","last_change":"2023-09-23T20:53:09.188782+0000","last_active":"2023-09-23T20:53:39.526605+0000","last_peered":"2023-09-23T20:53:39.526605+0000","last_clean":"2023-09-23T20:53:39.526605+0000","last_became_active":"2023-09-23T20:53:09.188563+0000","last_became_peered":"2023-09-23T20:53:09.188563+0000","last_unstale":"2023-09-23T20:53:39.526605+0000","last_undegraded":"2023-09-23T20:53:39.526605+0000","last_fullsized":"2023-09-23T20:53:39.526605+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:33.898108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.531807+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:53:39.531807+0000","last_peered":"2023-09-23T20:53:39.531807+0000","last_clean":"2023-09-23T20:53:39.531807+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:53:39.531807+0000","last_undegraded":"2023-09-23T20:53:39.531807+0000","last_fullsized":"2023-09-23T20:53:39.531807+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"214'12066","reported_seq":12142,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:48.889247+0000","last_change":"2023-09-23T20:53:10.403424+0000","last_active":"2023-09-23T20:53:48.889247+0000","last_peered":"2023-09-23T20:53:48.889247+0000","last_clean":"2023-09-23T20:52:30.157599+0000","last_became_active":"2023-09-23T20:53:10.359278+0000","last_became_peered":"2023-09-23T20:53:10.359278+0000","last_unstale":"2023-09-23T20:53:48.889247+0000","last_undegraded":"2023-09-23T20:53:10.196743+0000","last_fullsized":"2023-09-23T20:53:10.179018+0000","mapping_epoch":207,"log_start":"169'2000","ondisk_log_start":"169'2000","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:52.816379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64487424,"num_objects":984,"num_object_clones":0,"num_object_copies":2952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1922,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12066,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":984}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":175,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529751+0000","last_change":"2023-09-23T20:53:09.191673+0000","last_active":"2023-09-23T20:53:39.529751+0000","last_peered":"2023-09-23T20:53:39.529751+0000","last_clean":"2023-09-23T20:53:39.529751+0000","last_became_active":"2023-09-23T20:53:09.191277+0000","last_became_peered":"2023-09-23T20:53:09.191277+0000","last_unstale":"2023-09-23T20:53:39.529751+0000","last_undegraded":"2023-09-23T20:53:39.529751+0000","last_fullsized":"2023-09-23T20:53:39.529751+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:21.050943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":191,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.545545+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:53:40.545545+0000","last_peered":"2023-09-23T20:53:40.545545+0000","last_clean":"2023-09-23T20:53:40.545545+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:53:40.545545+0000","last_undegraded":"2023-09-23T20:53:40.545545+0000","last_fullsized":"2023-09-23T20:53:40.545545+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":128,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529785+0000","last_change":"2023-09-23T20:53:09.196041+0000","last_active":"2023-09-23T20:53:39.529785+0000","last_peered":"2023-09-23T20:53:39.529785+0000","last_clean":"2023-09-23T20:53:39.529785+0000","last_became_active":"2023-09-23T20:53:09.191615+0000","last_became_peered":"2023-09-23T20:53:09.191615+0000","last_unstale":"2023-09-23T20:53:39.529785+0000","last_undegraded":"2023-09-23T20:53:39.529785+0000","last_fullsized":"2023-09-23T20:53:39.529785+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:25.505589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"213'12140","reported_seq":12199,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:47.773346+0000","last_change":"2023-09-23T20:50:08.076706+0000","last_active":"2023-09-23T20:53:47.773346+0000","last_peered":"2023-09-23T20:53:47.773346+0000","last_clean":"2023-09-23T20:53:47.773346+0000","last_became_active":"2023-09-23T20:50:08.076360+0000","last_became_peered":"2023-09-23T20:50:08.076360+0000","last_unstale":"2023-09-23T20:53:47.773346+0000","last_undegraded":"2023-09-23T20:53:47.773346+0000","last_fullsized":"2023-09-23T20:53:47.773346+0000","mapping_epoch":164,"log_start":"169'2100","ondisk_log_start":"169'2100","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.596097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69206016,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12140,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":215,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.541813+0000","last_change":"2023-09-23T20:53:09.184802+0000","last_active":"2023-09-23T20:53:40.541813+0000","last_peered":"2023-09-23T20:53:40.541813+0000","last_clean":"2023-09-23T20:53:40.541813+0000","last_became_active":"2023-09-23T20:53:09.184501+0000","last_became_peered":"2023-09-23T20:53:09.184501+0000","last_unstale":"2023-09-23T20:53:40.541813+0000","last_undegraded":"2023-09-23T20:53:40.541813+0000","last_fullsized":"2023-09-23T20:53:40.541813+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:56.145028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":193,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606759+0000","last_change":"2023-09-23T20:51:06.033002+0000","last_active":"2023-09-23T20:53:39.606759+0000","last_peered":"2023-09-23T20:53:39.606759+0000","last_clean":"2023-09-23T20:53:39.606759+0000","last_became_active":"2023-09-23T20:51:06.032657+0000","last_became_peered":"2023-09-23T20:51:06.032657+0000","last_unstale":"2023-09-23T20:53:39.606759+0000","last_undegraded":"2023-09-23T20:53:39.606759+0000","last_fullsized":"2023-09-23T20:53:39.606759+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:07.293307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":183,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.542338+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:53:40.542338+0000","last_peered":"2023-09-23T20:53:40.542338+0000","last_clean":"2023-09-23T20:53:40.542338+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:53:40.542338+0000","last_undegraded":"2023-09-23T20:53:40.542338+0000","last_fullsized":"2023-09-23T20:53:40.542338+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"214'11808","reported_seq":11883,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:48.887656+0000","last_change":"2023-09-23T20:53:10.375283+0000","last_active":"2023-09-23T20:53:48.887656+0000","last_peered":"2023-09-23T20:53:48.887656+0000","last_clean":"2023-09-23T20:52:30.211455+0000","last_became_active":"2023-09-23T20:53:10.333927+0000","last_became_peered":"2023-09-23T20:53:10.333927+0000","last_unstale":"2023-09-23T20:53:48.887656+0000","last_undegraded":"2023-09-23T20:53:10.194479+0000","last_fullsized":"2023-09-23T20:53:10.178986+0000","mapping_epoch":207,"log_start":"168'1800","ondisk_log_start":"168'1800","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:47:13.773667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1872,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11808,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1002}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":171,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.531430+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:53:39.531430+0000","last_peered":"2023-09-23T20:53:39.531430+0000","last_clean":"2023-09-23T20:53:39.531430+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:53:39.531430+0000","last_undegraded":"2023-09-23T20:53:39.531430+0000","last_fullsized":"2023-09-23T20:53:39.531430+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529540+0000","last_change":"2023-09-23T20:53:09.187057+0000","last_active":"2023-09-23T20:53:39.529540+0000","last_peered":"2023-09-23T20:53:39.529540+0000","last_clean":"2023-09-23T20:53:39.529540+0000","last_became_active":"2023-09-23T20:53:09.186597+0000","last_became_peered":"2023-09-23T20:53:09.186597+0000","last_unstale":"2023-09-23T20:53:39.529540+0000","last_undegraded":"2023-09-23T20:53:39.529540+0000","last_fullsized":"2023-09-23T20:53:39.529540+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.376692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":170,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529764+0000","last_change":"2023-09-23T20:53:09.193262+0000","last_active":"2023-09-23T20:53:39.529764+0000","last_peered":"2023-09-23T20:53:39.529764+0000","last_clean":"2023-09-23T20:53:39.529764+0000","last_became_active":"2023-09-23T20:53:09.192941+0000","last_became_peered":"2023-09-23T20:53:09.192941+0000","last_unstale":"2023-09-23T20:53:39.529764+0000","last_undegraded":"2023-09-23T20:53:39.529764+0000","last_fullsized":"2023-09-23T20:53:39.529764+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:06.470020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"213'11541","reported_seq":11600,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:46.018516+0000","last_change":"2023-09-23T20:50:08.078559+0000","last_active":"2023-09-23T20:53:46.018516+0000","last_peered":"2023-09-23T20:53:46.018516+0000","last_clean":"2023-09-23T20:53:46.018516+0000","last_became_active":"2023-09-23T20:50:08.078294+0000","last_became_peered":"2023-09-23T20:50:08.078294+0000","last_unstale":"2023-09-23T20:53:46.018516+0000","last_undegraded":"2023-09-23T20:53:46.018516+0000","last_fullsized":"2023-09-23T20:53:46.018516+0000","mapping_epoch":164,"log_start":"176'3800","ondisk_log_start":"176'3800","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7741,"log_dups_size":0,"ondisk_log_size":7741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:11.281483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69926912,"num_objects":1067,"num_object_clones":0,"num_object_copies":3201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11541,"num_write_kb":403456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.541283+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:53:40.541283+0000","last_peered":"2023-09-23T20:53:40.541283+0000","last_clean":"2023-09-23T20:53:40.541283+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:53:40.541283+0000","last_undegraded":"2023-09-23T20:53:40.541283+0000","last_fullsized":"2023-09-23T20:53:40.541283+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":606,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.534010+0000","last_change":"2023-09-23T20:51:30.823817+0000","last_active":"2023-09-23T20:53:40.534010+0000","last_peered":"2023-09-23T20:53:40.534010+0000","last_clean":"2023-09-23T20:53:40.534010+0000","last_became_active":"2023-09-23T20:51:30.823372+0000","last_became_peered":"2023-09-23T20:51:30.823372+0000","last_unstale":"2023-09-23T20:53:40.534010+0000","last_undegraded":"2023-09-23T20:53:40.534010+0000","last_fullsized":"2023-09-23T20:53:40.534010+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:09.432277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":148,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526812+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:53:39.526812+0000","last_peered":"2023-09-23T20:53:39.526812+0000","last_clean":"2023-09-23T20:53:39.526812+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:53:39.526812+0000","last_undegraded":"2023-09-23T20:53:39.526812+0000","last_fullsized":"2023-09-23T20:53:39.526812+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":45,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.534394+0000","last_change":"2023-09-23T20:51:05.957381+0000","last_active":"2023-09-23T20:53:39.534394+0000","last_peered":"2023-09-23T20:53:39.534394+0000","last_clean":"2023-09-23T20:53:39.534394+0000","last_became_active":"2023-09-23T20:51:05.956991+0000","last_became_peered":"2023-09-23T20:51:05.956991+0000","last_unstale":"2023-09-23T20:53:39.534394+0000","last_undegraded":"2023-09-23T20:53:39.534394+0000","last_fullsized":"2023-09-23T20:53:39.534394+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:20.613174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"213'12054","reported_seq":14922,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:47.831355+0000","last_change":"2023-09-23T20:51:23.778886+0000","last_active":"2023-09-23T20:53:47.831355+0000","last_peered":"2023-09-23T20:53:47.831355+0000","last_clean":"2023-09-23T20:53:47.831355+0000","last_became_active":"2023-09-23T20:51:23.771838+0000","last_became_peered":"2023-09-23T20:51:23.771838+0000","last_unstale":"2023-09-23T20:53:47.831355+0000","last_undegraded":"2023-09-23T20:53:47.831355+0000","last_fullsized":"2023-09-23T20:53:47.831355+0000","mapping_epoch":183,"log_start":"169'2000","ondisk_log_start":"169'2000","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:54.532463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71696384,"num_objects":1094,"num_object_clones":0,"num_object_copies":3282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12054,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1391,"num_bytes_recovered":91160576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":212,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.533945+0000","last_change":"2023-09-23T20:53:09.183474+0000","last_active":"2023-09-23T20:53:40.533945+0000","last_peered":"2023-09-23T20:53:40.533945+0000","last_clean":"2023-09-23T20:53:40.533945+0000","last_became_active":"2023-09-23T20:53:09.183165+0000","last_became_peered":"2023-09-23T20:53:09.183165+0000","last_unstale":"2023-09-23T20:53:40.533945+0000","last_undegraded":"2023-09-23T20:53:40.533945+0000","last_fullsized":"2023-09-23T20:53:40.533945+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:11.886478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.615732+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:53:39.615732+0000","last_peered":"2023-09-23T20:53:39.615732+0000","last_clean":"2023-09-23T20:53:39.615732+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:53:39.615732+0000","last_undegraded":"2023-09-23T20:53:39.615732+0000","last_fullsized":"2023-09-23T20:53:39.615732+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":193,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618750+0000","last_change":"2023-09-23T20:53:09.181989+0000","last_active":"2023-09-23T20:53:39.618750+0000","last_peered":"2023-09-23T20:53:39.618750+0000","last_clean":"2023-09-23T20:53:39.618750+0000","last_became_active":"2023-09-23T20:53:09.181672+0000","last_became_peered":"2023-09-23T20:53:09.181672+0000","last_unstale":"2023-09-23T20:53:39.618750+0000","last_undegraded":"2023-09-23T20:53:39.618750+0000","last_fullsized":"2023-09-23T20:53:39.618750+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:39.497624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"214'11991","reported_seq":12071,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:53:48.906206+0000","last_change":"2023-09-23T20:53:10.407911+0000","last_active":"2023-09-23T20:53:48.906206+0000","last_peered":"2023-09-23T20:53:48.906206+0000","last_clean":"2023-09-23T20:52:30.212933+0000","last_became_active":"2023-09-23T20:53:10.358990+0000","last_became_peered":"2023-09-23T20:53:10.358990+0000","last_unstale":"2023-09-23T20:53:48.906206+0000","last_undegraded":"2023-09-23T20:53:10.195827+0000","last_fullsized":"2023-09-23T20:53:10.177309+0000","mapping_epoch":207,"log_start":"176'4200","ondisk_log_start":"176'4200","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:09.500397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63242240,"num_objects":965,"num_object_clones":0,"num_object_copies":2895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1955,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11991,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":965}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.536109+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:53:40.536109+0000","last_peered":"2023-09-23T20:53:40.536109+0000","last_clean":"2023-09-23T20:53:40.536109+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:53:40.536109+0000","last_undegraded":"2023-09-23T20:53:40.536109+0000","last_fullsized":"2023-09-23T20:53:40.536109+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":115,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529873+0000","last_change":"2023-09-23T20:53:09.199776+0000","last_active":"2023-09-23T20:53:39.529873+0000","last_peered":"2023-09-23T20:53:39.529873+0000","last_clean":"2023-09-23T20:53:39.529873+0000","last_became_active":"2023-09-23T20:53:09.199309+0000","last_became_peered":"2023-09-23T20:53:09.199309+0000","last_unstale":"2023-09-23T20:53:39.529873+0000","last_undegraded":"2023-09-23T20:53:39.529873+0000","last_fullsized":"2023-09-23T20:53:39.529873+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:32.904643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":149,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618511+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:53:39.618511+0000","last_peered":"2023-09-23T20:53:39.618511+0000","last_clean":"2023-09-23T20:53:39.618511+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:53:39.618511+0000","last_undegraded":"2023-09-23T20:53:39.618511+0000","last_fullsized":"2023-09-23T20:53:39.618511+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"214'11700","reported_seq":13035,"reported_epoch":214,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:53:48.926297+0000","last_change":"2023-09-23T20:53:38.485724+0000","last_active":"2023-09-23T20:53:48.926297+0000","last_peered":"2023-09-23T20:53:48.926297+0000","last_clean":"2023-09-23T20:52:30.141642+0000","last_became_active":"2023-09-23T20:53:10.309106+0000","last_became_peered":"2023-09-23T20:53:10.309106+0000","last_unstale":"2023-09-23T20:53:48.926297+0000","last_undegraded":"2023-09-23T20:53:48.926297+0000","last_fullsized":"2023-09-23T20:53:10.177303+0000","mapping_epoch":207,"log_start":"176'4000","ondisk_log_start":"176'4000","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7700,"log_dups_size":0,"ondisk_log_size":7700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:03.627658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863704,"num_objects":1006,"num_object_clones":0,"num_object_copies":3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1006,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11700,"num_write_kb":406529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":628,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1006}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":176,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526702+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:53:39.526702+0000","last_peered":"2023-09-23T20:53:39.526702+0000","last_clean":"2023-09-23T20:53:39.526702+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:53:39.526702+0000","last_undegraded":"2023-09-23T20:53:39.526702+0000","last_fullsized":"2023-09-23T20:53:39.526702+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":166,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.529916+0000","last_change":"2023-09-23T20:53:09.214852+0000","last_active":"2023-09-23T20:53:39.529916+0000","last_peered":"2023-09-23T20:53:39.529916+0000","last_clean":"2023-09-23T20:53:39.529916+0000","last_became_active":"2023-09-23T20:53:09.195778+0000","last_became_peered":"2023-09-23T20:53:09.195778+0000","last_unstale":"2023-09-23T20:53:39.529916+0000","last_undegraded":"2023-09-23T20:53:39.529916+0000","last_fullsized":"2023-09-23T20:53:39.529916+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.010784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526263+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:53:39.526263+0000","last_peered":"2023-09-23T20:53:39.526263+0000","last_clean":"2023-09-23T20:53:39.526263+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:53:39.526263+0000","last_undegraded":"2023-09-23T20:53:39.526263+0000","last_fullsized":"2023-09-23T20:53:39.526263+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"213'11724","reported_seq":11783,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:46.028974+0000","last_change":"2023-09-23T20:50:08.083439+0000","last_active":"2023-09-23T20:53:46.028974+0000","last_peered":"2023-09-23T20:53:46.028974+0000","last_clean":"2023-09-23T20:53:46.028974+0000","last_became_active":"2023-09-23T20:50:08.083061+0000","last_became_peered":"2023-09-23T20:50:08.083061+0000","last_unstale":"2023-09-23T20:53:46.028974+0000","last_undegraded":"2023-09-23T20:53:46.028974+0000","last_fullsized":"2023-09-23T20:53:46.028974+0000","mapping_epoch":164,"log_start":"176'4000","ondisk_log_start":"176'4000","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7724,"log_dups_size":0,"ondisk_log_size":7724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:41.569707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73007104,"num_objects":1114,"num_object_clones":0,"num_object_copies":3342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11724,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":45,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.534347+0000","last_change":"2023-09-23T20:51:05.957996+0000","last_active":"2023-09-23T20:53:39.534347+0000","last_peered":"2023-09-23T20:53:39.534347+0000","last_clean":"2023-09-23T20:53:39.534347+0000","last_became_active":"2023-09-23T20:51:05.957651+0000","last_became_peered":"2023-09-23T20:51:05.957651+0000","last_unstale":"2023-09-23T20:53:39.534347+0000","last_undegraded":"2023-09-23T20:53:39.534347+0000","last_fullsized":"2023-09-23T20:53:39.534347+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:05.872498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":181,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618607+0000","last_change":"2023-09-23T20:53:09.184461+0000","last_active":"2023-09-23T20:53:39.618607+0000","last_peered":"2023-09-23T20:53:39.618607+0000","last_clean":"2023-09-23T20:53:39.618607+0000","last_became_active":"2023-09-23T20:53:09.184067+0000","last_became_peered":"2023-09-23T20:53:09.184067+0000","last_unstale":"2023-09-23T20:53:39.618607+0000","last_undegraded":"2023-09-23T20:53:39.618607+0000","last_fullsized":"2023-09-23T20:53:39.618607+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:08.928487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":175,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606274+0000","last_change":"2023-09-23T20:51:05.959327+0000","last_active":"2023-09-23T20:53:39.606274+0000","last_peered":"2023-09-23T20:53:39.606274+0000","last_clean":"2023-09-23T20:53:39.606274+0000","last_became_active":"2023-09-23T20:51:05.958816+0000","last_became_peered":"2023-09-23T20:51:05.958816+0000","last_unstale":"2023-09-23T20:53:39.606274+0000","last_undegraded":"2023-09-23T20:53:39.606274+0000","last_fullsized":"2023-09-23T20:53:39.606274+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:52.674764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":151,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.618740+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:53:39.618740+0000","last_peered":"2023-09-23T20:53:39.618740+0000","last_clean":"2023-09-23T20:53:39.618740+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:53:39.618740+0000","last_undegraded":"2023-09-23T20:53:39.618740+0000","last_fullsized":"2023-09-23T20:53:39.618740+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":203,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.531576+0000","last_change":"2023-09-23T20:53:09.190345+0000","last_active":"2023-09-23T20:53:39.531576+0000","last_peered":"2023-09-23T20:53:39.531576+0000","last_clean":"2023-09-23T20:53:39.531576+0000","last_became_active":"2023-09-23T20:53:09.190072+0000","last_became_peered":"2023-09-23T20:53:09.190072+0000","last_unstale":"2023-09-23T20:53:39.531576+0000","last_undegraded":"2023-09-23T20:53:39.531576+0000","last_fullsized":"2023-09-23T20:53:39.531576+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:42.575345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":193,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.542529+0000","last_change":"2023-09-23T20:53:09.184633+0000","last_active":"2023-09-23T20:53:40.542529+0000","last_peered":"2023-09-23T20:53:40.542529+0000","last_clean":"2023-09-23T20:53:40.542529+0000","last_became_active":"2023-09-23T20:53:09.184285+0000","last_became_peered":"2023-09-23T20:53:09.184285+0000","last_unstale":"2023-09-23T20:53:40.542529+0000","last_undegraded":"2023-09-23T20:53:40.542529+0000","last_fullsized":"2023-09-23T20:53:40.542529+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:38.473632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":206,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606820+0000","last_change":"2023-09-23T20:53:09.191904+0000","last_active":"2023-09-23T20:53:39.606820+0000","last_peered":"2023-09-23T20:53:39.606820+0000","last_clean":"2023-09-23T20:53:39.606820+0000","last_became_active":"2023-09-23T20:53:09.191558+0000","last_became_peered":"2023-09-23T20:53:09.191558+0000","last_unstale":"2023-09-23T20:53:39.606820+0000","last_undegraded":"2023-09-23T20:53:39.606820+0000","last_fullsized":"2023-09-23T20:53:39.606820+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:10.469531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":187,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:53:40.545624+0000","last_change":"2023-09-23T20:51:05.969594+0000","last_active":"2023-09-23T20:53:40.545624+0000","last_peered":"2023-09-23T20:53:40.545624+0000","last_clean":"2023-09-23T20:53:40.545624+0000","last_became_active":"2023-09-23T20:51:05.959688+0000","last_became_peered":"2023-09-23T20:51:05.959688+0000","last_unstale":"2023-09-23T20:53:40.545624+0000","last_undegraded":"2023-09-23T20:53:40.545624+0000","last_fullsized":"2023-09-23T20:53:40.545624+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:59.980249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":189,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606593+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:53:39.606593+0000","last_peered":"2023-09-23T20:53:39.606593+0000","last_clean":"2023-09-23T20:53:39.606593+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:53:39.606593+0000","last_undegraded":"2023-09-23T20:53:39.606593+0000","last_fullsized":"2023-09-23T20:53:39.606593+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":188,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.527087+0000","last_change":"2023-09-23T20:51:05.999322+0000","last_active":"2023-09-23T20:53:39.527087+0000","last_peered":"2023-09-23T20:53:39.527087+0000","last_clean":"2023-09-23T20:53:39.527087+0000","last_became_active":"2023-09-23T20:51:05.998993+0000","last_became_peered":"2023-09-23T20:51:05.998993+0000","last_unstale":"2023-09-23T20:53:39.527087+0000","last_undegraded":"2023-09-23T20:53:39.527087+0000","last_fullsized":"2023-09-23T20:53:39.527087+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:47.495994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":185,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.526797+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:53:39.526797+0000","last_peered":"2023-09-23T20:53:39.526797+0000","last_clean":"2023-09-23T20:53:39.526797+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:53:39.526797+0000","last_undegraded":"2023-09-23T20:53:39.526797+0000","last_fullsized":"2023-09-23T20:53:39.526797+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":167,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.619026+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:53:39.619026+0000","last_peered":"2023-09-23T20:53:39.619026+0000","last_clean":"2023-09-23T20:53:39.619026+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:53:39.619026+0000","last_undegraded":"2023-09-23T20:53:39.619026+0000","last_fullsized":"2023-09-23T20:53:39.619026+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":209,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:53:39.606056+0000","last_change":"2023-09-23T20:53:09.192021+0000","last_active":"2023-09-23T20:53:39.606056+0000","last_peered":"2023-09-23T20:53:39.606056+0000","last_clean":"2023-09-23T20:53:39.606056+0000","last_became_active":"2023-09-23T20:53:09.191668+0000","last_became_peered":"2023-09-23T20:53:09.191668+0000","last_unstale":"2023-09-23T20:53:39.606056+0000","last_undegraded":"2023-09-23T20:53:39.606056+0000","last_fullsized":"2023-09-23T20:53:39.606056+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:10.164608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1108869144,"num_objects":16921,"num_object_clones":0,"num_object_copies":50763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13271,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16921,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188851,"num_write_kb":6584641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6127,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4251922432,"data_stored":4251910216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139567,"ondisk_log_size":139567,"up":48,"acting":40,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201581,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990908,"kb_used_data":587252,"kb_used_omap":0,"kb_used_meta":403648,"kb_avail":92754244,"statfs":{"total":95995035648,"available":94980345856,"internally_reserved":0,"allocated":601346048,"data_stored":600736964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":413335552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.3149999999999999,"15min":2.7890000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20499999999999999},"max":{"1min":4.9820000000000002,"5min":70.781999999999996,"15min":225.56999999999999},"last":4.3529999999999998},{"interface":"front","average":{"1min":3.327,"5min":2.661,"15min":3.129},"min":{"1min":0.307,"5min":0.29299999999999998,"15min":0.21299999999999999},"max":{"1min":14.035,"5min":70.876000000000005,"15min":225.61099999999999},"last":4.3390000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7439999999999998,"5min":3.8559999999999999,"15min":2.0920000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":24.016999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":4.827,"5min":6.7590000000000003,"15min":3.1160000000000001},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.991,"5min":288.60199999999998,"15min":288.60199999999998},"last":1.2629999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":11.965,"5min":3.5590000000000002,"15min":3.7320000000000002},"min":{"1min":0.46100000000000002,"5min":0.29399999999999998,"15min":0.22900000000000001},"max":{"1min":147.309,"5min":147.309,"15min":147.309},"last":4.4420000000000002},{"interface":"front","average":{"1min":11.414999999999999,"5min":3.262,"15min":3.0640000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":147.32400000000001,"5min":147.32400000000001,"15min":147.32400000000001},"last":4.4550000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":9.1080000000000005,"5min":5.194,"15min":8.5370000000000008},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":121.90900000000001,"5min":138.345,"15min":138.345},"last":4.8419999999999996},{"interface":"front","average":{"1min":10.446999999999999,"5min":5.5330000000000004,"15min":8.1259999999999994},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":121.748,"5min":133.76400000000001,"15min":133.76400000000001},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.5649999999999999,"15min":1.375},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.22800000000000001},"max":{"1min":6.9909999999999997,"5min":16.648,"15min":16.648},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.028,"5min":1.454,"15min":1.3180000000000001},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":8.9629999999999992,"5min":10.172000000000001,"15min":10.172000000000001},"last":0.498}]},{"osd":5,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.6520000000000001,"15min":1.718},"min":{"1min":0.35599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.8109999999999999,"5min":126.28,"15min":126.28},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.798,"5min":3.536,"15min":2.101},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.25600000000000001},"max":{"1min":9.3339999999999996,"5min":126.465,"15min":126.465},"last":0.435}]},{"osd":6,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.631,"15min":1.4450000000000001},"min":{"1min":0.40000000000000002,"5min":0.34300000000000003,"15min":0.28899999999999998},"max":{"1min":7.1040000000000001,"5min":16.678000000000001,"15min":16.678000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.484,"15min":1.262},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":8.7129999999999992,"5min":10.154,"15min":10.154},"last":0.46100000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463218,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505768,"kb_used_data":208160,"kb_used_omap":0,"kb_used_meta":297600,"kb_avail":93239384,"statfs":{"total":95995035648,"available":95477129216,"internally_reserved":0,"allocated":213155840,"data_stored":212566859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304742400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":5.9589999999999996,"5min":3.9089999999999998,"15min":2.7599999999999998},"min":{"1min":0.224,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":79.016000000000005,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":5.8209999999999997,"5min":3.6200000000000001,"15min":2.5979999999999999},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.20000000000000001},"max":{"1min":78.843000000000004,"5min":92.138000000000005,"15min":92.138000000000005},"last":1.9350000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":5.1440000000000001,"5min":3.6960000000000002,"15min":2.3639999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":35.262,"5min":96.623999999999995,"15min":96.623999999999995},"last":4.5780000000000003},{"interface":"front","average":{"1min":4.6790000000000003,"5min":3.7440000000000002,"15min":2.1930000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.223},"max":{"1min":35.350000000000001,"5min":96.254000000000005,"15min":96.254000000000005},"last":4.569}]},{"osd":2,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.8080000000000001,"15min":2.1589999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":8.2370000000000001,"5min":24.440999999999999,"15min":24.440999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.6880000000000002,"5min":1.9570000000000001,"15min":2.399},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.1109999999999998,"5min":23.916,"15min":23.916},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":22.062000000000001,"5min":7.4279999999999999,"15min":6.5590000000000002},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":250.804,"5min":250.804,"15min":250.804},"last":0.56799999999999995},{"interface":"front","average":{"1min":22.256,"5min":7.577,"15min":6.524},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":250.446,"5min":250.446,"15min":250.446},"last":7.7919999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.2410000000000001,"15min":1.111},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":9.1590000000000007,"5min":9.1590000000000007,"15min":9.1590000000000007},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.4359999999999999,"15min":1.3819999999999999},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.27000000000000002},"max":{"1min":7.5890000000000004,"5min":9.8650000000000002,"15min":10.058999999999999},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.2599999999999998,"15min":1.9370000000000001},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.26000000000000001},"max":{"1min":18.962,"5min":105.121,"15min":105.121},"last":0.32700000000000001},{"interface":"front","average":{"1min":3.2770000000000001,"5min":3.117,"15min":1.994},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":9.4469999999999992,"5min":106.30500000000001,"15min":106.30500000000001},"last":0.51200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:53:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6930000000000001,"5min":2.8370000000000002,"15min":1.7390000000000001},"min":{"1min":0.36599999999999999,"5min":0.33700000000000002,"15min":0.22600000000000001},"max":{"1min":9.4049999999999994,"5min":101.04600000000001,"15min":101.04600000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.962,"5min":2.5390000000000001,"15min":1.774},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32100000000000001},"max":{"1min":6.4210000000000003,"5min":101.081,"15min":101.081},"last":2.032}]}]},{"osd":1,"up_from":206,"seq":884763262985,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1695396,"kb_used_data":1560220,"kb_used_omap":5,"kb_used_meta":135162,"kb_avail":92049756,"statfs":{"total":95995035648,"available":94258950144,"internally_reserved":0,"allocated":1597665280,"data_stored":1596789946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":138406627},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3680000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.3540000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3979999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.3069999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.5499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.4249999999999998}]}]},{"osd":0,"up_from":92,"seq":395136991389,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":763988,"kb_used_data":278604,"kb_used_omap":5,"kb_used_meta":485306,"kb_avail":92981164,"statfs":{"total":95995035648,"available":95212711936,"internally_reserved":0,"allocated":285290496,"data_stored":284424838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":496953978},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0919999999999996,"5min":4.4889999999999999,"15min":2.9089999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":31.515000000000001,"5min":134.322,"15min":134.322},"last":0.32000000000000001},{"interface":"front","average":{"1min":4.4530000000000003,"5min":3.1400000000000001,"15min":2.5449999999999999},"min":{"1min":0.21199999999999999,"5min":0.19,"15min":0.19},"max":{"1min":31.48,"5min":59.570999999999998,"15min":59.570999999999998},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":7.5499999999999998,"5min":2.8170000000000002,"15min":2.3580000000000001},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":114.002,"5min":114.002,"15min":114.002},"last":0.26000000000000001},{"interface":"front","average":{"1min":7.5949999999999998,"5min":2.7829999999999999,"15min":3.9460000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":113.69499999999999,"5min":113.69499999999999,"15min":113.69499999999999},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":12.102,"5min":6.5439999999999996,"15min":4.4539999999999997},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":89.665999999999997,"5min":94.947999999999993,"15min":94.947999999999993},"last":0.371},{"interface":"front","average":{"1min":12.561,"5min":6.6660000000000004,"15min":4.6559999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":89.533000000000001,"5min":94.382999999999996,"15min":94.382999999999996},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.7669999999999999,"15min":1.839},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":5.4210000000000003,"5min":10.525,"15min":12.973000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.389,"15min":1.847},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5279999999999996,"5min":12.461,"15min":12.461},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":3.617,"15min":2.8039999999999998},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":18.146000000000001,"5min":124.40600000000001,"15min":124.40600000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.2450000000000001,"5min":2.8050000000000002,"15min":2.4049999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.5990000000000002,"5min":124.363,"15min":124.363},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":1.5349999999999999,"15min":2.0510000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.307},"max":{"1min":5.5519999999999996,"5min":12.510999999999999,"15min":14.037000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.9630000000000001,"5min":1.7549999999999999,"15min":1.7929999999999999},"min":{"1min":0.36599999999999999,"5min":0.35599999999999998,"15min":0.34799999999999998},"max":{"1min":10.026999999999999,"5min":10.026999999999999,"15min":20.399999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":2.3799999999999999,"15min":2.363},"min":{"1min":0.39600000000000002,"5min":0.35699999999999998,"15min":0.35099999999999998},"max":{"1min":6.46,"5min":60.037999999999997,"15min":60.037999999999997},"last":150.81999999999999},{"interface":"front","average":{"1min":1.7170000000000001,"5min":2.383,"15min":2.0760000000000001},"min":{"1min":0.46300000000000002,"5min":0.39800000000000002,"15min":0.38100000000000001},"max":{"1min":5.3399999999999999,"5min":60.012,"15min":60.012},"last":150.852}]}]},{"osd":2,"up_from":129,"seq":554050781279,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":535596,"kb_used_data":206436,"kb_used_omap":7,"kb_used_meta":329144,"kb_avail":93209556,"statfs":{"total":95995035648,"available":95446585344,"internally_reserved":0,"allocated":211390464,"data_stored":210561593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":337043537},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":2.4220000000000002,"15min":2.8530000000000002},"min":{"1min":0.218,"5min":0.182,"15min":0.17599999999999999},"max":{"1min":19.902000000000001,"5min":89.644999999999996,"15min":89.644999999999996},"last":5.6520000000000001},{"interface":"front","average":{"1min":3.3260000000000001,"5min":2.464,"15min":2.718},"min":{"1min":0.253,"5min":0.217,"15min":0.186},"max":{"1min":21.472000000000001,"5min":88.599999999999994,"15min":88.599999999999994},"last":3.7719999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.8439999999999999,"15min":2.9750000000000001},"min":{"1min":0.22500000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5879999999999992,"5min":91.325000000000003,"15min":91.325000000000003},"last":3.7080000000000002},{"interface":"front","average":{"1min":1.119,"5min":2.3599999999999999,"15min":2.996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2489999999999997,"5min":91.799999999999997,"15min":91.799999999999997},"last":1.3740000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:22 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":4.9960000000000004,"15min":2.7799999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.753,"5min":151.43799999999999,"15min":151.43799999999999},"last":1.4079999999999999},{"interface":"front","average":{"1min":2.8900000000000001,"5min":5.04,"15min":2.3580000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":23.725000000000001,"5min":152.20699999999999,"15min":152.20699999999999},"last":6.2210000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.415,"15min":1.7629999999999999},"min":{"1min":0.29999999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7960000000000003,"5min":8.7230000000000008,"15min":8.7230000000000008},"last":5.7880000000000003},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.3100000000000001,"15min":2.282},"min":{"1min":0.36599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.5350000000000001,"5min":8.5350000000000001,"15min":8.5350000000000001},"last":3.7469999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":3.2170000000000001,"15min":3.1869999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.5090000000000003,"5min":163.352,"15min":163.352},"last":3.794},{"interface":"front","average":{"1min":1.629,"5min":3.2810000000000001,"15min":3.0270000000000001},"min":{"1min":0.36699999999999999,"5min":0.316,"15min":0.30299999999999999},"max":{"1min":5.1909999999999998,"5min":163.089,"15min":163.089},"last":1.431}]},{"osd":6,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.367,"15min":2.6779999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.8920000000000003,"5min":6.3449999999999998,"15min":7.476},"last":1.4550000000000001},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.744,"15min":2.6600000000000001},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.6580000000000004,"5min":9.5220000000000002,"15min":12.595000000000001},"last":5.7519999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":3.4860000000000002,"15min":2.8820000000000001},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.6630000000000003,"5min":160.08099999999999,"15min":160.08099999999999},"last":5.7149999999999999},{"interface":"front","average":{"1min":3.008,"5min":3.427,"15min":3.0979999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.5559999999999992,"5min":160.13800000000001,"15min":160.13800000000001},"last":3.8220000000000001}]}]},{"osd":3,"up_from":148,"seq":635655159878,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846332,"kb_used_data":582132,"kb_used_omap":2,"kb_used_meta":264061,"kb_avail":92898820,"statfs":{"total":95995035648,"available":95128391680,"internally_reserved":0,"allocated":596103168,"data_stored":595261015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":270398517},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":7.391,"5min":5.2789999999999999,"15min":11.314},"min":{"1min":0.215,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":95.507999999999996,"5min":253.68100000000001,"15min":253.68100000000001},"last":3.2839999999999998},{"interface":"front","average":{"1min":7.1139999999999999,"5min":5.0640000000000001,"15min":10.493},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":95.259,"5min":253.46199999999999,"15min":253.46199999999999},"last":5.0469999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":5.0099999999999998,"15min":8.6150000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.0140000000000002,"5min":172.77099999999999,"15min":172.77099999999999},"last":3.137},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.7919999999999998,"15min":7.8689999999999998},"min":{"1min":0.25,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.692,"5min":172.36600000000001,"15min":172.36600000000001},"last":4.5250000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.3100000000000001,"15min":1.1279999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":3.3159999999999998,"5min":11.317,"15min":11.317},"last":3.1059999999999999},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.409,"15min":1.097},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.3049999999999997,"5min":8.8819999999999997,"15min":8.8819999999999997},"last":4.556}]},{"osd":4,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.5609999999999999,"15min":1.641},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.9749999999999996,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":4.0350000000000001},{"interface":"front","average":{"1min":1.27,"5min":1.571,"15min":2.0009999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.7930000000000001,"5min":16.408999999999999,"15min":16.408999999999999},"last":3.694}]},{"osd":5,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.847,"15min":1.399},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.2949999999999999,"5min":12.057,"15min":12.057},"last":4.5789999999999997},{"interface":"front","average":{"1min":2.1099999999999999,"5min":2.008,"15min":1.974},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":5.274,"5min":14.054,"15min":14.054},"last":4.0140000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.8859999999999999,"15min":1.9430000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6909999999999998,"5min":11.632,"15min":11.632},"last":3.9870000000000001},{"interface":"front","average":{"1min":2.0649999999999999,"5min":1.879,"15min":2.387},"min":{"1min":0.40300000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5309999999999997,"5min":16.451000000000001,"15min":16.451000000000001},"last":7.3540000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.024,"15min":1.7170000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7110000000000003,"5min":12.25,"15min":12.25},"last":5.6079999999999997},{"interface":"front","average":{"1min":2.3940000000000001,"5min":1.883,"15min":2.1949999999999998},"min":{"1min":0.42399999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.3639999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":4.0890000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986491,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557620,"kb_used_data":200620,"kb_used_omap":0,"kb_used_meta":356992,"kb_avail":93187532,"statfs":{"total":95995035648,"available":95424032768,"internally_reserved":0,"allocated":205434880,"data_stored":204833673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":365559808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":8.3989999999999991,"5min":3.831,"15min":2.6000000000000001},"min":{"1min":0.22600000000000001,"5min":0.216,"15min":0.214},"max":{"1min":146.25299999999999,"5min":146.25299999999999,"15min":146.25299999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":7.9160000000000004,"5min":3.7010000000000001,"15min":2.262},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22600000000000001},"max":{"1min":144.77000000000001,"5min":144.77000000000001,"15min":144.77000000000001},"last":0.90200000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":4.2939999999999996,"5min":7.4859999999999998,"15min":3.589},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":45.712000000000003,"5min":286.31099999999998,"15min":286.31099999999998},"last":0.68200000000000005},{"interface":"front","average":{"1min":4.6509999999999998,"5min":7.2949999999999999,"15min":4.1310000000000002},"min":{"1min":0.27900000000000003,"5min":0.25700000000000001,"15min":0.24299999999999999},"max":{"1min":45.460000000000001,"5min":285.97000000000003,"15min":285.97000000000003},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.1819999999999999,"15min":1.3009999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.25},"max":{"1min":8.2400000000000002,"5min":8.6639999999999997,"15min":8.6639999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.944,"5min":1.478,"15min":2.0899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.4509999999999996,"5min":7.4509999999999996,"15min":12.773999999999999},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:53:23 2023","interfaces":[{"interface":"back","average":{"1min":11.016,"5min":5.0640000000000001,"15min":6.7539999999999996},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":142.28399999999999,"5min":142.28399999999999,"15min":142.28399999999999},"last":6.6210000000000004},{"interface":"front","average":{"1min":11.807,"5min":5.3140000000000001,"15min":6.5659999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":142.34700000000001,"5min":142.34700000000001,"15min":142.34700000000001},"last":0.79300000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:52:50 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":1.0509999999999999,"15min":0.97999999999999998},"min":{"1min":0.28799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":9.2270000000000003},"last":0.318},{"interface":"front","average":{"1min":2.105,"5min":1.236,"15min":1.1319999999999999},"min":{"1min":0.29699999999999999,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":7.2960000000000003,"5min":7.2960000000000003,"15min":12.209},"last":0.505}]},{"osd":6,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":1.1100000000000001,"15min":1.135},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28000000000000003},"max":{"1min":5.2220000000000004,"5min":7.4109999999999996,"15min":16.271000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.337,"5min":0.99299999999999999,"15min":1.0700000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.30399999999999999},"max":{"1min":8.1609999999999996,"5min":8.1609999999999996,"15min":11.718},"last":0.41999999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:52:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":1.9410000000000001,"15min":1.458},"min":{"1min":0.35999999999999999,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":7.2619999999999996,"5min":73.268000000000001,"15min":73.268000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.129,"5min":1.9199999999999999,"15min":1.383},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":9.1679999999999993,"5min":69.760000000000005,"15min":69.760000000000005},"last":0.48599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724855,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1037876,"kb_used_data":581804,"kb_used_omap":0,"kb_used_meta":456064,"kb_avail":92707276,"statfs":{"total":95995035648,"available":94932250624,"internally_reserved":0,"allocated":595767296,"data_stored":595170159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":467009536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.7890000000000001,"15min":2.1299999999999999},"min":{"1min":0.25700000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":4.8680000000000003,"5min":124.873,"15min":124.873},"last":5.0300000000000002},{"interface":"front","average":{"1min":1.829,"5min":2.8119999999999998,"15min":1.9330000000000001},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":9.2789999999999999,"5min":123.554,"15min":123.554},"last":8.5429999999999993}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":11.34,"5min":4.9880000000000004,"15min":3.407},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":184.31200000000001,"5min":184.31200000000001,"15min":323.541},"last":9.766},{"interface":"front","average":{"1min":11.313000000000001,"5min":5.0149999999999997,"15min":3.6469999999999998},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":184.67400000000001,"5min":184.67400000000001,"15min":324.072},"last":5.0739999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.9279999999999999,"15min":2.2090000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.9059999999999997,"5min":12.585000000000001,"15min":13.394},"last":5.1219999999999999},{"interface":"front","average":{"1min":1.869,"5min":1.391,"15min":1.619},"min":{"1min":0.313,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.8929999999999998,"5min":7.8929999999999998,"15min":14.263},"last":8.5980000000000008}]},{"osd":3,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6240000000000001,"5min":5.4279999999999999,"15min":3.0510000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.5839999999999996,"5min":198.33099999999999,"15min":198.33099999999999},"last":9.7300000000000004},{"interface":"front","average":{"1min":2.2309999999999999,"5min":5.4930000000000003,"15min":2.8170000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.202,"5min":198.47300000000001,"15min":198.47300000000001},"last":5.2910000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:52:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.149,"15min":1.0449999999999999},"min":{"1min":0.34000000000000002,"5min":0.25900000000000001,"15min":0.216},"max":{"1min":10.906000000000001,"5min":10.906000000000001,"15min":10.906000000000001},"last":5.3230000000000004},{"interface":"front","average":{"1min":2.5739999999999998,"5min":1.2410000000000001,"15min":1.175},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":8.3940000000000001,"5min":8.3940000000000001,"15min":8.3940000000000001},"last":12.539}]},{"osd":6,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.238,"15min":1.1259999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":6.327,"5min":6.327,"15min":8.3010000000000002},"last":12.727},{"interface":"front","average":{"1min":2.7429999999999999,"5min":1.407,"15min":1.1279999999999999},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":8.6219999999999999,"5min":8.9870000000000001,"15min":9.6370000000000005},"last":9.7910000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:53:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.6719999999999999,"15min":1.4239999999999999},"min":{"1min":0.35399999999999998,"5min":0.33200000000000002,"15min":0.28199999999999997},"max":{"1min":12.669,"5min":12.669,"15min":14.217000000000001},"last":12.669},{"interface":"front","average":{"1min":2.7850000000000001,"5min":1.5780000000000001,"15min":1.1399999999999999},"min":{"1min":0.432,"5min":0.34200000000000003,"15min":0.28699999999999998},"max":{"1min":13.773999999999999,"5min":13.773999999999999,"15min":13.773999999999999},"last":8.5030000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":274530304,"data_stored":274530304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1589579776,"data_stored":1589575704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":209059840,"data_stored":209059840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":588578816,"data_stored":588578816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":197197824,"data_stored":197197824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":592252928,"data_stored":592248856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":206635008,"data_stored":206635008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":594087936,"data_stored":594083864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:53:52.510 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T20:53:52.603 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:52.645 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:52.798 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:52.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:52.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:52.905 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:52.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:52 smithi123 ceph-mon[193903]: from='client.45368 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:53.011 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:52 smithi121 ceph-mon[211526]: from='client.45368 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:53.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:52 smithi121 ceph-mon[219709]: from='client.45368 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:53:53.026 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:53.067 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:53:53.067 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:53:53.128 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:53:53.141 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:53.151 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:53:53.158 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:53.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:53.204 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:53:53.252 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:53:53.270 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:53.295 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:53:53.329 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:53:53.337 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:53.341 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:53.365 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:53:53.409 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.421 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:53:53.479 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:53.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:53.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:53.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:53.622 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:53 smithi121 ceph-mon[219709]: pgmap v684: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 980 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 13271/47085 objects degraded (28.185%); 0 B/s, 59 objects/s recovering 2023-09-23T20:53:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/406113306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:53.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:53 smithi121 ceph-mon[211526]: pgmap v684: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 980 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 13271/47085 objects degraded (28.185%); 0 B/s, 59 objects/s recovering 2023-09-23T20:53:53.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/406113306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:54.036 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:53:54.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:53 smithi123 ceph-mon[193903]: pgmap v684: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 980 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 13271/47085 objects degraded (28.185%); 0 B/s, 59 objects/s recovering 2023-09-23T20:53:54.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/406113306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:53:54.595 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:54.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:53:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:53:54] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T20:53:55.205 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:55.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:55.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:55.323 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:55.396 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:55 smithi121 ceph-mon[211526]: pgmap v685: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 859 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/41301 objects degraded (32.132%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:56.169 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:55 smithi121 ceph-mon[219709]: pgmap v685: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 859 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/41301 objects degraded (32.132%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:55 smithi123 ceph-mon[193903]: pgmap v685: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 859 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/41301 objects degraded (32.132%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:56.329 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:56.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:56.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:56.438 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:56.497 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:56.696 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:56.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:56.805 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:56.917 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:57.003 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:53:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:53:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:53:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:53:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136312 9.0 9:087a316d:::benchmark_data_smithi121_293770_object33424:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 86.771514097999997, 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.026362005, 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136312 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228794+0000", 2023-09-23T20:53:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228801+0000", 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6000000000000003e-06 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228812+0000", 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1517e-05 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228821+0000", 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.089e-06 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230287+0000", 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014660960000000001 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230299+0000", 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:53:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255083+0000", 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247833159999999 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255158+0000", 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5184000000000004e-05 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136315 9.0 9:07b4e2b9:::benchmark_data_smithi121_293770_object33427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 86.769503122000003, 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0248562560000001, 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136315 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230806+0000", 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230811+0000", 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.879e-06 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230823+0000", 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.151e-05 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230830+0000", 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8460000000000002e-06 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230876+0000", 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6560999999999997e-05 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230884+0000", 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8960000000000003e-06 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255603+0000", 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247184060000001 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255663+0000", 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0364e-05 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136335 9.0 9:00bf7afd:::benchmark_data_smithi121_293770_object33447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 86.743873652999994, 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0152258009999999, 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136335 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256434+0000", 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256453+0000", 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8349999999999999e-05 2023-09-23T20:53:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256484+0000", 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0908999999999998e-05 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256494+0000", 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0479000000000001e-05 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256538+0000", 2023-09-23T20:53:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3940999999999998e-05 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256547+0000", 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6689999999999995e-06 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255731+0000", 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99918387799999997 2023-09-23T20:53:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255863+0000", 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000131917 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255979+0000", 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011618899999999999 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.267419+0000", 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011440723999999999 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271524+0000", 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041050169999999999 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271584+0000", 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9577999999999998e-05 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271662+0000", 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7960999999999994e-05 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:150943 9.0 9:0c37a897:::benchmark_data_smithi121_293770_object48055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:48.789919+0000", 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 68.210391032000004, 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032519870999999999, 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 150943 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789919+0000", 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789919+0000", 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789917+0000", 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789925+0000", 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8359999999999992e-06 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789942+0000", 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7302000000000001e-05 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789949+0000", 2023-09-23T20:53:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6599999999999998e-06 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789996+0000", 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.74e-05 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790133+0000", 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013718500000000001 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790269+0000", 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013547599999999999 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.802229+0000", 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011959813 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822289+0000", 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020060906 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822346+0000", 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6920999999999999e-05 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822439+0000", 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2322000000000001e-05 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:150945 9.0 9:09d902a6:::benchmark_data_smithi121_293770_object48057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:48.789962+0000", 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 68.210347913000007, 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032658983000000003, 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 150945 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789962+0000", 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789962+0000", 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789962+0000", 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789964+0000", 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8040000000000002e-06 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789970+0000", 2023-09-23T20:53:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7309999999999998e-06 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.789974+0000", 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6370000000000001e-06 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.797827+0000", 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078530330000000006 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.797969+0000", 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014241199999999999 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.798098+0000", 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012912000000000001 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.806610+0000", 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085116270000000008 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822491+0000", 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015881560999999999 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822535+0000", 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3803000000000001e-05 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822621+0000", 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5678000000000002e-05 2023-09-23T20:53:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:150946 9.0 9:0b6d3e1c:::benchmark_data_smithi121_293770_object48058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e202)", 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:48.790006+0000", 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 68.210304047999998, 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032821134000000002, 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 150946 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790006+0000", 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790006+0000", 2023-09-23T20:53:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790005+0000", 2023-09-23T20:53:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790008+0000", 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8090000000000001e-06 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790017+0000", 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6629999999999999e-06 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.790021+0000", 2023-09-23T20:53:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0879999999999997e-06 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.802258+0000", 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012237385999999999 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.802372+0000", 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113936 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.802481+0000", 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010829500000000001 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.810865+0000", 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083842220000000002 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822691+0000", 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011826355 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822741+0000", 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9539000000000001e-05 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:48.822827+0000", 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6267999999999995e-05 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:171385 9.b e208/183 9:d0b9d239:::benchmark_data_smithi121_293770_object68497:head v 208'10901, mlcod=208'10897)", 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:18.372282+0000", 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 38.628028079000003, 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025165983999999999, 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.372282+0000", 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.372282+0000", 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.372281+0000", 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.372293+0000", 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1385e-05 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.372304+0000", 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1676e-05 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.372310+0000", 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6239999999999999e-06 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.376613+0000", 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043031409999999999 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.376646+0000", 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3445000000000003e-05 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.397408+0000", 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020761671999999998 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:18.397448+0000", 2023-09-23T20:53:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9675999999999998e-05 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:173737 9.b e209/183 9:d906a17a:::benchmark_data_smithi121_293770_object70849:head v 209'11050, mlcod=209'11047)", 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:22.072817+0000", 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 34.927493091999999, 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026524745999999998, 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.072817+0000", 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.072817+0000", 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.072816+0000", 2023-09-23T20:53:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.072828+0000", 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2566e-05 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.072845+0000", 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6827e-05 2023-09-23T20:53:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.072853+0000", 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5909999999999998e-06 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.074443+0000", 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001590845 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.074464+0000", 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0536000000000001e-05 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.099287+0000", 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024823495000000001 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.099342+0000", 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4101000000000003e-05 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:173779 9.1 e209/180 9:82a11e88:::benchmark_data_smithi121_293770_object70891:head v 209'10685, mlcod=209'10683)", 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:22.133839+0000", 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 34.866471335, 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.028486034, 2023-09-23T20:53:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133839+0000", 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133839+0000", 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133838+0000", 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133849+0000", 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1914e-05 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133870+0000", 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1058000000000001e-05 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133878+0000", 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.413e-06 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133946+0000", 2023-09-23T20:53:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8016999999999996e-05 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.133971+0000", 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4737999999999999e-05 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.162276+0000", 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.028305729000000002 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.162325+0000", 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8195999999999997e-05 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:173781 9.1 e209/180 9:8d89c04e:::benchmark_data_smithi121_293770_object70893:head v 209'10686, mlcod=209'10683)", 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:22.141659+0000", 2023-09-23T20:53:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 34.858651096000003, 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020691668, 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141659+0000", 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141659+0000", 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141658+0000", 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141669+0000", 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1901e-05 2023-09-23T20:53:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141688+0000", 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8368000000000001e-05 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141695+0000", 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4819999999999997e-06 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141761+0000", 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6006999999999993e-05 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.141786+0000", 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4372999999999999e-05 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.162329+0000", 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020542998 2023-09-23T20:53:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.162350+0000", 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1756e-05 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:174253 9.b e209/183 9:d248a14a:::benchmark_data_smithi121_293770_object71365:head v 209'11086, mlcod=209'11084)", 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:22.981806+0000", 2023-09-23T20:53:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 34.018504200999999, 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024804807000000002, 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981806+0000", 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981806+0000", 2023-09-23T20:53:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981805+0000", 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981815+0000", 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.022e-05 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981836+0000", 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1214999999999999e-05 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981843+0000", 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4680000000000003e-06 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981911+0000", 2023-09-23T20:53:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7965000000000004e-05 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.981932+0000", 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0404e-05 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:23.006569+0000", 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024637461999999999 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:23.006610+0000", 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1258999999999999e-05 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:174262 9.b e209/183 9:d6b774dc:::benchmark_data_smithi121_293770_object71374:head v 209'11087, mlcod=209'11084)", 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:22.986289+0000", 2023-09-23T20:53:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 34.014020895000002, 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020351883000000001, 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986289+0000", 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986289+0000", 2023-09-23T20:53:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986288+0000", 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986298+0000", 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4700000000000008e-06 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986315+0000", 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7136e-05 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986322+0000", 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1439999999999997e-06 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986816+0000", 2023-09-23T20:53:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00049438699999999999 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:22.986834+0000", 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7668000000000001e-05 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:23.006615+0000", 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019781435 2023-09-23T20:53:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:23.006641+0000", 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5548999999999999e-05 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:183429 9.0 e212/211 9:0342b256:::benchmark_data_smithi121_293770_object80541:head v 212'11477, mlcod=0'0)", 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:39.574328+0000", 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 17.425981579999998, 2023-09-23T20:53:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024980394999999999, 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574328+0000", 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574328+0000", 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574328+0000", 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574337+0000", 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0329999999999997e-06 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574352+0000", 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5199e-05 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574357+0000", 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4739999999999996e-06 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574405+0000", 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8149999999999998e-05 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.574433+0000", 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7248000000000001e-05 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.599282+0000", 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024849781000000001 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:39.599309+0000", 2023-09-23T20:53:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6293000000000001e-05 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:184625 9.b e213/183 9:d71708fe:::benchmark_data_smithi121_293770_object81737:head v 213'11748, mlcod=213'11742)", 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:40.928317+0000", 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 16.071992520999999, 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026684388999999999, 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.928317+0000", 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.928317+0000", 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.928317+0000", 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.928320+0000", 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.331e-06 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.928328+0000", 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.928330+0000", 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7190000000000001e-06 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.935808+0000", 2023-09-23T20:53:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0074776840000000001 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.935829+0000", 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0485000000000001e-05 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.954950+0000", 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019121873000000001 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:40.955002+0000", 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1332999999999998e-05 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:185199 9.b e213/183 9:d20ca6a3:::benchmark_data_smithi121_293770_object82311:head v 213'11787, mlcod=213'11784)", 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:41.918347+0000", 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.081962461, 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022618569000000002, 2023-09-23T20:53:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.918347+0000", 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.918347+0000", 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.918347+0000", 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.918353+0000", 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5560000000000004e-06 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.918368+0000", 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4722999999999999e-05 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.918374+0000", 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0970000000000002e-06 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.925559+0000", 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.007184742 2023-09-23T20:53:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.925580+0000", 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0771000000000002e-05 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.940921+0000", 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015341759 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.940966+0000", 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4659000000000001e-05 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:185216 9.b e213/183 9:d66c2035:::benchmark_data_smithi121_293770_object82328:head v 213'11791, mlcod=213'11784)", 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:41.941270+0000", 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.059040166999999, 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022042767000000001, 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.941270+0000", 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.941270+0000", 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.941269+0000", 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.941276+0000", 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9059999999999996e-06 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.941291+0000", 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5043e-05 2023-09-23T20:53:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.941297+0000", 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4219999999999997e-06 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.946299+0000", 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050014600000000001 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.946324+0000", 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5902999999999999e-05 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.963267+0000", 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016942299000000001 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:41.963312+0000", 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5698999999999997e-05 2023-09-23T20:53:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:185546 9.b e213/183 9:d94bad14:::benchmark_data_smithi121_293770_object82658:head v 213'11813, mlcod=213'11809)", 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:42.416841+0000", 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 14.583468573999999, 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023666474999999999, 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.416841+0000", 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.416841+0000", 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.416841+0000", 2023-09-23T20:53:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.416845+0000", 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4580000000000002e-06 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.416849+0000", 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.002e-06 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.416851+0000", 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7829999999999999e-06 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.422071+0000", 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0052194370000000004 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.422093+0000", 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1763000000000002e-05 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.440464+0000", 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018371769999999999 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:42.440508+0000", 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3495000000000003e-05 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:193046 9.b e214/183 9:d1ed5774:::benchmark_data_smithi121_293770_object90158:head v 214'12314, mlcod=214'12311)", 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:54.540520+0000", 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.459789915, 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020704633, 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.540520+0000", 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.540520+0000", 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.540519+0000", 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.540530+0000", 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0550999999999999e-05 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.540554+0000", 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3938e-05 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.540560+0000", 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9009999999999999e-06 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.541947+0000", 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001387342 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.541968+0000", 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1171000000000001e-05 2023-09-23T20:53:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.561136+0000", 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019167703000000001 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.561225+0000", 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8891e-05 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:193051 9.b e214/183 9:d38791db:::benchmark_data_smithi121_293770_object90163:head v 214'12316, mlcod=214'12312)", 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:54.552475+0000", 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.4478352050000001, 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022993481999999999, 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.552475+0000", 2023-09-23T20:53:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.552475+0000", 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.552474+0000", 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.552483+0000", 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8489999999999995e-06 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.552500+0000", 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7388999999999999e-05 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.552507+0000", 2023-09-23T20:53:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6239999999999996e-06 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.556309+0000", 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038019170000000001 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.556334+0000", 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5607000000000001e-05 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.575426+0000", 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019091736000000002 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:54.575468+0000", 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2249999999999997e-05 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:194028 9.b e214/183 9:d480f755:::benchmark_data_smithi121_293770_object91140:head v 214'12383, mlcod=214'12380)", 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:56.070724+0000", 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.92958555700000001, 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021886125999999999, 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070724+0000", 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070724+0000", 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070723+0000", 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:53:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070731+0000", 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1650000000000006e-06 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070759+0000", 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7993000000000001e-05 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070768+0000", 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4980000000000006e-06 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070827+0000", 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9312999999999999e-05 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.070850+0000", 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2676000000000001e-05 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.092571+0000", 2023-09-23T20:53:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021720877999999999 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:56.092610+0000", 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9674e-05 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:53:57.111 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:53:57.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13271/41301 objects degraded (32.132%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:57.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:56 smithi121 ceph-mon[211526]: pgmap v686: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 801 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/38514 objects degraded (34.458%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:57.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13271/41301 objects degraded (32.132%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:57.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:56 smithi121 ceph-mon[219709]: pgmap v686: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 801 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/38514 objects degraded (34.458%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:57.193 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:57.195 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13271/41301 objects degraded (32.132%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:53:57.196 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:56 smithi123 ceph-mon[193903]: pgmap v686: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 801 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/38514 objects degraded (34.458%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:57.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:57.251 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:53:57.301 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:57.339 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:57.401 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:57.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:57.509 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:57.548 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:57.611 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:57.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:57.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:57.715 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:53:57.784 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:57.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2957676238' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:57.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:57 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:57.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:57.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:57.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2342145665' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:57.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:57 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:57.887 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:53:57.894 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 9 pgp_num to 16 2023-09-23T20:53:57.901 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:53:58.087 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:53:58.092 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:53:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:58.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:53:58.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2957676238' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:58.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:58.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:58.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:58.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2342145665' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:58.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:58.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2957676238' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:58.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:53:58.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:53:58.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:53:58.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2342145665' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:58.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:57 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:53:58.202 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:53:58.266 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:58.403 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:53:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:58.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:53:58.517 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:58.570 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:58.654 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:53:58.870 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:58.870 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: failed to encode map e215 with expected crc 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: failed to encode map e215 with expected crc 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: failed to encode map e215 with expected crc 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: failed to encode map e215 with expected crc 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: failed to encode map e215 with expected crc 2023-09-23T20:53:58.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[211526]: pgmap v688: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 754 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/36228 objects degraded (36.632%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:53:58 smithi121 ceph-mon[219709]: pgmap v688: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 754 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/36228 objects degraded (36.632%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: failed to encode map e215 with expected crc 2023-09-23T20:53:59.182 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:53:58 smithi123 ceph-mon[193903]: pgmap v688: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 754 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13271/36228 objects degraded (36.632%); 0 B/s, 53 objects/s recovering 2023-09-23T20:53:59.204 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:53:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:59.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:53:59.309 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:53:59.367 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:53:59.774 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:53:59.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:53:59.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:53:59.877 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:53:59.948 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:00.163 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:00.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:00.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:00.274 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:00.343 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:00.505 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:00.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:00.614 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:00.675 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:00.921 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:00.927 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:54:00.927 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:197126 9.9 e215/164 9:974e1318:::benchmark_data_smithi121_293770_object94238:head v 215'12670, mlcod=215'12668)", 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:54:00.923330+0000", 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0041407980000000002, 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041713230000000002, 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923330+0000", 2023-09-23T20:54:00.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923330+0000", 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923330+0000", 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923335+0000", 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.203e-06 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923342+0000", 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3980000000000004e-06 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:54:00.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923346+0000", 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9720000000000003e-06 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923393+0000", 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7265999999999998e-05 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.923412+0000", 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8675999999999999e-05 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:54:00.931 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:197131 9.c e215/164)", 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:54:00.927522+0000", 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 4294967295.999949, 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3829000000000003e-05, 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.927522+0000", 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.927522+0000", 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.927522+0000", 2023-09-23T20:54:00.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.927532+0000", 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0226e-05 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.927549+0000", 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.713e-05 2023-09-23T20:54:00.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:00.927555+0000", 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3810000000000004e-06 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 2 2023-09-23T20:54:00.935 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:54:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:00.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:01.037 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:01.157 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:01.501 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:54:01.538 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:54:01.567 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:54:01.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:54:01.631 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:54:01.665 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:54:01.704 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:54:01.726 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:01.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:01.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:01.764 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:54:01.831 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:01.916 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:01 smithi123 ceph-mon[193903]: pgmap v689: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 669 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 13271/32142 objects degraded (41.289%); 0 B/s, 57 objects/s recovering 2023-09-23T20:54:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13271/32142 objects degraded (41.289%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/548635229' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:01.980 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:54:02.034 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:02.061 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:54:02.145 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:01 smithi121 ceph-mon[211526]: pgmap v689: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 669 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 13271/32142 objects degraded (41.289%); 0 B/s, 57 objects/s recovering 2023-09-23T20:54:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13271/32142 objects degraded (41.289%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/548635229' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:01 smithi121 ceph-mon[219709]: pgmap v689: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 669 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 13271/32142 objects degraded (41.289%); 0 B/s, 57 objects/s recovering 2023-09-23T20:54:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13271/32142 objects degraded (41.289%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/548635229' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:02.173 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:02.218 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:02.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:02.322 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:02.352 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:02.444 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:02.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:02.556 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:02.568 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:54:02.609 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:02.775 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:02.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:02.881 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:02.933 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:02.936 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.052 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:03.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:03.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:03.159 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:03.220 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.362 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:03.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:03.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:03.467 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:03.541 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.638 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:03.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:03.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:03.741 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:03.808 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:03 smithi123 ceph-mon[193903]: pgmap v690: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 593 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 13271/28500 objects degraded (46.565%); 0 B/s, 57 objects/s recovering 2023-09-23T20:54:03.831 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:03.901 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:03.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:03.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:04.005 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:04.029 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:03 smithi121 ceph-mon[211526]: pgmap v690: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 593 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 13271/28500 objects degraded (46.565%); 0 B/s, 57 objects/s recovering 2023-09-23T20:54:04.029 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:03 smithi121 ceph-mon[219709]: pgmap v690: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 593 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 13271/28500 objects degraded (46.565%); 0 B/s, 57 objects/s recovering 2023-09-23T20:54:04.031 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:04.091 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:04.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:04.199 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:04.262 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:54:04.262 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:04.262 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T20:54:04.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:54:04.276 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:04.378 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:04.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:04.483 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:04.565 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:04.677 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:04.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:04.745 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:04.746 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:04.782 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:04.786 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:54:04.880 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:04.934 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:54:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:54:04] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.43.0" 2023-09-23T20:54:05.109 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:05.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:05.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:05.179 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:05.220 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:05.251 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:05.313 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:05.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:05.419 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:05.478 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:05.736 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:05.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:54:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:05.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:05.852 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:05.860 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:54:05.944 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:06.096 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:06.171 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:05 smithi121 ceph-mon[211526]: pgmap v691: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 471 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13271/22650 objects degraded (58.592%); 0 B/s, 61 objects/s recovering 2023-09-23T20:54:06.171 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2732818917' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:06.171 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:06.171 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:05 smithi121 ceph-mon[219709]: pgmap v691: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 471 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13271/22650 objects degraded (58.592%); 0 B/s, 61 objects/s recovering 2023-09-23T20:54:06.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2732818917' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:06.172 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:06.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:05 smithi123 ceph-mon[193903]: pgmap v691: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 471 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13271/22650 objects degraded (58.592%); 0 B/s, 61 objects/s recovering 2023-09-23T20:54:06.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2732818917' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:06.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:06.241 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:06.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:06.327 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:54:06.355 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:06.383 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:06.447 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:06.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:06.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:06.550 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:06.644 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13271/22650 objects degraded (58.592%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13271/22650 objects degraded (58.592%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: failed to encode map e216 with expected crc 2023-09-23T20:54:06.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13271/22650 objects degraded (58.592%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:07.200 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:07.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:07.304 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:07.340 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:07.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:07.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:07.810 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:07.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:07 smithi121 ceph-mon[211526]: pgmap v693: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 414 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13271/19908 objects degraded (66.662%); 0 B/s, 69 objects/s recovering 2023-09-23T20:54:07.901 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:07 smithi123 ceph-mon[193903]: pgmap v693: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 414 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13271/19908 objects degraded (66.662%); 0 B/s, 69 objects/s recovering 2023-09-23T20:54:08.102 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:08.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:08.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:08.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:07 smithi121 ceph-mon[219709]: pgmap v693: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 414 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13271/19908 objects degraded (66.662%); 0 B/s, 69 objects/s recovering 2023-09-23T20:54:08.211 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:08.274 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:08.410 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:08.410 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:08.662 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:08.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:08.770 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:08.862 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:09.044 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:09.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:09.153 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:09.225 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:09.400 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:54:09.434 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:09.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:09.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:09.485 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:54:09.487 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:54:09.525 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:54:09.527 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:54:09.538 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:09.548 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:54:09.567 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:09.580 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:54:09.605 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:54:09.611 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:09.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:09.626 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:54:09.654 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:54:09.673 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:54:09.696 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:54:09.698 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.698 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.698 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.698 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.698 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.699 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.699 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.699 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:54:09.719 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:09.750 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: pgmap v694: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 367 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 13271/17661 objects degraded (75.143%); 0 B/s, 61 objects/s recovering 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: from='client.35412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:54:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/333411075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: pgmap v694: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 367 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 13271/17661 objects degraded (75.143%); 0 B/s, 61 objects/s recovering 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: from='client.35412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:54:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/333411075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:10.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: pgmap v694: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 367 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 13271/17661 objects degraded (75.143%); 0 B/s, 61 objects/s recovering 2023-09-23T20:54:10.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:54:10.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: failed to encode map e217 with expected crc 2023-09-23T20:54:10.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: from='client.35412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:54:10.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/333411075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:10.728 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 20:54:10 smithi123 conmon[172845]: logger=cleanup t=2023-09-23T20:54:10.262396026Z level=info msg="Completed cleanup jobs" duration=13.60895ms 2023-09-23T20:54:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[211526]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:54:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[211526]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[211526]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[211526]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[211526]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[211526]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[211526]: pgmap v697: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 288 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 13271/13851 objects degraded (95.813%); 0 B/s, 53 objects/s recovering 2023-09-23T20:54:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[219709]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:54:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[219709]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[219709]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[219709]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[219709]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[219709]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:10 smithi121 ceph-mon[219709]: pgmap v697: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 288 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 13271/13851 objects degraded (95.813%); 0 B/s, 53 objects/s recovering 2023-09-23T20:54:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:10 smithi123 ceph-mon[193903]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:54:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:10 smithi123 ceph-mon[193903]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:10 smithi123 ceph-mon[193903]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:10 smithi123 ceph-mon[193903]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:10 smithi123 ceph-mon[193903]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:10 smithi123 ceph-mon[193903]: failed to encode map e218 with expected crc 2023-09-23T20:54:11.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:10 smithi123 ceph-mon[193903]: pgmap v697: 89 pgs: 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 288 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 13271/13851 objects degraded (95.813%); 0 B/s, 53 objects/s recovering 2023-09-23T20:54:11.294 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:11.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:11.403 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:11.473 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":695,"stamp":"2023-09-23T20:54:08.888908+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":384767928,"num_objects":5887,"num_object_clones":0,"num_object_copies":17661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13271,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5887,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":200209,"num_write_kb":6587086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7051,"num_bytes_recovered":241381416,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139641,"ondisk_log_size":139641,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4722104,"kb_used_data":1970744,"kb_used_omap":21,"kb_used_meta":2751018,"kb_avail":745239112,"statfs":{"total":767960285184,"available":763124850688,"internally_reserved":0,"allocated":2018041856,"data_stored":2012128376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":2817042915},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1,1,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-316342272,"num_objects":-4827,"num_object_clones":0,"num_object_copies":-14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4827,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":317,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.143917"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":211,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918733+0000","last_change":"2023-09-23T20:53:09.184160+0000","last_active":"2023-09-23T20:53:57.918733+0000","last_peered":"2023-09-23T20:53:57.918733+0000","last_clean":"2023-09-23T20:53:57.918733+0000","last_became_active":"2023-09-23T20:53:09.183837+0000","last_became_peered":"2023-09-23T20:53:09.183837+0000","last_unstale":"2023-09-23T20:53:57.918733+0000","last_undegraded":"2023-09-23T20:53:57.918733+0000","last_fullsized":"2023-09-23T20:53:57.918733+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:03.396066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":171,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.897723+0000","last_change":"2023-09-23T20:53:09.205867+0000","last_active":"2023-09-23T20:53:57.897723+0000","last_peered":"2023-09-23T20:53:57.897723+0000","last_clean":"2023-09-23T20:53:57.897723+0000","last_became_active":"2023-09-23T20:53:09.205534+0000","last_became_peered":"2023-09-23T20:53:09.205534+0000","last_unstale":"2023-09-23T20:53:57.897723+0000","last_undegraded":"2023-09-23T20:53:57.897723+0000","last_fullsized":"2023-09-23T20:53:57.897723+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:23.682188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":137,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.879753+0000","last_change":"2023-09-23T20:53:09.200700+0000","last_active":"2023-09-23T20:53:57.879753+0000","last_peered":"2023-09-23T20:53:57.879753+0000","last_clean":"2023-09-23T20:53:57.879753+0000","last_became_active":"2023-09-23T20:53:09.200262+0000","last_became_peered":"2023-09-23T20:53:09.200262+0000","last_unstale":"2023-09-23T20:53:57.879753+0000","last_undegraded":"2023-09-23T20:53:57.879753+0000","last_fullsized":"2023-09-23T20:53:57.879753+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:04.656116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903514+0000","last_change":"2023-09-23T20:48:07.521825+0000","last_active":"2023-09-23T20:53:57.903514+0000","last_peered":"2023-09-23T20:53:57.903514+0000","last_clean":"2023-09-23T20:53:57.903514+0000","last_became_active":"2023-09-23T20:48:07.521427+0000","last_became_peered":"2023-09-23T20:48:07.521427+0000","last_unstale":"2023-09-23T20:53:57.903514+0000","last_undegraded":"2023-09-23T20:53:57.903514+0000","last_fullsized":"2023-09-23T20:53:57.903514+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:10.209326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.898359+0000","last_change":"2023-09-23T20:53:09.190070+0000","last_active":"2023-09-23T20:53:57.898359+0000","last_peered":"2023-09-23T20:53:57.898359+0000","last_clean":"2023-09-23T20:53:57.898359+0000","last_became_active":"2023-09-23T20:53:09.189785+0000","last_became_peered":"2023-09-23T20:53:09.189785+0000","last_unstale":"2023-09-23T20:53:57.898359+0000","last_undegraded":"2023-09-23T20:53:57.898359+0000","last_fullsized":"2023-09-23T20:53:57.898359+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.370133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":65,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.904896+0000","last_change":"2023-09-23T20:53:09.187404+0000","last_active":"2023-09-23T20:53:57.904896+0000","last_peered":"2023-09-23T20:53:57.904896+0000","last_clean":"2023-09-23T20:53:57.904896+0000","last_became_active":"2023-09-23T20:53:09.187078+0000","last_became_peered":"2023-09-23T20:53:09.187078+0000","last_unstale":"2023-09-23T20:53:57.904896+0000","last_undegraded":"2023-09-23T20:53:57.904896+0000","last_fullsized":"2023-09-23T20:53:57.904896+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:44.084084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":165,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903978+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:53:57.903978+0000","last_peered":"2023-09-23T20:53:57.903978+0000","last_clean":"2023-09-23T20:53:57.903978+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:53:57.903978+0000","last_undegraded":"2023-09-23T20:53:57.903978+0000","last_fullsized":"2023-09-23T20:53:57.903978+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":189,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.898288+0000","last_change":"2023-09-23T20:53:09.194688+0000","last_active":"2023-09-23T20:53:57.898288+0000","last_peered":"2023-09-23T20:53:57.898288+0000","last_clean":"2023-09-23T20:53:57.898288+0000","last_became_active":"2023-09-23T20:53:09.194399+0000","last_became_peered":"2023-09-23T20:53:09.194399+0000","last_unstale":"2023-09-23T20:53:57.898288+0000","last_undegraded":"2023-09-23T20:53:57.898288+0000","last_fullsized":"2023-09-23T20:53:57.898288+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:05.137531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":192,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.813917+0000","last_change":"2023-09-23T20:53:09.187702+0000","last_active":"2023-09-23T20:53:48.813917+0000","last_peered":"2023-09-23T20:53:48.813917+0000","last_clean":"2023-09-23T20:53:48.813917+0000","last_became_active":"2023-09-23T20:53:09.187353+0000","last_became_peered":"2023-09-23T20:53:09.187353+0000","last_unstale":"2023-09-23T20:53:48.813917+0000","last_undegraded":"2023-09-23T20:53:48.813917+0000","last_fullsized":"2023-09-23T20:53:48.813917+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:12.645048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.897641+0000","last_change":"2023-09-23T20:51:06.024444+0000","last_active":"2023-09-23T20:53:57.897641+0000","last_peered":"2023-09-23T20:53:57.897641+0000","last_clean":"2023-09-23T20:53:57.897641+0000","last_became_active":"2023-09-23T20:51:06.023727+0000","last_became_peered":"2023-09-23T20:51:06.023727+0000","last_unstale":"2023-09-23T20:53:57.897641+0000","last_undegraded":"2023-09-23T20:53:57.897641+0000","last_fullsized":"2023-09-23T20:53:57.897641+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:24.860774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":178,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808839+0000","last_change":"2023-09-23T20:48:07.526553+0000","last_active":"2023-09-23T20:53:48.808839+0000","last_peered":"2023-09-23T20:53:48.808839+0000","last_clean":"2023-09-23T20:53:48.808839+0000","last_became_active":"2023-09-23T20:48:07.526110+0000","last_became_peered":"2023-09-23T20:48:07.526110+0000","last_unstale":"2023-09-23T20:53:48.808839+0000","last_undegraded":"2023-09-23T20:53:48.808839+0000","last_fullsized":"2023-09-23T20:53:48.808839+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:03.406780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":143,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.882549+0000","last_change":"2023-09-23T20:53:09.188835+0000","last_active":"2023-09-23T20:53:57.882549+0000","last_peered":"2023-09-23T20:53:57.882549+0000","last_clean":"2023-09-23T20:53:57.882549+0000","last_became_active":"2023-09-23T20:53:09.188246+0000","last_became_peered":"2023-09-23T20:53:09.188246+0000","last_unstale":"2023-09-23T20:53:57.882549+0000","last_undegraded":"2023-09-23T20:53:57.882549+0000","last_fullsized":"2023-09-23T20:53:57.882549+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:27.239470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":221,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.920649+0000","last_change":"2023-09-23T20:51:05.966238+0000","last_active":"2023-09-23T20:53:57.920649+0000","last_peered":"2023-09-23T20:53:57.920649+0000","last_clean":"2023-09-23T20:53:57.920649+0000","last_became_active":"2023-09-23T20:51:05.959313+0000","last_became_peered":"2023-09-23T20:51:05.959313+0000","last_unstale":"2023-09-23T20:53:57.920649+0000","last_undegraded":"2023-09-23T20:53:57.920649+0000","last_fullsized":"2023-09-23T20:53:57.920649+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:02.228179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":212,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.919012+0000","last_change":"2023-09-23T20:53:09.191934+0000","last_active":"2023-09-23T20:53:57.919012+0000","last_peered":"2023-09-23T20:53:57.919012+0000","last_clean":"2023-09-23T20:53:57.919012+0000","last_became_active":"2023-09-23T20:53:09.191583+0000","last_became_peered":"2023-09-23T20:53:09.191583+0000","last_unstale":"2023-09-23T20:53:57.919012+0000","last_undegraded":"2023-09-23T20:53:57.919012+0000","last_fullsized":"2023-09-23T20:53:57.919012+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:45.729672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":177,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808532+0000","last_change":"2023-09-23T20:46:05.081502+0000","last_active":"2023-09-23T20:53:48.808532+0000","last_peered":"2023-09-23T20:53:48.808532+0000","last_clean":"2023-09-23T20:53:48.808532+0000","last_became_active":"2023-09-23T20:46:05.081038+0000","last_became_peered":"2023-09-23T20:46:05.081038+0000","last_unstale":"2023-09-23T20:53:48.808532+0000","last_undegraded":"2023-09-23T20:53:48.808532+0000","last_fullsized":"2023-09-23T20:53:48.808532+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:55.733800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":171,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.899052+0000","last_change":"2023-09-23T20:53:09.219437+0000","last_active":"2023-09-23T20:53:57.899052+0000","last_peered":"2023-09-23T20:53:57.899052+0000","last_clean":"2023-09-23T20:53:57.899052+0000","last_became_active":"2023-09-23T20:53:09.219177+0000","last_became_peered":"2023-09-23T20:53:09.219177+0000","last_unstale":"2023-09-23T20:53:57.899052+0000","last_undegraded":"2023-09-23T20:53:57.899052+0000","last_fullsized":"2023-09-23T20:53:57.899052+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:22.008304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":165,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903282+0000","last_change":"2023-09-23T20:48:07.521853+0000","last_active":"2023-09-23T20:53:57.903282+0000","last_peered":"2023-09-23T20:53:57.903282+0000","last_clean":"2023-09-23T20:53:57.903282+0000","last_became_active":"2023-09-23T20:48:07.521637+0000","last_became_peered":"2023-09-23T20:48:07.521637+0000","last_unstale":"2023-09-23T20:53:57.903282+0000","last_undegraded":"2023-09-23T20:53:57.903282+0000","last_fullsized":"2023-09-23T20:53:57.903282+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:08.687846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":174,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.810049+0000","last_change":"2023-09-23T20:48:07.522135+0000","last_active":"2023-09-23T20:53:48.810049+0000","last_peered":"2023-09-23T20:53:48.810049+0000","last_clean":"2023-09-23T20:53:48.810049+0000","last_became_active":"2023-09-23T20:48:07.521779+0000","last_became_peered":"2023-09-23T20:48:07.521779+0000","last_unstale":"2023-09-23T20:53:48.810049+0000","last_undegraded":"2023-09-23T20:53:48.810049+0000","last_fullsized":"2023-09-23T20:53:48.810049+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:04.881115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":178,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808232+0000","last_change":"2023-09-23T20:51:05.958217+0000","last_active":"2023-09-23T20:53:48.808232+0000","last_peered":"2023-09-23T20:53:48.808232+0000","last_clean":"2023-09-23T20:53:48.808232+0000","last_became_active":"2023-09-23T20:51:05.957874+0000","last_became_peered":"2023-09-23T20:51:05.957874+0000","last_unstale":"2023-09-23T20:53:48.808232+0000","last_undegraded":"2023-09-23T20:53:48.808232+0000","last_fullsized":"2023-09-23T20:53:48.808232+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:13.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":166,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903651+0000","last_change":"2023-09-23T20:48:07.522323+0000","last_active":"2023-09-23T20:53:57.903651+0000","last_peered":"2023-09-23T20:53:57.903651+0000","last_clean":"2023-09-23T20:53:57.903651+0000","last_became_active":"2023-09-23T20:48:07.521568+0000","last_became_peered":"2023-09-23T20:48:07.521568+0000","last_unstale":"2023-09-23T20:53:57.903651+0000","last_undegraded":"2023-09-23T20:53:57.903651+0000","last_fullsized":"2023-09-23T20:53:57.903651+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:31.285574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":58,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.904379+0000","last_change":"2023-09-23T20:51:06.038448+0000","last_active":"2023-09-23T20:53:57.904379+0000","last_peered":"2023-09-23T20:53:57.904379+0000","last_clean":"2023-09-23T20:53:57.904379+0000","last_became_active":"2023-09-23T20:51:06.005965+0000","last_became_peered":"2023-09-23T20:51:06.005965+0000","last_unstale":"2023-09-23T20:53:57.904379+0000","last_undegraded":"2023-09-23T20:53:57.904379+0000","last_fullsized":"2023-09-23T20:53:57.904379+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:13.397771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":171,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.898738+0000","last_change":"2023-09-23T20:53:09.232052+0000","last_active":"2023-09-23T20:53:57.898738+0000","last_peered":"2023-09-23T20:53:57.898738+0000","last_clean":"2023-09-23T20:53:57.898738+0000","last_became_active":"2023-09-23T20:53:09.231748+0000","last_became_peered":"2023-09-23T20:53:09.231748+0000","last_unstale":"2023-09-23T20:53:57.898738+0000","last_undegraded":"2023-09-23T20:53:57.898738+0000","last_fullsized":"2023-09-23T20:53:57.898738+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:04.683102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"216'12333","reported_seq":12395,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:07.767399+0000","last_change":"2023-09-23T20:50:08.077384+0000","last_active":"2023-09-23T20:54:07.767399+0000","last_peered":"2023-09-23T20:54:07.767399+0000","last_clean":"2023-09-23T20:54:07.767399+0000","last_became_active":"2023-09-23T20:50:08.077095+0000","last_became_peered":"2023-09-23T20:50:08.077095+0000","last_unstale":"2023-09-23T20:54:07.767399+0000","last_undegraded":"2023-09-23T20:54:07.767399+0000","last_fullsized":"2023-09-23T20:54:07.767399+0000","mapping_epoch":164,"log_start":"169'2300","ondisk_log_start":"169'2300","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10033,"log_dups_size":0,"ondisk_log_size":10033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:34.442024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18415616,"num_objects":281,"num_object_clones":0,"num_object_copies":843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12333,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":176,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808477+0000","last_change":"2023-09-23T20:40:53.853771+0000","last_active":"2023-09-23T20:53:48.808477+0000","last_peered":"2023-09-23T20:53:48.808477+0000","last_clean":"2023-09-23T20:53:48.808477+0000","last_became_active":"2023-09-23T20:40:53.853383+0000","last_became_peered":"2023-09-23T20:40:53.853383+0000","last_unstale":"2023-09-23T20:53:48.808477+0000","last_undegraded":"2023-09-23T20:53:48.808477+0000","last_fullsized":"2023-09-23T20:53:48.808477+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:34.977796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":177,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.809609+0000","last_change":"2023-09-23T20:51:05.962659+0000","last_active":"2023-09-23T20:53:48.809609+0000","last_peered":"2023-09-23T20:53:48.809609+0000","last_clean":"2023-09-23T20:53:48.809609+0000","last_became_active":"2023-09-23T20:51:05.962048+0000","last_became_peered":"2023-09-23T20:51:05.962048+0000","last_unstale":"2023-09-23T20:53:48.809609+0000","last_undegraded":"2023-09-23T20:53:48.809609+0000","last_fullsized":"2023-09-23T20:53:48.809609+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:03.530084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"216'12659","reported_seq":12733,"reported_epoch":216,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:06.010611+0000","last_change":"2023-09-23T20:53:10.348437+0000","last_active":"2023-09-23T20:54:06.010611+0000","last_peered":"2023-09-23T20:54:06.010611+0000","last_clean":"2023-09-23T20:52:30.157076+0000","last_became_active":"2023-09-23T20:53:10.306549+0000","last_became_peered":"2023-09-23T20:53:10.306549+0000","last_unstale":"2023-09-23T20:54:06.010611+0000","last_undegraded":"2023-09-23T20:53:10.193751+0000","last_fullsized":"2023-09-23T20:53:10.176070+0000","mapping_epoch":207,"log_start":"179'4925","ondisk_log_start":"179'4925","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7734,"log_dups_size":0,"ondisk_log_size":7734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:52.651633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22740992,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1960,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12659,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":347}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":194,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808676+0000","last_change":"2023-09-23T20:53:09.185607+0000","last_active":"2023-09-23T20:53:48.808676+0000","last_peered":"2023-09-23T20:53:48.808676+0000","last_clean":"2023-09-23T20:53:48.808676+0000","last_became_active":"2023-09-23T20:53:09.185271+0000","last_became_peered":"2023-09-23T20:53:09.185271+0000","last_unstale":"2023-09-23T20:53:48.808676+0000","last_undegraded":"2023-09-23T20:53:48.808676+0000","last_fullsized":"2023-09-23T20:53:48.808676+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:57.917928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":209,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918819+0000","last_change":"2023-09-23T20:53:09.191763+0000","last_active":"2023-09-23T20:53:57.918819+0000","last_peered":"2023-09-23T20:53:57.918819+0000","last_clean":"2023-09-23T20:53:57.918819+0000","last_became_active":"2023-09-23T20:53:09.191442+0000","last_became_peered":"2023-09-23T20:53:09.191442+0000","last_unstale":"2023-09-23T20:53:57.918819+0000","last_undegraded":"2023-09-23T20:53:57.918819+0000","last_fullsized":"2023-09-23T20:53:57.918819+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:02.859784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"215'12309","reported_seq":12368,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:54:03.854964+0000","last_change":"2023-09-23T20:50:08.078801+0000","last_active":"2023-09-23T20:54:03.854964+0000","last_peered":"2023-09-23T20:54:03.854964+0000","last_clean":"2023-09-23T20:54:03.854964+0000","last_became_active":"2023-09-23T20:50:08.078386+0000","last_became_peered":"2023-09-23T20:50:08.078386+0000","last_unstale":"2023-09-23T20:54:03.854964+0000","last_undegraded":"2023-09-23T20:54:03.854964+0000","last_fullsized":"2023-09-23T20:54:03.854964+0000","mapping_epoch":164,"log_start":"169'2300","ondisk_log_start":"169'2300","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10009,"log_dups_size":0,"ondisk_log_size":10009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:47.134724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28508160,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12309,"num_write_kb":407808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":187,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808752+0000","last_change":"2023-09-23T20:51:05.967804+0000","last_active":"2023-09-23T20:53:48.808752+0000","last_peered":"2023-09-23T20:53:48.808752+0000","last_clean":"2023-09-23T20:53:48.808752+0000","last_became_active":"2023-09-23T20:51:05.959914+0000","last_became_peered":"2023-09-23T20:51:05.959914+0000","last_unstale":"2023-09-23T20:53:48.808752+0000","last_undegraded":"2023-09-23T20:53:48.808752+0000","last_fullsized":"2023-09-23T20:53:48.808752+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:03.210361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":48,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.904591+0000","last_change":"2023-09-23T20:51:05.957335+0000","last_active":"2023-09-23T20:53:57.904591+0000","last_peered":"2023-09-23T20:53:57.904591+0000","last_clean":"2023-09-23T20:53:57.904591+0000","last_became_active":"2023-09-23T20:51:05.956858+0000","last_became_peered":"2023-09-23T20:51:05.956858+0000","last_unstale":"2023-09-23T20:53:57.904591+0000","last_undegraded":"2023-09-23T20:53:57.904591+0000","last_fullsized":"2023-09-23T20:53:57.904591+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:31.085919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"215'12236","reported_seq":12317,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:03.883453+0000","last_change":"2023-09-23T20:53:10.403698+0000","last_active":"2023-09-23T20:54:03.883453+0000","last_peered":"2023-09-23T20:54:03.883453+0000","last_clean":"2023-09-23T20:52:30.196736+0000","last_became_active":"2023-09-23T20:53:10.355510+0000","last_became_peered":"2023-09-23T20:53:10.355510+0000","last_unstale":"2023-09-23T20:54:03.883453+0000","last_undegraded":"2023-09-23T20:53:10.196118+0000","last_fullsized":"2023-09-23T20:53:10.178982+0000","mapping_epoch":207,"log_start":"169'2200","ondisk_log_start":"169'2200","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10036,"log_dups_size":0,"ondisk_log_size":10036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:55.710095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27787264,"num_objects":424,"num_object_clones":0,"num_object_copies":1272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1832,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12236,"num_write_kb":405120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":424}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":195,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.809861+0000","last_change":"2023-09-23T20:51:05.995979+0000","last_active":"2023-09-23T20:53:48.809861+0000","last_peered":"2023-09-23T20:53:48.809861+0000","last_clean":"2023-09-23T20:53:48.809861+0000","last_became_active":"2023-09-23T20:51:05.995591+0000","last_became_peered":"2023-09-23T20:51:05.995591+0000","last_unstale":"2023-09-23T20:53:48.809861+0000","last_undegraded":"2023-09-23T20:53:48.809861+0000","last_fullsized":"2023-09-23T20:53:48.809861+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:35.414294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":181,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903707+0000","last_change":"2023-09-23T20:53:09.185147+0000","last_active":"2023-09-23T20:53:57.903707+0000","last_peered":"2023-09-23T20:53:57.903707+0000","last_clean":"2023-09-23T20:53:57.903707+0000","last_became_active":"2023-09-23T20:53:09.184855+0000","last_became_peered":"2023-09-23T20:53:09.184855+0000","last_unstale":"2023-09-23T20:53:57.903707+0000","last_undegraded":"2023-09-23T20:53:57.903707+0000","last_fullsized":"2023-09-23T20:53:57.903707+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:45.699896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"216'12375","reported_seq":15301,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:06.056510+0000","last_change":"2023-09-23T20:51:15.811325+0000","last_active":"2023-09-23T20:54:06.056510+0000","last_peered":"2023-09-23T20:54:06.056510+0000","last_clean":"2023-09-23T20:54:06.056510+0000","last_became_active":"2023-09-23T20:51:15.805454+0000","last_became_peered":"2023-09-23T20:51:15.805454+0000","last_unstale":"2023-09-23T20:54:06.056510+0000","last_undegraded":"2023-09-23T20:54:06.056510+0000","last_fullsized":"2023-09-23T20:54:06.056510+0000","mapping_epoch":180,"log_start":"179'4600","ondisk_log_start":"179'4600","created":164,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7775,"log_dups_size":0,"ondisk_log_size":7775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:13.254228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21692416,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12375,"num_write_kb":406592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1418,"num_bytes_recovered":92930048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":179,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.898447+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:53:57.898447+0000","last_peered":"2023-09-23T20:53:57.898447+0000","last_clean":"2023-09-23T20:53:57.898447+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:53:57.898447+0000","last_undegraded":"2023-09-23T20:53:57.898447+0000","last_fullsized":"2023-09-23T20:53:57.898447+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":195,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.920573+0000","last_change":"2023-09-23T20:51:06.001664+0000","last_active":"2023-09-23T20:53:57.920573+0000","last_peered":"2023-09-23T20:53:57.920573+0000","last_clean":"2023-09-23T20:53:57.920573+0000","last_became_active":"2023-09-23T20:51:06.001352+0000","last_became_peered":"2023-09-23T20:51:06.001352+0000","last_unstale":"2023-09-23T20:53:57.920573+0000","last_undegraded":"2023-09-23T20:53:57.920573+0000","last_fullsized":"2023-09-23T20:53:57.920573+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:47.688374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"216'12573","reported_seq":18050,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:07.347061+0000","last_change":"2023-09-23T20:53:39.589168+0000","last_active":"2023-09-23T20:54:07.347061+0000","last_peered":"2023-09-23T20:54:07.347061+0000","last_clean":"2023-09-23T20:54:07.347061+0000","last_became_active":"2023-09-23T20:53:39.535053+0000","last_became_peered":"2023-09-23T20:53:39.535053+0000","last_unstale":"2023-09-23T20:54:07.347061+0000","last_undegraded":"2023-09-23T20:54:07.347061+0000","last_fullsized":"2023-09-23T20:54:07.347061+0000","mapping_epoch":211,"log_start":"182'5159","ondisk_log_start":"182'5159","created":164,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7414,"log_dups_size":0,"ondisk_log_size":7414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.122744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20643840,"num_objects":315,"num_object_clones":0,"num_object_copies":945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12573,"num_write_kb":412416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2684,"num_bytes_recovered":54329344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":65,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.904636+0000","last_change":"2023-09-23T20:53:09.236307+0000","last_active":"2023-09-23T20:53:57.904636+0000","last_peered":"2023-09-23T20:53:57.904636+0000","last_clean":"2023-09-23T20:53:57.904636+0000","last_became_active":"2023-09-23T20:53:09.235961+0000","last_became_peered":"2023-09-23T20:53:09.235961+0000","last_unstale":"2023-09-23T20:53:57.904636+0000","last_undegraded":"2023-09-23T20:53:57.904636+0000","last_fullsized":"2023-09-23T20:53:57.904636+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:54.763339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":209,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.919476+0000","last_change":"2023-09-23T20:53:09.184166+0000","last_active":"2023-09-23T20:53:57.919476+0000","last_peered":"2023-09-23T20:53:57.919476+0000","last_clean":"2023-09-23T20:53:57.919476+0000","last_became_active":"2023-09-23T20:53:09.183862+0000","last_became_peered":"2023-09-23T20:53:09.183862+0000","last_unstale":"2023-09-23T20:53:57.919476+0000","last_undegraded":"2023-09-23T20:53:57.919476+0000","last_fullsized":"2023-09-23T20:53:57.919476+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:40.482320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"215'12411","reported_seq":12494,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:03.832662+0000","last_change":"2023-09-23T20:53:10.383476+0000","last_active":"2023-09-23T20:54:03.832662+0000","last_peered":"2023-09-23T20:54:03.832662+0000","last_clean":"2023-09-23T20:52:30.214684+0000","last_became_active":"2023-09-23T20:53:10.343781+0000","last_became_peered":"2023-09-23T20:53:10.343781+0000","last_unstale":"2023-09-23T20:54:03.832662+0000","last_undegraded":"2023-09-23T20:53:10.193078+0000","last_fullsized":"2023-09-23T20:53:10.177310+0000","mapping_epoch":207,"log_start":"179'4700","ondisk_log_start":"179'4700","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7711,"log_dups_size":0,"ondisk_log_size":7711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:11.933025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28639232,"num_objects":437,"num_object_clones":0,"num_object_copies":1311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12411,"num_write_kb":411136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":437}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":176,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918625+0000","last_change":"2023-09-23T20:46:05.081244+0000","last_active":"2023-09-23T20:53:57.918625+0000","last_peered":"2023-09-23T20:53:57.918625+0000","last_clean":"2023-09-23T20:53:57.918625+0000","last_became_active":"2023-09-23T20:46:05.080963+0000","last_became_peered":"2023-09-23T20:46:05.080963+0000","last_unstale":"2023-09-23T20:53:57.918625+0000","last_undegraded":"2023-09-23T20:53:57.918625+0000","last_fullsized":"2023-09-23T20:53:57.918625+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:03.861562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":212,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.920283+0000","last_change":"2023-09-23T20:53:09.192221+0000","last_active":"2023-09-23T20:53:57.920283+0000","last_peered":"2023-09-23T20:53:57.920283+0000","last_clean":"2023-09-23T20:53:57.920283+0000","last_became_active":"2023-09-23T20:53:09.191599+0000","last_became_peered":"2023-09-23T20:53:09.191599+0000","last_unstale":"2023-09-23T20:53:57.920283+0000","last_undegraded":"2023-09-23T20:53:57.920283+0000","last_fullsized":"2023-09-23T20:53:57.920283+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:54.725504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"216'12621","reported_seq":12698,"reported_epoch":216,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:06.043358+0000","last_change":"2023-09-23T20:53:10.344228+0000","last_active":"2023-09-23T20:54:06.043358+0000","last_peered":"2023-09-23T20:54:06.043358+0000","last_clean":"2023-09-23T20:52:30.196344+0000","last_became_active":"2023-09-23T20:53:10.314804+0000","last_became_peered":"2023-09-23T20:53:10.314804+0000","last_unstale":"2023-09-23T20:54:06.043358+0000","last_undegraded":"2023-09-23T20:53:10.193272+0000","last_fullsized":"2023-09-23T20:53:10.176077+0000","mapping_epoch":207,"log_start":"179'4900","ondisk_log_start":"179'4900","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7721,"log_dups_size":0,"ondisk_log_size":7721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:48.249155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24444928,"num_objects":373,"num_object_clones":0,"num_object_copies":1119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12621,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":373}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":172,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.898206+0000","last_change":"2023-09-23T20:53:09.188782+0000","last_active":"2023-09-23T20:53:57.898206+0000","last_peered":"2023-09-23T20:53:57.898206+0000","last_clean":"2023-09-23T20:53:57.898206+0000","last_became_active":"2023-09-23T20:53:09.188563+0000","last_became_peered":"2023-09-23T20:53:09.188563+0000","last_unstale":"2023-09-23T20:53:57.898206+0000","last_undegraded":"2023-09-23T20:53:57.898206+0000","last_fullsized":"2023-09-23T20:53:57.898206+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:33.898108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":169,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.924920+0000","last_change":"2023-09-23T20:48:07.530143+0000","last_active":"2023-09-23T20:53:57.924920+0000","last_peered":"2023-09-23T20:53:57.924920+0000","last_clean":"2023-09-23T20:53:57.924920+0000","last_became_active":"2023-09-23T20:48:07.529807+0000","last_became_peered":"2023-09-23T20:48:07.529807+0000","last_unstale":"2023-09-23T20:53:57.924920+0000","last_undegraded":"2023-09-23T20:53:57.924920+0000","last_fullsized":"2023-09-23T20:53:57.924920+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:14.574071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"215'12612","reported_seq":12689,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:03.847442+0000","last_change":"2023-09-23T20:53:10.403424+0000","last_active":"2023-09-23T20:54:03.847442+0000","last_peered":"2023-09-23T20:54:03.847442+0000","last_clean":"2023-09-23T20:52:30.157599+0000","last_became_active":"2023-09-23T20:53:10.359278+0000","last_became_peered":"2023-09-23T20:53:10.359278+0000","last_unstale":"2023-09-23T20:54:03.847442+0000","last_undegraded":"2023-09-23T20:53:10.196743+0000","last_fullsized":"2023-09-23T20:53:10.179018+0000","mapping_epoch":207,"log_start":"170'2600","ondisk_log_start":"170'2600","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10012,"log_dups_size":0,"ondisk_log_size":10012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:20:52.816379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28704768,"num_objects":438,"num_object_clones":0,"num_object_copies":1314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1922,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12612,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":438}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":178,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.880096+0000","last_change":"2023-09-23T20:53:09.191673+0000","last_active":"2023-09-23T20:53:57.880096+0000","last_peered":"2023-09-23T20:53:57.880096+0000","last_clean":"2023-09-23T20:53:57.880096+0000","last_became_active":"2023-09-23T20:53:09.191277+0000","last_became_peered":"2023-09-23T20:53:09.191277+0000","last_unstale":"2023-09-23T20:53:57.880096+0000","last_undegraded":"2023-09-23T20:53:57.880096+0000","last_fullsized":"2023-09-23T20:53:57.880096+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:21.050943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":192,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.810093+0000","last_change":"2023-09-23T20:48:07.522544+0000","last_active":"2023-09-23T20:53:48.810093+0000","last_peered":"2023-09-23T20:53:48.810093+0000","last_clean":"2023-09-23T20:53:48.810093+0000","last_became_active":"2023-09-23T20:48:07.522199+0000","last_became_peered":"2023-09-23T20:48:07.522199+0000","last_unstale":"2023-09-23T20:53:48.810093+0000","last_undegraded":"2023-09-23T20:53:48.810093+0000","last_fullsized":"2023-09-23T20:53:48.810093+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:44.256737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":131,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.882614+0000","last_change":"2023-09-23T20:53:09.196041+0000","last_active":"2023-09-23T20:53:57.882614+0000","last_peered":"2023-09-23T20:53:57.882614+0000","last_clean":"2023-09-23T20:53:57.882614+0000","last_became_active":"2023-09-23T20:53:09.191615+0000","last_became_peered":"2023-09-23T20:53:09.191615+0000","last_unstale":"2023-09-23T20:53:57.882614+0000","last_undegraded":"2023-09-23T20:53:57.882614+0000","last_fullsized":"2023-09-23T20:53:57.882614+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:25.505589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"216'12930","reported_seq":12992,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:07.840040+0000","last_change":"2023-09-23T20:50:08.076706+0000","last_active":"2023-09-23T20:54:07.840040+0000","last_peered":"2023-09-23T20:54:07.840040+0000","last_clean":"2023-09-23T20:54:07.840040+0000","last_became_active":"2023-09-23T20:50:08.076360+0000","last_became_peered":"2023-09-23T20:50:08.076360+0000","last_unstale":"2023-09-23T20:54:07.840040+0000","last_undegraded":"2023-09-23T20:54:07.840040+0000","last_fullsized":"2023-09-23T20:54:07.840040+0000","mapping_epoch":164,"log_start":"170'2900","ondisk_log_start":"170'2900","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10030,"log_dups_size":0,"ondisk_log_size":10030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:15.596097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17432576,"num_objects":266,"num_object_clones":0,"num_object_copies":798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12930,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":216,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.810142+0000","last_change":"2023-09-23T20:53:09.184802+0000","last_active":"2023-09-23T20:53:48.810142+0000","last_peered":"2023-09-23T20:53:48.810142+0000","last_clean":"2023-09-23T20:53:48.810142+0000","last_became_active":"2023-09-23T20:53:09.184501+0000","last_became_peered":"2023-09-23T20:53:09.184501+0000","last_unstale":"2023-09-23T20:53:48.810142+0000","last_undegraded":"2023-09-23T20:53:48.810142+0000","last_fullsized":"2023-09-23T20:53:48.810142+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:56.145028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":196,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918931+0000","last_change":"2023-09-23T20:51:06.033002+0000","last_active":"2023-09-23T20:53:57.918931+0000","last_peered":"2023-09-23T20:53:57.918931+0000","last_clean":"2023-09-23T20:53:57.918931+0000","last_became_active":"2023-09-23T20:51:06.032657+0000","last_became_peered":"2023-09-23T20:51:06.032657+0000","last_unstale":"2023-09-23T20:53:57.918931+0000","last_undegraded":"2023-09-23T20:53:57.918931+0000","last_fullsized":"2023-09-23T20:53:57.918931+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:07.293307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":184,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.809747+0000","last_change":"2023-09-23T20:48:07.524306+0000","last_active":"2023-09-23T20:53:48.809747+0000","last_peered":"2023-09-23T20:53:48.809747+0000","last_clean":"2023-09-23T20:53:48.809747+0000","last_became_active":"2023-09-23T20:48:07.523967+0000","last_became_peered":"2023-09-23T20:48:07.523967+0000","last_unstale":"2023-09-23T20:53:48.809747+0000","last_undegraded":"2023-09-23T20:53:48.809747+0000","last_fullsized":"2023-09-23T20:53:48.809747+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:54.158061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"215'12418","reported_seq":12494,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:03.868428+0000","last_change":"2023-09-23T20:53:10.375283+0000","last_active":"2023-09-23T20:54:03.868428+0000","last_peered":"2023-09-23T20:54:03.868428+0000","last_clean":"2023-09-23T20:52:30.211455+0000","last_became_active":"2023-09-23T20:53:10.333927+0000","last_became_peered":"2023-09-23T20:53:10.333927+0000","last_unstale":"2023-09-23T20:54:03.868428+0000","last_undegraded":"2023-09-23T20:53:10.194479+0000","last_fullsized":"2023-09-23T20:53:10.178986+0000","mapping_epoch":207,"log_start":"169'2400","ondisk_log_start":"169'2400","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:47:13.773667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25690112,"num_objects":392,"num_object_clones":0,"num_object_copies":1176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1872,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12418,"num_write_kb":409920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":392}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":174,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.925784+0000","last_change":"2023-09-23T20:48:07.519509+0000","last_active":"2023-09-23T20:53:57.925784+0000","last_peered":"2023-09-23T20:53:57.925784+0000","last_clean":"2023-09-23T20:53:57.925784+0000","last_became_active":"2023-09-23T20:48:07.519189+0000","last_became_peered":"2023-09-23T20:48:07.519189+0000","last_unstale":"2023-09-23T20:53:57.925784+0000","last_undegraded":"2023-09-23T20:53:57.925784+0000","last_fullsized":"2023-09-23T20:53:57.925784+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:54.661223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":143,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.882150+0000","last_change":"2023-09-23T20:53:09.187057+0000","last_active":"2023-09-23T20:53:57.882150+0000","last_peered":"2023-09-23T20:53:57.882150+0000","last_clean":"2023-09-23T20:53:57.882150+0000","last_became_active":"2023-09-23T20:53:09.186597+0000","last_became_peered":"2023-09-23T20:53:09.186597+0000","last_unstale":"2023-09-23T20:53:57.882150+0000","last_undegraded":"2023-09-23T20:53:57.882150+0000","last_fullsized":"2023-09-23T20:53:57.882150+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:41.376692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":173,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.880508+0000","last_change":"2023-09-23T20:53:09.193262+0000","last_active":"2023-09-23T20:53:57.880508+0000","last_peered":"2023-09-23T20:53:57.880508+0000","last_clean":"2023-09-23T20:53:57.880508+0000","last_became_active":"2023-09-23T20:53:09.192941+0000","last_became_peered":"2023-09-23T20:53:09.192941+0000","last_unstale":"2023-09-23T20:53:57.880508+0000","last_undegraded":"2023-09-23T20:53:57.880508+0000","last_fullsized":"2023-09-23T20:53:57.880508+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:06.470020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"216'12261","reported_seq":12323,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:06.041179+0000","last_change":"2023-09-23T20:50:08.078559+0000","last_active":"2023-09-23T20:54:06.041179+0000","last_peered":"2023-09-23T20:54:06.041179+0000","last_clean":"2023-09-23T20:54:06.041179+0000","last_became_active":"2023-09-23T20:50:08.078294+0000","last_became_peered":"2023-09-23T20:50:08.078294+0000","last_unstale":"2023-09-23T20:54:06.041179+0000","last_undegraded":"2023-09-23T20:54:06.041179+0000","last_fullsized":"2023-09-23T20:54:06.041179+0000","mapping_epoch":164,"log_start":"179'4500","ondisk_log_start":"179'4500","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7761,"log_dups_size":0,"ondisk_log_size":7761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:11.281483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22740992,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12261,"num_write_kb":403456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":179,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.809808+0000","last_change":"2023-09-23T20:46:05.088787+0000","last_active":"2023-09-23T20:53:48.809808+0000","last_peered":"2023-09-23T20:53:48.809808+0000","last_clean":"2023-09-23T20:53:48.809808+0000","last_became_active":"2023-09-23T20:46:05.088497+0000","last_became_peered":"2023-09-23T20:46:05.088497+0000","last_unstale":"2023-09-23T20:53:48.809808+0000","last_undegraded":"2023-09-23T20:53:48.809808+0000","last_fullsized":"2023-09-23T20:53:48.809808+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:53.638539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":607,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808015+0000","last_change":"2023-09-23T20:51:30.823817+0000","last_active":"2023-09-23T20:53:48.808015+0000","last_peered":"2023-09-23T20:53:48.808015+0000","last_clean":"2023-09-23T20:53:48.808015+0000","last_became_active":"2023-09-23T20:51:30.823372+0000","last_became_peered":"2023-09-23T20:51:30.823372+0000","last_unstale":"2023-09-23T20:53:48.808015+0000","last_undegraded":"2023-09-23T20:53:48.808015+0000","last_fullsized":"2023-09-23T20:53:48.808015+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:09.432277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":151,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.897747+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:53:57.897747+0000","last_peered":"2023-09-23T20:53:57.897747+0000","last_clean":"2023-09-23T20:53:57.897747+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:53:57.897747+0000","last_undegraded":"2023-09-23T20:53:57.897747+0000","last_fullsized":"2023-09-23T20:53:57.897747+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":48,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.904721+0000","last_change":"2023-09-23T20:51:05.957381+0000","last_active":"2023-09-23T20:53:57.904721+0000","last_peered":"2023-09-23T20:53:57.904721+0000","last_clean":"2023-09-23T20:53:57.904721+0000","last_became_active":"2023-09-23T20:51:05.956991+0000","last_became_peered":"2023-09-23T20:51:05.956991+0000","last_unstale":"2023-09-23T20:53:57.904721+0000","last_undegraded":"2023-09-23T20:53:57.904721+0000","last_fullsized":"2023-09-23T20:53:57.904721+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:20.613174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"216'12840","reported_seq":15711,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:07.844491+0000","last_change":"2023-09-23T20:51:23.778886+0000","last_active":"2023-09-23T20:54:07.844491+0000","last_peered":"2023-09-23T20:54:07.844491+0000","last_clean":"2023-09-23T20:54:07.844491+0000","last_became_active":"2023-09-23T20:51:23.771838+0000","last_became_peered":"2023-09-23T20:51:23.771838+0000","last_unstale":"2023-09-23T20:54:07.844491+0000","last_undegraded":"2023-09-23T20:54:07.844491+0000","last_fullsized":"2023-09-23T20:54:07.844491+0000","mapping_epoch":183,"log_start":"170'2800","ondisk_log_start":"170'2800","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:54.532463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20185088,"num_objects":308,"num_object_clones":0,"num_object_copies":924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12840,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1391,"num_bytes_recovered":91160576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":213,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.807989+0000","last_change":"2023-09-23T20:53:09.183474+0000","last_active":"2023-09-23T20:53:48.807989+0000","last_peered":"2023-09-23T20:53:48.807989+0000","last_clean":"2023-09-23T20:53:48.807989+0000","last_became_active":"2023-09-23T20:53:09.183165+0000","last_became_peered":"2023-09-23T20:53:09.183165+0000","last_unstale":"2023-09-23T20:53:48.807989+0000","last_undegraded":"2023-09-23T20:53:48.807989+0000","last_fullsized":"2023-09-23T20:53:48.807989+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:11.886478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":166,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.904417+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:53:57.904417+0000","last_peered":"2023-09-23T20:53:57.904417+0000","last_clean":"2023-09-23T20:53:57.904417+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:53:57.904417+0000","last_undegraded":"2023-09-23T20:53:57.904417+0000","last_fullsized":"2023-09-23T20:53:57.904417+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":197,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903577+0000","last_change":"2023-09-23T20:53:09.181989+0000","last_active":"2023-09-23T20:53:57.903577+0000","last_peered":"2023-09-23T20:53:57.903577+0000","last_clean":"2023-09-23T20:53:57.903577+0000","last_became_active":"2023-09-23T20:53:09.181672+0000","last_became_peered":"2023-09-23T20:53:09.181672+0000","last_unstale":"2023-09-23T20:53:57.903577+0000","last_undegraded":"2023-09-23T20:53:57.903577+0000","last_fullsized":"2023-09-23T20:53:57.903577+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:39.497624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"215'12541","reported_seq":12622,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:54:03.835379+0000","last_change":"2023-09-23T20:53:10.407911+0000","last_active":"2023-09-23T20:54:03.835379+0000","last_peered":"2023-09-23T20:54:03.835379+0000","last_clean":"2023-09-23T20:52:30.212933+0000","last_became_active":"2023-09-23T20:53:10.358990+0000","last_became_peered":"2023-09-23T20:53:10.358990+0000","last_unstale":"2023-09-23T20:54:03.835379+0000","last_undegraded":"2023-09-23T20:53:10.195827+0000","last_fullsized":"2023-09-23T20:53:10.177309+0000","mapping_epoch":207,"log_start":"179'4800","ondisk_log_start":"179'4800","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7741,"log_dups_size":0,"ondisk_log_size":7741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:09.500397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27197440,"num_objects":415,"num_object_clones":0,"num_object_copies":1245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1955,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12541,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":415}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":179,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.808795+0000","last_change":"2023-09-23T20:40:53.854562+0000","last_active":"2023-09-23T20:53:48.808795+0000","last_peered":"2023-09-23T20:53:48.808795+0000","last_clean":"2023-09-23T20:53:48.808795+0000","last_became_active":"2023-09-23T20:40:53.854054+0000","last_became_peered":"2023-09-23T20:40:53.854054+0000","last_unstale":"2023-09-23T20:53:48.808795+0000","last_undegraded":"2023-09-23T20:53:48.808795+0000","last_fullsized":"2023-09-23T20:53:48.808795+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:19.520254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":118,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.880246+0000","last_change":"2023-09-23T20:53:09.199776+0000","last_active":"2023-09-23T20:53:57.880246+0000","last_peered":"2023-09-23T20:53:57.880246+0000","last_clean":"2023-09-23T20:53:57.880246+0000","last_became_active":"2023-09-23T20:53:09.199309+0000","last_became_peered":"2023-09-23T20:53:09.199309+0000","last_unstale":"2023-09-23T20:53:57.880246+0000","last_undegraded":"2023-09-23T20:53:57.880246+0000","last_fullsized":"2023-09-23T20:53:57.880246+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:32.904643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":152,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903573+0000","last_change":"2023-09-23T20:40:53.872683+0000","last_active":"2023-09-23T20:53:57.903573+0000","last_peered":"2023-09-23T20:53:57.903573+0000","last_clean":"2023-09-23T20:53:57.903573+0000","last_became_active":"2023-09-23T20:40:53.868521+0000","last_became_peered":"2023-09-23T20:40:53.868521+0000","last_unstale":"2023-09-23T20:53:57.903573+0000","last_undegraded":"2023-09-23T20:53:57.903573+0000","last_fullsized":"2023-09-23T20:53:57.903573+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.708605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"215'12296","reported_seq":15420,"reported_epoch":215,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:54:03.893985+0000","last_change":"2023-09-23T20:53:38.485724+0000","last_active":"2023-09-23T20:54:03.893985+0000","last_peered":"2023-09-23T20:54:03.893985+0000","last_clean":"2023-09-23T20:52:30.141642+0000","last_became_active":"2023-09-23T20:53:10.309106+0000","last_became_peered":"2023-09-23T20:53:10.309106+0000","last_unstale":"2023-09-23T20:54:03.893985+0000","last_undegraded":"2023-09-23T20:54:03.893985+0000","last_fullsized":"2023-09-23T20:53:10.177303+0000","mapping_epoch":207,"log_start":"179'4600","ondisk_log_start":"179'4600","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7696,"log_dups_size":0,"ondisk_log_size":7696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:03.627658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26804248,"num_objects":410,"num_object_clones":0,"num_object_copies":1230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":410,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12296,"num_write_kb":406529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1522,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":410}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":179,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.899175+0000","last_change":"2023-09-23T20:48:07.522373+0000","last_active":"2023-09-23T20:53:57.899175+0000","last_peered":"2023-09-23T20:53:57.899175+0000","last_clean":"2023-09-23T20:53:57.899175+0000","last_became_active":"2023-09-23T20:48:07.522023+0000","last_became_peered":"2023-09-23T20:48:07.522023+0000","last_unstale":"2023-09-23T20:53:57.899175+0000","last_undegraded":"2023-09-23T20:53:57.899175+0000","last_fullsized":"2023-09-23T20:53:57.899175+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:02.255156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":169,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.881513+0000","last_change":"2023-09-23T20:53:09.214852+0000","last_active":"2023-09-23T20:53:57.881513+0000","last_peered":"2023-09-23T20:53:57.881513+0000","last_clean":"2023-09-23T20:53:57.881513+0000","last_became_active":"2023-09-23T20:53:09.195778+0000","last_became_peered":"2023-09-23T20:53:09.195778+0000","last_unstale":"2023-09-23T20:53:57.881513+0000","last_undegraded":"2023-09-23T20:53:57.881513+0000","last_fullsized":"2023-09-23T20:53:57.881513+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:37.010784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":177,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:05.825583+0000","last_change":"2023-09-23T20:48:07.534063+0000","last_active":"2023-09-23T20:54:05.825583+0000","last_peered":"2023-09-23T20:54:05.825583+0000","last_clean":"2023-09-23T20:54:05.825583+0000","last_became_active":"2023-09-23T20:48:07.533758+0000","last_became_peered":"2023-09-23T20:48:07.533758+0000","last_unstale":"2023-09-23T20:54:05.825583+0000","last_undegraded":"2023-09-23T20:54:05.825583+0000","last_fullsized":"2023-09-23T20:54:05.825583+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:21.573366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"216'12494","reported_seq":12555,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:06.043029+0000","last_change":"2023-09-23T20:50:08.083439+0000","last_active":"2023-09-23T20:54:06.043029+0000","last_peered":"2023-09-23T20:54:06.043029+0000","last_clean":"2023-09-23T20:54:06.043029+0000","last_became_active":"2023-09-23T20:50:08.083061+0000","last_became_peered":"2023-09-23T20:50:08.083061+0000","last_unstale":"2023-09-23T20:54:06.043029+0000","last_undegraded":"2023-09-23T20:54:06.043029+0000","last_fullsized":"2023-09-23T20:54:06.043029+0000","mapping_epoch":164,"log_start":"179'4800","ondisk_log_start":"179'4800","created":164,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:50:07.057840+0000","last_clean_scrub_stamp":"2023-09-23T20:50:07.057840+0000","objects_scrubbed":0,"log_size":7694,"log_dups_size":0,"ondisk_log_size":7694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:41.569707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22544384,"num_objects":344,"num_object_clones":0,"num_object_copies":1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12494,"num_write_kb":410816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":48,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.904860+0000","last_change":"2023-09-23T20:51:05.957996+0000","last_active":"2023-09-23T20:53:57.904860+0000","last_peered":"2023-09-23T20:53:57.904860+0000","last_clean":"2023-09-23T20:53:57.904860+0000","last_became_active":"2023-09-23T20:51:05.957651+0000","last_became_peered":"2023-09-23T20:51:05.957651+0000","last_unstale":"2023-09-23T20:53:57.904860+0000","last_undegraded":"2023-09-23T20:53:57.904860+0000","last_fullsized":"2023-09-23T20:53:57.904860+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:05.872498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":185,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903444+0000","last_change":"2023-09-23T20:53:09.184461+0000","last_active":"2023-09-23T20:53:57.903444+0000","last_peered":"2023-09-23T20:53:57.903444+0000","last_clean":"2023-09-23T20:53:57.903444+0000","last_became_active":"2023-09-23T20:53:09.184067+0000","last_became_peered":"2023-09-23T20:53:09.184067+0000","last_unstale":"2023-09-23T20:53:57.903444+0000","last_undegraded":"2023-09-23T20:53:57.903444+0000","last_fullsized":"2023-09-23T20:53:57.903444+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:08.928487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":178,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918703+0000","last_change":"2023-09-23T20:51:05.959327+0000","last_active":"2023-09-23T20:53:57.918703+0000","last_peered":"2023-09-23T20:53:57.918703+0000","last_clean":"2023-09-23T20:53:57.918703+0000","last_became_active":"2023-09-23T20:51:05.958816+0000","last_became_peered":"2023-09-23T20:51:05.958816+0000","last_unstale":"2023-09-23T20:53:57.918703+0000","last_undegraded":"2023-09-23T20:53:57.918703+0000","last_fullsized":"2023-09-23T20:53:57.918703+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:52.674764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":154,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903791+0000","last_change":"2023-09-23T20:40:53.856885+0000","last_active":"2023-09-23T20:53:57.903791+0000","last_peered":"2023-09-23T20:53:57.903791+0000","last_clean":"2023-09-23T20:53:57.903791+0000","last_became_active":"2023-09-23T20:40:53.856291+0000","last_became_peered":"2023-09-23T20:40:53.856291+0000","last_unstale":"2023-09-23T20:53:57.903791+0000","last_undegraded":"2023-09-23T20:53:57.903791+0000","last_fullsized":"2023-09-23T20:53:57.903791+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:51.917473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":206,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.924583+0000","last_change":"2023-09-23T20:53:09.190345+0000","last_active":"2023-09-23T20:53:57.924583+0000","last_peered":"2023-09-23T20:53:57.924583+0000","last_clean":"2023-09-23T20:53:57.924583+0000","last_became_active":"2023-09-23T20:53:09.190072+0000","last_became_peered":"2023-09-23T20:53:09.190072+0000","last_unstale":"2023-09-23T20:53:57.924583+0000","last_undegraded":"2023-09-23T20:53:57.924583+0000","last_fullsized":"2023-09-23T20:53:57.924583+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:42.575345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":194,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.809911+0000","last_change":"2023-09-23T20:53:09.184633+0000","last_active":"2023-09-23T20:53:48.809911+0000","last_peered":"2023-09-23T20:53:48.809911+0000","last_clean":"2023-09-23T20:53:48.809911+0000","last_became_active":"2023-09-23T20:53:09.184285+0000","last_became_peered":"2023-09-23T20:53:09.184285+0000","last_unstale":"2023-09-23T20:53:48.809911+0000","last_undegraded":"2023-09-23T20:53:48.809911+0000","last_fullsized":"2023-09-23T20:53:48.809911+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:38.473632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":209,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918930+0000","last_change":"2023-09-23T20:53:09.191904+0000","last_active":"2023-09-23T20:53:57.918930+0000","last_peered":"2023-09-23T20:53:57.918930+0000","last_clean":"2023-09-23T20:53:57.918930+0000","last_became_active":"2023-09-23T20:53:09.191558+0000","last_became_peered":"2023-09-23T20:53:09.191558+0000","last_unstale":"2023-09-23T20:53:57.918930+0000","last_undegraded":"2023-09-23T20:53:57.918930+0000","last_fullsized":"2023-09-23T20:53:57.918930+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:10.469531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":188,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T20:53:48.810143+0000","last_change":"2023-09-23T20:51:05.969594+0000","last_active":"2023-09-23T20:53:48.810143+0000","last_peered":"2023-09-23T20:53:48.810143+0000","last_clean":"2023-09-23T20:53:48.810143+0000","last_became_active":"2023-09-23T20:51:05.959688+0000","last_became_peered":"2023-09-23T20:51:05.959688+0000","last_unstale":"2023-09-23T20:53:48.810143+0000","last_undegraded":"2023-09-23T20:53:48.810143+0000","last_fullsized":"2023-09-23T20:53:48.810143+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:59.980249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":192,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918812+0000","last_change":"2023-09-23T20:48:07.517634+0000","last_active":"2023-09-23T20:53:57.918812+0000","last_peered":"2023-09-23T20:53:57.918812+0000","last_clean":"2023-09-23T20:53:57.918812+0000","last_became_active":"2023-09-23T20:48:07.516717+0000","last_became_peered":"2023-09-23T20:48:07.516717+0000","last_unstale":"2023-09-23T20:53:57.918812+0000","last_undegraded":"2023-09-23T20:53:57.918812+0000","last_fullsized":"2023-09-23T20:53:57.918812+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:43.557782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":192,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:54:05.825433+0000","last_change":"2023-09-23T20:51:05.999322+0000","last_active":"2023-09-23T20:54:05.825433+0000","last_peered":"2023-09-23T20:54:05.825433+0000","last_clean":"2023-09-23T20:54:05.825433+0000","last_became_active":"2023-09-23T20:51:05.998993+0000","last_became_peered":"2023-09-23T20:51:05.998993+0000","last_unstale":"2023-09-23T20:54:05.825433+0000","last_undegraded":"2023-09-23T20:54:05.825433+0000","last_fullsized":"2023-09-23T20:54:05.825433+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:47.495994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":188,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.897884+0000","last_change":"2023-09-23T20:48:07.522432+0000","last_active":"2023-09-23T20:53:57.897884+0000","last_peered":"2023-09-23T20:53:57.897884+0000","last_clean":"2023-09-23T20:53:57.897884+0000","last_became_active":"2023-09-23T20:48:07.521681+0000","last_became_peered":"2023-09-23T20:48:07.521681+0000","last_unstale":"2023-09-23T20:53:57.897884+0000","last_undegraded":"2023-09-23T20:53:57.897884+0000","last_fullsized":"2023-09-23T20:53:57.897884+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:11.540447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":170,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.903861+0000","last_change":"2023-09-23T20:46:05.076292+0000","last_active":"2023-09-23T20:53:57.903861+0000","last_peered":"2023-09-23T20:53:57.903861+0000","last_clean":"2023-09-23T20:53:57.903861+0000","last_became_active":"2023-09-23T20:46:05.075964+0000","last_became_peered":"2023-09-23T20:46:05.075964+0000","last_unstale":"2023-09-23T20:53:57.903861+0000","last_undegraded":"2023-09-23T20:53:57.903861+0000","last_fullsized":"2023-09-23T20:53:57.903861+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:43.706625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":212,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:53:57.918606+0000","last_change":"2023-09-23T20:53:09.192021+0000","last_active":"2023-09-23T20:53:57.918606+0000","last_peered":"2023-09-23T20:53:57.918606+0000","last_clean":"2023-09-23T20:53:57.918606+0000","last_became_active":"2023-09-23T20:53:09.191668+0000","last_became_peered":"2023-09-23T20:53:09.191668+0000","last_unstale":"2023-09-23T20:53:57.918606+0000","last_undegraded":"2023-09-23T20:53:57.918606+0000","last_fullsized":"2023-09-23T20:53:57.918606+0000","mapping_epoch":206,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:10.164608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":384172056,"num_objects":5863,"num_object_clones":0,"num_object_copies":17589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13271,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5863,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":199909,"num_write_kb":6584641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7021,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1962225664,"data_stored":1962213448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139425,"ondisk_log_size":139425,"up":48,"acting":40,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201584,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":653524,"kb_used_data":240908,"kb_used_omap":0,"kb_used_meta":412480,"kb_avail":93091628,"statfs":{"total":95995035648,"available":95325827072,"internally_reserved":0,"allocated":246689792,"data_stored":246072736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":422379520},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.3149999999999999,"15min":2.7890000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20499999999999999},"max":{"1min":4.9820000000000002,"5min":70.781999999999996,"15min":225.56999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":3.327,"5min":2.661,"15min":3.129},"min":{"1min":0.307,"5min":0.29299999999999998,"15min":0.21299999999999999},"max":{"1min":14.035,"5min":70.876000000000005,"15min":225.61099999999999},"last":6.484}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7439999999999998,"5min":3.8559999999999999,"15min":2.0920000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":24.016999999999999,"5min":128.97999999999999,"15min":128.97999999999999},"last":0.625},{"interface":"front","average":{"1min":4.827,"5min":6.7590000000000003,"15min":3.1160000000000001},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.991,"5min":288.60199999999998,"15min":288.60199999999998},"last":0.69999999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":11.965,"5min":3.5590000000000002,"15min":3.7320000000000002},"min":{"1min":0.46100000000000002,"5min":0.29399999999999998,"15min":0.22900000000000001},"max":{"1min":147.309,"5min":147.309,"15min":147.309},"last":0.59899999999999998},{"interface":"front","average":{"1min":11.414999999999999,"5min":3.262,"15min":3.0640000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":147.32400000000001,"5min":147.32400000000001,"15min":147.32400000000001},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":9.1080000000000005,"5min":5.194,"15min":8.5370000000000008},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":121.90900000000001,"5min":138.345,"15min":138.345},"last":0.34000000000000002},{"interface":"front","average":{"1min":10.446999999999999,"5min":5.5330000000000004,"15min":8.1259999999999994},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":121.748,"5min":133.76400000000001,"15min":133.76400000000001},"last":6.6890000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9259999999999999,"5min":1.5649999999999999,"15min":1.375},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.22800000000000001},"max":{"1min":6.9909999999999997,"5min":16.648,"15min":16.648},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.028,"5min":1.454,"15min":1.3180000000000001},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":8.9629999999999992,"5min":10.172000000000001,"15min":10.172000000000001},"last":0.32200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.6520000000000001,"15min":1.718},"min":{"1min":0.35599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.8109999999999999,"5min":126.28,"15min":126.28},"last":0.371},{"interface":"front","average":{"1min":2.798,"5min":3.536,"15min":2.101},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.25600000000000001},"max":{"1min":9.3339999999999996,"5min":126.465,"15min":126.465},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:53:47 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.631,"15min":1.4450000000000001},"min":{"1min":0.40000000000000002,"5min":0.34300000000000003,"15min":0.28899999999999998},"max":{"1min":7.1040000000000001,"5min":16.678000000000001,"15min":16.678000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.484,"15min":1.262},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":8.7129999999999992,"5min":10.154,"15min":10.154},"last":0.65200000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463222,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":346444,"kb_used_data":55492,"kb_used_omap":0,"kb_used_meta":290944,"kb_avail":93398708,"statfs":{"total":95995035648,"available":95640276992,"internally_reserved":0,"allocated":56823808,"data_stored":56222869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":297926656},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":5.9589999999999996,"5min":3.9089999999999998,"15min":2.7599999999999998},"min":{"1min":0.224,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":79.016000000000005,"5min":92.164000000000001,"15min":92.164000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":5.8209999999999997,"5min":3.6200000000000001,"15min":2.5979999999999999},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.20000000000000001},"max":{"1min":78.843000000000004,"5min":92.138000000000005,"15min":92.138000000000005},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":5.1440000000000001,"5min":3.6960000000000002,"15min":2.3639999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":35.262,"5min":96.623999999999995,"15min":96.623999999999995},"last":13.509},{"interface":"front","average":{"1min":4.6790000000000003,"5min":3.7440000000000002,"15min":2.1930000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.223},"max":{"1min":35.350000000000001,"5min":96.254000000000005,"15min":96.254000000000005},"last":13.528}]},{"osd":2,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":1.8080000000000001,"15min":2.1589999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":8.2370000000000001,"5min":24.440999999999999,"15min":24.440999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.6880000000000002,"5min":1.9570000000000001,"15min":2.399},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.1109999999999998,"5min":23.916,"15min":23.916},"last":0.371}]},{"osd":3,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":22.062000000000001,"5min":7.4279999999999999,"15min":6.5590000000000002},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":250.804,"5min":250.804,"15min":250.804},"last":18.693000000000001},{"interface":"front","average":{"1min":22.256,"5min":7.577,"15min":6.524},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":250.446,"5min":250.446,"15min":250.446},"last":18.204999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.2410000000000001,"15min":1.111},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":9.1590000000000007,"5min":9.1590000000000007,"15min":9.1590000000000007},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.4359999999999999,"15min":1.3819999999999999},"min":{"1min":0.32000000000000001,"5min":0.313,"15min":0.27000000000000002},"max":{"1min":7.5890000000000004,"5min":9.8650000000000002,"15min":10.058999999999999},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:53:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.2599999999999998,"15min":1.9370000000000001},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.26000000000000001},"max":{"1min":18.962,"5min":105.121,"15min":105.121},"last":0.42999999999999999},{"interface":"front","average":{"1min":3.2770000000000001,"5min":3.117,"15min":1.994},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":9.4469999999999992,"5min":106.30500000000001,"15min":106.30500000000001},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:53:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6930000000000001,"5min":2.8370000000000002,"15min":1.7390000000000001},"min":{"1min":0.36599999999999999,"5min":0.33700000000000002,"15min":0.22600000000000001},"max":{"1min":9.4049999999999994,"5min":101.04600000000001,"15min":101.04600000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.962,"5min":2.5390000000000001,"15min":1.774},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.32100000000000001},"max":{"1min":6.4210000000000003,"5min":101.081,"15min":101.081},"last":0.47999999999999998}]}]},{"osd":1,"up_from":206,"seq":884763262989,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1189968,"kb_used_data":1036744,"kb_used_omap":5,"kb_used_meta":153210,"kb_avail":92555184,"statfs":{"total":95995035648,"available":94776508416,"internally_reserved":0,"allocated":1061625856,"data_stored":1060731374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":156887779},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.927}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":10.898}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":9.1099999999999994}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.9649999999999999}]}]},{"osd":0,"up_from":92,"seq":395136991393,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":563840,"kb_used_data":82616,"kb_used_omap":5,"kb_used_meta":481210,"kb_avail":93181312,"statfs":{"total":95995035648,"available":95417663488,"internally_reserved":0,"allocated":84598784,"data_stored":83713978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":492759674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0919999999999996,"5min":4.4889999999999999,"15min":2.9089999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":31.515000000000001,"5min":134.322,"15min":134.322},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.4530000000000003,"5min":3.1400000000000001,"15min":2.5449999999999999},"min":{"1min":0.21199999999999999,"5min":0.19,"15min":0.19},"max":{"1min":31.48,"5min":59.570999999999998,"15min":59.570999999999998},"last":5.8049999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":7.5499999999999998,"5min":2.8170000000000002,"15min":2.3580000000000001},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":114.002,"5min":114.002,"15min":114.002},"last":1.3069999999999999},{"interface":"front","average":{"1min":7.5949999999999998,"5min":2.7829999999999999,"15min":3.9460000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":113.69499999999999,"5min":113.69499999999999,"15min":113.69499999999999},"last":4.6050000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":12.102,"5min":6.5439999999999996,"15min":4.4539999999999997},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":89.665999999999997,"5min":94.947999999999993,"15min":94.947999999999993},"last":2.4809999999999999},{"interface":"front","average":{"1min":12.561,"5min":6.6660000000000004,"15min":4.6559999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":89.533000000000001,"5min":94.382999999999996,"15min":94.382999999999996},"last":4.6980000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":1.7669999999999999,"15min":1.839},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":5.4210000000000003,"5min":10.525,"15min":12.973000000000001},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.389,"15min":1.847},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.5279999999999996,"5min":12.461,"15min":12.461},"last":0.33800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":3.617,"15min":2.8039999999999998},"min":{"1min":0.35399999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":18.146000000000001,"5min":124.40600000000001,"15min":124.40600000000001},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.2450000000000001,"5min":2.8050000000000002,"15min":2.4049999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.5990000000000002,"5min":124.363,"15min":124.363},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":1.5349999999999999,"15min":2.0510000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.307},"max":{"1min":5.5519999999999996,"5min":12.510999999999999,"15min":14.037000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":2.9630000000000001,"5min":1.7549999999999999,"15min":1.7929999999999999},"min":{"1min":0.36599999999999999,"5min":0.35599999999999998,"15min":0.34799999999999998},"max":{"1min":10.026999999999999,"5min":10.026999999999999,"15min":20.399999999999999},"last":4.6740000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":2.3799999999999999,"15min":2.363},"min":{"1min":0.39600000000000002,"5min":0.35699999999999998,"15min":0.35099999999999998},"max":{"1min":6.46,"5min":60.037999999999997,"15min":60.037999999999997},"last":0.37},{"interface":"front","average":{"1min":1.7170000000000001,"5min":2.383,"15min":2.0760000000000001},"min":{"1min":0.46300000000000002,"5min":0.39800000000000002,"15min":0.38100000000000001},"max":{"1min":5.3399999999999999,"5min":60.012,"15min":60.012},"last":4.6479999999999997}]}]},{"osd":2,"up_from":129,"seq":554050781283,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":393748,"kb_used_data":56396,"kb_used_omap":7,"kb_used_meta":337336,"kb_avail":93351404,"statfs":{"total":95995035648,"available":95591837696,"internally_reserved":0,"allocated":57749504,"data_stored":56904910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":345432145},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":2.4220000000000002,"15min":2.8530000000000002},"min":{"1min":0.218,"5min":0.182,"15min":0.17599999999999999},"max":{"1min":19.902000000000001,"5min":89.644999999999996,"15min":89.644999999999996},"last":6.8869999999999996},{"interface":"front","average":{"1min":3.3260000000000001,"5min":2.464,"15min":2.718},"min":{"1min":0.253,"5min":0.217,"15min":0.186},"max":{"1min":21.472000000000001,"5min":88.599999999999994,"15min":88.599999999999994},"last":8.3420000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.8439999999999999,"15min":2.9750000000000001},"min":{"1min":0.22500000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5879999999999992,"5min":91.325000000000003,"15min":91.325000000000003},"last":8.2940000000000005},{"interface":"front","average":{"1min":1.119,"5min":2.3599999999999999,"15min":2.996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2489999999999997,"5min":91.799999999999997,"15min":91.799999999999997},"last":0.60299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:53:22 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":4.9960000000000004,"15min":2.7799999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":21.753,"5min":151.43799999999999,"15min":151.43799999999999},"last":3.2010000000000001},{"interface":"front","average":{"1min":2.8900000000000001,"5min":5.04,"15min":2.3580000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":23.725000000000001,"5min":152.20699999999999,"15min":152.20699999999999},"last":3.258}]},{"osd":4,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.415,"15min":1.7629999999999999},"min":{"1min":0.29999999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7960000000000003,"5min":8.7230000000000008,"15min":8.7230000000000008},"last":3.294},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.3100000000000001,"15min":2.282},"min":{"1min":0.36599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.5350000000000001,"5min":8.5350000000000001,"15min":8.5350000000000001},"last":8.3190000000000008}]},{"osd":5,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":3.2170000000000001,"15min":3.1869999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.5090000000000003,"5min":163.352,"15min":163.352},"last":8.2669999999999995},{"interface":"front","average":{"1min":1.629,"5min":3.2810000000000001,"15min":3.0270000000000001},"min":{"1min":0.36699999999999999,"5min":0.316,"15min":0.30299999999999999},"max":{"1min":5.1909999999999998,"5min":163.089,"15min":163.089},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.367,"15min":2.6779999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.8920000000000003,"5min":6.3449999999999998,"15min":7.476},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.744,"15min":2.6600000000000001},"min":{"1min":0.34499999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.6580000000000004,"5min":9.5220000000000002,"15min":12.595000000000001},"last":2.4460000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:53:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4129999999999998,"5min":3.4860000000000002,"15min":2.8820000000000001},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.6630000000000003,"5min":160.08099999999999,"15min":160.08099999999999},"last":2.488},{"interface":"front","average":{"1min":3.008,"5min":3.427,"15min":3.0979999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.5559999999999992,"5min":160.13800000000001,"15min":160.13800000000001},"last":8.2170000000000005}]}]},{"osd":3,"up_from":148,"seq":635655159882,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462620,"kb_used_data":198548,"kb_used_omap":2,"kb_used_meta":264061,"kb_avail":93282532,"statfs":{"total":95995035648,"available":95521312768,"internally_reserved":0,"allocated":203313152,"data_stored":202455837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":270398517},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":7.391,"5min":5.2789999999999999,"15min":11.314},"min":{"1min":0.215,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":95.507999999999996,"5min":253.68100000000001,"15min":253.68100000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":7.1139999999999999,"5min":5.0640000000000001,"15min":10.493},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":95.259,"5min":253.46199999999999,"15min":253.46199999999999},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":5.0099999999999998,"15min":8.6150000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":7.0140000000000002,"5min":172.77099999999999,"15min":172.77099999999999},"last":6.516},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.7919999999999998,"15min":7.8689999999999998},"min":{"1min":0.25,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.692,"5min":172.36600000000001,"15min":172.36600000000001},"last":3.548}]},{"osd":2,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.3100000000000001,"15min":1.1279999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":3.3159999999999998,"5min":11.317,"15min":11.317},"last":0.315},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.409,"15min":1.097},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.3049999999999997,"5min":8.8819999999999997,"15min":8.8819999999999997},"last":0.27200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.5609999999999999,"15min":1.641},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":5.9749999999999996,"5min":8.4749999999999996,"15min":8.4749999999999996},"last":1.179},{"interface":"front","average":{"1min":1.27,"5min":1.571,"15min":2.0009999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.7930000000000001,"5min":16.408999999999999,"15min":16.408999999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0339999999999998,"5min":1.847,"15min":1.399},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.2949999999999999,"5min":12.057,"15min":12.057},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.1099999999999999,"5min":2.008,"15min":1.974},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":5.274,"5min":14.054,"15min":14.054},"last":2.0539999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.8859999999999999,"15min":1.9430000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6909999999999998,"5min":11.632,"15min":11.632},"last":19.568000000000001},{"interface":"front","average":{"1min":2.0649999999999999,"5min":1.879,"15min":2.387},"min":{"1min":0.40300000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5309999999999997,"5min":16.451000000000001,"15min":16.451000000000001},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:53:20 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.024,"15min":1.7170000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.7110000000000003,"5min":12.25,"15min":12.25},"last":0.432},{"interface":"front","average":{"1min":2.3940000000000001,"5min":1.883,"15min":2.1949999999999998},"min":{"1min":0.42399999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.3639999999999999,"5min":11.561999999999999,"15min":11.561999999999999},"last":0.42099999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986495,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":407832,"kb_used_data":56720,"kb_used_omap":0,"kb_used_meta":351104,"kb_avail":93337320,"statfs":{"total":95995035648,"available":95577415680,"internally_reserved":0,"allocated":58081280,"data_stored":57468115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":359530496},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":8.3989999999999991,"5min":3.831,"15min":2.6000000000000001},"min":{"1min":0.22600000000000001,"5min":0.216,"15min":0.214},"max":{"1min":146.25299999999999,"5min":146.25299999999999,"15min":146.25299999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":7.9160000000000004,"5min":3.7010000000000001,"15min":2.262},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22600000000000001},"max":{"1min":144.77000000000001,"5min":144.77000000000001,"15min":144.77000000000001},"last":4.7199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:54:08 2023","interfaces":[{"interface":"back","average":{"1min":6.0999999999999996,"5min":6.0999999999999996,"15min":6.0999999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":75.445999999999998,"5min":75.445999999999998,"15min":75.445999999999998},"last":2.0219999999999998},{"interface":"front","average":{"1min":6.4199999999999999,"5min":6.4199999999999999,"15min":6.4199999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.734999999999999,"5min":71.734999999999999,"15min":71.734999999999999},"last":3.1040000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.1819999999999999,"15min":1.3009999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.25},"max":{"1min":8.2400000000000002,"5min":8.6639999999999997,"15min":8.6639999999999997},"last":3.6619999999999999},{"interface":"front","average":{"1min":1.944,"5min":1.478,"15min":2.0899999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.4509999999999996,"5min":7.4509999999999996,"15min":12.773999999999999},"last":3.0230000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:23 2023","interfaces":[{"interface":"back","average":{"1min":11.016,"5min":5.0640000000000001,"15min":6.7539999999999996},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":142.28399999999999,"5min":142.28399999999999,"15min":142.28399999999999},"last":2.391},{"interface":"front","average":{"1min":11.807,"5min":5.3140000000000001,"15min":6.5659999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":142.34700000000001,"5min":142.34700000000001,"15min":142.34700000000001},"last":3.5739999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:53:54 2023","interfaces":[{"interface":"back","average":{"1min":4.7409999999999997,"5min":1.665,"15min":1.2130000000000001},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":72.304000000000002,"5min":72.304000000000002,"15min":72.304000000000002},"last":5.2089999999999996},{"interface":"front","average":{"1min":5.3659999999999997,"5min":2.0070000000000001,"15min":1.419},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":72.290000000000006,"5min":72.290000000000006,"15min":72.290000000000006},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 20:53:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":1.1100000000000001,"15min":1.135},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28000000000000003},"max":{"1min":5.2220000000000004,"5min":7.4109999999999996,"15min":16.271000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.337,"5min":0.99299999999999999,"15min":1.0700000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.30399999999999999},"max":{"1min":8.1609999999999996,"5min":8.1609999999999996,"15min":11.718},"last":0.91400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:53:54 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":1.966,"15min":1.5189999999999999},"min":{"1min":0.372,"5min":0.34300000000000003,"15min":0.313},"max":{"1min":10.872,"5min":73.268000000000001,"15min":73.268000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.9930000000000001,"5min":2.0299999999999998,"15min":1.4319999999999999},"min":{"1min":0.33600000000000002,"5min":0.33400000000000002,"15min":0.33300000000000002},"max":{"1min":11.916,"5min":69.760000000000005,"15min":69.760000000000005},"last":3.3319999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724858,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":704128,"kb_used_data":243320,"kb_used_omap":0,"kb_used_meta":460672,"kb_avail":93041024,"statfs":{"total":95995035648,"available":95274008576,"internally_reserved":0,"allocated":249159680,"data_stored":248558557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":471728128},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.7890000000000001,"15min":2.1299999999999999},"min":{"1min":0.25700000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":4.8680000000000003,"5min":124.873,"15min":124.873},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.829,"5min":2.8119999999999998,"15min":1.9330000000000001},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":9.2789999999999999,"5min":123.554,"15min":123.554},"last":0.66600000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:52:09 2023","interfaces":[{"interface":"back","average":{"1min":11.34,"5min":4.9880000000000004,"15min":3.407},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":184.31200000000001,"5min":184.31200000000001,"15min":323.541},"last":0.745},{"interface":"front","average":{"1min":11.313000000000001,"5min":5.0149999999999997,"15min":3.6469999999999998},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":184.67400000000001,"5min":184.67400000000001,"15min":324.072},"last":0.75900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.9279999999999999,"15min":2.2090000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":4.9059999999999997,"5min":12.585000000000001,"15min":13.394},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.869,"5min":1.391,"15min":1.619},"min":{"1min":0.313,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.8929999999999998,"5min":7.8929999999999998,"15min":14.263},"last":0.51300000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6240000000000001,"5min":5.4279999999999999,"15min":3.0510000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":8.5839999999999996,"5min":198.33099999999999,"15min":198.33099999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.2309999999999999,"5min":5.4930000000000003,"15min":2.8170000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.202,"5min":198.47300000000001,"15min":198.47300000000001},"last":0.59799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:53:51 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":1.228,"15min":1.0109999999999999},"min":{"1min":0.29399999999999998,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":5.3230000000000004,"5min":10.906000000000001,"15min":10.906000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.8500000000000001,"5min":1.5509999999999999,"15min":1.2529999999999999},"min":{"1min":0.30099999999999999,"5min":0.28299999999999997,"15min":0.26000000000000001},"max":{"1min":12.539,"5min":12.539,"15min":12.539},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:53:16 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.238,"15min":1.1259999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":6.327,"5min":6.327,"15min":8.3010000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.7429999999999999,"5min":1.407,"15min":1.1279999999999999},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":8.6219999999999999,"5min":8.9870000000000001,"15min":9.6370000000000005},"last":0.36099999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:53:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.6719999999999999,"15min":1.4239999999999999},"min":{"1min":0.35399999999999998,"5min":0.33200000000000002,"15min":0.28199999999999997},"max":{"1min":12.669,"5min":12.669,"15min":14.217000000000001},"last":13.83},{"interface":"front","average":{"1min":2.7850000000000001,"5min":1.5780000000000001,"15min":1.1399999999999999},"min":{"1min":0.432,"5min":0.34200000000000003,"15min":0.28699999999999998},"max":{"1min":13.773999999999999,"5min":13.773999999999999,"15min":13.773999999999999},"last":15.634}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":75169792,"data_stored":75169792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1057820672,"data_stored":1057816600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":55050240,"data_stored":55050240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":185335808,"data_stored":185335808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":49545216,"data_stored":49545216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":246878208,"data_stored":246874136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":49610752,"data_stored":49610752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":242814976,"data_stored":242810904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:54:11.480 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T20:54:11.481 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:11.707 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:11.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:11.760 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:11.782 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:11.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:11.862 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:12.069 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:12.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:12.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:12.084 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13271/13851 objects degraded (95.813%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:12.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13271/13851 objects degraded (95.813%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:12.184 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13271/13851 objects degraded (95.813%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:12.258 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:12.375 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:12.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:12.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:12.480 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:12.543 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:12.628 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:12.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:12.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:12.740 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:12.822 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:13.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:13.407 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:13.407 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:13 smithi121 ceph-mon[211526]: pgmap v698: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 226 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 11381/10881 objects degraded (104.595%); 0 B/s, 73 objects/s recovering 2023-09-23T20:54:13.439 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:54:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:13 smithi121 ceph-mon[219709]: pgmap v698: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 226 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 11381/10881 objects degraded (104.595%); 0 B/s, 73 objects/s recovering 2023-09-23T20:54:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:13.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:13 smithi123 ceph-mon[193903]: pgmap v698: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 226 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 11381/10881 objects degraded (104.595%); 0 B/s, 73 objects/s recovering 2023-09-23T20:54:14.564 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:14.598 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:14.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:14.718 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:14.758 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:14.759 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:14.783 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:54:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:54:14] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-09-23T20:54:14.826 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:14.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:14.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:14.941 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:14.993 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.067 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:15.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:15.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:15.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:15.308 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.643 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 102885 objects 2023-09-23T20:54:15.643 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :151.131 2023-09-23T20:54:15.744 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:15.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:15.848 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:15.877 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:15.965 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:54:15.965 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:54:15.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.965 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205707 9.e e218/207 9:7e760b73:::benchmark_data_smithi121_293770_object102819:head v 218'12951, mlcod=198'8593)", 2023-09-23T20:54:15.965 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.538420+0000", 2023-09-23T20:54:15.965 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.038432754, 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038467526000000002, 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538420+0000", 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538420+0000", 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538420+0000", 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538424+0000", 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5730000000000001e-06 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538432+0000", 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5900000000000002e-06 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538436+0000", 2023-09-23T20:54:15.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2189999999999998e-06 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544005+0000", 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0055694469999999999 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544037+0000", 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1241000000000001e-05 2023-09-23T20:54:15.969 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205709 9.6 e218/207 9:6ff7d07f:::benchmark_data_smithi121_293770_object102821:head v 218'12655, mlcod=198'8378)", 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.538577+0000", 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.038275243, 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038388512999999999, 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:15.970 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538577+0000", 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538577+0000", 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538577+0000", 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538585+0000", 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2640000000000008e-06 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538600+0000", 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5511000000000001e-05 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.538608+0000", 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7400000000000004e-06 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.549169+0000", 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010560945 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.549194+0000", 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5083e-05 2023-09-23T20:54:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205714 9.e e218/207 9:7e60b702:::benchmark_data_smithi121_293770_object102826:head v 218'12952, mlcod=198'8593)", 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.542221+0000", 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.034631784999999998, 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.034803784999999997, 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.542221+0000", 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.542221+0000", 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.542219+0000", 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.542233+0000", 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4302e-05 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.542249+0000", 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5982e-05 2023-09-23T20:54:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.542260+0000", 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0869999999999999e-05 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.553388+0000", 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011127259 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.553415+0000", 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7637999999999999e-05 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205718 9.6 e218/207 9:6a62ded2:::benchmark_data_smithi121_293770_object102830:head v 218'12656, mlcod=198'8378)", 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.544446+0000", 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.032406650000000002, 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032642135000000003, 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544446+0000", 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544446+0000", 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544445+0000", 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544455+0000", 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9420000000000007e-06 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544465+0000", 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0006e-05 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.544471+0000", 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2020000000000001e-06 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566566+0000", 2023-09-23T20:54:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022095114999999999 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566599+0000", 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2450999999999998e-05 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205723 9.6 e218/207 9:60231312:::benchmark_data_smithi121_293770_object102835:head v 218'12657, mlcod=198'8378)", 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.555402+0000", 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.021450918999999999, 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.021754493, 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.555402+0000", 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.555402+0000", 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.555401+0000", 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:54:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.555409+0000", 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9039999999999998e-06 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.555420+0000", 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.08e-05 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.555428+0000", 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5639999999999999e-06 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.572986+0000", 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017558141999999999 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.573320+0000", 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00033419500000000002 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205725 9.f e218/207 9:fbff7492:::benchmark_data_smithi121_293770_object102837:head v 218'13047, mlcod=198'8616)", 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.560754+0000", 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.016098944, 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016458621999999999, 2023-09-23T20:54:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560754+0000", 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560754+0000", 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560752+0000", 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560766+0000", 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3909e-05 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560781+0000", 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4192999999999999e-05 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560789+0000", 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5879999999999998e-06 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560852+0000", 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3227999999999998e-05 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.560879+0000", 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6174e-05 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205732 9.6 e218/207)", 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.566828+0000", 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.010024965, 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010459487, 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566828+0000", 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566828+0000", 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566827+0000", 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566839+0000", 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.183e-05 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566849+0000", 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8679999999999994e-06 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.566856+0000", 2023-09-23T20:54:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9489999999999999e-06 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205730 9.f e218/207 9:f36dbbe4:::benchmark_data_smithi121_293770_object102842:head v 218'13048, mlcod=198'8616)", 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.570129+0000", 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0067237850000000004, 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072131180000000001, 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570129+0000", 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570129+0000", 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570128+0000", 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570138+0000", 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0071000000000001e-05 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570152+0000", 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4019000000000001e-05 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570164+0000", 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1959e-05 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570814+0000", 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00064992299999999995 2023-09-23T20:54:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.570837+0000", 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3297000000000001e-05 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:205733 9.4 e218/207 9:242cf782:::benchmark_data_smithi121_293770_object102845:head v 218'13002, mlcod=198'8557)", 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:15.572939+0000", 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0039136379999999997, 2023-09-23T20:54:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0044674529999999997, 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.572939+0000", 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.572939+0000", 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.572938+0000", 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.572945+0000", 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5410000000000003e-06 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.572955+0000", 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.011e-05 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.572962+0000", 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1090000000000003e-06 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.573023+0000", 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1226e-05 2023-09-23T20:54:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:15.573049+0000", 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5826999999999999e-05 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 9 2023-09-23T20:54:16.003 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:54:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:15 smithi121 ceph-mon[211526]: pgmap v699: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 116 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 11381/5613 objects degraded (202.761%); 0 B/s, 63 objects/s recovering 2023-09-23T20:54:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:15 smithi121 ceph-mon[219709]: pgmap v699: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 116 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 11381/5613 objects degraded (202.761%); 0 B/s, 63 objects/s recovering 2023-09-23T20:54:16.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:15 smithi123 ceph-mon[193903]: pgmap v699: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 116 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 11381/5613 objects degraded (202.761%); 0 B/s, 63 objects/s recovering 2023-09-23T20:54:17.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11381/5613 objects degraded (202.761%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:17.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/33283291' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:17.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:16 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:17.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11381/5613 objects degraded (202.761%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:17.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/33283291' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:17.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:16 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11381/5613 objects degraded (202.761%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/33283291' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:16 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:54:17.726 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:54:18.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[219709]: pgmap v700: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 57 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11381/2805 objects degraded (405.740%); 0 B/s, 97 objects/s recovering 2023-09-23T20:54:18.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:18.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:18.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[219709]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:54:18.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[211526]: pgmap v700: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 57 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11381/2805 objects degraded (405.740%); 0 B/s, 97 objects/s recovering 2023-09-23T20:54:18.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:18.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:18.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:17 smithi121 ceph-mon[211526]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:54:18.070 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:18.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:18.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:18.175 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:18.206 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:17 smithi123 ceph-mon[193903]: pgmap v700: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 57 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11381/2805 objects degraded (405.740%); 0 B/s, 97 objects/s recovering 2023-09-23T20:54:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:17 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:17 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:54:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:17 smithi123 ceph-mon[193903]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:54:18.927 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:54:19.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:18 smithi121 ceph-mon[211526]: failed to encode map e219 with expected crc 2023-09-23T20:54:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:18 smithi121 ceph-mon[219709]: failed to encode map e219 with expected crc 2023-09-23T20:54:19.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:18 smithi123 ceph-mon[193903]: failed to encode map e219 with expected crc 2023-09-23T20:54:19.563 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:19.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:19.667 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:19.694 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:20.092 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:19 smithi121 ceph-mon[219709]: failed to encode map e219 with expected crc 2023-09-23T20:54:20.092 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:19 smithi121 ceph-mon[219709]: pgmap v702: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 30 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 11381/1500 objects degraded (758.733%); 0 B/s, 51 objects/s recovering 2023-09-23T20:54:20.093 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:19 smithi121 ceph-mon[211526]: failed to encode map e219 with expected crc 2023-09-23T20:54:20.093 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:19 smithi121 ceph-mon[211526]: pgmap v702: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 30 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 11381/1500 objects degraded (758.733%); 0 B/s, 51 objects/s recovering 2023-09-23T20:54:20.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:19 smithi123 ceph-mon[193903]: failed to encode map e219 with expected crc 2023-09-23T20:54:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:19 smithi123 ceph-mon[193903]: pgmap v702: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 30 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 11381/1500 objects degraded (758.733%); 0 B/s, 51 objects/s recovering 2023-09-23T20:54:20.494 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:20.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:20.582 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:20.599 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:20.632 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:20.916 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:20.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:20.988 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:54:21.008 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:54:21.021 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:21.029 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:54:21.052 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:54:21.053 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:21.076 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:54:21.095 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:54:21.103 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:21.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:21.115 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:54:21.133 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.135 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:54:21.179 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 9 pgp_num to 16 2023-09-23T20:54:21.210 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:21.244 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:21.975 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: pgmap v703: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 11381/72 objects degraded (15806.944%); 0 B/s, 45 objects/s recovering 2023-09-23T20:54:21.975 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1966299634' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:54:21.975 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:54:21.975 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1557411066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:21.975 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11381/72 objects degraded (15806.944%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:21.975 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: failed to encode map e220 with expected crc 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[211526]: failed to encode map e220 with expected crc 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: pgmap v703: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 11381/72 objects degraded (15806.944%); 0 B/s, 45 objects/s recovering 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1966299634' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1557411066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11381/72 objects degraded (15806.944%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:54:21.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:54:21.977 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: failed to encode map e220 with expected crc 2023-09-23T20:54:21.977 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:21 smithi121 ceph-mon[219709]: failed to encode map e220 with expected crc 2023-09-23T20:54:22.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:54:22.153 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:22.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:22.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: pgmap v703: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 11381/72 objects degraded (15806.944%); 0 B/s, 45 objects/s recovering 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1966299634' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1557411066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11381/72 objects degraded (15806.944%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:54:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: failed to encode map e220 with expected crc 2023-09-23T20:54:22.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:21 smithi123 ceph-mon[193903]: failed to encode map e220 with expected crc 2023-09-23T20:54:22.261 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:22.306 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:22.351 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:22.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:22.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:22.483 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:22.921 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T20:54:22.921 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T20:54:22.951 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:22.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:23.057 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:23.069 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[211526]: failed to encode map e220 with expected crc 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[211526]: failed to encode map e220 with expected crc 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[211526]: failed to encode map e221 with expected crc 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[211526]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[219709]: failed to encode map e220 with expected crc 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[219709]: failed to encode map e220 with expected crc 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[219709]: failed to encode map e221 with expected crc 2023-09-23T20:54:23.084 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:22 smithi121 ceph-mon[219709]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:54:23.086 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:23.145 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:23.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:22 smithi123 ceph-mon[193903]: failed to encode map e220 with expected crc 2023-09-23T20:54:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:22 smithi123 ceph-mon[193903]: failed to encode map e220 with expected crc 2023-09-23T20:54:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:22 smithi123 ceph-mon[193903]: failed to encode map e221 with expected crc 2023-09-23T20:54:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:22 smithi123 ceph-mon[193903]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:54:23.250 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:23.279 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:23.432 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:23.432 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:23.433 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:23.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:23.537 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:23.566 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:23.712 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:23.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:23.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:23.844 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:23.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:23 smithi121 ceph-mon[211526]: failed to encode map e221 with expected crc 2023-09-23T20:54:23.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:23 smithi121 ceph-mon[211526]: pgmap v706: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 11381/72 objects degraded (15806.944%); 0 B/s, 238 objects/s recovering 2023-09-23T20:54:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:23 smithi121 ceph-mon[219709]: failed to encode map e221 with expected crc 2023-09-23T20:54:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:23 smithi121 ceph-mon[219709]: pgmap v706: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 11381/72 objects degraded (15806.944%); 0 B/s, 238 objects/s recovering 2023-09-23T20:54:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:23 smithi123 ceph-mon[193903]: failed to encode map e221 with expected crc 2023-09-23T20:54:24.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:23 smithi123 ceph-mon[193903]: pgmap v706: 89 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 11381/72 objects degraded (15806.944%); 0 B/s, 238 objects/s recovering 2023-09-23T20:54:24.289 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:24.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:24.393 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:24.421 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:24.706 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:24.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:24.729 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:54:24.788 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:24.810 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:24.847 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:24.952 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:54:24.976 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:54:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:54:24] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-09-23T20:54:25.012 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:25.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:25.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:25.118 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:25.153 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:25.323 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:25.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:25.426 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:25.452 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:25.497 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:25.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:25.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:25.604 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:25.631 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:25 smithi121 ceph-mon[211526]: pgmap v707: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 9509/72 objects degraded (13206.944%); 0 B/s, 329 objects/s recovering 2023-09-23T20:54:25.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:25 smithi121 ceph-mon[219709]: pgmap v707: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 9509/72 objects degraded (13206.944%); 0 B/s, 329 objects/s recovering 2023-09-23T20:54:26.029 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:26.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:26.133 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:26.165 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:26.222 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:25 smithi123 ceph-mon[193903]: pgmap v707: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 9509/72 objects degraded (13206.944%); 0 B/s, 329 objects/s recovering 2023-09-23T20:54:26.316 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:26.326 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:26.342 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:26.351 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:26.396 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:26.502 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:26.534 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:26.589 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:26.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:26.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:26.693 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:26.727 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:27.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 9509/72 objects degraded (13206.944%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:27.052 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[211526]: failed to encode map e222 with expected crc 2023-09-23T20:54:27.052 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[211526]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:54:27.052 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2883843568' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:27.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 9509/72 objects degraded (13206.944%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:27.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[219709]: failed to encode map e222 with expected crc 2023-09-23T20:54:27.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[219709]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:54:27.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2883843568' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:27.223 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_4' removed 2023-09-23T20:54:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 9509/72 objects degraded (13206.944%), 5 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:54:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:26 smithi123 ceph-mon[193903]: failed to encode map e222 with expected crc 2023-09-23T20:54:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:26 smithi123 ceph-mon[193903]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:54:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2883843568' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:27.240 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:54:27.350 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:27.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:27.467 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:27.494 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:27.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: failed to encode map e222 with expected crc 2023-09-23T20:54:27.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: failed to encode map e222 with expected crc 2023-09-23T20:54:27.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: pgmap v709: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9509/72 objects degraded (13206.944%); 0 B/s, 415 objects/s recovering 2023-09-23T20:54:27.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2355892881' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:54:27.896 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2883843568' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: failed to encode map e222 with expected crc 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: failed to encode map e222 with expected crc 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: pgmap v709: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9509/72 objects degraded (13206.944%); 0 B/s, 415 objects/s recovering 2023-09-23T20:54:27.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2355892881' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:54:27.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:54:27.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:54:27.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2883843568' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:27.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:54:27.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:54:27.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:27.898 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:27.921 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:54:27.922 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:27.922 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:54:27.922 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.095933 2023-09-23T20:54:27.922 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0.09593300059992138 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: failed to encode map e222 with expected crc 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: failed to encode map e222 with expected crc 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: pgmap v709: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9509/72 objects degraded (13206.944%); 0 B/s, 415 objects/s recovering 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2355892881' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2883843568' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:54:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:54:28.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:28.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:28.945 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:28.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:28.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:29.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:29.081 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:29.129 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:29.132 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:28 smithi123 ceph-mon[193903]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.132 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:28 smithi123 ceph-mon[193903]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.132 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:28 smithi123 ceph-mon[193903]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.132 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:28 smithi123 ceph-mon[193903]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.132 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:28 smithi123 ceph-mon[193903]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:29.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:29.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[211526]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[211526]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[211526]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[211526]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[211526]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[219709]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[219709]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[219709]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[219709]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:28 smithi121 ceph-mon[219709]: failed to encode map e223 with expected crc 2023-09-23T20:54:29.234 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:29.264 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:29.406 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:54:29.425 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:54:29.443 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:54:29.463 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:54:29.482 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:54:29.501 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:54:29.520 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:54:29.539 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:54:29.540 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.541 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.541 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.541 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.541 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.541 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.541 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.541 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:54:29.837 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:54:29.837 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:54:29.837 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T20:54:29.838 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T20:54:30.062 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:30.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[211526]: pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9509/72 objects degraded (13206.944%), 5 pgs degraded, 6 pgs undersized) 2023-09-23T20:54:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T20:54:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2144602987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[219709]: pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9509/72 objects degraded (13206.944%), 5 pgs degraded, 6 pgs undersized) 2023-09-23T20:54:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T20:54:30.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2144602987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:30.166 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:30.192 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:30.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:29 smithi123 ceph-mon[193903]: pgmap v711: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:29 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9509/72 objects degraded (13206.944%), 5 pgs degraded, 6 pgs undersized) 2023-09-23T20:54:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:29 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T20:54:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2144602987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:30.537 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:30.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:30.642 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:30.662 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:30.671 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:30.690 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:30.775 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:30.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:30.880 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:54:30.880 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:30.912 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:30.959 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:30.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:30.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:166463 9.0 9.42c9e230 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:08.188245+0000", 2023-09-23T20:54:30.995 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 82.802861562000004, 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98743947099999996, 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 166463 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:30.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.188245+0000", 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.188245+0000", 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.188243+0000", 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:30.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.188251+0000", 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6559999999999992e-06 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.188262+0000", 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1508999999999999e-05 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.188286+0000", 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3601e-05 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.192994+0000", 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047082449999999998 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.193006+0000", 2023-09-23T20:54:30.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.153e-05 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:09.175676+0000", 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98267080500000004 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:09.175684+0000", 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9440000000000005e-06 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.000 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:166468 9.0 9.25c6440 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:08.192328+0000", 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 82.798778881000004, 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.983374686, 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.001 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 166468 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.192328+0000", 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.192328+0000", 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.192326+0000", 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.192333+0000", 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6529999999999997e-06 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.003 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.192345+0000", 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2294e-05 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.192356+0000", 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.064e-05 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.193029+0000", 2023-09-23T20:54:31.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00067328099999999997 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.193043+0000", 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4341999999999999e-05 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:09.175697+0000", 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98265406399999999 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.005 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:09.175702+0000", 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.87e-06 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:166479 9.0 9.4ba88fc0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:08.212309+0000", 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 82.778797576000002, 2023-09-23T20:54:31.006 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96340960600000003, 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 166479 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212309+0000", 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212309+0000", 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212307+0000", 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212315+0000", 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9240000000000007e-06 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212330+0000", 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4465e-05 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212337+0000", 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3080000000000002e-06 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212385+0000", 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8522000000000001e-05 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:08.212394+0000", 2023-09-23T20:54:31.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0820000000000005e-06 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:09.175714+0000", 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96331922199999998 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:09.175718+0000", 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7949999999999998e-06 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.011 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183429 9.0 9:0342b256:::benchmark_data_smithi121_293770_object80541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.553444+0000", 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.437662113999998, 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.050896018, 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.012 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183429 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553444+0000", 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553444+0000", 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553442+0000", 2023-09-23T20:54:31.014 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553454+0000", 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1804000000000001e-05 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553469+0000", 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5109999999999999e-05 2023-09-23T20:54:31.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553479+0000", 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5689999999999994e-06 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553536+0000", 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6740000000000002e-05 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.553544+0000", 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.242e-06 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.573526+0000", 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0199826000000001 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.573677+0000", 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000150314 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.574127+0000", 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00045048400000000002 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.601925+0000", 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027797961999999999 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.018 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.603909+0000", 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019835439999999998 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604205+0000", 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00029630500000000002 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604294+0000", 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9077000000000001e-05 2023-09-23T20:54:31.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604340+0000", 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6279999999999997e-05 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183443 9.0 9:02f2768a:::benchmark_data_smithi121_293770_object80555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.582263+0000", 2023-09-23T20:54:31.020 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.408843470999997, 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0222440129999999, 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183443 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.582263+0000", 2023-09-23T20:54:31.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.582263+0000", 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.582261+0000", 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.582289+0000", 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8268000000000001e-05 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.582307+0000", 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7540000000000001e-05 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.582316+0000", 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5040000000000008e-06 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.585159+0000", 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002842453 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.585171+0000", 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1615e-05 2023-09-23T20:54:31.024 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.589292+0000", 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.004121109 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.589441+0000", 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014910499999999999 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.589587+0000", 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000146718 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.601969+0000", 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012381094 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.603951+0000", 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019825229999999999 2023-09-23T20:54:31.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604406+0000", 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00045510899999999998 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604458+0000", 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2213999999999998e-05 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604507+0000", 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8572000000000002e-05 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183493 9.0 9:09ca936c:::benchmark_data_smithi121_293770_object80605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.654547+0000", 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.336559297999997, 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.97208736699999998, 2023-09-23T20:54:31.028 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183493 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654547+0000", 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654547+0000", 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654545+0000", 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654552+0000", 2023-09-23T20:54:31.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.674e-06 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654565+0000", 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3285000000000001e-05 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654574+0000", 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0399999999999998e-06 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654628+0000", 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3600999999999997e-05 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.654638+0000", 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9709999999999999e-06 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.591618+0000", 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.93697981799999996 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.032 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.591719+0000", 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000100765 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.591795+0000", 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6508999999999995e-05 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.601977+0000", 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010182190000000001 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604010+0000", 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020332530000000001 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626551+0000", 2023-09-23T20:54:31.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022540144000000002 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626595+0000", 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4855000000000003e-05 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626635+0000", 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9032000000000002e-05 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.035 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183502 9.0 9:01abbf3f:::benchmark_data_smithi121_293770_object80614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.664156+0000", 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.326950498999999, 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96257715499999996, 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.036 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183502 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664156+0000", 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664156+0000", 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664154+0000", 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664162+0000", 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9379999999999992e-06 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664179+0000", 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7516999999999998e-05 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664187+0000", 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6699999999999994e-06 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664247+0000", 2023-09-23T20:54:31.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9704000000000003e-05 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.664258+0000", 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1059e-05 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.593609+0000", 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.929351276 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.040 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.593735+0000", 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000126117 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.593870+0000", 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013429699999999999 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.603918+0000", 2023-09-23T20:54:31.041 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010048286999999999 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604072+0000", 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015425500000000001 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626668+0000", 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022595482 2023-09-23T20:54:31.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626704+0000", 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6310999999999998e-05 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626733+0000", 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9227e-05 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.043 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183550 9.0 9:088c1e86:::benchmark_data_smithi121_293770_object80662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.736284+0000", 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.254822021999999, 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89054461299999998, 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.044 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183550 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736284+0000", 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736284+0000", 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736283+0000", 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736293+0000", 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9289999999999993e-06 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736309+0000", 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6232999999999999e-05 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736318+0000", 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7949999999999996e-06 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736372+0000", 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3869000000000002e-05 2023-09-23T20:54:31.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.736379+0000", 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4479999999999997e-06 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.595663+0000", 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.85928429699999997 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.595809+0000", 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000145162 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.595972+0000", 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000163518 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.603922+0000", 2023-09-23T20:54:31.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079498639999999992 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604115+0000", 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000193282 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626764+0000", 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022648964000000001 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626799+0000", 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4613000000000002e-05 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626829+0000", 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0165999999999999e-05 2023-09-23T20:54:31.051 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183566 9.0 9:0f0ece37:::benchmark_data_smithi121_293770_object80678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.758382+0000", 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.232724429999998, 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86854301899999997, 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.052 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183566 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758382+0000", 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758382+0000", 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758380+0000", 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.054 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758389+0000", 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4200000000000007e-06 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758406+0000", 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7909000000000001e-05 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758416+0000", 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.38e-06 2023-09-23T20:54:31.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758483+0000", 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7121999999999999e-05 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758494+0000", 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0777e-05 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.599129+0000", 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.84063561399999998 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.599216+0000", 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7139999999999996e-05 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.599324+0000", 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107559 2023-09-23T20:54:31.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.603965+0000", 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046408209999999998 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604154+0000", 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018946800000000001 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626859+0000", 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022705158999999999 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626895+0000", 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5312999999999999e-05 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626925+0000", 2023-09-23T20:54:31.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0258999999999999e-05 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183567 9.0 9:01652130:::benchmark_data_smithi121_293770_object80679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.758432+0000", 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.232674535999998, 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86858606100000002, 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.060 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183567 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758432+0000", 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758432+0000", 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758431+0000", 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758434+0000", 2023-09-23T20:54:31.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8609999999999998e-06 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758480+0000", 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5297999999999999e-05 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758487+0000", 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1289999999999998e-06 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758517+0000", 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9876999999999999e-05 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.758523+0000", 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5930000000000002e-06 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.602010+0000", 2023-09-23T20:54:31.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.84348679199999999 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.602092+0000", 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2464000000000002e-05 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.602174+0000", 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1246000000000001e-05 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604162+0000", 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019880480000000001 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604560+0000", 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039867100000000002 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626956+0000", 2023-09-23T20:54:31.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022395475000000001 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.626990+0000", 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4025e-05 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627018+0000", 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8036999999999999e-05 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.067 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183612 9.0 9:004c6cb8:::benchmark_data_smithi121_293770_object80724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.828550+0000", 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.162556524999999, 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79856276199999998, 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.068 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183612 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828550+0000", 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828550+0000", 2023-09-23T20:54:31.069 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828548+0000", 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828557+0000", 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8939999999999999e-06 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828574+0000", 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6778000000000001e-05 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828583+0000", 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7800000000000006e-06 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828650+0000", 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6866000000000007e-05 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.828664+0000", 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4823e-05 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604607+0000", 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.77594237799999999 2023-09-23T20:54:31.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604753+0000", 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014604200000000001 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.604889+0000", 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013647400000000001 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.073 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.615257+0000", 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010367968 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.615275+0000", 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7697999999999999e-05 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627049+0000", 2023-09-23T20:54:31.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011774237999999999 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627085+0000", 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5478000000000002e-05 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627113+0000", 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7994e-05 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.075 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183627 9.0 9:009bf688:::benchmark_data_smithi121_293770_object80739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.852365+0000", 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.138741947, 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.77484390999999997, 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.076 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183627 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852365+0000", 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852365+0000", 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852363+0000", 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.078 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852372+0000", 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9239999999999996e-06 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852389+0000", 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7893999999999998e-05 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852400+0000", 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0358e-05 2023-09-23T20:54:31.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852472+0000", 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1891999999999995e-05 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.852485+0000", 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3066e-05 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.608366+0000", 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.75588097300000001 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.608525+0000", 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000159221 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.608682+0000", 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015741600000000001 2023-09-23T20:54:31.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.615280+0000", 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065975039999999997 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.620771+0000", 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054916080000000003 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.082 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627144+0000", 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063721070000000001 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627179+0000", 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.506e-05 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627208+0000", 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9791e-05 2023-09-23T20:54:31.083 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183697 9.0 9:01bce3ee:::benchmark_data_smithi121_293770_object80809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.967046+0000", 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.024059989999998, 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.66029594999999996, 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.084 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183697 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967046+0000", 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967046+0000", 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967045+0000", 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967051+0000", 2023-09-23T20:54:31.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3999999999999997e-06 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967065+0000", 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3361e-05 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967073+0000", 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9680000000000006e-06 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.087 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967150+0000", 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7711000000000002e-05 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.967162+0000", 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1137e-05 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.612127+0000", 2023-09-23T20:54:31.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.64496558199999998 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.612253+0000", 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000126222 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.612385+0000", 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013142300000000001 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.089 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.620781+0000", 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0083961969999999993 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627243+0000", 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064618990000000001 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627278+0000", 2023-09-23T20:54:31.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5389999999999998e-05 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627313+0000", 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4949e-05 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627342+0000", 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9164999999999999e-05 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.091 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183704 9.0 9:03868d23:::benchmark_data_smithi121_293770_object80816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:38.977571+0000", 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.013535263999998, 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.66149503399999998, 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.092 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183704 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977571+0000", 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977571+0000", 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977570+0000", 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977576+0000", 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3779999999999998e-06 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977589+0000", 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2500999999999999e-05 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977597+0000", 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9039999999999992e-06 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977654+0000", 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7376000000000001e-05 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:38.977667+0000", 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3317e-05 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.615303+0000", 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63763563499999998 2023-09-23T20:54:31.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.615430+0000", 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012708000000000001 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.615554+0000", 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012356800000000001 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627376+0000", 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011822605 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638845+0000", 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011468742000000001 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638977+0000", 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013168000000000001 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.639024+0000", 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7315999999999999e-05 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.639066+0000", 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2372e-05 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183728 9.0 9:0925e652:::benchmark_data_smithi121_293770_object80840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:39.016618+0000", 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 51.974488673000003, 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.62910453300000002, 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183728 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016618+0000", 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016618+0000", 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016616+0000", 2023-09-23T20:54:31.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016624+0000", 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7840000000000004e-06 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016637+0000", 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3222e-05 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016645+0000", 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7349999999999996e-06 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016701+0000", 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6471999999999997e-05 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.016712+0000", 2023-09-23T20:54:31.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0973e-05 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.618050+0000", 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.60133806999999995 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.618177+0000", 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012636599999999999 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.618317+0000", 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013994100000000001 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638833+0000", 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.020516816 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638928+0000", 2023-09-23T20:54:31.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4629000000000005e-05 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645636+0000", 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067075850000000003 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645681+0000", 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5896000000000002e-05 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645722+0000", 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0887999999999999e-05 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183733 9.0 9:005e1062:::benchmark_data_smithi121_293770_object80845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:39.026528+0000", 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 51.964577959000003, 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.61929567200000002, 2023-09-23T20:54:31.107 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183733 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026528+0000", 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026528+0000", 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026527+0000", 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026534+0000", 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3969999999999997e-06 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026548+0000", 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3679999999999999e-05 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026554+0000", 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3899999999999998e-06 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026613+0000", 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8278e-05 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.026625+0000", 2023-09-23T20:54:31.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1810999999999999e-05 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.620800+0000", 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.59417524799999999 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.620898+0000", 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7782000000000007e-05 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.620986+0000", 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8781000000000003e-05 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638880+0000", 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017893643000000001 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638934+0000", 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4098000000000003e-05 2023-09-23T20:54:31.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645760+0000", 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068255299999999998 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645792+0000", 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2725999999999998e-05 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645824+0000", 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1838000000000002e-05 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183745 9.0 9:0ac869de:::benchmark_data_smithi121_293770_object80857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:39.049178+0000", 2023-09-23T20:54:31.115 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 51.941928138999998, 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.59674073900000002, 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183745 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049178+0000", 2023-09-23T20:54:31.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049178+0000", 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049177+0000", 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049190+0000", 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3563e-05 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049208+0000", 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7706e-05 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049217+0000", 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0580000000000004e-06 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049307+0000", 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9801000000000003e-05 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.049320+0000", 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2928000000000001e-05 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.623657+0000", 2023-09-23T20:54:31.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.57433763000000004 2023-09-23T20:54:31.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.623761+0000", 2023-09-23T20:54:31.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010415499999999999 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.623844+0000", 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3121000000000006e-05 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638920+0000", 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015075361000000001 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.638940+0000", 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0338000000000001e-05 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645859+0000", 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069184229999999999 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645891+0000", 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2277000000000003e-05 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645919+0000", 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8191000000000002e-05 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183768 9.0 9:0366c102:::benchmark_data_smithi121_293770_object80880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:39.114150+0000", 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 51.876956679999999, 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.53188292699999995, 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183768 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.125 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114150+0000", 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114150+0000", 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114148+0000", 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114155+0000", 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3810000000000004e-06 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114167+0000", 2023-09-23T20:54:31.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2112e-05 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114173+0000", 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2899999999999999e-06 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114233+0000", 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9673999999999998e-05 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.114246+0000", 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3444999999999999e-05 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627397+0000", 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.51315144099999999 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627517+0000", 2023-09-23T20:54:31.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119121 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.627636+0000", 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119726 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.639108+0000", 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011471927 2023-09-23T20:54:31.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.642324+0000", 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032155640000000002 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645958+0000", 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036340980000000001 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.645997+0000", 2023-09-23T20:54:31.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9403999999999997e-05 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.646033+0000", 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5395999999999998e-05 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:183774 9.0 9:064ebf65:::benchmark_data_smithi121_293770_object80886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:39.142706+0000", 2023-09-23T20:54:31.132 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 51.848400257999998, 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.50956941899999997, 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 183774 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142706+0000", 2023-09-23T20:54:31.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142706+0000", 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142705+0000", 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:31.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142710+0000", 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.186e-06 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142722+0000", 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2072000000000001e-05 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142728+0000", 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0619999999999997e-06 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142774+0000", 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6125999999999998e-05 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.142783+0000", 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8750000000000006e-06 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.630568+0000", 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.48778522000000002 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.630680+0000", 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000111771 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:54:31.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.630793+0000", 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112897 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.639153+0000", 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.008359722 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.642534+0000", 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003380826 2023-09-23T20:54:31.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.652164+0000", 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096298749999999995 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.652223+0000", 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9746e-05 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:39.652276+0000", 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2299999999999997e-05 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:184798 9.6 e213/207 9:61fc159e:::benchmark_data_smithi121_293770_object81910:head v 213'11335, mlcod=198'8378)", 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:41.140325+0000", 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 49.850781120999997, 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23167979599999999, 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:31.140 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140325+0000", 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140325+0000", 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140325+0000", 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140329+0000", 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7730000000000001e-06 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:31.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140335+0000", 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6250000000000001e-06 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140340+0000", 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.267e-06 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140600+0000", 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00025996800000000002 2023-09-23T20:54:31.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.140620+0000", 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0367e-05 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.371963+0000", 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23134315599999999 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:31.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:41.372005+0000", 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1928999999999998e-05 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:31.145 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:54:31.145 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:31.175 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.221 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:31.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:31.329 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:31.356 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.399 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:31.506 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:31.533 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.574 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:31.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:31.680 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:31.706 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.747 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:31.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:31.856 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:31.881 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:31.917 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:54:31.922 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:31.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:32.027 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:32.054 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:32.096 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:32.101 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:31 smithi123 ceph-mon[193903]: pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:32.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:32.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:31 smithi121 ceph-mon[211526]: pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:32.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:31 smithi121 ceph-mon[219709]: pgmap v712: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:32.203 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:32.230 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:32.525 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:54:32.843 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:32 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:54:32.843 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:32 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:32.928 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:32.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:32.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:33.032 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:33.060 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:33.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:32 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:54:33.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:32 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:33.149 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:33.156 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:33.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:33.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:33.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:32 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:54:33.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:32 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:33.261 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:33.286 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:33.289 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:33.299 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:33.486 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:33.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:33.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:33.592 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:33.620 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:33.665 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:33.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:33.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:33.770 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:33.802 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:33.900 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:33.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:33.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:34.004 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:34.033 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:34.078 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:34.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:34.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:34.185 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:34.208 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:34 smithi123 ceph-mon[193903]: pgmap v713: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:34.211 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:34.256 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:34.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:34.298 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:34 smithi121 ceph-mon[211526]: pgmap v713: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:34.299 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:34 smithi121 ceph-mon[219709]: pgmap v713: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:34.365 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:34.396 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:34.764 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_5' created 2023-09-23T20:54:34.766 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.4 primary-affinity to 0.095933 (862872) 2023-09-23T20:54:35.062 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:54:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:54:34] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.43.0" 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3354897144' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3312268298' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.09593300059992138}]: dispatch 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: pgmap v714: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3312268298' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.09593300059992138}]': finished 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:54:35.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[211526]: failed to encode map e224 with expected crc 2023-09-23T20:54:35.071 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:35.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:35.178 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:35.209 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:35.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3354897144' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:54:35.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:54:35.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3312268298' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.09593300059992138}]: dispatch 2023-09-23T20:54:35.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: pgmap v714: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:35.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:54:35.320 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3312268298' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.09593300059992138}]': finished 2023-09-23T20:54:35.320 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:54:35.320 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:35 smithi121 ceph-mon[219709]: failed to encode map e224 with expected crc 2023-09-23T20:54:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3354897144' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:54:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:54:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3312268298' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.09593300059992138}]: dispatch 2023-09-23T20:54:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: pgmap v714: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:54:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3312268298' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.09593300059992138}]': finished 2023-09-23T20:54:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:54:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:35 smithi123 ceph-mon[193903]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.159 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:36.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:36.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:36.290 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: failed to encode map e225 with expected crc 2023-09-23T20:54:36.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[219709]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: failed to encode map e225 with expected crc 2023-09-23T20:54:36.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:36 smithi121 ceph-mon[211526]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:54:36.385 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: failed to encode map e224 with expected crc 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: failed to encode map e225 with expected crc 2023-09-23T20:54:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:36 smithi123 ceph-mon[193903]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:54:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:36.491 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:36.517 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:36.559 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:36.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:36.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:36.695 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:36.738 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:36.844 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:36.870 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:36.912 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:36.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:36.926 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T20:54:37.020 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:37.045 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:37.087 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:37.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:37.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:37.193 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:37.223 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: pgmap v717: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: failed to encode map e226 with expected crc 2023-09-23T20:54:37.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[211526]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: pgmap v717: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: failed to encode map e226 with expected crc 2023-09-23T20:54:37.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:37 smithi121 ceph-mon[219709]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:54:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: failed to encode map e225 with expected crc 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: pgmap v717: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: failed to encode map e226 with expected crc 2023-09-23T20:54:37.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:37 smithi123 ceph-mon[193903]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:54:37.552 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:54:37.553 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:54:37.553 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T20:54:37.553 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:54:37.571 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:37.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:37.675 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:37.707 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:37.720 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:54:37.749 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:54:37.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:54:37.788 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:54:37.807 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:54:37.828 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:54:37.851 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:54:37.861 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:37.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:37.874 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.875 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:54:37.966 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:37.994 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[211526]: failed to encode map e226 with expected crc 2023-09-23T20:54:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3272507964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1426155499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[219709]: failed to encode map e226 with expected crc 2023-09-23T20:54:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3272507964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1426155499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:38.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:38 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:38 smithi123 ceph-mon[193903]: failed to encode map e226 with expected crc 2023-09-23T20:54:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3272507964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1426155499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:38 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:54:38.485 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:54:38.815 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:54:38.992 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:38.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:39.097 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:39.126 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: failed to encode map e227 with expected crc 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[211526]: pgmap v720: 89 pgs: 4 unknown, 85 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: failed to encode map e227 with expected crc 2023-09-23T20:54:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:54:39.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:54:39.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:39 smithi121 ceph-mon[219709]: pgmap v720: 89 pgs: 4 unknown, 85 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: failed to encode map e226 with expected crc 2023-09-23T20:54:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: failed to encode map e227 with expected crc 2023-09-23T20:54:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:54:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:54:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:39 smithi123 ceph-mon[193903]: pgmap v720: 89 pgs: 4 unknown, 85 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:39.709 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:39.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:39.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:39.774 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:39.798 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:39.817 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:39.849 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:39.897 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:39.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:39.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:40.003 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:40.030 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:40.130 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:40.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:40.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:40 smithi123 ceph-mon[193903]: failed to encode map e227 with expected crc 2023-09-23T20:54:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:40 smithi123 ceph-mon[193903]: failed to encode map e227 with expected crc 2023-09-23T20:54:40.234 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:40.266 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:40.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:40 smithi121 ceph-mon[219709]: failed to encode map e227 with expected crc 2023-09-23T20:54:40.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:40 smithi121 ceph-mon[219709]: failed to encode map e227 with expected crc 2023-09-23T20:54:40.296 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:40 smithi121 ceph-mon[211526]: failed to encode map e227 with expected crc 2023-09-23T20:54:40.296 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:40 smithi121 ceph-mon[211526]: failed to encode map e227 with expected crc 2023-09-23T20:54:40.314 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:40.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:40.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:40.420 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:40.449 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:40.494 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:40.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:40.599 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:54:40.628 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:41 smithi121 ceph-mon[211526]: failed to encode map e227 with expected crc 2023-09-23T20:54:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:41 smithi121 ceph-mon[211526]: pgmap v721: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:41 smithi121 ceph-mon[219709]: failed to encode map e227 with expected crc 2023-09-23T20:54:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:41 smithi121 ceph-mon[219709]: pgmap v721: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:41.465 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:54:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:41 smithi123 ceph-mon[193903]: failed to encode map e227 with expected crc 2023-09-23T20:54:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:41 smithi123 ceph-mon[193903]: pgmap v721: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:41.710 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:54:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:41.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:54:41.731 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:54:41.816 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:41.848 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:41.896 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:41.900 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:54:41.900 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:41.900 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:54:41.900 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T20:54:41.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:54:41.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:41.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:42.003 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:42.019 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:54:41 smithi121 systemd[1]: Stopping Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:54:42.031 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.076 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:42.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:42.184 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:42.210 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.221 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:42.252 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:42.359 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:42.391 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.440 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:42.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:42.545 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:42.572 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.601 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:54:42.601 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:54:42.613 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:42.721 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:42.750 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:42.898 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:42.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:42.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:43.003 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:43.030 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:43.075 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:43.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:43.127 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:43.179 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:43.206 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:43.402 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T20:54:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2194887426' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:54:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:54:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[219709]: pgmap v722: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:43.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2194887426' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:54:43.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:54:43.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:43 smithi121 ceph-mon[211526]: pgmap v722: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2194887426' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:54:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:43 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:54:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:43 smithi123 ceph-mon[193903]: pgmap v722: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:44.602 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[211526]: failed to encode map e228 with expected crc 2023-09-23T20:54:44.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[211526]: failed to encode map e228 with expected crc 2023-09-23T20:54:44.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:54:44.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[211526]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:54:44.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[211526]: osd.1 marked itself down and dead 2023-09-23T20:54:44.603 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:54:44 smithi121 conmon[331554]: 2023-09-23T20:54:44.207+0000 7fceccdb7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:54:44.603 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:54:44 smithi121 conmon[331554]: 2023-09-23T20:54:44.207+0000 7fceccdb7700 -1 osd.1 228 *** Got signal Terminated *** 2023-09-23T20:54:44.603 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:54:44 smithi121 conmon[331554]: 2023-09-23T20:54:44.207+0000 7fceccdb7700 -1 osd.1 228 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:54:44.604 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[219709]: failed to encode map e228 with expected crc 2023-09-23T20:54:44.604 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[219709]: failed to encode map e228 with expected crc 2023-09-23T20:54:44.604 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:54:44.604 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[219709]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:54:44.604 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:44 smithi121 ceph-mon[219709]: osd.1 marked itself down and dead 2023-09-23T20:54:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:44 smithi123 ceph-mon[193903]: failed to encode map e228 with expected crc 2023-09-23T20:54:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:44 smithi123 ceph-mon[193903]: failed to encode map e228 with expected crc 2023-09-23T20:54:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:44 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:54:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:44 smithi123 ceph-mon[193903]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:54:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:44 smithi123 ceph-mon[193903]: osd.1 marked itself down and dead 2023-09-23T20:54:44.892 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:54:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:54:44] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.43.0" 2023-09-23T20:54:45.000 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:54:45.113 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:45.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:45.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:45.249 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:45.663 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:54:45 smithi121 bash[357876]: 47e41b5525ab93df0b6493030d551d69c9082934ed6df0977d43df6d053eb377 2023-09-23T20:54:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[211526]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:54:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[211526]: failed to encode map e229 with expected crc 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[211526]: failed to encode map e229 with expected crc 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[211526]: osdmap e229: 8 total, 7 up, 8 in 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[211526]: pgmap v725: 89 pgs: 11 stale+active+clean, 78 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[219709]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[219709]: failed to encode map e229 with expected crc 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[219709]: failed to encode map e229 with expected crc 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[219709]: osdmap e229: 8 total, 7 up, 8 in 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[219709]: pgmap v725: 89 pgs: 11 stale+active+clean, 78 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:45.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:45 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:45 smithi123 ceph-mon[193903]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:54:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:45 smithi123 ceph-mon[193903]: failed to encode map e229 with expected crc 2023-09-23T20:54:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:45 smithi123 ceph-mon[193903]: failed to encode map e229 with expected crc 2023-09-23T20:54:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:45 smithi123 ceph-mon[193903]: osdmap e229: 8 total, 7 up, 8 in 2023-09-23T20:54:45.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:45 smithi123 ceph-mon[193903]: pgmap v725: 89 pgs: 11 stale+active+clean, 78 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:45.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:45 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:45.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:45 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:45.956 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T20:54:45.965 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:54:45.965 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:54:46.026 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T20:54:46.027 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:54:46.027 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_358375 2023-09-23T20:54:46.027 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:54:46.027 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: osdmap e230: 8 total, 7 up, 8 in 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:54:46.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:54:46.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:46 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:46.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: osdmap e230: 8 total, 7 up, 8 in 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:54:46.751 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: osdmap e230: 8 total, 7 up, 8 in 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: failed to encode map e230 with expected crc 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:54:46.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:54:46.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:46.819 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:46.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:46.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:46.925 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:46.968 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:47.029 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 3123 3107 193.828 194.188 0.00339998 0.00514106 2023-09-23T20:54:47.047 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:47.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:47.058 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:47.083 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:54:47.124 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:54:47.156 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:47.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:54:47.212 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:54:47.227 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:47.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:54:47.287 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:54:47.321 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:54:47.321 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:54:47.321 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:54:47.322 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:54:47.322 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:54:47.322 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:54:47.322 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:54:47.322 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:54:47.322 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:54:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:47 smithi123 ceph-mon[193903]: pgmap v727: 89 pgs: 9 active+undersized, 9 stale+active+clean, 71 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1527657600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:47 smithi121 ceph-mon[211526]: pgmap v727: 89 pgs: 9 active+undersized, 9 stale+active+clean, 71 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1527657600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:47 smithi121 ceph-mon[219709]: pgmap v727: 89 pgs: 9 active+undersized, 9 stale+active+clean, 71 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:54:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1527657600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:48.029 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 6580 6564 204.926 216.062 0.00348925 0.0048707 2023-09-23T20:54:48.793 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:54:49.028 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 9973 9957 207.298 212.062 0.00460128 0.0048164 2023-09-23T20:54:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2432774715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:49.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:49.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2432774715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:49.180 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:49.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:49.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:49.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2432774715' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:54:49.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:49.335 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:49.410 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:49.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:49.516 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:49.572 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: pgmap v728: 89 pgs: 4 active+undersized+degraded, 24 active+undersized, 2 stale+active+clean, 59 active+clean; 373 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 1484/17937 objects degraded (8.273%) 2023-09-23T20:54:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 1484/17937 objects degraded (8.273%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2432774715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: osdmap e231: 8 total, 7 up, 8 in 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[211526]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: pgmap v728: 89 pgs: 4 active+undersized+degraded, 24 active+undersized, 2 stale+active+clean, 59 active+clean; 373 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 1484/17937 objects degraded (8.273%) 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 1484/17937 objects degraded (8.273%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:49.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2432774715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: osdmap e231: 8 total, 7 up, 8 in 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: failed to encode map e231 with expected crc 2023-09-23T20:54:49.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:49 smithi121 ceph-mon[219709]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.028 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 12790 12774 199.489 176.062 0.00403319 0.00500718 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: pgmap v728: 89 pgs: 4 active+undersized+degraded, 24 active+undersized, 2 stale+active+clean, 59 active+clean; 373 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 1484/17937 objects degraded (8.273%) 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 1484/17937 objects degraded (8.273%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2432774715' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:54:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: osdmap e231: 8 total, 7 up, 8 in 2023-09-23T20:54:50.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:49 smithi123 ceph-mon[193903]: failed to encode map e231 with expected crc 2023-09-23T20:54:50.475 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:54:50.476 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:54:50.581 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:50.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:50.689 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:50.742 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.029 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 15947 15931 199.05 197.312 0.00330949 0.00501815 2023-09-23T20:54:51.464 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:51.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:51.570 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:51.613 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.722 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:51.829 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:51.873 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:51 smithi123 ceph-mon[193903]: pgmap v730: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 527 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 1486/25359 objects degraded (5.860%) 2023-09-23T20:54:51.879 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:51.980 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:51.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:52.034 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 18651 18635 193.888 169 0.00861349 0.00515231 2023-09-23T20:54:52.087 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19278 8.1 8:842dafa6:::benchmark_data_smithi121_252781_object19277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997279+0000", 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 592.08653768299996, 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.267984131, 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.089 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19278 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997279+0000", 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997279+0000", 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997278+0000", 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997298+0000", 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9559999999999999e-05 2023-09-23T20:54:52.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997306+0000", 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.14e-06 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997311+0000", 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5140000000000001e-06 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997468+0000", 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000157114 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997474+0000", 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.913e-06 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.254448+0000", 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.256973422 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.254599+0000", 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015122399999999999 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.254776+0000", 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017681899999999999 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.258239+0000", 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034635690000000001 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.259023+0000", 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00078313199999999997 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.264771+0000", 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057481930000000004 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.264821+0000", 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0098e-05 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265263+0000", 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00044190400000000002 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:19294 8.1 8:8216d95b:::benchmark_data_smithi121_252781_object19293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 592.08650023600001, 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2685024899999999, 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 19294 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997316+0000", 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997334+0000", 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7994000000000001e-05 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997339+0000", 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997344+0000", 2023-09-23T20:54:52.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2279999999999998e-06 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997493+0000", 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014906399999999999 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:44:59.997500+0000", 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2759999999999997e-06 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.100 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.255558+0000", 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2580583510000001 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.255704+0000", 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000145931 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.255845+0000", 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014099100000000001 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.258428+0000", 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002583049 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.259031+0000", 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00060272199999999998 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265363+0000", 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063324569999999997 2023-09-23T20:54:52.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265422+0000", 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8442000000000001e-05 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:45:01.265819+0000", 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039688400000000002 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95051 8.e 8:737847a6:::benchmark_data_smithi121_252781_object95050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.060916+0000", 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 528.02290070900006, 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2772001780000002, 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95051 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060916+0000", 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060916+0000", 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060915+0000", 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060935+0000", 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9998000000000001e-05 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060942+0000", 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1359999999999999e-06 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060948+0000", 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4689999999999996e-06 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070165+0000", 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092176889999999994 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070178+0000", 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2893e-05 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070395+0000", 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0002171209999999 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070408+0000", 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2425e-05 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.321120+0000", 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.250712338 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.321280+0000", 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016026500000000001 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.321453+0000", 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017232799999999999 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329458+0000", 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080051189999999998 2023-09-23T20:54:52.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329467+0000", 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6409999999999997e-06 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.336925+0000", 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074573410000000001 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.337008+0000", 2023-09-23T20:54:52.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3193000000000001e-05 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.338116+0000", 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0011077330000000001 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95053 8.e 8:7ad97c2c:::benchmark_data_smithi121_252781_object95052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 528.02286382600005, 2023-09-23T20:54:52.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2798300080000002, 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95053 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060952+0000", 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060972+0000", 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0196000000000001e-05 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.060992+0000", 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0049999999999999e-05 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.061000+0000", 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3749999999999997e-06 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070202+0000", 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092024120000000001 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.070210+0000", 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8509999999999999e-06 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070431+0000", 2023-09-23T20:54:52.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0002214380000001 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070440+0000", 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2379999999999997e-06 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.326020+0000", 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.255580801 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.326163+0000", 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014299500000000001 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.326332+0000", 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168989 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333147+0000", 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068145599999999999 2023-09-23T20:54:52.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.338206+0000", 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050593840000000001 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.339743+0000", 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001536765 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.339804+0000", 2023-09-23T20:54:52.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0646000000000002e-05 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.340782+0000", 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00097870100000000001 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95061 8.e 8:74014da4:::benchmark_data_smithi121_252781_object95060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.077670+0000", 2023-09-23T20:54:52.119 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 528.00614660400004, 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.270089563, 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95061 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077670+0000", 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077670+0000", 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077668+0000", 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077698+0000", 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9169000000000001e-05 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077712+0000", 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4589999999999999e-05 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077720+0000", 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.14e-06 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077774+0000", 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3924000000000001e-05 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.077787+0000", 2023-09-23T20:54:52.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.296e-05 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070461+0000", 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.992673625 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070469+0000", 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9300000000000003e-06 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329493+0000", 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2590239009999999 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329630+0000", 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000137701 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.329759+0000", 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000128572 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.336891+0000", 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071319629999999998 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.340812+0000", 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0039211530000000001 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.346685+0000", 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0058733609999999997 2023-09-23T20:54:52.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.346742+0000", 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6841e-05 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.347759+0000", 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001016996 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:95065 8.e 8:75a28b09:::benchmark_data_smithi121_252781_object95064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:46:04.082096+0000", 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 528.00172072400005, 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2666930719999998, 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 95065 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.128 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082096+0000", 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082096+0000", 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082095+0000", 2023-09-23T20:54:52.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082112+0000", 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7521e-05 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082124+0000", 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1742999999999999e-05 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082132+0000", 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9570000000000002e-06 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082182+0000", 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0015999999999997e-05 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:04.082191+0000", 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7900000000000005e-06 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070491+0000", 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98829999700000004 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:05.070498+0000", 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5549999999999997e-06 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333194+0000", 2023-09-23T20:54:52.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2626954370000001 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333342+0000", 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148679 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.333508+0000", 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000165308 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.341077+0000", 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075696369999999997 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.343593+0000", 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025160489999999998 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.347824+0000", 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042309499999999998 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.347898+0000", 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3291e-05 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:46:06.348789+0000", 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089107699999999999 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.135 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:186211 8.c 8:3aecfc72:::benchmark_data_smithi121_252781_object77036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:06.495533+0000", 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 405.58828362999998, 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1146059930000001, 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.136 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 186211 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495533+0000", 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495533+0000", 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495532+0000", 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495535+0000", 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.869e-06 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495540+0000", 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4280000000000004e-06 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495545+0000", 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5820000000000003e-06 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495659+0000", 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114418 2023-09-23T20:54:52.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495791+0000", 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013153700000000001 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 7", 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.495928+0000", 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013725 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.504256+0000", 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0083274019999999994 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.510300+0000", 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0060443040000000003 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.510331+0000", 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0778000000000002e-05 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.512602+0000", 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.00227154 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.513103+0000", 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050099699999999999 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600139+0000", 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.087035502 2023-09-23T20:54:52.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600301+0000", 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000162118 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.610027+0000", 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097265790000000008 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.610139+0000", 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000111156 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:186219 8.c 8:3b0c0d24:::benchmark_data_smithi121_252781_object77044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:06.514378+0000", 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 405.56943871499999, 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0982041680000001, 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.144 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 186219 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514378+0000", 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514378+0000", 2023-09-23T20:54:52.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514376+0000", 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514383+0000", 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.584e-06 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514395+0000", 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2095e-05 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514404+0000", 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0799999999999995e-06 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514455+0000", 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0735000000000001e-05 2023-09-23T20:54:52.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:06.514467+0000", 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2680000000000001e-05 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.513156+0000", 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99868875800000001 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:07.513168+0000", 2023-09-23T20:54:52.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1637000000000001e-05 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600577+0000", 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0874094480000001 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600669+0000", 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1785000000000003e-05 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:54:52.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.600751+0000", 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2384000000000003e-05 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.606048+0000", 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052968579999999998 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.608509+0000", 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002460907 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.612298+0000", 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037888959999999999 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.612357+0000", 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8764999999999997e-05 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:08.612582+0000", 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00022486899999999999 2023-09-23T20:54:52.151 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57325 9.b 9:d45aa29e:::benchmark_data_smithi121_293770_object57324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.096490+0000", 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 244.98732605999999, 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1300021230000001, 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.152 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57325 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096490+0000", 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096490+0000", 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096488+0000", 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096544+0000", 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6341999999999997e-05 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.154 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096578+0000", 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3216999999999997e-05 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096589+0000", 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1024e-05 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096651+0000", 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2706999999999994e-05 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.155 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096803+0000", 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015159799999999999 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096949+0000", 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000146267 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.099329+0000", 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023796609999999999 2023-09-23T20:54:52.156 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101731+0000", 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024025470000000001 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.110981+0000", 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092501100000000006 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.110997+0000", 2023-09-23T20:54:52.157 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5291999999999999e-05 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106431+0000", 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99543410600000004 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106445+0000", 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3692e-05 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.158 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.226393+0000", 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1199482549999999 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.226492+0000", 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9518999999999994e-05 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.159 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57369 9.b 9:da8768ce:::benchmark_data_smithi121_293770_object57368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.166112+0000", 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 244.91770421499999, 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.080740686, 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57369 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.160 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166112+0000", 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166112+0000", 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166111+0000", 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:52.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166157+0000", 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5760000000000002e-05 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166167+0000", 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0291e-05 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166174+0000", 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7240000000000003e-06 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166228+0000", 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4268999999999998e-05 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166242+0000", 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4525999999999999e-05 2023-09-23T20:54:52.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106469+0000", 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.94022618300000005 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106477+0000", 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0709999999999995e-06 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.229385+0000", 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1229085350000001 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.229579+0000", 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019396100000000001 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.229909+0000", 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00033003399999999999 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.165 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234210+0000", 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0043013569999999996 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241345+0000", 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071342230000000003 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243943+0000", 2023-09-23T20:54:52.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025985019999999999 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.244003+0000", 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0109000000000003e-05 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.246853+0000", 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028494240000000001 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.167 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57386 9.b 9:d32d2c48:::benchmark_data_smithi121_293770_object57385:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.180559+0000", 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 244.90325696799999, 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0718573739999999, 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57386 2023-09-23T20:54:52.168 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180559+0000", 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180559+0000", 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.169 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180557+0000", 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180583+0000", 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5701999999999999e-05 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180602+0000", 2023-09-23T20:54:52.170 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8366e-05 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180611+0000", 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9350000000000006e-06 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180673+0000", 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1908000000000004e-05 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.171 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180688+0000", 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4799e-05 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106499+0000", 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92581111500000002 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106507+0000", 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.43e-06 2023-09-23T20:54:52.172 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231970+0000", 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.125463221 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.232077+0000", 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106759 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:54:52.173 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.232212+0000", 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013490799999999999 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234246+0000", 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002034826 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.246919+0000", 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012672951 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249731+0000", 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028111719999999998 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249795+0000", 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4807000000000005e-05 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252417+0000", 2023-09-23T20:54:52.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026213289999999999 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57391 9.b 9:da3371e4:::benchmark_data_smithi121_293770_object57390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.183791+0000", 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 244.90002520600001, 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0859722039999999, 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.176 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57391 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183791+0000", 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183791+0000", 2023-09-23T20:54:52.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183790+0000", 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183838+0000", 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8195999999999997e-05 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183858+0000", 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9692999999999999e-05 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183867+0000", 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7189999999999996e-06 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183941+0000", 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4167999999999999e-05 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183958+0000", 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7071e-05 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106686+0000", 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92272719299999995 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106694+0000", 2023-09-23T20:54:52.180 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6060000000000003e-06 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234279+0000", 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1275844180000001 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234430+0000", 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015113799999999999 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:54:52.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234598+0000", 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168276 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243796+0000", 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091980459999999997 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252432+0000", 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086362450000000007 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.267140+0000", 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01470714 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.267194+0000", 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4255000000000001e-05 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.269763+0000", 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025694730000000001 2023-09-23T20:54:52.183 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57396 9.b 9:d278708d:::benchmark_data_smithi121_293770_object57395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.189786+0000", 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 244.89403057800001, 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0825362539999999, 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.184 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57396 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189786+0000", 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189786+0000", 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.185 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189784+0000", 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189825+0000", 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0500999999999997e-05 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189839+0000", 2023-09-23T20:54:52.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3815000000000001e-05 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189848+0000", 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5889999999999992e-06 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189915+0000", 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7034999999999998e-05 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189929+0000", 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4034e-05 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107460+0000", 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.91753040399999997 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107470+0000", 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0519999999999999e-05 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.188 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236631+0000", 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1291604989999999 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236763+0000", 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013231000000000001 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236906+0000", 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014250299999999999 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.189 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243834+0000", 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069287050000000003 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252439+0000", 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086044020000000006 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.269825+0000", 2023-09-23T20:54:52.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017385838000000001 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.269875+0000", 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0822999999999998e-05 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.272322+0000", 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024465889999999999 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.191 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57397 9.b 9:d3fe843c:::benchmark_data_smithi121_293770_object57396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 244.89282002499999, 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.084046802, 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57397 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.192 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:54:52.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191048+0000", 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2011000000000001e-05 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191057+0000", 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0659999999999993e-06 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191064+0000", 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8260000000000004e-06 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191098+0000", 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4050000000000001e-05 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191104+0000", 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9869999999999996e-06 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.195 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107494+0000", 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.91639063600000004 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107502+0000", 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.238908+0000", 2023-09-23T20:54:52.196 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.131405518 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239063+0000", 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015518300000000001 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239231+0000", 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168252 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.197 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243871+0000", 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046402170000000003 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252445+0000", 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085733660000000007 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.272373+0000", 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019928461000000001 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.198 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.272431+0000", 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7723000000000003e-05 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.275043+0000", 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026121289999999999 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.199 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57398 9.b 9:de1ab5d1:::benchmark_data_smithi121_293770_object57397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 244.89264791799999, 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.089526835, 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57398 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.200 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:54:52.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191210+0000", 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2212000000000002e-05 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191221+0000", 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1318000000000001e-05 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.203 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191228+0000", 2023-09-23T20:54:52.203 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5339999999999996e-06 2023-09-23T20:54:52.203 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.203 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191253+0000", 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5191e-05 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191257+0000", 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2169999999999996e-06 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.204 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107526+0000", 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.91626845800000001 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107535+0000", 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7690000000000008e-06 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241373+0000", 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.133838047 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241524+0000", 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015076400000000001 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241701+0000", 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017661599999999999 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243909+0000", 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002208485 2023-09-23T20:54:52.206 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252451+0000", 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085418309999999997 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.278101+0000", 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025650435999999999 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.278155+0000", 2023-09-23T20:54:52.207 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3356999999999998e-05 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.280695+0000", 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025403700000000001 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175048 9.9 9:98e06086:::benchmark_data_smithi121_293770_object72160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 87.873954204, 2023-09-23T20:54:52.208 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43337474100000001, 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175048 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.209 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209865+0000", 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9799999999999998e-06 2023-09-23T20:54:52.210 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209882+0000", 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7142e-05 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209888+0000", 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.691e-06 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226284+0000", 2023-09-23T20:54:52.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01639606 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226451+0000", 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000167028 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226596+0000", 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014432099999999999 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.212 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232724+0000", 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061281840000000001 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.235331+0000", 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026065340000000002 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643145+0000", 2023-09-23T20:54:52.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.40781475099999998 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643197+0000", 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1427000000000001e-05 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643237+0000", 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0119e-05 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.214 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175054 9.5 9:a0da263b:::benchmark_data_smithi121_293770_object72166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.213383+0000", 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 87.870433457000004, 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41714561900000002, 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.215 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175054 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.213383+0000", 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.213383+0000", 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.213381+0000", 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.213389+0000", 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9340000000000006e-06 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.213403+0000", 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3730999999999999e-05 2023-09-23T20:54:52.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.213412+0000", 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8400000000000001e-06 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.215861+0000", 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024494759999999999 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.216045+0000", 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000183828 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.216925+0000", 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00088027999999999999 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.230647+0000", 2023-09-23T20:54:52.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013721130999999999 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232420+0000", 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0017738820000000001 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.630425+0000", 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39800505400000002 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.220 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.630482+0000", 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6362e-05 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.630528+0000", 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6567999999999999e-05 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175062 9.5 9:af0bbe63:::benchmark_data_smithi121_293770_object72174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.223421+0000", 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 87.860395259000001, 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41800432799999998, 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.222 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175062 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223421+0000", 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223421+0000", 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.223 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223420+0000", 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223426+0000", 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4750000000000001e-06 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223440+0000", 2023-09-23T20:54:52.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3971000000000001e-05 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223447+0000", 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5379999999999997e-06 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223496+0000", 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8513000000000001e-05 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223640+0000", 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014410900000000001 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223777+0000", 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000137279 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232561+0000", 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0087839670000000002 2023-09-23T20:54:52.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232568+0000", 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9580000000000001e-06 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641331+0000", 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.4087635 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641388+0000", 2023-09-23T20:54:52.227 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6258000000000002e-05 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641425+0000", 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7688999999999998e-05 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175077 9.9 9:9d65ef46:::benchmark_data_smithi121_293770_object72189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.256170+0000", 2023-09-23T20:54:52.228 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 87.827645821999994, 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.38723719499999998, 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175077 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256170+0000", 2023-09-23T20:54:52.229 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256170+0000", 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256169+0000", 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.230 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256176+0000", 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3080000000000002e-06 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256194+0000", 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7558999999999999e-05 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256203+0000", 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6469999999999993e-06 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256254+0000", 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0484e-05 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256407+0000", 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000152996 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.256543+0000", 2023-09-23T20:54:52.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013633699999999999 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.260737+0000", 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041939250000000003 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.261331+0000", 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00059431399999999998 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643331+0000", 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.38199953599999997 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643367+0000", 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6452999999999998e-05 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643408+0000", 2023-09-23T20:54:52.234 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0365999999999999e-05 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:196331 9.5 9:a9740c3c:::benchmark_data_smithi121_293770_object93443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:59.620499+0000", 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.463317488000001, 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23859586499999999, 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:54:52.235 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 196331 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.620499+0000", 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:54:52.236 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.620499+0000", 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.620498+0000", 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.620501+0000", 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6189999999999998e-06 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.620507+0000", 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7100000000000004e-06 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.620513+0000", 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2750000000000001e-06 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:54:52.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.630325+0000", 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098123559999999995 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.630460+0000", 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013444600000000001 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.630540+0000", 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9888999999999994e-05 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.634648+0000", 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041086550000000001 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.638534+0000", 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0038857190000000002 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.859000+0000", 2023-09-23T20:54:52.240 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22046627599999999 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.859051+0000", 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1088000000000003e-05 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:59.859095+0000", 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3204000000000002e-05 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.241 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.242 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:54:52.242 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:54:52.242 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:54:52.242 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:51 smithi121 ceph-mon[211526]: pgmap v730: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 527 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 1486/25359 objects degraded (5.860%) 2023-09-23T20:54:52.242 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:51 smithi121 ceph-mon[219709]: pgmap v730: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 527 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 1486/25359 objects degraded (5.860%) 2023-09-23T20:54:52.255 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:52.363 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:52.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:52.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:52.469 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:52.476 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:54:52.528 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:52.632 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:52.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:52.737 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:52.787 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:52.880 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:52.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:52.988 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:53.015 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:53.033 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 21012 20996 187.275 147.562 0.00346507 0.00529998 2023-09-23T20:54:53.594 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:54:53.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:53 smithi121 ceph-mon[211526]: pgmap v731: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 527 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 1486/25359 objects degraded (5.860%) 2023-09-23T20:54:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:53 smithi121 ceph-mon[219709]: pgmap v731: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 527 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 1486/25359 objects degraded (5.860%) 2023-09-23T20:54:54.034 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 23198 23182 180.947 136.625 0.00196335 0.0054717 2023-09-23T20:54:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:53 smithi123 ceph-mon[193903]: pgmap v731: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 527 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 1486/25359 objects degraded (5.860%) 2023-09-23T20:54:54.235 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:54.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:54.300 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:54:54.340 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:54.384 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:54.475 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:54.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:54.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:54.497 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T20:54:54.587 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:54.597 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:54:54.597 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T20:54:54.633 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:54.656 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:54.712 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:54.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:54.745 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:54.782 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:54:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:54:54] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.43.0" 2023-09-23T20:54:54.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:54.874 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:54 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4170/66765 objects degraded (6.246%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:55.031 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:54 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4170/66765 objects degraded (6.246%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:55.031 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:54 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4170/66765 objects degraded (6.246%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:54:55.035 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 25155 25139 174.405 122.312 0.0101694 0.00572756 2023-09-23T20:54:55.111 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:54:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:55.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:54:55.217 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:54:55.266 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:55.327 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:54:55.366 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:54:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:55.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:54:55.473 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:55.531 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:56.035 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 27517 27501 171.728 147.625 0.00166084 0.00580203 2023-09-23T20:54:56.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:55 smithi121 ceph-mon[211526]: pgmap v732: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 4170/66765 objects degraded (6.246%) 2023-09-23T20:54:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:55 smithi121 ceph-mon[219709]: pgmap v732: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 4170/66765 objects degraded (6.246%) 2023-09-23T20:54:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:55 smithi123 ceph-mon[193903]: pgmap v732: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 4170/66765 objects degraded (6.246%) 2023-09-23T20:54:56.718 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:56.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:56.824 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:56.851 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:57.035 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 15 29534 29519 167.585 126.125 0.00500863 0.00596264 2023-09-23T20:54:57.309 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:57.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:57.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:57 smithi121 ceph-mon[211526]: pgmap v733: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 4170/66765 objects degraded (6.246%) 2023-09-23T20:54:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:57 smithi121 ceph-mon[219709]: pgmap v733: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 4170/66765 objects degraded (6.246%) 2023-09-23T20:54:57.413 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:57.478 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:57.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:57 smithi123 ceph-mon[193903]: pgmap v733: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 4170/66765 objects degraded (6.246%) 2023-09-23T20:54:57.697 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:57.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:57.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:57.803 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:57.853 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:57.949 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:57.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:57.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:58.036 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 31704 31688 164.902 135.562 0.0038751 0.00605954 2023-09-23T20:54:58.060 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:58.115 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:58.202 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:58.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:58.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:58.307 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:54:58.333 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:58.434 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:54:58.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:58.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:54:58.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:54:58.607 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:58.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:54:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:54:58.777 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:54:58.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:58.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:54:58.883 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:54:58.934 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.019 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:54:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:59.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:54:59.037 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 33427 33411 160.503 107.688 0.00869396 0.00622449 2023-09-23T20:54:59.126 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:54:59.127 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:54:59.173 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:54:59.188 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.210 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:54:59.263 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:54:59.292 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:54:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:59.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:54:59.302 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:54:59.336 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:54:59.373 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:54:59.373 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:54:59.373 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:54:59.374 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:54:59.374 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:54:59.374 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:54:59.374 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:54:59.374 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:54:59.374 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:54:59.402 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:54:59.447 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.469 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:59 smithi121 ceph-mon[211526]: pgmap v734: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 6099/89943 objects degraded (6.781%) 2023-09-23T20:54:59.470 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:54:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1472808647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:59.470 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:59 smithi121 ceph-mon[219709]: pgmap v734: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 6099/89943 objects degraded (6.781%) 2023-09-23T20:54:59.470 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:54:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1472808647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:54:59.541 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:54:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:54:59.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:54:59.598 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:54:59.598 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:54:59.598 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:54:59.598 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.767426 2023-09-23T20:54:59.598 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0.7674256641262868 2023-09-23T20:54:59.652 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:54:59.714 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:54:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:59 smithi123 ceph-mon[193903]: pgmap v734: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 6099/89943 objects degraded (6.781%) 2023-09-23T20:54:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:54:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1472808647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:00.037 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 35144 35128 156.706 107.312 0.0085372 0.00637704 2023-09-23T20:55:00.156 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:00.156 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:00.180 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:00.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:00.285 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:00.336 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:00.437 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:55:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3784191952' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3784191952' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3784191952' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:01.038 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 36701 36685 152.748 97.3125 0.00647739 0.00653448 2023-09-23T20:55:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: osdmap e232: 8 total, 7 up, 8 in 2023-09-23T20:55:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: pgmap v736: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 6099/97767 objects degraded (6.238%) 2023-09-23T20:55:01.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6099/89943 objects degraded (6.781%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:01.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: osdmap e232: 8 total, 7 up, 8 in 2023-09-23T20:55:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: pgmap v736: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 6099/97767 objects degraded (6.238%) 2023-09-23T20:55:01.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6099/89943 objects degraded (6.781%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: failed to encode map e232 with expected crc 2023-09-23T20:55:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: osdmap e232: 8 total, 7 up, 8 in 2023-09-23T20:55:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: pgmap v736: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 6099/97767 objects degraded (6.238%) 2023-09-23T20:55:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6099/89943 objects degraded (6.781%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:01.795 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:55:01.803 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:01.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:01.839 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:01.840 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:01.910 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:01.956 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:01.982 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:02.035 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:02.037 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 38673 38657 150.905 123.25 0.00244052 0.00661811 2023-09-23T20:55:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:02.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:02.141 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:02.189 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:02.288 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:02.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:02.396 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:02.448 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:02.682 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:02.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:02.787 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:02.844 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:02.937 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:02.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:03.039 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 40288 40272 147.966 100.938 0.00608063 0.00671692 2023-09-23T20:55:03.046 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:03.084 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:03.125 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:03.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:03.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:03.232 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:03.278 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:03.554 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:03.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:03.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:03.607 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:03.662 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:03.708 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:03.764 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:55:03.802 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:03.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:03.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:03.841 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:55:03.914 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:03 smithi123 ceph-mon[193903]: pgmap v737: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 6099/97767 objects degraded (6.238%) 2023-09-23T20:55:04.006 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:04.044 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 41735 41719 144.717 90.4375 0.00496918 0.00690432 2023-09-23T20:55:04.080 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:03 smithi121 ceph-mon[211526]: pgmap v737: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 6099/97767 objects degraded (6.238%) 2023-09-23T20:55:04.081 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:03 smithi121 ceph-mon[219709]: pgmap v737: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 6099/97767 objects degraded (6.238%) 2023-09-23T20:55:04.583 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:04.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:04.688 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:04.751 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:04.850 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:04.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:04.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:04.886 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:55:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:55:04] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2023-09-23T20:55:04.886 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3492991157' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]: dispatch 2023-09-23T20:55:04.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3492991157' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]: dispatch 2023-09-23T20:55:04.888 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]: dispatch 2023-09-23T20:55:04.898 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.0 primary-affinity to 0.767426 (8502942) 2023-09-23T20:55:04.959 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:04.992 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.044 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 43290 43274 142.217 97.1875 0.00272679 0.00697992 2023-09-23T20:55:05.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]: dispatch 2023-09-23T20:55:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3492991157' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]: dispatch 2023-09-23T20:55:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]: dispatch 2023-09-23T20:55:05.299 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:05.402 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:05.432 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.478 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:05.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:05.583 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:05.610 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.652 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:05.762 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:05.788 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:05.887 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:05.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:05.992 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:06.026 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:06.044 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:55:06.045006+0000 min lat: 0.00106683 max lat: 1.02878 avg lat: 0.00735494 2023-09-23T20:55:06.044 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:55:06.044 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 43531 43515 135.864 15.0625 0.00764726 0.00735494 2023-09-23T20:55:06.059 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[211526]: pgmap v738: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 7706/123264 objects degraded (6.252%) 2023-09-23T20:55:06.059 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[211526]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.059 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[211526]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.059 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]': finished 2023-09-23T20:55:06.060 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[211526]: osdmap e233: 8 total, 7 up, 8 in 2023-09-23T20:55:06.060 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[211526]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.060 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[219709]: pgmap v738: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 7706/123264 objects degraded (6.252%) 2023-09-23T20:55:06.060 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[219709]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.060 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[219709]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.060 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]': finished 2023-09-23T20:55:06.061 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[219709]: osdmap e233: 8 total, 7 up, 8 in 2023-09-23T20:55:06.061 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:05 smithi121 ceph-mon[219709]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.065 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:06.167 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:06.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:06.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:05 smithi123 ceph-mon[193903]: pgmap v738: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 7706/123264 objects degraded (6.252%) 2023-09-23T20:55:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:05 smithi123 ceph-mon[193903]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:05 smithi123 ceph-mon[193903]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.7674256641262868}]': finished 2023-09-23T20:55:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:05 smithi123 ceph-mon[193903]: osdmap e233: 8 total, 7 up, 8 in 2023-09-23T20:55:06.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:05 smithi123 ceph-mon[193903]: failed to encode map e233 with expected crc 2023-09-23T20:55:06.277 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:06.334 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:06.359 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:06.494 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:06.494 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:06.494 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:06.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:06.598 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:06.656 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:06.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: osdmap e234: 8 total, 7 up, 8 in 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7706/123264 objects degraded (6.252%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[219709]: pgmap v741: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 7706/123264 objects degraded (6.252%) 2023-09-23T20:55:06.899 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: osdmap e234: 8 total, 7 up, 8 in 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: failed to encode map e234 with expected crc 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7706/123264 objects degraded (6.252%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:06.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:06 smithi121 ceph-mon[211526]: pgmap v741: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 7706/123264 objects degraded (6.252%) 2023-09-23T20:55:07.044 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 45276 45260 134.587 109.062 0.00919208 0.00742556 2023-09-23T20:55:07.110 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:07.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:07.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: failed to encode map e234 with expected crc 2023-09-23T20:55:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: failed to encode map e234 with expected crc 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: osdmap e234: 8 total, 7 up, 8 in 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: failed to encode map e234 with expected crc 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: failed to encode map e234 with expected crc 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: failed to encode map e234 with expected crc 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: failed to encode map e234 with expected crc 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: failed to encode map e234 with expected crc 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7706/123264 objects degraded (6.252%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:06 smithi123 ceph-mon[193903]: pgmap v741: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 7706/123264 objects degraded (6.252%) 2023-09-23T20:55:07.264 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:07.350 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:07.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:07.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:07.455 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:07.481 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:07.567 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:07.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:07.681 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:07.707 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:08.046 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 46586 46570 132.193 81.875 0.00738628 0.00756012 2023-09-23T20:55:08.257 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:08.264 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:55:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:08.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:08.369 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:08.459 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:08.790 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:08.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:08.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:08.895 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:08.938 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:09.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 48131 48115 130.644 96.5625 0.00684304 0.00764989 2023-09-23T20:55:09.278 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:09.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:09.384 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:09.444 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:09.630 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:09.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:09.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:09.678 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:09.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:09.788 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:09.864 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:09.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:09.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:09.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:09 smithi121 ceph-mon[211526]: pgmap v742: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s; 9024/138042 objects degraded (6.537%) 2023-09-23T20:55:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:09 smithi121 ceph-mon[219709]: pgmap v742: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s; 9024/138042 objects degraded (6.537%) 2023-09-23T20:55:09.973 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:10.043 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:10.044 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 49706 49690 129.303 98.4375 0.0129161 0.00772834 2023-09-23T20:55:10.142 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:10.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:09 smithi123 ceph-mon[193903]: pgmap v742: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s; 9024/138042 objects degraded (6.537%) 2023-09-23T20:55:10.247 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:10.275 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:55:10.320 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:55:10.324 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:10.365 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:55:10.403 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:55:10.443 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:55:10.478 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:55:10.516 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:10.517 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:55:10.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:55:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:10.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:10.624 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:10.673 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:10.749 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:10.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:10.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:10.856 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:10.856 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:55:10.867 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:55:10.867 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:10.868 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T20:55:10.868 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:55:10.868 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:55:10.868 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T20:55:10.913 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:11.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 51287 51271 128.084 98.8125 0.00879729 0.00780232 2023-09-23T20:55:11.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1871104863' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/536676547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:11.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1871104863' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:11.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/536676547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1871104863' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/536676547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:11.526 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:11.987 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:11.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:11.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:12.002 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:11 smithi123 ceph-mon[193903]: pgmap v743: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 9024/142677 objects degraded (6.325%) 2023-09-23T20:55:12.002 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:11 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:12.003 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:11 smithi123 ceph-mon[193903]: failed to encode map e235 with expected crc 2023-09-23T20:55:12.003 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:11 smithi123 ceph-mon[193903]: failed to encode map e235 with expected crc 2023-09-23T20:55:12.003 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1871104863' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:12.003 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:11 smithi123 ceph-mon[193903]: osdmap e235: 8 total, 7 up, 8 in 2023-09-23T20:55:12.003 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 9024/142677 objects degraded (6.325%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:12.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 52687 52671 126.524 87.5 0.0214719 0.00789838 2023-09-23T20:55:12.067 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:12.068 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:12.095 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:12.162 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:12.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[211526]: pgmap v743: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 9024/142677 objects degraded (6.325%) 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[211526]: failed to encode map e235 with expected crc 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[211526]: failed to encode map e235 with expected crc 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1871104863' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[211526]: osdmap e235: 8 total, 7 up, 8 in 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 9024/142677 objects degraded (6.325%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[219709]: pgmap v743: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s; 9024/142677 objects degraded (6.325%) 2023-09-23T20:55:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[219709]: failed to encode map e235 with expected crc 2023-09-23T20:55:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[219709]: failed to encode map e235 with expected crc 2023-09-23T20:55:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1871104863' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[219709]: osdmap e235: 8 total, 7 up, 8 in 2023-09-23T20:55:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 9024/142677 objects degraded (6.325%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:12.262 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:12.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:12.370 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:12.397 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:12.438 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:12.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:12.541 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:12.585 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:12.850 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:12.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:12.956 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:13.014 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:13.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 53989 53973 124.852 81.375 0.0272195 0.00800469 2023-09-23T20:55:13.112 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:13.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:13.228 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:13.283 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b7ed88ab:::benchmark_data_smithi121_252781_object87596:head, version: 125'5492, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b7ed88ab:::benchmark_data_smithi121_252781_object87596:head@125'5492, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 452.67504963599998, 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17604561799999999, 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.284 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605288+0000", 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605345+0000", 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6996000000000001e-05 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.605357+0000", 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2374e-05 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778323+0000", 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.172965539 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778421+0000", 2023-09-23T20:55:13.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8530999999999996e-05 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.778437+0000", 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5136999999999999e-05 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:40.781334+0000", 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028975680000000001 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.288 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(8.d 141/130 [PushOp(8:b02b4366:::benchmark_data_smithi121_252781_object90395:head, version: 125'5662, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b02b4366:::benchmark_data_smithi121_252781_object90395:head@125'5662, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 450.35433469999998, 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22766398199999999, 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926003+0000", 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.925997+0000", 2023-09-23T20:55:13.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999938 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:42.926050+0000", 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2899999999999998e-05 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150250+0000", 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22419977199999999 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150290+0000", 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9712000000000002e-05 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150341+0000", 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1696000000000001e-05 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.150349+0000", 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7409999999999992e-06 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:43.153667+0000", 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033184040000000001 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.293 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172419 8.f 8:f3e775cc:::benchmark_data_smithi121_252781_object63244:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 446.16327866199998, 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0270669539999999, 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172419 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.294 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117059+0000", 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.295 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117058+0000", 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117065+0000", 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9940000000000003e-06 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117075+0000", 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8649999999999996e-06 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117083+0000", 2023-09-23T20:55:13.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7399999999999993e-06 2023-09-23T20:55:13.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117315+0000", 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00023133999999999999 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117325+0000", 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0008e-05 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144044+0000", 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.026719535 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144126+0000", 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2231000000000006e-05 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.299 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172440 8.f 8:fcfd2d26:::benchmark_data_smithi121_252781_object63265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 446.16324107100002, 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047514855, 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172440 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117097+0000", 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117099+0000", 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117106+0000", 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2210000000000002e-06 2023-09-23T20:55:13.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117111+0000", 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2360000000000004e-06 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117343+0000", 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000232597 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117349+0000", 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1930000000000001e-06 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144377+0000", 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.027027844 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144494+0000", 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000117108 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.144624+0000", 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000130677 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.155446+0000", 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010821377 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164470+0000", 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090247639999999994 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164526+0000", 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5665999999999999e-05 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.306 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164612+0000", 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.577e-05 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:172441 8.f 8:f67e9cac:::benchmark_data_smithi121_252781_object63266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 446.16322298799997, 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047671113, 2023-09-23T20:55:13.307 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172441 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:55:13.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117115+0000", 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117118+0000", 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8109999999999999e-06 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117122+0000", 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7160000000000002e-06 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117127+0000", 2023-09-23T20:55:13.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4100000000000001e-06 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117369+0000", 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024246900000000001 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:47.117375+0000", 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3000000000000001e-06 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.311 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148014+0000", 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.030639455 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148124+0000", 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110089 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.148227+0000", 2023-09-23T20:55:13.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010324599999999999 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.155452+0000", 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072244780000000003 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164664+0000", 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092118040000000005 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164708+0000", 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4780999999999997e-05 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:48.164786+0000", 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7847000000000002e-05 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.314 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:177178 8.f 8:f0621e84:::benchmark_data_smithi121_252781_object68003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 438.926528416, 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36433712000000001, 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:13.315 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 177178 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353810+0000", 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353808+0000", 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353817+0000", 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6180000000000001e-06 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353835+0000", 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7601e-05 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353845+0000", 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0268000000000001e-05 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.353898+0000", 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3173000000000001e-05 2023-09-23T20:55:13.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.354023+0000", 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000124456 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.354142+0000", 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000118789 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.357799+0000", 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036574469999999999 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718014+0000", 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36021456600000001 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718061+0000", 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7530000000000001e-05 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.718147+0000", 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5612999999999995e-05 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.321 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:177186 8.f 8:f634efb7:::benchmark_data_smithi121_252781_object68011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 438.91360128999997, 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.361305761, 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 177186 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.322 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366737+0000", 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366735+0000", 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366742+0000", 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8569999999999997e-06 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366756+0000", 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3405e-05 2023-09-23T20:55:13.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366765+0000", 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7969999999999995e-06 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366817+0000", 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2021000000000002e-05 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366923+0000", 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106098 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1", 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.366996+0000", 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2354999999999994e-05 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.370015+0000", 2023-09-23T20:55:13.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030194229999999998 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.727915+0000", 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35789987899999998 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.727963+0000", 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7817e-05 2023-09-23T20:55:13.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:47:54.728043+0000", 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9654999999999995e-05 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185557 8.4 e146/134 8:23b68925:::benchmark_data_smithi121_252781_object76382:head v 146'11606, mlcod=146'11603)", 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:55:13.328 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 427.78788960999998, 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33023901100000003, 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.329 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492448+0000", 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.330 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492455+0000", 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5970000000000003e-06 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492464+0000", 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9279999999999999e-06 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492471+0000", 2023-09-23T20:55:13.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4699999999999999e-06 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492508+0000", 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7024999999999999e-05 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.492525+0000", 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6776e-05 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822566+0000", 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33004159199999999 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822688+0000", 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121403 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185566 8.d e146/143 8:bb91d518:::benchmark_data_smithi121_252781_object76391:head v 146'11646, mlcod=146'11643)", 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 427.78531800100001, 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32767350499999998, 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495020+0000", 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495025+0000", 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6550000000000003e-06 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495035+0000", 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.057e-05 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495043+0000", 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9699999999999999e-06 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495079+0000", 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6300999999999997e-05 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.495094+0000", 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4430000000000001e-05 2023-09-23T20:55:13.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822604+0000", 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32751017900000001 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822694+0000", 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9649999999999997e-05 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.338 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185568 8.4 e146/134 8:2156ec6b:::benchmark_data_smithi121_252781_object76393:head v 146'11607, mlcod=146'11604)", 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 427.779697226, 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33877399600000002, 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500641+0000", 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500640+0000", 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.500649+0000", 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0450000000000006e-06 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822380+0000", 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32173094000000002 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822388+0000", 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1999999999999994e-06 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.824928+0000", 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025395029999999998 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.824942+0000", 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4331999999999999e-05 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.839373+0000", 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01443056 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.839415+0000", 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2320000000000001e-05 2023-09-23T20:55:13.343 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.44585.0:185570 8.a e147/145 8:550a95bc:::benchmark_data_smithi121_252781_object76395:head v 147'11761, mlcod=146'11759)", 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 427.77858592699999, 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32342548900000001, 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.345 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501752+0000", 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.501756+0000", 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9689999999999996e-06 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822281+0000", 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32052530800000001 2023-09-23T20:55:13.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822294+0000", 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2822e-05 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822388+0000", 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4215999999999994e-05 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.822411+0000", 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3161999999999999e-05 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.825145+0000", 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027331389999999999 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:05.825178+0000", 2023-09-23T20:55:13.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3142999999999998e-05 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:172848 9.0 9:0b58a0a9:::benchmark_data_smithi121_293770_object69960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:20.648779+0000", 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 112.631559518, 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.18863866000000001, 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.349 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172848 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.648779+0000", 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.350 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.648779+0000", 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.648777+0000", 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.648785+0000", 2023-09-23T20:55:13.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9379999999999992e-06 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.648798+0000", 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3084e-05 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.648807+0000", 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5960000000000004e-06 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.352 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.649099+0000", 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00029241800000000002 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.649239+0000", 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013996000000000001 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.649375+0000", 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000135647 2023-09-23T20:55:13.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837036+0000", 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.18766140000000001 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837049+0000", 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3331e-05 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837127+0000", 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8057000000000002e-05 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837182+0000", 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4440000000000001e-05 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837417+0000", 2023-09-23T20:55:13.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000235417 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:172858 9.0 9:05fb70b5:::benchmark_data_smithi121_293770_object69970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:20.653497+0000", 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 112.626840649, 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.184148495, 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.356 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172858 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653497+0000", 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653497+0000", 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653496+0000", 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653504+0000", 2023-09-23T20:55:13.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4239999999999993e-06 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653518+0000", 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3543000000000001e-05 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653526+0000", 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3299999999999999e-06 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653576+0000", 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0008e-05 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653710+0000", 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000133912 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.653826+0000", 2023-09-23T20:55:13.360 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115635 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837056+0000", 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.18323003900000001 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837096+0000", 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0373000000000001e-05 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837491+0000", 2023-09-23T20:55:13.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039465000000000002 2023-09-23T20:55:13.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837531+0000", 2023-09-23T20:55:13.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9792999999999997e-05 2023-09-23T20:55:13.363 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:13.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.363 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837646+0000", 2023-09-23T20:55:13.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115406 2023-09-23T20:55:13.363 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:172862 9.0 9:069cbd7e:::benchmark_data_smithi121_293770_object69974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:20.658571+0000", 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 112.62176681, 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17925797800000001, 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.364 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172862 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658571+0000", 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.365 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658571+0000", 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658570+0000", 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658578+0000", 2023-09-23T20:55:13.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2709999999999992e-06 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658588+0000", 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.04e-05 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658596+0000", 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5129999999999999e-06 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658919+0000", 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0003235 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.658999+0000", 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9680999999999998e-05 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:55:13.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.659083+0000", 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4448000000000002e-05 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837062+0000", 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17797808400000001 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837460+0000", 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039854199999999999 2023-09-23T20:55:13.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837687+0000", 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00022695500000000001 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837724+0000", 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6473000000000001e-05 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837829+0000", 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010576100000000001 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:172857 9.2 e208/207 9:43089d06:::benchmark_data_smithi121_293770_object69969:head v 208'10887, mlcod=198'8603)", 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:20.666331+0000", 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 112.614007555, 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.18667431600000001, 2023-09-23T20:55:13.371 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666331+0000", 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666331+0000", 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666329+0000", 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666347+0000", 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8187999999999999e-05 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666361+0000", 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4134999999999999e-05 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666370+0000", 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8240000000000006e-06 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666533+0000", 2023-09-23T20:55:13.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000163277 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.666561+0000", 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8424000000000001e-05 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.852965+0000", 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.186403664 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.375 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.853005+0000", 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9746000000000002e-05 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:172867 9.0 9:07b1d958:::benchmark_data_smithi121_293770_object69979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:20.670312+0000", 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 112.610026234, 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.20131882200000001, 2023-09-23T20:55:13.376 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 172867 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.670312+0000", 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.670312+0000", 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.670310+0000", 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.670318+0000", 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0849999999999997e-06 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.836873+0000", 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.166554753 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.836885+0000", 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2299e-05 2023-09-23T20:55:13.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837854+0000", 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00096898700000000004 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.837993+0000", 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000138722 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.838119+0000", 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012600799999999999 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.847979+0000", 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00985953 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.864473+0000", 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016494601000000001 2023-09-23T20:55:13.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.871352+0000", 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068789200000000002 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.871407+0000", 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5158000000000003e-05 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.871631+0000", 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00022332499999999999 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45101.0:172870 9.2 e208/207 9:4fd3c177:::benchmark_data_smithi121_293770_object69982:head v 208'10888, mlcod=198'8603)", 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:20.675351+0000", 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 112.60498717599999, 2023-09-23T20:55:13.383 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17767091199999999, 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.675351+0000", 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.675351+0000", 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.675350+0000", 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.675362+0000", 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2756e-05 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.836850+0000", 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.161487358 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.836867+0000", 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7533e-05 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.840975+0000", 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041079100000000002 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.841001+0000", 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5850000000000002e-05 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.853009+0000", 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012007814 2023-09-23T20:55:13.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:20.853022+0000", 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2943e-05 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:20992 10.5 10:a4ff5a44:::benchmark_data_smithi121_358375_object20991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:54:52.983356+0000", 2023-09-23T20:55:13.388 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 20.296981963, 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17750719100000001, 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 20992 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983356+0000", 2023-09-23T20:55:13.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983356+0000", 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983355+0000", 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983384+0000", 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9164999999999999e-05 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983411+0000", 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7676999999999999e-05 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.391 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983422+0000", 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0815999999999999e-05 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983482+0000", 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0055e-05 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983602+0000", 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119646 2023-09-23T20:55:13.392 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.983706+0000", 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000104046 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.984523+0000", 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00081675499999999995 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.393 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:52.984863+0000", 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00034028200000000002 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:53.160765+0000", 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.175901905 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:53.160821+0000", 2023-09-23T20:55:13.394 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5807999999999998e-05 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:54:53.160863+0000", 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2641000000000001e-05 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:53048 10.5 10:ad397d7b:::benchmark_data_smithi121_358375_object53047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:55:13.395 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:12.295245+0000", 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.98509280099999996, 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17128188899999999, 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 53048 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.396 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295245+0000", 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295245+0000", 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295244+0000", 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:13.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295279+0000", 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5465e-05 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295292+0000", 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2554999999999999e-05 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.398 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295300+0000", 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8650000000000001e-06 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295374+0000", 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4102000000000003e-05 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.399 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295544+0000", 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017043700000000001 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295658+0000", 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000113359 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.298223+0000", 2023-09-23T20:55:13.400 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025653590000000001 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.298245+0000", 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2204e-05 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.466441+0000", 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16819594700000001 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.466490+0000", 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8757000000000001e-05 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.466527+0000", 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7128000000000001e-05 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:53055 10.5 10:a3f0884b:::benchmark_data_smithi121_358375_object53054:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:12.295315+0000", 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.98502276499999997, 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17170591900000001, 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 53055 2023-09-23T20:55:13.403 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295315+0000", 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295315+0000", 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295315+0000", 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295333+0000", 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8649999999999999e-05 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.405 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295341+0000", 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1369999999999996e-06 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.295348+0000", 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1559999999999998e-06 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:13.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.296992+0000", 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016447650000000001 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.297074+0000", 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1700999999999996e-05 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.297160+0000", 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5828000000000006e-05 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.407 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.298345+0000", 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0011851660000000001 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.298628+0000", 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00028318700000000001 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.466937+0000", 2023-09-23T20:55:13.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.168308397 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.466986+0000", 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9461e-05 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:12.467021+0000", 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5067000000000002e-05 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.409 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.410 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.410 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:13.410 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:13.410 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:55:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:13.411 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:13.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:13.511 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:13.565 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:13.566 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:13.566 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:13 smithi121 ceph-mon[211526]: pgmap v745: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 9024/142677 objects degraded (6.325%) 2023-09-23T20:55:13.567 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:13.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:13.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:13 smithi121 ceph-mon[219709]: pgmap v745: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 9024/142677 objects degraded (6.325%) 2023-09-23T20:55:13.571 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:13.633 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:13.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:13.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:13 smithi123 ceph-mon[193903]: pgmap v745: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 9024/142677 objects degraded (6.325%) 2023-09-23T20:55:13.736 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:13.762 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:13.804 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:13.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:13.907 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:13.952 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:14.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 55184 55168 123.061 74.6875 0.0110218 0.00812102 2023-09-23T20:55:14.069 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:55:14.253 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:14.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:14.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:14.279 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:55:14.368 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:14.425 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:14.515 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:14.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:14.621 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:14.685 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:14.697 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:14.782 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:14.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:14.891 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:14.897 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:55:14.942 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:55:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:55:14] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T20:55:14.943 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:15.023 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:15.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:15.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:15.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 56568 56552 121.801 86.5 0.0122108 0.0082062 2023-09-23T20:55:15.126 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:15.187 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:15.325 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:15.490 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:15.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:15.601 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:15.633 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:15.729 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:15.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:15.833 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:15.861 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:15.905 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:15.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:15 smithi123 ceph-mon[193903]: pgmap v746: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 10401/164319 objects degraded (6.330%) 2023-09-23T20:55:15.990 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:55:15.991 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "" 2023-09-23T20:55:15.991 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:55:16.008 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:16.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 57578 57562 119.846 63.125 0.0112897 0.00834 2023-09-23T20:55:16.071 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:15 smithi121 ceph-mon[211526]: pgmap v746: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 10401/164319 objects degraded (6.330%) 2023-09-23T20:55:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:15 smithi121 ceph-mon[219709]: pgmap v746: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 10401/164319 objects degraded (6.330%) 2023-09-23T20:55:16.280 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:16.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:16.389 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:16.450 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.531 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:16.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:16.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:16.635 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:16.661 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:16.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:16.805 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:16.864 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10401/164319 objects degraded (6.330%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:17.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 58683 58667 118.209 69.0625 0.00491014 0.00845497 2023-09-23T20:55:17.068 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:17.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10401/164319 objects degraded (6.330%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:17.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:17.076 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10401/164319 objects degraded (6.330%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:17.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:17.142 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:55:17.174 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:55:17.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:17.208 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:55:17.232 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:17.250 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:55:17.279 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:55:17.314 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:55:17.349 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:55:17.350 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:55:17.889 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:18.011 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:18.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:18.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:18.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 59889 59873 116.871 75.375 0.01108 0.00855279 2023-09-23T20:55:18.100 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:17 smithi121 ceph-mon[211526]: pgmap v747: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 10401/164319 objects degraded (6.330%) 2023-09-23T20:55:18.100 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2042621138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:18.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:17 smithi121 ceph-mon[219709]: pgmap v747: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 10401/164319 objects degraded (6.330%) 2023-09-23T20:55:18.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2042621138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:18.120 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:18.177 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:17 smithi123 ceph-mon[193903]: pgmap v747: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 10401/164319 objects degraded (6.330%) 2023-09-23T20:55:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2042621138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:18.612 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:18.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:18.719 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:18.774 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:18.785 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:18.871 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:18.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:18.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:18.981 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:19.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 61261 61245 115.927 85.75 0.00727163 0.00862131 2023-09-23T20:55:19.050 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:19.130 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:19.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:19.244 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:19.305 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:19.411 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:19.411 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:19.449 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:19.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:19.557 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:19.611 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:19.656 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:19.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:19.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:19.769 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:19.795 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:19.836 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:19.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:19.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:19.944 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:19 smithi123 ceph-mon[193903]: pgmap v748: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 11499/177621 objects degraded (6.474%) 2023-09-23T20:55:20.006 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:20.045 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 62301 62285 114.43 65 0.0105589 0.00873485 2023-09-23T20:55:20.122 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:20.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:20.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:19 smithi121 ceph-mon[211526]: pgmap v748: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 11499/177621 objects degraded (6.474%) 2023-09-23T20:55:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:19 smithi121 ceph-mon[219709]: pgmap v748: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 11499/177621 objects degraded (6.474%) 2023-09-23T20:55:20.235 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:20.285 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:20.612 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:20.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:20.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:20.721 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:20.784 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:20.787 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:55:20.838 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:55:20.911 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:20.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:20.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:21.023 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:21.046 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 63711 63695 113.679 88.125 0.0116174 0.00879244 2023-09-23T20:55:21.075 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:21.161 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:21.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:21.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:21.270 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:21.310 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:21.385 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:21.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:21.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:21.471 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:21.493 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:21.544 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:21.574 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:21.631 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:21.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:21.741 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:21.799 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:21.923 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:21.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:21.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:21.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:21 smithi123 ceph-mon[193903]: pgmap v749: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 11499/182127 objects degraded (6.314%) 2023-09-23T20:55:21.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11499/182127 objects degraded (6.314%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:21.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2185881843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:21.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:22.029 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[219709]: pgmap v749: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 11499/182127 objects degraded (6.314%) 2023-09-23T20:55:22.029 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11499/182127 objects degraded (6.314%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:22.029 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2185881843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:22.029 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:22.030 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[211526]: pgmap v749: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 11499/182127 objects degraded (6.314%) 2023-09-23T20:55:22.030 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11499/182127 objects degraded (6.314%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:22.030 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2185881843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:22.030 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:55:22.033 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:22.046 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 64919 64903 112.618 75.5 0.0164494 0.00887413 2023-09-23T20:55:22.089 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:22.192 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:55:22.192 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:22.192 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 7, 6] 2023-09-23T20:55:22.193 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T20:55:22.214 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:55:22.381 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:22.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:22.493 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:22.551 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:23.046 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 66039 66023 111.467 70 0.00993612 0.00896726 2023-09-23T20:55:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[211526]: failed to encode map e236 with expected crc 2023-09-23T20:55:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[211526]: osdmap e236: 8 total, 7 up, 8 in 2023-09-23T20:55:23.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[211526]: failed to encode map e236 with expected crc 2023-09-23T20:55:23.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:23.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[219709]: failed to encode map e236 with expected crc 2023-09-23T20:55:23.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:23.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[219709]: osdmap e236: 8 total, 7 up, 8 in 2023-09-23T20:55:23.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:22 smithi121 ceph-mon[219709]: failed to encode map e236 with expected crc 2023-09-23T20:55:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:22 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:22 smithi123 ceph-mon[193903]: failed to encode map e236 with expected crc 2023-09-23T20:55:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:22 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:55:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:22 smithi123 ceph-mon[193903]: osdmap e236: 8 total, 7 up, 8 in 2023-09-23T20:55:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:22 smithi123 ceph-mon[193903]: failed to encode map e236 with expected crc 2023-09-23T20:55:23.724 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:23.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:23.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:23.828 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:23.902 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:24.011 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:24.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:23 smithi121 ceph-mon[211526]: failed to encode map e236 with expected crc 2023-09-23T20:55:24.014 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:23 smithi121 ceph-mon[211526]: pgmap v751: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 11499/182127 objects degraded (6.314%) 2023-09-23T20:55:24.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:23 smithi121 ceph-mon[219709]: failed to encode map e236 with expected crc 2023-09-23T20:55:24.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:23 smithi121 ceph-mon[219709]: pgmap v751: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 11499/182127 objects degraded (6.314%) 2023-09-23T20:55:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:24.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:24.046 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 67295 67279 110.599 78.5 0.0104265 0.0090371 2023-09-23T20:55:24.116 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:24.166 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:23 smithi123 ceph-mon[193903]: failed to encode map e236 with expected crc 2023-09-23T20:55:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:23 smithi123 ceph-mon[193903]: pgmap v751: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 11499/182127 objects degraded (6.314%) 2023-09-23T20:55:25.048 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 15 68585 68570 109.826 80.6875 0.0162349 0.00909977 2023-09-23T20:55:25.077 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:55:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:55:24] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T20:55:25.145 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:25.175 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:25.289 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:25.357 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:25.435 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:55:25.472 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:25.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:25.491 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:55:25.530 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:55:25.559 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:55:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:55:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T20:55:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T20:55:25.560 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:55:25.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:55:25.578 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:25.594 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:55:25.632 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:25.635 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:55:25.673 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:55:25.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:55:25.704 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:25.811 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:25.856 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:25 smithi121 ceph-mon[219709]: pgmap v752: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 12595/200523 objects degraded (6.281%) 2023-09-23T20:55:25.856 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1816155550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:25.866 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:25.971 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:25 smithi123 ceph-mon[193903]: pgmap v752: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 12595/200523 objects degraded (6.281%) 2023-09-23T20:55:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1816155550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:25.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:26.049 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:55:26.049378+0000 min lat: 0.00106683 max lat: 1.02878 avg lat: 0.00916499 2023-09-23T20:55:26.049 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:55:26.049 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 69858 69842 109.068 79.5 0.00754659 0.00916499 2023-09-23T20:55:26.080 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:26.148 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:26.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:25 smithi121 ceph-mon[211526]: pgmap v752: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 12595/200523 objects degraded (6.281%) 2023-09-23T20:55:26.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1816155550' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:26.190 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:26.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:26.296 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:26.352 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:26.785 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:55:27.049 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 71056 71040 108.234 74.875 0.0105699 0.00923441 2023-09-23T20:55:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12595/200523 objects degraded (6.281%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:27.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12595/200523 objects degraded (6.281%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:27.183 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:55:27.213 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:27.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12595/200523 objects degraded (6.281%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:27.323 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:27.392 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:27.493 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:27.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:27.602 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:27.640 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:27.718 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:27.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:27.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:27.825 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:27.875 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:27.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:27 smithi121 ceph-mon[211526]: pgmap v753: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 12595/200523 objects degraded (6.281%) 2023-09-23T20:55:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:27.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:27 smithi121 ceph-mon[219709]: pgmap v753: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 12595/200523 objects degraded (6.281%) 2023-09-23T20:55:27.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:27.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:28.049 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 72183 72167 107.334 70.4375 0.0116938 0.00931254 2023-09-23T20:55:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:28.083 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:28.100 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:28.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:28.207 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:28.214 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:27 smithi123 ceph-mon[193903]: pgmap v753: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s; 12595/200523 objects degraded (6.281%) 2023-09-23T20:55:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:28.267 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:28.302 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:28.363 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:28.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:28.470 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:28.541 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:28.641 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:28.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:28.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:28.755 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:28.821 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:28.930 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:28.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:28.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:29.038 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:29.052 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 73337 73321 106.509 72.125 0.0136934 0.00938334 2023-09-23T20:55:29.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:29 smithi121 conmon[211495]: 2023-09-23T20:55:29.066+0000 7fdc50c66700 -1 mon.a@0(leader).osd e236 definitely_dead 0 2023-09-23T20:55:29.106 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:29.906 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.7. 2023-09-23T20:55:30.052 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 74279 74263 105.427 58.875 0.00325093 0.00943377 2023-09-23T20:55:30.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:55:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:29 smithi121 ceph-mon[219709]: pgmap v754: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 13643/214152 objects degraded (6.371%) 2023-09-23T20:55:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1128969602' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:55:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:29 smithi121 ceph-mon[219709]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T20:55:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:29 smithi121 ceph-mon[211526]: pgmap v754: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 13643/214152 objects degraded (6.371%) 2023-09-23T20:55:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1128969602' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:55:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:29 smithi121 ceph-mon[211526]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T20:55:30.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:29 smithi123 ceph-mon[193903]: pgmap v754: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 13643/214152 objects degraded (6.371%) 2023-09-23T20:55:30.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1128969602' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T20:55:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:29 smithi123 ceph-mon[193903]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T20:55:30.231 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:30.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:30.335 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:30.363 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:30.406 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:30.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:30.512 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.052 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 74279 74263 103.085 0 - 0.00943377 2023-09-23T20:55:31.084 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[219709]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[219709]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1128969602' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T20:55:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[219709]: osdmap e237: 8 total, 7 up, 7 in 2023-09-23T20:55:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[219709]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[219709]: pgmap v756: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 13643/218724 objects degraded (6.238%) 2023-09-23T20:55:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[219709]: osdmap e238: 8 total, 7 up, 7 in 2023-09-23T20:55:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[211526]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[211526]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1128969602' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T20:55:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[211526]: osdmap e237: 8 total, 7 up, 7 in 2023-09-23T20:55:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[211526]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[211526]: pgmap v756: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 13643/218724 objects degraded (6.238%) 2023-09-23T20:55:31.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:30 smithi121 ceph-mon[211526]: osdmap e238: 8 total, 7 up, 7 in 2023-09-23T20:55:31.187 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:31.220 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:30 smithi123 ceph-mon[193903]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:30 smithi123 ceph-mon[193903]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1128969602' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T20:55:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:30 smithi123 ceph-mon[193903]: osdmap e237: 8 total, 7 up, 7 in 2023-09-23T20:55:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:30 smithi123 ceph-mon[193903]: failed to encode map e237 with expected crc 2023-09-23T20:55:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:30 smithi123 ceph-mon[193903]: pgmap v756: 89 pgs: 6 active+undersized+degraded, 29 active+undersized, 54 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 13643/218724 objects degraded (6.238%) 2023-09-23T20:55:31.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:30 smithi123 ceph-mon[193903]: osdmap e238: 8 total, 7 up, 7 in 2023-09-23T20:55:31.419 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:31.522 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:31.548 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.590 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:31.697 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:31.722 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.764 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:31.796 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:31.869 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:31.901 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:31.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: failed to encode map e238 with expected crc 2023-09-23T20:55:31.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: failed to encode map e238 with expected crc 2023-09-23T20:55:31.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: failed to encode map e238 with expected crc 2023-09-23T20:55:31.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: failed to encode map e238 with expected crc 2023-09-23T20:55:31.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: failed to encode map e238 with expected crc 2023-09-23T20:55:31.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: failed to encode map e238 with expected crc 2023-09-23T20:55:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: failed to encode map e238 with expected crc 2023-09-23T20:55:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13643/218724 objects degraded (6.238%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[211526]: osdmap e239: 8 total, 7 up, 7 in 2023-09-23T20:55:31.972 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:31.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:31.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:32.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 74288 74272 100.856 0.28125 0.00484444 0.0096796 2023-09-23T20:55:32.079 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:32.159 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13643/218724 objects degraded (6.238%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:32.160 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:31 smithi121 ceph-mon[219709]: osdmap e239: 8 total, 7 up, 7 in 2023-09-23T20:55:32.169 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: failed to encode map e238 with expected crc 2023-09-23T20:55:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13643/218724 objects degraded (6.238%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:32.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:31 smithi123 ceph-mon[193903]: osdmap e239: 8 total, 7 up, 7 in 2023-09-23T20:55:32.277 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:55:32.277 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:55:32.277 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194759 8.b 8:d1bf2064:::benchmark_data_smithi121_252781_object85584:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 432.10297084600001, 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0533581320000001, 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.278 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194759 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170901+0000", 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.279 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170899+0000", 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170910+0000", 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1206000000000001e-05 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.280 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170925+0000", 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4596999999999999e-05 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170936+0000", 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1445e-05 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171104+0000", 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016754899999999999 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171116+0000", 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1989e-05 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224175+0000", 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0530593850000001 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.282 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224259+0000", 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3837000000000004e-05 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194764 8.b 8:d2640955:::benchmark_data_smithi121_252781_object85589:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 432.10291606700002, 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.053600919, 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:55:32.283 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194764 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.284 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170956+0000", 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170955+0000", 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170961+0000", 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2619999999999998e-06 2023-09-23T20:55:32.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170968+0000", 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.345e-06 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170973+0000", 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3410000000000002e-06 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171165+0000", 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019139 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171173+0000", 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6260000000000001e-06 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224498+0000", 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0533245570000001 2023-09-23T20:55:32.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224557+0000", 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8866000000000002e-05 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194773 8.b 8:dae7dbf1:::benchmark_data_smithi121_252781_object85598:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 432.10289303600001, 2023-09-23T20:55:32.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067740498, 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194773 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.289 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170979+0000", 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170978+0000", 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170981+0000", 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.249e-06 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170986+0000", 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6590000000000004e-06 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170991+0000", 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7709999999999997e-06 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171203+0000", 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021228600000000001 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171210+0000", 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0199999999999997e-06 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224626+0000", 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0534161630000001 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224718+0000", 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1296000000000001e-05 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238636+0000", 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013917802 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238719+0000", 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3721999999999996e-05 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194787 8.b 8:dd8b32fd:::benchmark_data_smithi121_252781_object85612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 432.10287637599998, 2023-09-23T20:55:32.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067236472, 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194787 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.295 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170995+0000", 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.170997+0000", 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2220000000000001e-06 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171003+0000", 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4620000000000004e-06 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171008+0000", 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9250000000000003e-06 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171302+0000", 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000294331 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.171311+0000", 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7250000000000003e-06 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224751+0000", 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.053440331 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224842+0000", 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1292000000000006e-05 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.224927+0000", 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5044999999999996e-05 2023-09-23T20:55:32.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237978+0000", 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013050277000000001 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237987+0000", 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3090000000000001e-06 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238081+0000", 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4129999999999995e-05 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238135+0000", 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4212999999999997e-05 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238232+0000", 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6571999999999996e-05 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.301 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194805 8.b 8:de52f621:::benchmark_data_smithi121_252781_object85630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 432.08577835800003, 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0503357229999999, 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.302 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194805 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188093+0000", 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188092+0000", 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188100+0000", 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.729e-06 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188113+0000", 2023-09-23T20:55:32.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3016999999999999e-05 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188122+0000", 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3230000000000004e-06 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188182+0000", 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9775000000000003e-05 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.188194+0000", 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2452e-05 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227071+0000", 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.038876744 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227205+0000", 2023-09-23T20:55:32.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013409199999999999 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.227299+0000", 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3601e-05 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.237991+0000", 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010692421000000001 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238032+0000", 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0800000000000002e-05 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238293+0000", 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000260938 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238331+0000", 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8381999999999999e-05 2023-09-23T20:55:32.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238429+0000", 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7999999999999997e-05 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194818 8.b 8:db0a1c54:::benchmark_data_smithi121_252781_object85643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:55:32.309 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 432.06791952700002, 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0410233019999999, 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194818 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:55:32.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205952+0000", 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205951+0000", 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.311 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205957+0000", 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205975+0000", 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.766e-05 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.205984+0000", 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4099999999999997e-06 2023-09-23T20:55:32.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.206051+0000", 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6910999999999997e-05 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.206063+0000", 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2242000000000001e-05 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230211+0000", 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.024147707 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230377+0000", 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000165635 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.230528+0000", 2023-09-23T20:55:32.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015100899999999999 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238042+0000", 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075139159999999998 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238773+0000", 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00073170099999999997 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246813+0000", 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080395229999999998 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246870+0000", 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6894000000000001e-05 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.246976+0000", 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000105787 2023-09-23T20:55:32.316 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:194827 8.b 8:d1592b5b:::benchmark_data_smithi121_252781_object85652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 432.05096442299998, 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0286707319999999, 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.317 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 194827 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222907+0000", 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222906+0000", 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222913+0000", 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3719999999999999e-06 2023-09-23T20:55:32.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222926+0000", 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3294e-05 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222935+0000", 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4789999999999996e-06 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.222996+0000", 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1631000000000007e-05 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:20.223009+0000", 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2109e-05 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.234721+0000", 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0117129 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.234865+0000", 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014378500000000001 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.235011+0000", 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014604 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238786+0000", 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003774576 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.238821+0000", 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5635e-05 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251421+0000", 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012599552 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251476+0000", 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5387999999999999e-05 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:21.251578+0000", 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000101727 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216597 8.0 8:06510814:::benchmark_data_smithi121_252781_object107422:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.56193313, 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.054665387, 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216597 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711939+0000", 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711937+0000", 2023-09-23T20:55:32.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711944+0000", 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.25e-06 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711957+0000", 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2999e-05 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.711966+0000", 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9719999999999998e-06 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.712019+0000", 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2799000000000001e-05 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.712028+0000", 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.003e-06 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766511+0000", 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.054482731 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766604+0000", 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3139000000000004e-05 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216620 8.0 8:04259452:::benchmark_data_smithi121_252781_object107445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.53862891, 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0569858000000001, 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216620 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.330 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735243+0000", 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735241+0000", 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735249+0000", 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6429999999999995e-06 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735279+0000", 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9805e-05 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735289+0000", 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9820000000000003e-06 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735397+0000", 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107877 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.735410+0000", 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2962000000000001e-05 2023-09-23T20:55:32.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.766912+0000", 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.031502782 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.767011+0000", 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8770999999999993e-05 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.767472+0000", 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000460922 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785103+0000", 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017630777 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787562+0000", 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024589770000000002 2023-09-23T20:55:32.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792167+0000", 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046052189999999998 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792202+0000", 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5182000000000003e-05 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792229+0000", 2023-09-23T20:55:32.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6404999999999999e-05 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216632 8.0 8:09792f9f:::benchmark_data_smithi121_252781_object107457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.528841887, 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0473433670000001, 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.337 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216632 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745030+0000", 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745028+0000", 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745036+0000", 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1319999999999994e-06 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745053+0000", 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6854000000000002e-05 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745062+0000", 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4940000000000003e-06 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745125+0000", 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3314999999999999e-05 2023-09-23T20:55:32.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.745136+0000", 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0546e-05 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782661+0000", 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.037525241 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782813+0000", 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015243099999999999 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.782964+0000", 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000150925 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792128+0000", 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091633930000000006 2023-09-23T20:55:32.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792279+0000", 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015110899999999999 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792323+0000", 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3788000000000002e-05 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792350+0000", 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7915e-05 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792373+0000", 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2816e-05 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216637 8.0 8:0d71f1da:::benchmark_data_smithi121_252781_object107462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.344 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.51944447800003, 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0470203279999999, 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216637 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.345 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754427+0000", 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754426+0000", 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:32.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754433+0000", 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5839999999999997e-06 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754450+0000", 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7129000000000002e-05 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754458+0000", 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1689999999999992e-06 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754544+0000", 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5140000000000001e-05 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754557+0000", 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3107e-05 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785135+0000", 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0305782050000001 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785220+0000", 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4860999999999999e-05 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.785315+0000", 2023-09-23T20:55:32.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5088999999999997e-05 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792290+0000", 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069754370000000001 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792437+0000", 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014619000000000001 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.350 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801364+0000", 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089271820000000005 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801415+0000", 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1214999999999997e-05 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801448+0000", 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.277e-05 2023-09-23T20:55:32.351 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216642 8.0 8:0cd41a68:::benchmark_data_smithi121_252781_object107467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.51939815200001, 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047113078, 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.352 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216642 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754474+0000", 2023-09-23T20:55:32.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754473+0000", 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754476+0000", 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8870000000000001e-06 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754483+0000", 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.985e-06 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754488+0000", 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.977e-06 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754584+0000", 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6200999999999996e-05 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.754593+0000", 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5310000000000001e-06 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787595+0000", 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.033002559 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787703+0000", 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107431 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.787777+0000", 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4598000000000006e-05 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792296+0000", 2023-09-23T20:55:32.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045180970000000004 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792476+0000", 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000180669 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801501+0000", 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090245069999999993 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801549+0000", 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8742000000000002e-05 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801587+0000", 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7317000000000002e-05 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216645 8.0 8:049490ab:::benchmark_data_smithi121_252781_object107470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.51056296000002, 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.038903957, 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216645 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.360 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763309+0000", 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763307+0000", 2023-09-23T20:55:32.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763315+0000", 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0779999999999998e-06 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763331+0000", 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6860000000000001e-05 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.763341+0000", 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4250000000000004e-06 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.765814+0000", 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024728929999999999 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.765825+0000", 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1243000000000001e-05 2023-09-23T20:55:32.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.789907+0000", 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.024082245 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.790014+0000", 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106702 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.364 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.790102+0000", 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7682999999999994e-05 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792526+0000", 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024248820000000002 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801599+0000", 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090724640000000006 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802118+0000", 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00051864700000000001 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802173+0000", 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5547000000000002e-05 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.802213+0000", 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9702e-05 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216663 8.0 8:0ce516a0:::benchmark_data_smithi121_252781_object107488:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.488693748, 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.035871486, 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216663 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785178+0000", 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785177+0000", 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785184+0000", 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8469999999999998e-06 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785199+0000", 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5784999999999999e-05 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785209+0000", 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7880000000000001e-06 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785281+0000", 2023-09-23T20:55:32.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2002000000000006e-05 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.785295+0000", 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3675e-05 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792572+0000", 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0072767949999999 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792688+0000", 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116554 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.792802+0000", 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011395099999999999 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801610+0000", 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088076600000000001 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.814161+0000", 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012550716 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.820977+0000", 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068159379999999997 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821010+0000", 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3442000000000003e-05 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821050+0000", 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9595999999999998e-05 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.374 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.44585.0:216676 8.0 8:0d0e61e5:::benchmark_data_smithi121_252781_object107501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e157)", 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 401.47036948300001, 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.017644091, 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.44585", 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 216676 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.375 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803502+0000", 2023-09-23T20:55:32.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803507+0000", 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8500000000000002e-06 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803521+0000", 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4942e-05 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803529+0000", 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1300000000000003e-06 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803579+0000", 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0317e-05 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:50.803589+0000", 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8409999999999995e-06 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795218+0000", 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99162894599999996 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795317+0000", 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9674999999999997e-05 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.795433+0000", 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011584 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.801615+0000", 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061815459999999996 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.820910+0000", 2023-09-23T20:55:32.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01929556 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821082+0000", 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017190599999999999 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821115+0000", 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2373000000000003e-05 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:48:51.821146+0000", 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1846e-05 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57331 9.1 9:8abe57f8:::benchmark_data_smithi121_293770_object57330:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 285.17768180399997, 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1261466759999998, 2023-09-23T20:55:32.382 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57331 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.383 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096190+0000", 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096188+0000", 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096226+0000", 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7861999999999998e-05 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096239+0000", 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2727e-05 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096289+0000", 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0037999999999998e-05 2023-09-23T20:55:32.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.100438+0000", 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041489939999999996 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.100451+0000", 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2942000000000001e-05 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093896+0000", 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99344541600000003 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093910+0000", 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3175e-05 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212636+0000", 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.118726068 2023-09-23T20:55:32.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212786+0000", 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015021899999999999 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.212927+0000", 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014154 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.388 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219297+0000", 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063694299999999997 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219311+0000", 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3786e-05 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219344+0000", 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3769000000000001e-05 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.389 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.219408+0000", 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3341000000000002e-05 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222337+0000", 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029289189999999999 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.390 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57332 9.1 9:830fc147:::benchmark_data_smithi121_293770_object57331:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 285.17251177100002, 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1239288190000001, 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57332 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.391 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101360+0000", 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101359+0000", 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.392 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101380+0000", 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1217e-05 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101396+0000", 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5953000000000001e-05 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.393 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101405+0000", 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7679999999999997e-06 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101456+0000", 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1297e-05 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101470+0000", 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4127e-05 2023-09-23T20:55:32.394 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093968+0000", 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.992497713 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.093975+0000", 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5719999999999997e-06 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.395 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217098+0000", 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1231222729999999 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217225+0000", 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000127253 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.217358+0000", 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013342700000000001 2023-09-23T20:55:32.396 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222449+0000", 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050906379999999998 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222491+0000", 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1764999999999997e-05 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.397 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222529+0000", 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8318999999999997e-05 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.222585+0000", 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6302999999999999e-05 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225289+0000", 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027034759999999998 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.398 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57335 9.1 9:8de699c5:::benchmark_data_smithi121_293770_object57334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 285.17235348200001, 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1379105439999999, 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:32.399 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57335 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.400 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101518+0000", 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101541+0000", 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3251000000000002e-05 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101553+0000", 2023-09-23T20:55:32.401 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2714999999999999e-05 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101561+0000", 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8099999999999998e-06 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101611+0000", 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9898000000000002e-05 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.402 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101618+0000", 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3380000000000001e-06 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094001+0000", 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99238338000000004 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094008+0000", 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7530000000000004e-06 2023-09-23T20:55:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225332+0000", 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1313241270000001 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225481+0000", 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000149269 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:55:32.404 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.225643+0000", 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016234999999999999 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.230166+0000", 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045225370000000001 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231616+0000", 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0014502429999999999 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.405 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236427+0000", 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048111459999999997 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236496+0000", 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8417999999999994e-05 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239429+0000", 2023-09-23T20:55:32.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002933125 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57345 9.1 9:89ab31a2:::benchmark_data_smithi121_293770_object57344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 285.14607183999999, 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1243519640000001, 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.407 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57345 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127800+0000", 2023-09-23T20:55:32.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127798+0000", 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127828+0000", 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9895000000000001e-05 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.409 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127846+0000", 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8281e-05 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127857+0000", 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.081e-05 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127899+0000", 2023-09-23T20:55:32.410 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2271999999999998e-05 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.127908+0000", 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5469999999999997e-06 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094072+0000", 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96616466300000003 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.411 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.094083+0000", 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.098e-05 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227666+0000", 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.133582614 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227790+0000", 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012393300000000001 2023-09-23T20:55:32.412 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.227940+0000", 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000149989 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231704+0000", 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037640120000000002 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.413 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239483+0000", 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0077796499999999999 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249105+0000", 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096215469999999994 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249162+0000", 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7080999999999998e-05 2023-09-23T20:55:32.414 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252152+0000", 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002989916 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:136306 9.2 9:41ee0b2f:::benchmark_data_smithi121_293770_object33418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:52:30.175927+0000", 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 182.09794451499999, 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.067013811, 2023-09-23T20:55:32.415 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 136306 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175927+0000", 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.416 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175927+0000", 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175926+0000", 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175932+0000", 2023-09-23T20:55:32.417 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6309999999999997e-06 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175943+0000", 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0485e-05 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.175959+0000", 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5865000000000001e-05 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.418 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.180074+0000", 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004115563 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.180213+0000", 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013848699999999999 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.180353+0000", 2023-09-23T20:55:32.419 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139915 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.191511+0000", 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01115794 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.195631+0000", 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041203580000000002 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.420 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.229020+0000", 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.033389094000000001 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:30.229031+0000", 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0849999999999999e-05 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:31.242859+0000", 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.013828344 2023-09-23T20:55:32.421 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:52:31.242941+0000", 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1901000000000001e-05 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:55:32.422 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:55:32.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:32.423 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:32.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:32.524 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:32.581 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.053 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 75123 75107 99.8208 52.1875 0.00920016 0.00998834 2023-09-23T20:55:33.054 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:55:33.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1299100654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:33.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:32 smithi123 ceph-mon[193903]: pgmap v759: 89 pgs: 5 remapped+peering, 6 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13643/218724 objects degraded (6.238%); 14 B/s, 0 objects/s recovering 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1299100654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:33.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[211526]: pgmap v759: 89 pgs: 5 remapped+peering, 6 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13643/218724 objects degraded (6.238%); 14 B/s, 0 objects/s recovering 2023-09-23T20:55:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: failed to encode map e239 with expected crc 2023-09-23T20:55:33.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1299100654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:33.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:32 smithi121 ceph-mon[219709]: pgmap v759: 89 pgs: 5 remapped+peering, 6 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 13643/218724 objects degraded (6.238%); 14 B/s, 0 objects/s recovering 2023-09-23T20:55:33.471 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:33.576 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:33.605 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.651 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:33.757 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:33.822 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:33.919 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:33.923 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:33 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:55:33.923 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:33 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:33.923 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:33 smithi123 ceph-mon[193903]: failed to encode map e240 with expected crc 2023-09-23T20:55:33.923 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:33 smithi123 ceph-mon[193903]: failed to encode map e240 with expected crc 2023-09-23T20:55:33.923 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1299100654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:33.923 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:33 smithi123 ceph-mon[193903]: osdmap e240: 8 total, 7 up, 7 in 2023-09-23T20:55:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:33.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:34.036 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:34.052 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 76060 76044 98.9615 58.5625 0.0103745 0.0101005 2023-09-23T20:55:34.095 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:34.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:55:34.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:34.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[211526]: failed to encode map e240 with expected crc 2023-09-23T20:55:34.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[211526]: failed to encode map e240 with expected crc 2023-09-23T20:55:34.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1299100654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:34.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[211526]: osdmap e240: 8 total, 7 up, 7 in 2023-09-23T20:55:34.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:55:34.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:34.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[219709]: failed to encode map e240 with expected crc 2023-09-23T20:55:34.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[219709]: failed to encode map e240 with expected crc 2023-09-23T20:55:34.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1299100654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:34.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:33 smithi121 ceph-mon[219709]: osdmap e240: 8 total, 7 up, 7 in 2023-09-23T20:55:34.235 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:34.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:34.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:34.343 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:34.415 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:34.877 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:55:35.026 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:35.028 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:34 smithi121 ceph-mon[211526]: pgmap v761: 89 pgs: 12 active+clean+remapped, 5 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 553 op/s; 18720/226842 objects degraded (8.252%); 14067/226842 objects misplaced (6.201%); 1.1 MiB/s, 2 keys/s, 22 objects/s recovering 2023-09-23T20:55:35.028 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:34 smithi121 ceph-mon[219709]: pgmap v761: 89 pgs: 12 active+clean+remapped, 5 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 553 op/s; 18720/226842 objects degraded (8.252%); 14067/226842 objects misplaced (6.201%); 1.1 MiB/s, 2 keys/s, 22 objects/s recovering 2023-09-23T20:55:35.029 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:55:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:55:34] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.43.0" 2023-09-23T20:55:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:35.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:35.053 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 76901 76885 98.015 52.5625 0.0204951 0.0101964 2023-09-23T20:55:35.130 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:35.196 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:34 smithi123 ceph-mon[193903]: pgmap v761: 89 pgs: 12 active+clean+remapped, 5 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 553 op/s; 18720/226842 objects degraded (8.252%); 14067/226842 objects misplaced (6.201%); 1.1 MiB/s, 2 keys/s, 22 objects/s recovering 2023-09-23T20:55:35.208 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:35.321 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:35.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:35.437 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:35.500 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:35.547 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:55:35.579 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:55:35.608 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:55:35.624 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:35.654 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:55:35.657 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:35.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:35.696 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:55:35.732 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:55:35.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:35.768 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:55:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:55:35.814 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.043 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:36.053 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 77811 77795 97.1924 56.875 0.00675871 0.0102839 2023-09-23T20:55:36.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:36.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:36.056 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/184193468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:36.058 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/184193468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:36.157 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:36.221 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/184193468' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:36.298 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:36.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:36.403 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:36.431 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.475 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:36.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:36.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:36.529 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:55:36.529 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:36.529 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 3, 4, 5, 6] 2023-09-23T20:55:36.529 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T20:55:36.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:36.605 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.682 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:36.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:36.786 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:36.841 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:36.990 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:36.997 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18720/226842 objects degraded (8.252%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:36.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:36 smithi121 ceph-mon[219709]: pgmap v762: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 28 MiB/s wr, 450 op/s; 18720/226842 objects degraded (8.252%); 14067/226842 objects misplaced (6.201%); 907 KiB/s, 1 keys/s, 18 objects/s recovering 2023-09-23T20:55:36.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:36.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:36.999 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18720/226842 objects degraded (8.252%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:36.999 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:36 smithi121 ceph-mon[211526]: pgmap v762: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 28 MiB/s wr, 450 op/s; 18720/226842 objects degraded (8.252%); 14067/226842 objects misplaced (6.201%); 907 KiB/s, 1 keys/s, 18 objects/s recovering 2023-09-23T20:55:37.053 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 78590 78574 96.2416 48.6875 0.0312523 0.0103856 2023-09-23T20:55:37.093 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:37.094 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:37.099 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:37.149 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:37.203 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:55:37.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18720/226842 objects degraded (8.252%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:37.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:36 smithi123 ceph-mon[193903]: pgmap v762: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 28 MiB/s wr, 450 op/s; 18720/226842 objects degraded (8.252%); 14067/226842 objects misplaced (6.201%); 907 KiB/s, 1 keys/s, 18 objects/s recovering 2023-09-23T20:55:37.232 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:37.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:37.247 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:37.335 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:37.395 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:37.485 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:37.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:37.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:37.605 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:37.700 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:37.937 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:37 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:55:38.053 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 79486 79470 95.4679 56 0.01096 0.0104699 2023-09-23T20:55:38.094 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:38.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:37 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:55:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:37 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:55:38.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:38.253 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:38.352 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:38.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:38.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:38.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:38.486 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:38.532 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:38.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:38.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:38.638 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:38.702 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:39.053 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 80346 80330 94.6805 53.75 0.0120406 0.0105572 2023-09-23T20:55:39.078 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:39.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:39.094 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:55:39.125 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:39.191 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:38 smithi123 ceph-mon[193903]: pgmap v763: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 19467/238092 objects degraded (8.176%); 14184/238092 objects misplaced (5.957%); 2.0 MiB/s, 1 keys/s, 35 objects/s recovering 2023-09-23T20:55:39.236 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:39.325 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:39.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:39.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:38 smithi121 ceph-mon[219709]: pgmap v763: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 19467/238092 objects degraded (8.176%); 14184/238092 objects misplaced (5.957%); 2.0 MiB/s, 1 keys/s, 35 objects/s recovering 2023-09-23T20:55:39.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:38 smithi121 ceph-mon[211526]: pgmap v763: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 817 op/s; 19467/238092 objects degraded (8.176%); 14184/238092 objects misplaced (5.957%); 2.0 MiB/s, 1 keys/s, 35 objects/s recovering 2023-09-23T20:55:39.430 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:39.514 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:40.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 81232 81216 93.9521 55.375 0.0243421 0.0106376 2023-09-23T20:55:40.272 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:40.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:40.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:40.379 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:40.456 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.2. 2023-09-23T20:55:40.515 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:40.630 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:40.631 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:40 smithi121 conmon[211495]: 2023-09-23T20:55:40.370+0000 7fdc50c66700 -1 mon.a@0(leader).osd e240 definitely_dead 0 2023-09-23T20:55:40.631 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2148072348' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:55:40.631 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:55:40.631 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:40 smithi121 ceph-mon[211526]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:55:40.631 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2148072348' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:55:40.632 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:55:40.632 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:40 smithi121 ceph-mon[219709]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:55:40.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:40.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:40.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2148072348' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:55:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:55:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:40 smithi123 ceph-mon[193903]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:55:40.733 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:40.761 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:41.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 81622 81606 92.6875 24.375 0.00278812 0.0106682 2023-09-23T20:55:41.346 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:41.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:41.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:41.450 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:41.478 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: osdmap e241: 8 total, 7 up, 6 in 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: pgmap v765: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 875 op/s; 19467/239742 objects degraded (8.120%); 14005/239742 objects misplaced (5.842%); 3.4 MiB/s, 1 keys/s, 57 objects/s recovering 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 19467/238092 objects degraded (8.176%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[219709]: osdmap e242: 8 total, 7 up, 6 in 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: osdmap e241: 8 total, 7 up, 6 in 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: pgmap v765: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 875 op/s; 19467/239742 objects degraded (8.120%); 14005/239742 objects misplaced (5.842%); 3.4 MiB/s, 1 keys/s, 57 objects/s recovering 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 19467/238092 objects degraded (8.176%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:41.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:41 smithi121 ceph-mon[211526]: osdmap e242: 8 total, 7 up, 6 in 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: osdmap e241: 8 total, 7 up, 6 in 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: failed to encode map e241 with expected crc 2023-09-23T20:55:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: pgmap v765: 89 pgs: 17 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 41 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 875 op/s; 19467/239742 objects degraded (8.120%); 14005/239742 objects misplaced (5.842%); 3.4 MiB/s, 1 keys/s, 57 objects/s recovering 2023-09-23T20:55:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 19467/238092 objects degraded (8.176%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:41 smithi123 ceph-mon[193903]: osdmap e242: 8 total, 7 up, 6 in 2023-09-23T20:55:42.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 81624 81608 91.0353 0.125 0.00326614 0.0106794 2023-09-23T20:55:42.118 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:42.118 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:42.391 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:42.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: osdmap e243: 8 total, 7 up, 6 in 2023-09-23T20:55:42.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:42.494 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:42.545 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:42.614 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: osdmap e243: 8 total, 7 up, 6 in 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: failed to encode map e242 with expected crc 2023-09-23T20:55:42.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: osdmap e243: 8 total, 7 up, 6 in 2023-09-23T20:55:42.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:42.727 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:42.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:42.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:42.834 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:42.859 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:42.901 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:42.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:42.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:43.004 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:43.029 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 81751 81735 89.578 7.9375 0.0045464 0.0110284 2023-09-23T20:55:43.071 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:43.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:43.174 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:43.199 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.371 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:43.424 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:55:43.473 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:55:43.474 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:43.502 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.547 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:43.656 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:43.681 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.681 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.681 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.681 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.681 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.682 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: pgmap v768: 89 pgs: 2 remapped+peering, 2 peering, 3 unknown, 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 37 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 19467/239742 objects degraded (8.120%); 14005/239742 objects misplaced (5.842%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:55:43.682 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1142464214' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:43.682 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:43 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:43.684 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:43.732 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:43.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:43.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:43.790 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:43.841 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:43.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: pgmap v768: 89 pgs: 2 remapped+peering, 2 peering, 3 unknown, 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 37 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 19467/239742 objects degraded (8.120%); 14005/239742 objects misplaced (5.842%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1142464214' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: failed to encode map e243 with expected crc 2023-09-23T20:55:43.873 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: pgmap v768: 89 pgs: 2 remapped+peering, 2 peering, 3 unknown, 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 22 active+undersized, 37 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 19467/239742 objects degraded (8.120%); 14005/239742 objects misplaced (5.842%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:55:43.873 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1142464214' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:43.873 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:43 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:55:43.879 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:44.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 81751 81735 88.0342 0 - 0.0110284 2023-09-23T20:55:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:44 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T20:55:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:44 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:44 smithi123 ceph-mon[193903]: failed to encode map e244 with expected crc 2023-09-23T20:55:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:44 smithi123 ceph-mon[193903]: failed to encode map e244 with expected crc 2023-09-23T20:55:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:44 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:44.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:44 smithi123 ceph-mon[193903]: osdmap e244: 8 total, 7 up, 6 in 2023-09-23T20:55:44.741 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T20:55:44.741 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:44.741 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[211526]: failed to encode map e244 with expected crc 2023-09-23T20:55:44.741 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[211526]: failed to encode map e244 with expected crc 2023-09-23T20:55:44.741 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:44.741 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[211526]: osdmap e244: 8 total, 7 up, 6 in 2023-09-23T20:55:44.744 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-09-23T20:55:44.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:55:44.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[219709]: failed to encode map e244 with expected crc 2023-09-23T20:55:44.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[219709]: failed to encode map e244 with expected crc 2023-09-23T20:55:44.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:55:44.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:44 smithi121 ceph-mon[219709]: osdmap e244: 8 total, 7 up, 6 in 2023-09-23T20:55:45.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 81751 81735 86.5426 0 - 0.0110284 2023-09-23T20:55:45.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:55:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:55:44] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-09-23T20:55:45.321 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:45.322 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:45.639 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:45.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:45.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:45 smithi123 ceph-mon[193903]: pgmap v770: 89 pgs: 2 remapped+peering, 2 peering, 3 unknown, 20 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 33 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 19 MiB/s wr, 297 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T20:55:45.744 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:45.771 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:45 smithi121 ceph-mon[211526]: pgmap v770: 89 pgs: 2 remapped+peering, 2 peering, 3 unknown, 20 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 33 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 19 MiB/s wr, 297 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T20:55:45.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:45 smithi121 ceph-mon[219709]: pgmap v770: 89 pgs: 2 remapped+peering, 2 peering, 3 unknown, 20 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 33 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 19 MiB/s wr, 297 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T20:55:45.776 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:45.923 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:45.924 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:55:45.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:45.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:46.027 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:46.053 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:46.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:55:46.055353+0000 min lat: 0.00106683 max lat: 2.33494 avg lat: 0.0110284 2023-09-23T20:55:46.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:55:46.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 81751 81735 85.1008 0 - 0.0110284 2023-09-23T20:55:46.102 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:46.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:46.207 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:46.236 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:46.281 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:46.285 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:46.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:46.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:46.393 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:46.423 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:46.480 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:46.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:46.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:46.584 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:46.614 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2948064984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 49159/243621 objects degraded (20.178%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:46 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:55:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2948064984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 49159/243621 objects degraded (20.178%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:46 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:55:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2948064984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:55:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 49159/243621 objects degraded (20.178%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:55:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:46 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:55:46.962 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:46.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:47.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 81751 81735 83.7062 0 - 0.0110284 2023-09-23T20:55:47.066 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:47.098 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:47.205 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:47.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:47.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:47.309 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:47.338 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:47.387 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:47.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:47.495 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:47.530 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6] out_osds: [7, 2] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:55:47.531 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:55:47.531 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:55:47.531 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.421706 2023-09-23T20:55:47.531 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.4217063994701832 2023-09-23T20:55:47.532 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:47.589 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:47.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:47.601 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:47 smithi123 ceph-mon[193903]: pgmap v771: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 13 MiB/s wr, 215 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.3 MiB/s, 22 objects/s recovering 2023-09-23T20:55:47.625 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:55:47.693 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:47.722 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:47.767 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:47.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:47.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:47.871 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:47.898 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:47 smithi121 ceph-mon[211526]: pgmap v771: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 13 MiB/s wr, 215 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.3 MiB/s, 22 objects/s recovering 2023-09-23T20:55:47.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:47 smithi121 ceph-mon[219709]: pgmap v771: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 13 MiB/s wr, 215 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.3 MiB/s, 22 objects/s recovering 2023-09-23T20:55:47.945 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:47.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:47.969 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:47.969 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:48.018 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:48.050 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:48.054 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 81751 81735 82.3566 0 - 0.0110284 2023-09-23T20:55:48.089 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:48.126 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:48.147 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:48.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:48.251 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:48.283 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:48.332 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:48.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:48.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:48.437 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:48.464 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:48.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:48 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive) 2023-09-23T20:55:48.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:48 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive) 2023-09-23T20:55:48.664 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:48.666 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:48 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive) 2023-09-23T20:55:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:48.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:48.716 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:55:48.768 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:48.796 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:48.952 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:48.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:49.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 81751 81735 81.0498 0 - 0.0110284 2023-09-23T20:55:49.056 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:49.083 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:49.227 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:49.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:49.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:49.241 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:49.331 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:49.357 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:49.458 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:49.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:49.562 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:49.596 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:49 smithi121 ceph-mon[211526]: pgmap v772: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 11 MiB/s wr, 170 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:55:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:49 smithi121 ceph-mon[219709]: pgmap v772: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 11 MiB/s wr, 170 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:55:49.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:49 smithi123 ceph-mon[193903]: pgmap v772: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 11 MiB/s wr, 170 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:55:50.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 81751 81735 79.7838 0 - 0.0110284 2023-09-23T20:55:51.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 81751 81735 78.5568 0 - 0.0110284 2023-09-23T20:55:51.298 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:55:51.298 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:55:51.647 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:51.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:51.752 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:51.780 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:51.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:51 smithi121 ceph-mon[211526]: pgmap v773: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 9.4 MiB/s wr, 150 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 963 KiB/s, 15 objects/s recovering 2023-09-23T20:55:51.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 49159/243621 objects degraded (20.178%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:55:51.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:51 smithi121 ceph-mon[219709]: pgmap v773: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 9.4 MiB/s wr, 150 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 963 KiB/s, 15 objects/s recovering 2023-09-23T20:55:51.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 49159/243621 objects degraded (20.178%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:55:51.922 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:55:51.981 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:51.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:52.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 81751 81735 77.367 0 - 0.0110284 2023-09-23T20:55:52.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:52.118 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:51 smithi123 ceph-mon[193903]: pgmap v773: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 9.4 MiB/s wr, 150 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 963 KiB/s, 15 objects/s recovering 2023-09-23T20:55:52.119 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 49159/243621 objects degraded (20.178%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:55:52.121 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:52.369 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:52.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:52.392 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:52.477 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:52.511 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:52.825 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:52.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:52.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:52.930 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:52.959 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 81751 81735 76.2127 0 - 0.0110284 2023-09-23T20:55:53.309 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:53.414 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:55:53.444 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.540 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:55:53.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:55:53.647 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:55:53.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:53.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:53.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:53 smithi121 ceph-mon[219709]: pgmap v774: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 8.1 MiB/s wr, 129 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 826 KiB/s, 13 objects/s recovering 2023-09-23T20:55:53.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:53.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:53.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:53 smithi121 ceph-mon[211526]: pgmap v774: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 8.1 MiB/s wr, 129 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 826 KiB/s, 13 objects/s recovering 2023-09-23T20:55:53.674 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:53.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:53.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:53 smithi123 ceph-mon[193903]: pgmap v774: 89 pgs: 25 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 18 active+undersized, 35 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 8.1 MiB/s wr, 129 op/s; 49159/243621 objects degraded (20.178%); 10257/243621 objects misplaced (4.210%); 826 KiB/s, 13 objects/s recovering 2023-09-23T20:55:53.768 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:55:53.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:53.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:55:53.873 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:53.885 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:55:53.909 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:53.911 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:55:53.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:55:53.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:55:54.025 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:55:54.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 81845 81829 75.1786 0.534091 0.0389201 0.0132959 2023-09-23T20:55:54.060 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:55:54.068 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:55:54.068 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:55:54.068 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T20:55:54.068 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T20:55:54.068 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:55:54.093 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:54.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:55:54.101 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:55:54.102 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:55:54.197 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:54.265 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:54.390 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:54.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:54.461 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:55:54.464 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.5 primary-affinity to 0.421706 (8276362) 2023-09-23T20:55:54.498 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:54.532 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4194061769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4009038381' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]: dispatch 2023-09-23T20:55:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]: dispatch 2023-09-23T20:55:54.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4194061769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:54.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:54.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4009038381' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]: dispatch 2023-09-23T20:55:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]: dispatch 2023-09-23T20:55:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4194061769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:55:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4009038381' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]: dispatch 2023-09-23T20:55:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]: dispatch 2023-09-23T20:55:54.922 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:55:55.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 82191 82175 74.4027 21.625 0.00564657 0.0133366 2023-09-23T20:55:55.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:55:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:55:54] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-09-23T20:55:55.268 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:55.270 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:55.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:55.372 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:55.400 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:55.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]': finished 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: osdmap e245: 8 total, 7 up, 6 in 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.611 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: pgmap v776: 89 pgs: 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 3.5 MiB/s wr, 56 op/s; 49683/245322 objects degraded (20.252%); 13862/245322 objects misplaced (5.651%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]': finished 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: osdmap e245: 8 total, 7 up, 6 in 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.612 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: pgmap v776: 89 pgs: 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 3.5 MiB/s wr, 56 op/s; 49683/245322 objects degraded (20.252%); 13862/245322 objects misplaced (5.651%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:55.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:55.649 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:55.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.4217063994701832}]': finished 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: osdmap e245: 8 total, 7 up, 6 in 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: failed to encode map e245 with expected crc 2023-09-23T20:55:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: pgmap v776: 89 pgs: 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 3.5 MiB/s wr, 56 op/s; 49683/245322 objects degraded (20.252%); 13862/245322 objects misplaced (5.651%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:55:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:55.752 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:55.780 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:55.877 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:55.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:55.982 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:56.009 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:56.055 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 82218 82202 73.3642 1.6875 0.00528386 0.0133801 2023-09-23T20:55:56.512 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:55:56.512 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:55:56.702 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:56 smithi123 ceph-mon[193903]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:56 smithi123 ceph-mon[193903]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:56 smithi123 ceph-mon[193903]: osdmap e246: 8 total, 7 up, 6 in 2023-09-23T20:55:56.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:56 smithi123 ceph-mon[193903]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:56 smithi123 ceph-mon[193903]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:56 smithi123 ceph-mon[193903]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 49683/245322 objects degraded (20.252%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:55:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[211526]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[211526]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[211526]: osdmap e246: 8 total, 7 up, 6 in 2023-09-23T20:55:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[211526]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[211526]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[211526]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 49683/245322 objects degraded (20.252%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:55:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[219709]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[219709]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[219709]: osdmap e246: 8 total, 7 up, 6 in 2023-09-23T20:55:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[219709]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[219709]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[219709]: failed to encode map e246 with expected crc 2023-09-23T20:55:56.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 49683/245322 objects degraded (20.252%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:55:56.978 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:55:56.978 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:55:56.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:55:56.978 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:55:56.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95048 8.4 8:2454d6a3:::benchmark_data_smithi121_252781_object95047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 592.91000250000002, 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2752492069999999, 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:56.979 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95048 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:56.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064132+0000", 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064130+0000", 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064169+0000", 2023-09-23T20:55:56.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8658999999999997e-05 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064181+0000", 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2089999999999999e-05 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:56.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064188+0000", 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2400000000000001e-06 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064346+0000", 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015842800000000001 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064356+0000", 2023-09-23T20:55:56.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.01e-05 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072299+0000", 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0079425179999999 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072311+0000", 2023-09-23T20:55:56.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1701e-05 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.324944+0000", 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.252633109 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.325108+0000", 2023-09-23T20:55:56.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016413699999999999 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.325270+0000", 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000162201 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333069+0000", 2023-09-23T20:55:56.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077994290000000001 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333084+0000", 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4813e-05 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.338377+0000", 2023-09-23T20:55:56.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0052931949999999997 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.338442+0000", 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4246999999999997e-05 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.339381+0000", 2023-09-23T20:55:56.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00093899200000000004 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95052 8.d 8:bca20ec2:::benchmark_data_smithi121_252781_object95051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 592.90993406799998, 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2831711729999999, 2023-09-23T20:55:56.989 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95052 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:56.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064200+0000", 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:56.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064199+0000", 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064234+0000", 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4929999999999999e-05 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064242+0000", 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8450000000000003e-06 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064248+0000", 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8989999999999998e-06 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:56.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064373+0000", 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012531700000000001 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.064383+0000", 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4509999999999998e-06 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072655+0000", 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.008272584 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072666+0000", 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0275000000000001e-05 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336059+0000", 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2633933260000001 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336211+0000", 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000152299 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.336390+0000", 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017826800000000001 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.340844+0000", 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044546439999999998 2023-09-23T20:55:56.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.340887+0000", 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2695999999999999e-05 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346536+0000", 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056488220000000004 2023-09-23T20:55:56.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346600+0000", 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3899000000000006e-05 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.347371+0000", 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00077147299999999995 2023-09-23T20:55:56.999 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:95057 8.4 8:2a7799b6:::benchmark_data_smithi121_252781_object95056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 592.90347879000001, 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2767475730000002, 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.000 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 95057 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:55:57.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070655+0000", 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070654+0000", 2023-09-23T20:55:57.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070684+0000", 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0334000000000001e-05 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070699+0000", 2023-09-23T20:55:57.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4211e-05 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070706+0000", 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7249999999999997e-06 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070765+0000", 2023-09-23T20:55:57.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8112999999999997e-05 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:04.070775+0000", 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0766e-05 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072335+0000", 2023-09-23T20:55:57.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0015602290000001 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:05.072343+0000", 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0609999999999998e-06 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329624+0000", 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2572813840000001 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329748+0000", 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000123678 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.329878+0000", 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013038500000000001 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333185+0000", 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033074290000000002 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.333622+0000", 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043623000000000002 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346585+0000", 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012963585 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.346641+0000", 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6227999999999998e-05 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:46:06.347403+0000", 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00076131300000000001 2023-09-23T20:55:57.010 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186212 8.8 8:10804386:::benchmark_data_smithi121_252781_object77037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 470.47864407499998, 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1681942240000001, 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.011 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186212 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495489+0000", 2023-09-23T20:55:57.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495493+0000", 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1779999999999999e-06 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495500+0000", 2023-09-23T20:55:57.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7600000000000002e-06 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495504+0000", 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9720000000000003e-06 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500662+0000", 2023-09-23T20:55:57.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051571189999999999 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500891+0000", 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022905000000000001 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 7", 2023-09-23T20:55:57.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.501010+0000", 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119268 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509849+0000", 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088391219999999996 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509859+0000", 2023-09-23T20:55:57.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0032e-05 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513758+0000", 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0038989819999999 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513773+0000", 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5116999999999999e-05 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602445+0000", 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0886714449999999 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602553+0000", 2023-09-23T20:55:57.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010843199999999999 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663605+0000", 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.061052150999999999 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663684+0000", 2023-09-23T20:55:57.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9040000000000002e-05 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186226 8.8 8:109cf42b:::benchmark_data_smithi121_252781_object77051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 470.44451162500002, 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0872477360000001, 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.021 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186226 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:55:57.022 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529620+0000", 2023-09-23T20:55:57.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529629+0000", 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8680000000000005e-06 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529649+0000", 2023-09-23T20:55:57.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9874e-05 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529657+0000", 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6680000000000001e-06 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529724+0000", 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7138000000000007e-05 2023-09-23T20:55:57.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529737+0000", 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3083e-05 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513799+0000", 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98406165899999998 2023-09-23T20:55:57.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513808+0000", 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8489999999999995e-06 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602617+0000", 2023-09-23T20:55:57.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0888096490000001 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602726+0000", 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010896 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602846+0000", 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011963099999999999 2023-09-23T20:55:57.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616554+0000", 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013708016 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616622+0000", 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7794999999999997e-05 2023-09-23T20:55:57.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616656+0000", 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4397999999999998e-05 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616718+0000", 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1920000000000003e-05 2023-09-23T20:55:57.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616870+0000", 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015218 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186232 8.8 8:18ce4fcc:::benchmark_data_smithi121_252781_object77057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:55:57.031 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 470.44121510500003, 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1309886809999998, 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186232 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.032 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532918+0000", 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532923+0000", 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7319999999999998e-06 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532936+0000", 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.293e-05 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532942+0000", 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6980000000000003e-06 2023-09-23T20:55:57.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532993+0000", 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1447999999999999e-05 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532999+0000", 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3389999999999998e-06 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513832+0000", 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98083224099999999 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513840+0000", 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1070000000000005e-06 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607582+0000", 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0937428 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607747+0000", 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016451500000000001 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.608823+0000", 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001075559 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616564+0000", 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077410530000000003 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616924+0000", 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00036078199999999998 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663714+0000", 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.046789848000000002 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663776+0000", 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2114999999999997e-05 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663908+0000", 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013124799999999999 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186215 8.0 8:00f6bf9b:::benchmark_data_smithi121_252781_object77040:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 469.45985720800002, 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.162298345, 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.042 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186215 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514275+0000", 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514284+0000", 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9570000000000008e-06 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514297+0000", 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2663000000000001e-05 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514305+0000", 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2970000000000004e-06 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514439+0000", 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013414199999999999 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514449+0000", 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3139999999999993e-06 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649698+0000", 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1352495979999999 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:55:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649809+0000", 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110911 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676465+0000", 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026655729999999999 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676575+0000", 2023-09-23T20:55:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110305 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186222 8.0 8:0eef9874:::benchmark_data_smithi121_252781_object77047:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 469.45979961199998, 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1614129209999999, 2023-09-23T20:55:57.049 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186222 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514339+0000", 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1680000000000003e-06 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514348+0000", 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0140000000000004e-06 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514353+0000", 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514474+0000", 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121105 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514482+0000", 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4039999999999998e-06 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649859+0000", 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.135377012 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649954+0000", 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4914999999999995e-05 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.650033+0000", 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9120000000000001e-05 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663245+0000", 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013211926000000001 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674215+0000", 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010970126 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675506+0000", 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001290914 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675559+0000", 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2643999999999999e-05 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675747+0000", 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000188815 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.058 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186230 8.0 8:041ac878:::benchmark_data_smithi121_252781_object77055:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 469.459774235, 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1616748569999999, 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.059 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186230 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514359+0000", 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514363+0000", 2023-09-23T20:55:57.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8360000000000002e-06 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514370+0000", 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4899999999999997e-06 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514374+0000", 2023-09-23T20:55:57.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6750000000000001e-06 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514504+0000", 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012982599999999999 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514512+0000", 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5959999999999998e-06 2023-09-23T20:55:57.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654104+0000", 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.139592663 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654250+0000", 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014595400000000001 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654394+0000", 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143442 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663683+0000", 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092891689999999999 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674249+0000", 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010566002 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675814+0000", 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015649539999999999 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675875+0000", 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1224999999999997e-05 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676035+0000", 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015948000000000001 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186231 8.0 8:0bcd4c07:::benchmark_data_smithi121_252781_object77056:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 469.4597536, 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.168650363, 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.068 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186231 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514383+0000", 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514388+0000", 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5969999999999999e-06 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514393+0000", 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.16e-06 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514533+0000", 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140841 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514540+0000", 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9890000000000003e-06 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659037+0000", 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.144496156 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659183+0000", 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014606600000000001 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659334+0000", 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015159700000000001 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663874+0000", 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045394600000000004 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674301+0000", 2023-09-23T20:55:57.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010427439 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682815+0000", 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085137149999999998 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682865+0000", 2023-09-23T20:55:57.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0510999999999998e-05 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.683031+0000", 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016547400000000001 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209245 8.8 8:1443b4f6:::benchmark_data_smithi121_252781_object100070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.24575979500003, 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.08464988, 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209245 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728373+0000", 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728379+0000", 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5089999999999999e-06 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728395+0000", 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5710999999999999e-05 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728402+0000", 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7190000000000004e-06 2023-09-23T20:55:57.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728795+0000", 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039319799999999999 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728805+0000", 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6690000000000001e-06 2023-09-23T20:55:57.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757335+0000", 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0285302249999999 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757427+0000", 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1818999999999996e-05 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757509+0000", 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1922000000000006e-05 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768732+0000", 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011223713 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812940+0000", 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.044207659000000003 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812972+0000", 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1633e-05 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813001+0000", 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9300000000000001e-05 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813024+0000", 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3244999999999999e-05 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.086 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209248 8.8 8:136e166f:::benchmark_data_smithi121_252781_object100073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.24572333200001, 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.084736245, 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.087 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209248 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728410+0000", 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728413+0000", 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.756e-06 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728419+0000", 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3940000000000001e-06 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728423+0000", 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9400000000000004e-06 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728828+0000", 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00040448300000000002 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728835+0000", 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8469999999999998e-06 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761382+0000", 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0325472419999999 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761496+0000", 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113714 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761583+0000", 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7459000000000004e-05 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.772112+0000", 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010529221 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812957+0000", 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040844889000000002 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813061+0000", 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000104036 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813109+0000", 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7704999999999998e-05 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813147+0000", 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8046000000000001e-05 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136323 9.6 9:6c612c1e:::benchmark_data_smithi121_293770_object33435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:55:57.096 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 206.75946811099999, 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0778323409999999, 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136323 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.097 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214665+0000", 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214669+0000", 2023-09-23T20:55:57.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2310000000000001e-06 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214675+0000", 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6250000000000001e-06 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214680+0000", 2023-09-23T20:55:57.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.301e-06 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214759+0000", 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9574999999999996e-05 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214869+0000", 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010993399999999999 2023-09-23T20:55:57.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214978+0000", 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000108773 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224530+0000", 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095521080000000001 2023-09-23T20:55:57.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224538+0000", 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0970000000000006e-06 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.283176+0000", 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0586380129999999 2023-09-23T20:55:57.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.283513+0000", 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000336769 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292448+0000", 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0089352670000000002 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292498+0000", 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0102000000000002e-05 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74225 10.9 10:9b7c5c1b:::benchmark_data_smithi121_358375_object74224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:55:57.105 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.710286+0000", 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.263847704, 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3339274059999999, 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74225 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.106 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710286+0000", 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710286+0000", 2023-09-23T20:55:57.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710282+0000", 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999957 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710348+0000", 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6082999999999997e-05 2023-09-23T20:55:57.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710359+0000", 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1491000000000001e-05 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710364+0000", 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.206e-06 2023-09-23T20:55:57.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710407+0000", 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3637000000000003e-05 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710560+0000", 2023-09-23T20:55:57.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015250499999999999 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710732+0000", 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017187699999999999 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.718766+0000", 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080340979999999999 2023-09-23T20:55:57.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.727854+0000", 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0090881409999999992 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885871+0000", 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.15801679499999999 2023-09-23T20:55:57.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885883+0000", 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1973999999999999e-05 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899218+0000", 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.013335289 2023-09-23T20:55:57.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899229+0000", 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1124000000000001e-05 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.044111+0000", 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1448817710000001 2023-09-23T20:55:57.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.044214+0000", 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102935 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.115 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74232 10.9 10:9163f21b:::benchmark_data_smithi121_358375_object74231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.716015+0000", 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.258119331, 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.329761704, 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74232 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.116 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716015+0000", 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716015+0000", 2023-09-23T20:55:57.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716013+0000", 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716234+0000", 2023-09-23T20:55:57.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022076299999999999 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716248+0000", 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3532e-05 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716259+0000", 2023-09-23T20:55:57.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1138000000000001e-05 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716302+0000", 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2920000000000002e-05 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716418+0000", 2023-09-23T20:55:57.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115862 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716541+0000", 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012331199999999999 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.718820+0000", 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022795559999999999 2023-09-23T20:55:57.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.729911+0000", 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011090645 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885918+0000", 2023-09-23T20:55:57.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.15600684300000001 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885925+0000", 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3590000000000002e-06 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899269+0000", 2023-09-23T20:55:57.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0133434050000001 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899275+0000", 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0410000000000003e-06 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045708+0000", 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.146433093 2023-09-23T20:55:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045776+0000", 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8604999999999998e-05 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.125 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74256 10.9 10:9d9325d5:::benchmark_data_smithi121_358375_object74255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.874623+0000", 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.099510797000001, 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1713063940000001, 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74256 2023-09-23T20:55:57.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874623+0000", 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874623+0000", 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874622+0000", 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874668+0000", 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5856000000000003e-05 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874680+0000", 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2401999999999999e-05 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874685+0000", 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6380000000000003e-06 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874732+0000", 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6709999999999998e-05 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874873+0000", 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014162800000000001 2023-09-23T20:55:57.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.875031+0000", 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015807899999999999 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.877600+0000", 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025683099999999999 2023-09-23T20:55:57.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.878245+0000", 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00064534499999999997 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.892611+0000", 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014365562 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.892621+0000", 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0226e-05 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899301+0000", 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0066803090000001 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899307+0000", 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6520000000000003e-06 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045830+0000", 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.146523298 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045930+0000", 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9457999999999995e-05 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74271 10.9 10:993b4897:::benchmark_data_smithi121_358375_object74270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.909039+0000", 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.065095287999998, 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1620222980000001, 2023-09-23T20:55:57.135 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74271 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909039+0000", 2023-09-23T20:55:57.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909039+0000", 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909037+0000", 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909098+0000", 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0850999999999997e-05 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909118+0000", 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0117999999999999e-05 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909126+0000", 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2110000000000001e-06 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909186+0000", 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0157e-05 2023-09-23T20:55:57.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909198+0000", 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1755e-05 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899327+0000", 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.990129386 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899333+0000", 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6910000000000002e-06 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045959+0000", 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.14662636 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.046111+0000", 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015205500000000001 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.046294+0000", 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000182938 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.056140+0000", 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0098462289999999997 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061791+0000", 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056506589999999997 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.067344+0000", 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055534979999999996 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.067403+0000", 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9017000000000001e-05 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071061+0000", 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036576600000000001 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74276 10.9 10:916b3091:::benchmark_data_smithi121_358375_object74275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.915396+0000", 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.058737659999998, 2023-09-23T20:55:57.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.159243085, 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74276 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915396+0000", 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915396+0000", 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915395+0000", 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915445+0000", 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9669000000000002e-05 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915456+0000", 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1321000000000001e-05 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915461+0000", 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0590000000000002e-06 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915508+0000", 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7253999999999999e-05 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915517+0000", 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1599999999999998e-06 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899353+0000", 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98383633000000004 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899359+0000", 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3860000000000004e-06 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048395+0000", 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1490360150000001 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048548+0000", 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015308499999999999 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:57.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048724+0000", 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017566700000000001 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061802+0000", 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013078227 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061864+0000", 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2067000000000001e-05 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071113+0000", 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092486760000000008 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071163+0000", 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9515999999999998e-05 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074639+0000", 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003476884 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74277 10.9 10:902456f8:::benchmark_data_smithi121_358375_object74276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.058263549999999, 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1624961859999998, 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74277 2023-09-23T20:55:57.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:55:57.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915934+0000", 2023-09-23T20:55:57.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4109999999999994e-05 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915944+0000", 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7089999999999997e-06 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915948+0000", 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.33e-06 2023-09-23T20:55:57.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915982+0000", 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4347000000000002e-05 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915990+0000", 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8930000000000005e-06 2023-09-23T20:55:57.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899380+0000", 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98338946299999996 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899385+0000", 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4439999999999999e-06 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.050951+0000", 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1515660750000001 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.051065+0000", 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113613 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.051231+0000", 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016582399999999999 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061811+0000", 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010579965 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061902+0000", 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1799000000000007e-05 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074825+0000", 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012922417 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074866+0000", 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0949999999999999e-05 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078367+0000", 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035008359999999998 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74279 10.9 10:9ea30411:::benchmark_data_smithi121_358375_object74278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.920027+0000", 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.054106935, 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.161850388, 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:55:57.166 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74279 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920027+0000", 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:55:57.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920027+0000", 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920026+0000", 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920073+0000", 2023-09-23T20:55:57.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6397000000000003e-05 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920085+0000", 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.287e-05 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920091+0000", 2023-09-23T20:55:57.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1490000000000002e-06 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920130+0000", 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9684999999999997e-05 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920138+0000", 2023-09-23T20:55:57.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5689999999999999e-06 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899415+0000", 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97927704699999996 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:55:57.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899421+0000", 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9020000000000004e-06 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053407+0000", 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1539860749999999 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053558+0000", 2023-09-23T20:55:57.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000151216 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053723+0000", 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165048 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061817+0000", 2023-09-23T20:55:57.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080936069999999992 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061971+0000", 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000154582 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078458+0000", 2023-09-23T20:55:57.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016486923000000001 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078518+0000", 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0068000000000002e-05 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.081877+0000", 2023-09-23T20:55:57.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033591329999999998 2023-09-23T20:55:57.176 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.176 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.176 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.176 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:55:57.176 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:55:57.176 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:55:57.177 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 82420 82404 72.509 12.625 0.00402423 0.0135888 2023-09-23T20:55:57.275 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:57.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:57.380 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:57.408 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:57.578 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:57.615 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.615 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: osdmap e247: 8 total, 7 up, 6 in 2023-09-23T20:55:57.615 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.615 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:55:57.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:55:57.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:57.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: pgmap v779: 89 pgs: 2 peering, 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.2 MiB/s wr, 99 op/s; 34939/245418 objects degraded (14.237%); 13862/245418 objects misplaced (5.648%); 12 MiB/s, 187 objects/s recovering 2023-09-23T20:55:57.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:57.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: osdmap e247: 8 total, 7 up, 6 in 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: pgmap v779: 89 pgs: 2 peering, 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.2 MiB/s wr, 99 op/s; 34939/245418 objects degraded (14.237%); 13862/245418 objects misplaced (5.648%); 12 MiB/s, 187 objects/s recovering 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:57.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:57.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:57.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:57.716 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.717 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: osdmap e247: 8 total, 7 up, 6 in 2023-09-23T20:55:57.717 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.717 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.717 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: failed to encode map e247 with expected crc 2023-09-23T20:55:57.717 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:55:57.718 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:55:57.718 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:57.718 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: pgmap v779: 89 pgs: 2 peering, 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.2 MiB/s wr, 99 op/s; 34939/245418 objects degraded (14.237%); 13862/245418 objects misplaced (5.648%); 12 MiB/s, 187 objects/s recovering 2023-09-23T20:55:57.718 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:55:57.718 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:55:57.720 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:57.730 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:55:57.749 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:57.798 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:57.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:57.904 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:57.931 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:57.978 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:57.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:58.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 82422 82406 71.504 0.125 0.00327211 0.0135886 2023-09-23T20:55:58.083 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:58.100 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:55:58.111 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:58.157 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:58.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:58.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:58.264 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:58.290 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:58.337 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:58.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:58.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:55:58.441 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:58.469 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:58.514 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:55:58.715 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:58.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:58.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:58.819 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:55:58.849 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:58.904 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:55:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:58.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:55:58.913 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:55:59.010 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:55:59.038 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:59.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 82422 82406 70.5248 0 - 0.0135886 2023-09-23T20:55:59.266 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:55:59.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:59.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:55:59.373 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:55:59.401 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:59.448 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:55:59.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:59.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:55:59.555 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:55:59.582 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:59.636 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:55:59.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:59.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:55:59.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:55:59.712 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:55:59.712 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:55:59.712 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T20:55:59.712 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:55:59.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:55:59.772 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:55:59.968 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:55:59.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:55:59.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:00.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 82422 82406 69.5721 0 - 0.0135886 2023-09-23T20:56:00.071 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:00.095 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:55:59 smithi123 ceph-mon[193903]: pgmap v780: 89 pgs: 2 peering, 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 10 MiB/s wr, 160 op/s; 30615/246510 objects degraded (12.419%); 17910/246510 objects misplaced (7.265%); 14 MiB/s, 221 objects/s recovering 2023-09-23T20:56:00.097 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:55:59 smithi121 ceph-mon[211526]: pgmap v780: 89 pgs: 2 peering, 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 10 MiB/s wr, 160 op/s; 30615/246510 objects degraded (12.419%); 17910/246510 objects misplaced (7.265%); 14 MiB/s, 221 objects/s recovering 2023-09-23T20:56:00.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:55:59 smithi121 ceph-mon[219709]: pgmap v780: 89 pgs: 2 peering, 1 active+undersized+degraded+remapped, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 10 MiB/s wr, 160 op/s; 30615/246510 objects degraded (12.419%); 17910/246510 objects misplaced (7.265%); 14 MiB/s, 221 objects/s recovering 2023-09-23T20:56:00.341 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:00.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:00.444 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:00.446 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:56:00.512 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.556 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:00.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:00.666 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:00.693 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.741 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:00.847 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:00.851 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:00.878 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:00.932 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:00.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:01.037 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:01.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 82422 82406 68.6447 0 - 0.0135886 2023-09-23T20:56:01.065 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:01.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:01.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:01.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:01.214 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:01.243 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:01.290 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:01.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:01.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:01.395 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:01.423 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:01.567 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:01.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:01.641 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:01.671 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:01.705 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:01.783 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:01.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:01.888 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:01.902 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:01 smithi121 ceph-mon[219709]: pgmap v781: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.2 MiB/s wr, 99 op/s; 34072/247218 objects degraded (13.782%); 14489/247218 objects misplaced (5.861%); 2.1 MiB/s, 32 objects/s recovering 2023-09-23T20:56:01.902 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 34072/247218 objects degraded (13.782%), 10 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:01.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:01 smithi121 ceph-mon[211526]: pgmap v781: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.2 MiB/s wr, 99 op/s; 34072/247218 objects degraded (13.782%); 14489/247218 objects misplaced (5.861%); 2.1 MiB/s, 32 objects/s recovering 2023-09-23T20:56:01.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 34072/247218 objects degraded (13.782%), 10 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:01.914 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:02.008 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:02.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:02.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:02.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 82422 82406 67.7417 0 - 0.0135886 2023-09-23T20:56:02.113 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:02.146 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:01 smithi123 ceph-mon[193903]: pgmap v781: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 6.2 MiB/s wr, 99 op/s; 34072/247218 objects degraded (13.782%); 14489/247218 objects misplaced (5.861%); 2.1 MiB/s, 32 objects/s recovering 2023-09-23T20:56:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 34072/247218 objects degraded (13.782%), 10 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:02.300 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:02.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:02.403 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:02.433 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:02.477 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:56:02.477 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:56:02.477 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T20:56:02.477 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T20:56:02.477 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:56:02.479 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:02.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:02.582 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:02.610 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:02.620 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6] out_osds: [7, 2] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:56:02.620 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:02.620 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:56:02.620 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:56:02.706 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:02.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:02.712 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:56:02.735 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:56:02.810 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:02.838 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1236781590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1236781590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:03.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 82422 82406 66.8622 0 - 0.0135886 2023-09-23T20:56:03.104 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:03.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:03.154 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1236781590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:03.171 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:56:03.208 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:03.239 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:03.526 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:03.541 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:03.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:03.565 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:03.648 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:03.678 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:03.785 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:03.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:03.890 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:03.924 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:04.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:03 smithi121 ceph-mon[219709]: pgmap v782: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.9 MiB/s wr, 78 op/s; 34072/247218 objects degraded (13.782%); 14489/247218 objects misplaced (5.861%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:56:04.053 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:03 smithi121 ceph-mon[211526]: pgmap v782: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.9 MiB/s wr, 78 op/s; 34072/247218 objects degraded (13.782%); 14489/247218 objects misplaced (5.861%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:56:04.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 82422 82406 66.0053 0 - 0.0135886 2023-09-23T20:56:04.081 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:04.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:04.122 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:03 smithi123 ceph-mon[193903]: pgmap v782: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.9 MiB/s wr, 78 op/s; 34072/247218 objects degraded (13.782%); 14489/247218 objects misplaced (5.861%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:56:04.187 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:04.214 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:04.257 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:04.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:04.361 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:04.388 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:04.597 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:04.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:04.700 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:04.733 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:04.857 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:56:04.880 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:04.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:04.987 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:05.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:05.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 82422 82406 65.17 0 - 0.0135886 2023-09-23T20:56:05.062 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:05.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:05.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:05.124 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:56:05.124 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:56:05.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136312 9.0 9:087a316d:::benchmark_data_smithi121_293770_object33424:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 214.892672546, 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.026362005, 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:56:05.125 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136312 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228794+0000", 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228801+0000", 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6000000000000003e-06 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228812+0000", 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1517e-05 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228821+0000", 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.089e-06 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230287+0000", 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014660960000000001 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230299+0000", 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255083+0000", 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247833159999999 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255158+0000", 2023-09-23T20:56:05.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5184000000000004e-05 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136315 9.0 9:07b4e2b9:::benchmark_data_smithi121_293770_object33427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 214.89066156999999, 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0248562560000001, 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.130 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136315 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:56:05.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230806+0000", 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230811+0000", 2023-09-23T20:56:05.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.879e-06 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230823+0000", 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.151e-05 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230830+0000", 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8460000000000002e-06 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230876+0000", 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6560999999999997e-05 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230884+0000", 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8960000000000003e-06 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255603+0000", 2023-09-23T20:56:05.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247184060000001 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255663+0000", 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0364e-05 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136335 9.0 9:00bf7afd:::benchmark_data_smithi121_293770_object33447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:56:05.135 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 214.865032101, 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0152258009999999, 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136335 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:56:05.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256434+0000", 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256453+0000", 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8349999999999999e-05 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256484+0000", 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0908999999999998e-05 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256494+0000", 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0479000000000001e-05 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256538+0000", 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3940999999999998e-05 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256547+0000", 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6689999999999995e-06 2023-09-23T20:56:05.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255731+0000", 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99918387799999997 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255863+0000", 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000131917 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T20:56:05.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255979+0000", 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011618899999999999 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.267419+0000", 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011440723999999999 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271524+0000", 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041050169999999999 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271584+0000", 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9577999999999998e-05 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271662+0000", 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7960999999999994e-05 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:20990 10.d 10:bc9bb19b:::benchmark_data_smithi121_358375_object20989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:54:52.982047+0000", 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 72.139421532, 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.176620527, 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 20990 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.143 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982047+0000", 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982047+0000", 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982046+0000", 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982120+0000", 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3212000000000003e-05 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982129+0000", 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4399999999999994e-06 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982140+0000", 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0609e-05 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982191+0000", 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1471000000000002e-05 2023-09-23T20:56:05.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982304+0000", 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011323999999999999 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.982436+0000", 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013189699999999999 2023-09-23T20:56:05.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983559+0000", 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001122266 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.984786+0000", 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012279019999999999 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.158602+0000", 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17381511999999999 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.158640+0000", 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8890000000000002e-05 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.158667+0000", 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6837000000000001e-05 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.149 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:20996 10.b 10:ddb71fff:::benchmark_data_smithi121_358375_object20995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:54:52.983639+0000", 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 72.137829507000006, 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17627100700000001, 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.150 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 20996 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983639+0000", 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983639+0000", 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983638+0000", 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983725+0000", 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6908000000000002e-05 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983737+0000", 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1742999999999999e-05 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983743+0000", 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6359999999999997e-06 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983786+0000", 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3260000000000003e-05 2023-09-23T20:56:05.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983906+0000", 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011948800000000001 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.984044+0000", 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013844700000000001 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985626+0000", 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015812770000000001 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.159751+0000", 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17412514200000001 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.159810+0000", 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9218000000000001e-05 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.159910+0000", 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9807000000000002e-05 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:21001 10.d 10:b0fcb8aa:::benchmark_data_smithi121_358375_object21000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:54:52.983937+0000", 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 72.137531742999997, 2023-09-23T20:56:05.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17702852099999999, 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 21001 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983937+0000", 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983937+0000", 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983936+0000", 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983969+0000", 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2580999999999999e-05 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983978+0000", 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6479999999999998e-06 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.983984+0000", 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8880000000000002e-06 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.984806+0000", 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000821619 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.984914+0000", 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000108402 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985072+0000", 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015764800000000001 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.987581+0000", 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025090490000000002 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.988724+0000", 2023-09-23T20:56:05.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001143219 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.160880+0000", 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.172155898 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.160929+0000", 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9048999999999997e-05 2023-09-23T20:56:05.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.160965+0000", 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6001000000000003e-05 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:21002 10.d 10:bd7bc5e8:::benchmark_data_smithi121_358375_object21001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-09-23T20:56:05.163 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:54:52.985661+0000", 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 72.135807169000003, 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17540602399999999, 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 21002 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985661+0000", 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985661+0000", 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985660+0000", 2023-09-23T20:56:05.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985692+0000", 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2047e-05 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985704+0000", 2023-09-23T20:56:05.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2182e-05 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.985711+0000", 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3489999999999997e-06 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.986774+0000", 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001063236 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.986872+0000", 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7948000000000005e-05 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.986975+0000", 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103491 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.988739+0000", 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0017635730000000001 2023-09-23T20:56:05.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:52.989024+0000", 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00028551500000000002 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.160999+0000", 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17197464900000001 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.161035+0000", 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5655999999999999e-05 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:54:53.161067+0000", 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2601000000000002e-05 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:53056 10.a 10:56c76c61:::benchmark_data_smithi121_358375_object53055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:12.295720+0000", 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 52.825747988000003, 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.16838745099999999, 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 53056 2023-09-23T20:56:05.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.295720+0000", 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.295720+0000", 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.295720+0000", 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.295752+0000", 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2258999999999997e-05 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.295759+0000", 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2189999999999998e-06 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.295764+0000", 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5040000000000004e-06 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.297765+0000", 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020015369999999998 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.300357+0000", 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025916310000000001 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.300492+0000", 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000135313 2023-09-23T20:56:05.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.302596+0000", 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021033670000000001 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.305565+0000", 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002968923 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.464030+0000", 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.15846507600000001 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.464074+0000", 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4048000000000002e-05 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:12.464108+0000", 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4068e-05 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.177 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54863 10.a 10:565aacae:::benchmark_data_smithi121_358375_object54862:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.606766+0000", 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 51.514702227000001, 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24719792199999999, 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.178 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54863 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606766+0000", 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606766+0000", 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606765+0000", 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606858+0000", 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3129999999999998e-05 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606867+0000", 2023-09-23T20:56:05.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8130000000000002e-06 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606871+0000", 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7170000000000002e-06 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606931+0000", 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0034999999999997e-05 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607084+0000", 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015297299999999999 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607228+0000", 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014409000000000001 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610026+0000", 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027982810000000001 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.611999+0000", 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019722400000000001 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853860+0000", 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.241861883 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853919+0000", 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8319999999999997e-05 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853964+0000", 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5309000000000003e-05 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.184 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54867 10.d 10:ba9afdf3:::benchmark_data_smithi121_358375_object54866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.610576+0000", 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 51.510892173000002, 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24739549799999999, 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.185 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54867 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610576+0000", 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610576+0000", 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610575+0000", 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610607+0000", 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1170999999999997e-05 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610613+0000", 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8569999999999997e-06 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610618+0000", 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1930000000000003e-06 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610669+0000", 2023-09-23T20:56:05.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1564999999999998e-05 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610789+0000", 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119528 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610914+0000", 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000124776 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612991+0000", 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020775300000000002 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.615082+0000", 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020910730000000002 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857885+0000", 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24280277 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857934+0000", 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9243999999999998e-05 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857972+0000", 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7534999999999999e-05 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.191 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54870 10.d 10:beed2a01:::benchmark_data_smithi121_358375_object54869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.612938+0000", 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 51.508530215, 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.246329456, 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.192 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54870 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612938+0000", 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612938+0000", 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612937+0000", 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613166+0000", 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022887800000000001 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613179+0000", 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3349e-05 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613187+0000", 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2450000000000001e-06 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613229+0000", 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2012999999999999e-05 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613362+0000", 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013293800000000001 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613508+0000", 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014622999999999999 2023-09-23T20:56:05.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.616317+0000", 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028088649999999998 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.616844+0000", 2023-09-23T20:56:05.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00052692799999999997 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.859173+0000", 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24232903 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.859220+0000", 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7658999999999999e-05 2023-09-23T20:56:05.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.859268+0000", 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7193e-05 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57388 10.d 10:b72d9c21:::benchmark_data_smithi121_358375_object57387:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.686295+0000", 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 49.435172848000001, 2023-09-23T20:56:05.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.245134888, 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57388 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686295+0000", 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686295+0000", 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686295+0000", 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686339+0000", 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4079000000000003e-05 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686347+0000", 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8790000000000003e-06 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686352+0000", 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4790000000000004e-06 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686392+0000", 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9817999999999997e-05 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686472+0000", 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0037999999999995e-05 2023-09-23T20:56:05.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686553+0000", 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0502999999999998e-05 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.688144+0000", 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015914119999999999 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.691819+0000", 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036744120000000002 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.931351+0000", 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23953239400000001 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.931395+0000", 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4261999999999997e-05 2023-09-23T20:56:05.205 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.931430+0000", 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5182999999999998e-05 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57390 10.a 10:5f660332:::benchmark_data_smithi121_358375_object57389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.206 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.686368+0000", 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 49.435100532, 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24707572799999999, 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57390 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686368+0000", 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686368+0000", 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686367+0000", 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686395+0000", 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7129000000000001e-05 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686401+0000", 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7329999999999997e-06 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686408+0000", 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.246e-06 2023-09-23T20:56:05.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686460+0000", 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2568999999999998e-05 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686571+0000", 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110541 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.686699+0000", 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128455 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.688816+0000", 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002117383 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.693022+0000", 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042059360000000004 2023-09-23T20:56:05.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933360+0000", 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24033731799999999 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933407+0000", 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7710999999999998e-05 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933444+0000", 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6115999999999998e-05 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57392 10.f 10:f63c67b4:::benchmark_data_smithi121_358375_object57391:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.690189+0000", 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 49.431279683, 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.237790907, 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.213 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57392 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690189+0000", 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690189+0000", 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690188+0000", 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690221+0000", 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3281999999999997e-05 2023-09-23T20:56:05.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690232+0000", 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1659e-05 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690238+0000", 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.84e-06 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690276+0000", 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7945999999999999e-05 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690398+0000", 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121817 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690530+0000", 2023-09-23T20:56:05.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013215300000000001 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.692647+0000", 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021171110000000001 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.692798+0000", 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000150371 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927903+0000", 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.235104966 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927942+0000", 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8930000000000002e-05 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927980+0000", 2023-09-23T20:56:05.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7985000000000003e-05 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57397 10.f 10:f58b51e2:::benchmark_data_smithi121_358375_object57396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.694113+0000", 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 49.427355147, 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23771592499999999, 2023-09-23T20:56:05.220 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57397 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694113+0000", 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694113+0000", 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694112+0000", 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694171+0000", 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8239999999999998e-05 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694188+0000", 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7638999999999998e-05 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694195+0000", 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7519999999999999e-06 2023-09-23T20:56:05.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694373+0000", 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000178247 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694498+0000", 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000124621 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:56:05.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694622+0000", 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000124053 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.696787+0000", 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021652720000000002 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.696865+0000", 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7667e-05 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.931743+0000", 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23487841000000001 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.931791+0000", 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7564000000000001e-05 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.931829+0000", 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8365999999999998e-05 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:58207 10.e 10:7c929052:::benchmark_data_smithi121_358375_object58206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.630068117, 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.26084283200000002, 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.227 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 58207 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491437+0000", 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7049999999999999e-05 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491445+0000", 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.909e-06 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491450+0000", 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.074e-06 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.493204+0000", 2023-09-23T20:56:05.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001754579 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.493354+0000", 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014998499999999999 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.493457+0000", 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102198 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.495328+0000", 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0018714059999999999 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.497890+0000", 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025619940000000002 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752164+0000", 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25427398499999998 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752206+0000", 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2197999999999998e-05 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752243+0000", 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6906999999999997e-05 2023-09-23T20:56:05.233 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:58217 10.e 10:7b3f15f5:::benchmark_data_smithi121_358375_object58216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:16.499231+0000", 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.622236899999997, 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25558374, 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.234 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 58217 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499231+0000", 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499231+0000", 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499230+0000", 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499290+0000", 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9568999999999998e-05 2023-09-23T20:56:05.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499304+0000", 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3618000000000001e-05 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499310+0000", 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2779999999999999e-06 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499359+0000", 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9128999999999997e-05 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499466+0000", 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107271 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499601+0000", 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000134399 2023-09-23T20:56:05.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752289+0000", 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25268851599999997 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752331+0000", 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1819000000000001e-05 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.754726+0000", 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002394551 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.754780+0000", 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4574000000000003e-05 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.754815+0000", 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5052000000000003e-05 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.240 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:58218 10.0 10:080c1c3d:::benchmark_data_smithi121_358375_object58217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:16.499333+0000", 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.622135206999999, 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25737310699999999, 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:56:05.241 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 58218 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499333+0000", 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499333+0000", 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499333+0000", 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499364+0000", 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0858000000000002e-05 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.244 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.244 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499371+0000", 2023-09-23T20:56:05.244 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6070000000000002e-06 2023-09-23T20:56:05.244 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.244 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.244 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499376+0000", 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.237e-06 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499414+0000", 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7154999999999999e-05 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499528+0000", 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011468600000000001 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4", 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499657+0000", 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128319 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752168+0000", 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25251165399999997 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.756574+0000", 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044055320000000002 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.756624+0000", 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0575000000000003e-05 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.756706+0000", 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1945000000000002e-05 2023-09-23T20:56:05.247 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop_reply(client.45470.0:58207 10.e e235/224 ondisk, result = 0)", 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:16.499772+0000", 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.621696514999996, 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25247557900000001, 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.248 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499772+0000", 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499772+0000", 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499771+0000", 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.499775+0000", 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6119999999999999e-06 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.750182+0000", 2023-09-23T20:56:05.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25040714200000003 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.750194+0000", 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1496e-05 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752152+0000", 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019583930000000001 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752159+0000", 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7630000000000003e-06 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752247+0000", 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8654999999999997e-05 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop_reply(client.45470.0:58218 10.0 e235/229 ondisk, result = 0)", 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:16.501500+0000", 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.619968544000002, 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25067011300000003, 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.501500+0000", 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.501500+0000", 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.501499+0000", 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.501504+0000", 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.374e-06 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.750205+0000", 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24870140199999999 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.750213+0000", 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4060000000000005e-06 2023-09-23T20:56:05.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752155+0000", 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001942018 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752162+0000", 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8299999999999998e-06 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:56:05.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752170+0000", 2023-09-23T20:56:05.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6279999999999996e-06 2023-09-23T20:56:05.257 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.257 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.257 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.257 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:56:05.257 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:56:05.257 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:56:05.257 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:05.258 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:56:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:56:04] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-09-23T20:56:05.258 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/825762842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:05.259 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/825762842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:05.259 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/825762842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:05.288 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:05.845 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:56:05.981 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:05.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:05.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:06.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T20:56:06.057170+0000 min lat: 0.00106683 max lat: 13.5629 avg lat: 0.0135886 2023-09-23T20:56:06.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:56:06.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 82422 82406 64.3556 0 - 0.0135886 2023-09-23T20:56:06.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:06.111 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:05 smithi123 ceph-mon[193903]: pgmap v783: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.2 MiB/s wr, 67 op/s; 34072/247218 objects degraded (13.782%); 14011/247218 objects misplaced (5.667%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:06.111 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:05 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:06.111 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:05 smithi123 ceph-mon[193903]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.111 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/825762842' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:06.111 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:05 smithi123 ceph-mon[193903]: osdmap e248: 8 total, 7 up, 6 in 2023-09-23T20:56:06.111 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:05 smithi123 ceph-mon[193903]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.112 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:05 smithi123 ceph-mon[193903]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.113 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:06.158 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[211526]: pgmap v783: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.2 MiB/s wr, 67 op/s; 34072/247218 objects degraded (13.782%); 14011/247218 objects misplaced (5.667%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[211526]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/825762842' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[211526]: osdmap e248: 8 total, 7 up, 6 in 2023-09-23T20:56:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[211526]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[211526]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[219709]: pgmap v783: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 4.2 MiB/s wr, 67 op/s; 34072/247218 objects degraded (13.782%); 14011/247218 objects misplaced (5.667%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[219709]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/825762842' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[219709]: osdmap e248: 8 total, 7 up, 6 in 2023-09-23T20:56:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[219709]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:05 smithi121 ceph-mon[219709]: failed to encode map e248 with expected crc 2023-09-23T20:56:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:06.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:06.201 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:06.266 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:06.292 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:06.336 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:06.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:06.440 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:06.468 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:06.512 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:06.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:06.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:06.615 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:06.643 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:06.839 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:06.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:06.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:06.943 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:06.969 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.012 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:07.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 82422 82406 63.5613 0 - 0.0135886 2023-09-23T20:56:07.116 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:07.141 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.180 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:56:07.213 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:56:07.231 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:07.234 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:56:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:07.256 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:56:07.276 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:56:07.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:56:07.276 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:56:07.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:56:07.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:56:07.277 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:56:07.277 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:56:07.277 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:56:07.277 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:56:07.336 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:07.383 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:07 smithi123 ceph-mon[193903]: pgmap v785: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 3.7 MiB/s wr, 59 op/s; 34072/247218 objects degraded (13.782%); 14011/247218 objects misplaced (5.667%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:56:07.494 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:07.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:07 smithi121 ceph-mon[211526]: pgmap v785: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 3.7 MiB/s wr, 59 op/s; 34072/247218 objects degraded (13.782%); 14011/247218 objects misplaced (5.667%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:56:07.496 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:07 smithi121 ceph-mon[219709]: pgmap v785: 89 pgs: 2 peering, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 3.7 MiB/s wr, 59 op/s; 34072/247218 objects degraded (13.782%); 14011/247218 objects misplaced (5.667%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:56:07.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:07.598 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:07.631 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.734 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:07.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:07.749 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:56:07.752 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:56:07.809 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:07.809 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:07.838 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:07.874 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:07.929 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:07.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:08.034 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:08.056 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 15 82628 82613 62.9441 1.29375 0.00387206 0.0156867 2023-09-23T20:56:08.072 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:08.119 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:08.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:08.223 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:08.252 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:08 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 34072/247218 objects degraded (13.782%), 10 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:08 smithi123 ceph-mon[193903]: from='client.35592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:56:08.635 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:56:08.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:08 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 34072/247218 objects degraded (13.782%), 10 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:08.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:08 smithi121 ceph-mon[211526]: from='client.35592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:56:08.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:08 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 34072/247218 objects degraded (13.782%), 10 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:08.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:08 smithi121 ceph-mon[219709]: from='client.35592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:56:09.011 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:09.047 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:56:09.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 82801 82785 62.3154 10.75 0.00353949 0.0159777 2023-09-23T20:56:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:09 smithi121 ceph-mon[211526]: pgmap v786: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.9 MiB/s wr, 45 op/s; 48880/247884 objects degraded (19.719%); 13533/247884 objects misplaced (5.459%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:56:09.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:09 smithi121 ceph-mon[219709]: pgmap v786: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.9 MiB/s wr, 45 op/s; 48880/247884 objects degraded (19.719%); 13533/247884 objects misplaced (5.459%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:56:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:09 smithi123 ceph-mon[193903]: pgmap v786: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.9 MiB/s wr, 45 op/s; 48880/247884 objects degraded (19.719%); 13533/247884 objects misplaced (5.459%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:56:09.845 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:56:09.845 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:56:09.972 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:09.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:09.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:10.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 83066 83050 61.7708 16.5625 0.00383059 0.0161164 2023-09-23T20:56:10.079 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:10.108 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:10.119 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":785,"stamp":"2023-09-23T20:56:06.782953+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5399582624,"num_objects":82406,"num_object_clones":0,"num_object_copies":247218,"num_objects_missing_on_primary":0,"num_objects_missing":5128,"num_objects_degraded":34072,"num_objects_misplaced":14011,"num_objects_unfound":0,"num_objects_dirty":82406,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":82682,"num_write_kb":5274893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2189,"num_bytes_recovered":142885062,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82598,"ondisk_log_size":82598,"up":217,"acting":241,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":257,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14576200,"kb_used_data":12234072,"kb_used_omap":13,"kb_used_meta":2342066,"kb_avail":547894712,"statfs":{"total":575970213888,"available":561044185088,"internally_reserved":0,"allocated":12527689728,"data_stored":12522652879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13934,"internal_metadata":2398275986},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":39321600,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-867,"num_objects_misplaced":149,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":600,"num_write_kb":38400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":446,"num_bytes_recovered":29294592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002822"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":17,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.455294+0000","last_change":"2023-09-23T20:55:42.207544+0000","last_active":"2023-09-23T20:55:55.455294+0000","last_peered":"2023-09-23T20:55:55.455294+0000","last_clean":"2023-09-23T20:55:55.455294+0000","last_became_active":"2023-09-23T20:55:42.207090+0000","last_became_peered":"2023-09-23T20:55:42.207090+0000","last_unstale":"2023-09-23T20:55:55.455294+0000","last_undegraded":"2023-09-23T20:55:55.455294+0000","last_fullsized":"2023-09-23T20:55:55.455294+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:57.313464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,2],"avail_no_missing":["0","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:55:54.449998+0000","last_change":"2023-09-23T20:54:45.421354+0000","last_active":"2023-09-23T20:55:54.449998+0000","last_peered":"2023-09-23T20:55:54.449998+0000","last_clean":"2023-09-23T20:54:43.398928+0000","last_became_active":"2023-09-23T20:54:45.421354+0000","last_became_peered":"2023-09-23T20:54:45.421354+0000","last_unstale":"2023-09-23T20:55:54.449998+0000","last_undegraded":"2023-09-23T20:55:54.449998+0000","last_fullsized":"2023-09-23T20:54:45.408903+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:11.407307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":144,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:55:54.448530+0000","last_change":"2023-09-23T20:54:45.423525+0000","last_active":"2023-09-23T20:55:54.448530+0000","last_peered":"2023-09-23T20:55:54.448530+0000","last_clean":"2023-09-23T20:47:29.289772+0000","last_became_active":"2023-09-23T20:54:45.423525+0000","last_became_peered":"2023-09-23T20:54:45.423525+0000","last_unstale":"2023-09-23T20:55:54.448530+0000","last_undegraded":"2023-09-23T20:55:54.448530+0000","last_fullsized":"2023-09-23T20:54:45.408320+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:58.200608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":209,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.453945+0000","last_change":"2023-09-23T20:55:31.903190+0000","last_active":"2023-09-23T20:55:55.453945+0000","last_peered":"2023-09-23T20:55:55.453945+0000","last_clean":"2023-09-23T20:55:55.453945+0000","last_became_active":"2023-09-23T20:55:31.902712+0000","last_became_peered":"2023-09-23T20:55:31.902712+0000","last_unstale":"2023-09-23T20:55:55.453945+0000","last_undegraded":"2023-09-23T20:55:55.453945+0000","last_fullsized":"2023-09-23T20:55:55.453945+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.104530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":226,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:55:54.448212+0000","last_change":"2023-09-23T20:54:45.411781+0000","last_active":"2023-09-23T20:55:54.448212+0000","last_peered":"2023-09-23T20:55:54.448212+0000","last_clean":"2023-09-23T20:54:43.396052+0000","last_became_active":"2023-09-23T20:54:45.411781+0000","last_became_peered":"2023-09-23T20:54:45.411781+0000","last_unstale":"2023-09-23T20:55:54.448212+0000","last_undegraded":"2023-09-23T20:55:54.448212+0000","last_fullsized":"2023-09-23T20:54:45.407690+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:21.658470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.459556+0000","last_change":"2023-09-23T20:54:45.416360+0000","last_active":"2023-09-23T20:55:55.459556+0000","last_peered":"2023-09-23T20:55:55.459556+0000","last_clean":"2023-09-23T20:54:43.443472+0000","last_became_active":"2023-09-23T20:54:45.416360+0000","last_became_peered":"2023-09-23T20:54:45.416360+0000","last_unstale":"2023-09-23T20:55:55.459556+0000","last_undegraded":"2023-09-23T20:55:55.459556+0000","last_fullsized":"2023-09-23T20:54:45.413642+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:59.097081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":197,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.454857+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:55:55.454857+0000","last_peered":"2023-09-23T20:55:55.454857+0000","last_clean":"2023-09-23T20:55:55.454857+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:55:55.454857+0000","last_undegraded":"2023-09-23T20:55:55.454857+0000","last_fullsized":"2023-09-23T20:55:55.454857+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":226,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:55:54.448177+0000","last_change":"2023-09-23T20:54:45.416346+0000","last_active":"2023-09-23T20:55:54.448177+0000","last_peered":"2023-09-23T20:55:54.448177+0000","last_clean":"2023-09-23T20:54:43.396018+0000","last_became_active":"2023-09-23T20:54:45.416346+0000","last_became_peered":"2023-09-23T20:54:45.416346+0000","last_unstale":"2023-09-23T20:55:54.448177+0000","last_undegraded":"2023-09-23T20:55:54.448177+0000","last_fullsized":"2023-09-23T20:54:45.407419+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:16.361022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":243,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.457179+0000","last_change":"2023-09-23T20:55:31.899600+0000","last_active":"2023-09-23T20:55:55.457179+0000","last_peered":"2023-09-23T20:55:55.457179+0000","last_clean":"2023-09-23T20:55:55.457179+0000","last_became_active":"2023-09-23T20:55:31.899087+0000","last_became_peered":"2023-09-23T20:55:31.899087+0000","last_unstale":"2023-09-23T20:55:55.457179+0000","last_undegraded":"2023-09-23T20:55:55.457179+0000","last_fullsized":"2023-09-23T20:55:55.457179+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:35.651074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":192,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.447691+0000","last_change":"2023-09-23T20:55:30.899971+0000","last_active":"2023-09-23T20:55:54.447691+0000","last_peered":"2023-09-23T20:55:54.447691+0000","last_clean":"2023-09-23T20:55:54.447691+0000","last_became_active":"2023-09-23T20:55:30.899749+0000","last_became_peered":"2023-09-23T20:55:30.899749+0000","last_unstale":"2023-09-23T20:55:54.447691+0000","last_undegraded":"2023-09-23T20:55:54.447691+0000","last_fullsized":"2023-09-23T20:55:54.447691+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:55.333451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":37,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.449954+0000","last_change":"2023-09-23T20:55:30.897624+0000","last_active":"2023-09-23T20:55:54.449954+0000","last_peered":"2023-09-23T20:55:54.449954+0000","last_clean":"2023-09-23T20:55:54.449954+0000","last_became_active":"2023-09-23T20:55:30.894260+0000","last_became_peered":"2023-09-23T20:55:30.894260+0000","last_unstale":"2023-09-23T20:55:54.449954+0000","last_undegraded":"2023-09-23T20:55:54.449954+0000","last_fullsized":"2023-09-23T20:55:54.449954+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:10.446888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.455002+0000","last_change":"2023-09-23T20:55:42.209459+0000","last_active":"2023-09-23T20:55:55.455002+0000","last_peered":"2023-09-23T20:55:55.455002+0000","last_clean":"2023-09-23T20:55:55.455002+0000","last_became_active":"2023-09-23T20:55:42.209101+0000","last_became_peered":"2023-09-23T20:55:42.209101+0000","last_unstale":"2023-09-23T20:55:55.455002+0000","last_undegraded":"2023-09-23T20:55:55.455002+0000","last_fullsized":"2023-09-23T20:55:55.455002+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:42.946017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,2],"avail_no_missing":["6","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":216,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.455165+0000","last_change":"2023-09-23T20:55:41.250513+0000","last_active":"2023-09-23T20:55:55.455165+0000","last_peered":"2023-09-23T20:55:55.455165+0000","last_clean":"2023-09-23T20:55:55.455165+0000","last_became_active":"2023-09-23T20:55:41.233339+0000","last_became_peered":"2023-09-23T20:55:41.233339+0000","last_unstale":"2023-09-23T20:55:55.455165+0000","last_undegraded":"2023-09-23T20:55:55.455165+0000","last_fullsized":"2023-09-23T20:55:55.455165+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:41.270084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":236,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.455293+0000","last_change":"2023-09-23T20:54:45.422759+0000","last_active":"2023-09-23T20:55:55.455293+0000","last_peered":"2023-09-23T20:55:55.455293+0000","last_clean":"2023-09-23T20:54:43.442133+0000","last_became_active":"2023-09-23T20:54:45.422759+0000","last_became_peered":"2023-09-23T20:54:45.422759+0000","last_unstale":"2023-09-23T20:55:55.455293+0000","last_undegraded":"2023-09-23T20:55:55.455293+0000","last_fullsized":"2023-09-23T20:54:45.413736+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:09.737985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":37,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.455270+0000","last_change":"2023-09-23T20:55:41.240912+0000","last_active":"2023-09-23T20:55:55.455270+0000","last_peered":"2023-09-23T20:55:55.455270+0000","last_clean":"2023-09-23T20:55:55.455270+0000","last_became_active":"2023-09-23T20:55:41.220592+0000","last_became_peered":"2023-09-23T20:55:41.220592+0000","last_unstale":"2023-09-23T20:55:55.455270+0000","last_undegraded":"2023-09-23T20:55:55.455270+0000","last_fullsized":"2023-09-23T20:55:55.455270+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:35.664132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":228,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.449632+0000","last_change":"2023-09-23T20:55:42.211336+0000","last_active":"2023-09-23T20:55:54.449632+0000","last_peered":"2023-09-23T20:55:54.449632+0000","last_clean":"2023-09-23T20:55:54.449632+0000","last_became_active":"2023-09-23T20:55:42.210877+0000","last_became_peered":"2023-09-23T20:55:42.210877+0000","last_unstale":"2023-09-23T20:55:54.449632+0000","last_undegraded":"2023-09-23T20:55:54.449632+0000","last_fullsized":"2023-09-23T20:55:54.449632+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:10.301573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,2],"avail_no_missing":["3","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":112,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.448147+0000","last_change":"2023-09-23T20:55:31.915425+0000","last_active":"2023-09-23T20:55:54.448147+0000","last_peered":"2023-09-23T20:55:54.448147+0000","last_clean":"2023-09-23T20:55:54.448147+0000","last_became_active":"2023-09-23T20:55:31.914985+0000","last_became_peered":"2023-09-23T20:55:31.914985+0000","last_unstale":"2023-09-23T20:55:54.448147+0000","last_undegraded":"2023-09-23T20:55:54.448147+0000","last_fullsized":"2023-09-23T20:55:54.448147+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:38.814363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":210,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.461471+0000","last_change":"2023-09-23T20:55:55.460665+0000","last_active":"2023-09-23T20:55:55.461471+0000","last_peered":"2023-09-23T20:55:55.461471+0000","last_clean":"2023-09-23T20:55:55.461471+0000","last_became_active":"2023-09-23T20:55:55.459517+0000","last_became_peered":"2023-09-23T20:55:55.459517+0000","last_unstale":"2023-09-23T20:55:55.461471+0000","last_undegraded":"2023-09-23T20:55:55.461471+0000","last_fullsized":"2023-09-23T20:55:55.461471+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:15.515153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.454921+0000","last_change":"2023-09-23T20:55:30.904289+0000","last_active":"2023-09-23T20:55:55.454921+0000","last_peered":"2023-09-23T20:55:55.454921+0000","last_clean":"2023-09-23T20:55:55.454921+0000","last_became_active":"2023-09-23T20:55:30.902684+0000","last_became_peered":"2023-09-23T20:55:30.902684+0000","last_unstale":"2023-09-23T20:55:55.454921+0000","last_undegraded":"2023-09-23T20:55:55.454921+0000","last_fullsized":"2023-09-23T20:55:55.454921+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:00.833399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":216,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.454171+0000","last_change":"2023-09-23T20:55:41.224947+0000","last_active":"2023-09-23T20:55:55.454171+0000","last_peered":"2023-09-23T20:55:55.454171+0000","last_clean":"2023-09-23T20:55:55.454171+0000","last_became_active":"2023-09-23T20:55:41.224589+0000","last_became_peered":"2023-09-23T20:55:41.224589+0000","last_unstale":"2023-09-23T20:55:55.454171+0000","last_undegraded":"2023-09-23T20:55:55.454171+0000","last_fullsized":"2023-09-23T20:55:55.454171+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:03.755330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":89,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.454176+0000","last_change":"2023-09-23T20:51:06.038448+0000","last_active":"2023-09-23T20:55:55.454176+0000","last_peered":"2023-09-23T20:55:55.454176+0000","last_clean":"2023-09-23T20:55:55.454176+0000","last_became_active":"2023-09-23T20:51:06.005965+0000","last_became_peered":"2023-09-23T20:51:06.005965+0000","last_unstale":"2023-09-23T20:55:55.454176+0000","last_undegraded":"2023-09-23T20:55:55.454176+0000","last_fullsized":"2023-09-23T20:55:55.454176+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:13.397771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":228,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.448203+0000","last_change":"2023-09-23T20:55:42.404489+0000","last_active":"2023-09-23T20:55:54.448203+0000","last_peered":"2023-09-23T20:55:54.448203+0000","last_clean":"2023-09-23T20:55:54.448203+0000","last_became_active":"2023-09-23T20:55:42.404027+0000","last_became_peered":"2023-09-23T20:55:42.404027+0000","last_unstale":"2023-09-23T20:55:54.448203+0000","last_undegraded":"2023-09-23T20:55:54.448203+0000","last_fullsized":"2023-09-23T20:55:54.448203+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:25.940348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,2],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"247'5240","reported_seq":5279,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T20:55:57.062154+0000","last_change":"2023-09-23T20:55:55.476703+0000","last_active":"2023-09-23T20:55:57.062154+0000","last_peered":"2023-09-23T20:55:57.062154+0000","last_clean":"2023-09-23T20:55:57.062154+0000","last_became_active":"2023-09-23T20:55:55.467711+0000","last_became_peered":"2023-09-23T20:55:55.467711+0000","last_unstale":"2023-09-23T20:55:57.062154+0000","last_undegraded":"2023-09-23T20:55:57.062154+0000","last_fullsized":"2023-09-23T20:55:57.062154+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5240,"log_dups_size":0,"ondisk_log_size":5240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:31.924695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343408640,"num_objects":5240,"num_object_clones":0,"num_object_copies":15720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5240,"num_write_kb":335360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":33,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.454383+0000","last_change":"2023-09-23T20:55:31.901311+0000","last_active":"2023-09-23T20:55:55.454383+0000","last_peered":"2023-09-23T20:55:55.454383+0000","last_clean":"2023-09-23T20:55:55.454383+0000","last_became_active":"2023-09-23T20:55:31.900886+0000","last_became_peered":"2023-09-23T20:55:31.900886+0000","last_unstale":"2023-09-23T20:55:55.454383+0000","last_undegraded":"2023-09-23T20:55:55.454383+0000","last_fullsized":"2023-09-23T20:55:55.454383+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:59.648440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":219,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.453693+0000","last_change":"2023-09-23T20:55:30.898681+0000","last_active":"2023-09-23T20:55:55.453693+0000","last_peered":"2023-09-23T20:55:55.453693+0000","last_clean":"2023-09-23T20:55:55.453693+0000","last_became_active":"2023-09-23T20:55:30.898355+0000","last_became_peered":"2023-09-23T20:55:30.898355+0000","last_unstale":"2023-09-23T20:55:55.453693+0000","last_undegraded":"2023-09-23T20:55:55.453693+0000","last_fullsized":"2023-09-23T20:55:55.453693+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:00.787233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"247'4997","reported_seq":5028,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T20:55:56.915086+0000","last_change":"2023-09-23T20:54:35.777545+0000","last_active":"2023-09-23T20:55:56.915086+0000","last_peered":"2023-09-23T20:55:56.915086+0000","last_clean":"2023-09-23T20:55:56.915086+0000","last_became_active":"2023-09-23T20:54:35.776552+0000","last_became_peered":"2023-09-23T20:54:35.776552+0000","last_unstale":"2023-09-23T20:55:56.915086+0000","last_undegraded":"2023-09-23T20:55:56.915086+0000","last_fullsized":"2023-09-23T20:55:56.915086+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":4997,"log_dups_size":0,"ondisk_log_size":4997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:29.974409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327483392,"num_objects":4997,"num_object_clones":0,"num_object_copies":14991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4997,"num_write_kb":319808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":25,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.453252+0000","last_change":"2023-09-23T20:55:31.917022+0000","last_active":"2023-09-23T20:55:55.453252+0000","last_peered":"2023-09-23T20:55:55.453252+0000","last_clean":"2023-09-23T20:55:55.453252+0000","last_became_active":"2023-09-23T20:55:31.902259+0000","last_became_peered":"2023-09-23T20:55:31.902259+0000","last_unstale":"2023-09-23T20:55:55.453252+0000","last_undegraded":"2023-09-23T20:55:55.453252+0000","last_fullsized":"2023-09-23T20:55:55.453252+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:01.919191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":218,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:55:54.448379+0000","last_change":"2023-09-23T20:54:45.422415+0000","last_active":"2023-09-23T20:55:54.448379+0000","last_peered":"2023-09-23T20:55:54.448379+0000","last_clean":"2023-09-23T20:52:20.810428+0000","last_became_active":"2023-09-23T20:54:45.422415+0000","last_became_peered":"2023-09-23T20:54:45.422415+0000","last_unstale":"2023-09-23T20:55:54.448379+0000","last_undegraded":"2023-09-23T20:55:54.448379+0000","last_fullsized":"2023-09-23T20:54:45.407861+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:56.793872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"247'5165","reported_seq":5197,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T20:55:56.968548+0000","last_change":"2023-09-23T20:54:35.769971+0000","last_active":"2023-09-23T20:55:56.968548+0000","last_peered":"2023-09-23T20:55:56.968548+0000","last_clean":"2023-09-23T20:55:56.968548+0000","last_became_active":"2023-09-23T20:54:35.769661+0000","last_became_peered":"2023-09-23T20:54:35.769661+0000","last_unstale":"2023-09-23T20:55:56.968548+0000","last_undegraded":"2023-09-23T20:55:56.968548+0000","last_fullsized":"2023-09-23T20:55:56.968548+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5165,"log_dups_size":0,"ondisk_log_size":5165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:55.146768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338493440,"num_objects":5165,"num_object_clones":0,"num_object_copies":15495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5165,"num_write_kb":330560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":37,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.455198+0000","last_change":"2023-09-23T20:55:30.908676+0000","last_active":"2023-09-23T20:55:55.455198+0000","last_peered":"2023-09-23T20:55:55.455198+0000","last_clean":"2023-09-23T20:55:55.455198+0000","last_became_active":"2023-09-23T20:55:30.904030+0000","last_became_peered":"2023-09-23T20:55:30.904030+0000","last_unstale":"2023-09-23T20:55:55.455198+0000","last_undegraded":"2023-09-23T20:55:55.455198+0000","last_fullsized":"2023-09-23T20:55:55.455198+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:27.738667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.455407+0000","last_change":"2023-09-23T20:55:42.211657+0000","last_active":"2023-09-23T20:55:55.455407+0000","last_peered":"2023-09-23T20:55:55.455407+0000","last_clean":"2023-09-23T20:55:55.455407+0000","last_became_active":"2023-09-23T20:55:42.210369+0000","last_became_peered":"2023-09-23T20:55:42.210369+0000","last_unstale":"2023-09-23T20:55:55.455407+0000","last_undegraded":"2023-09-23T20:55:55.455407+0000","last_fullsized":"2023-09-23T20:55:55.455407+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:18.225705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"247'5172","reported_seq":5217,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:55:57.030741+0000","last_change":"2023-09-23T20:55:53.897679+0000","last_active":"2023-09-23T20:55:57.030741+0000","last_peered":"2023-09-23T20:55:57.030741+0000","last_clean":"2023-09-23T20:55:40.433781+0000","last_became_active":"2023-09-23T20:55:53.857032+0000","last_became_peered":"2023-09-23T20:55:53.857032+0000","last_unstale":"2023-09-23T20:55:57.030741+0000","last_undegraded":"2023-09-23T20:55:53.771924+0000","last_fullsized":"2023-09-23T20:55:53.767064+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5172,"log_dups_size":0,"ondisk_log_size":5172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:55.822896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338952192,"num_objects":5172,"num_object_clones":0,"num_object_copies":15516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5112,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":331008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":5112},{"shards":"0,3","objects":60}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":161,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.461062+0000","last_change":"2023-09-23T20:55:55.460433+0000","last_active":"2023-09-23T20:55:55.461062+0000","last_peered":"2023-09-23T20:55:55.461062+0000","last_clean":"2023-09-23T20:55:55.461062+0000","last_became_active":"2023-09-23T20:55:55.459883+0000","last_became_peered":"2023-09-23T20:55:55.459883+0000","last_unstale":"2023-09-23T20:55:55.461062+0000","last_undegraded":"2023-09-23T20:55:55.461062+0000","last_fullsized":"2023-09-23T20:55:55.461062+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:59.444482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":219,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.452758+0000","last_change":"2023-09-23T20:54:45.424467+0000","last_active":"2023-09-23T20:55:55.452758+0000","last_peered":"2023-09-23T20:55:55.452758+0000","last_clean":"2023-09-23T20:54:43.442816+0000","last_became_active":"2023-09-23T20:54:45.424467+0000","last_became_peered":"2023-09-23T20:54:45.424467+0000","last_unstale":"2023-09-23T20:55:55.452758+0000","last_undegraded":"2023-09-23T20:55:55.452758+0000","last_fullsized":"2023-09-23T20:54:45.412983+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:59.019227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"247'5149","reported_seq":5181,"reported_epoch":247,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:55:56.979717+0000","last_change":"2023-09-23T20:55:05.900894+0000","last_active":"2023-09-23T20:55:56.979717+0000","last_peered":"2023-09-23T20:55:56.979717+0000","last_clean":"2023-09-23T20:54:34.751733+0000","last_became_active":"2023-09-23T20:55:05.900894+0000","last_became_peered":"2023-09-23T20:55:05.900894+0000","last_unstale":"2023-09-23T20:55:56.979717+0000","last_undegraded":"2023-09-23T20:55:05.898276+0000","last_fullsized":"2023-09-23T20:55:05.898079+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5149,"log_dups_size":0,"ondisk_log_size":5149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:10.125193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337444864,"num_objects":5149,"num_object_clones":0,"num_object_copies":15447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5149,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5149,"num_write_kb":329536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":5149}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":210,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.448877+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:55:54.448877+0000","last_peered":"2023-09-23T20:55:54.448877+0000","last_clean":"2023-09-23T20:55:54.448877+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:55:54.448877+0000","last_undegraded":"2023-09-23T20:55:54.448877+0000","last_fullsized":"2023-09-23T20:55:54.448877+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":181,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.448936+0000","last_change":"2023-09-23T20:54:35.776908+0000","last_active":"2023-09-23T20:55:54.448936+0000","last_peered":"2023-09-23T20:55:54.448936+0000","last_clean":"2023-09-23T20:55:54.448936+0000","last_became_active":"2023-09-23T20:54:35.776549+0000","last_became_peered":"2023-09-23T20:54:35.776549+0000","last_unstale":"2023-09-23T20:55:54.448936+0000","last_undegraded":"2023-09-23T20:55:54.448936+0000","last_fullsized":"2023-09-23T20:55:54.448936+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.593595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"244'5279","reported_seq":5317,"reported_epoch":246,"state":"peering","last_fresh":"2023-09-23T20:55:55.461445+0000","last_change":"2023-09-23T20:55:55.452546+0000","last_active":"2023-09-23T20:55:55.452546+0000","last_peered":"2023-09-23T20:55:54.424531+0000","last_clean":"2023-09-23T20:55:40.431047+0000","last_became_active":"2023-09-23T20:55:42.470547+0000","last_became_peered":"2023-09-23T20:55:42.470547+0000","last_unstale":"2023-09-23T20:55:55.461445+0000","last_undegraded":"2023-09-23T20:55:55.461445+0000","last_fullsized":"2023-09-23T20:55:55.461445+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5279,"log_dups_size":0,"ondisk_log_size":5279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:46.726643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345964544,"num_objects":5279,"num_object_clones":0,"num_object_copies":15837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5279,"num_write_kb":337856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.455525+0000","last_change":"2023-09-23T20:54:45.422582+0000","last_active":"2023-09-23T20:55:55.455525+0000","last_peered":"2023-09-23T20:55:55.455525+0000","last_clean":"2023-09-23T20:54:43.441969+0000","last_became_active":"2023-09-23T20:54:45.422582+0000","last_became_peered":"2023-09-23T20:54:45.422582+0000","last_unstale":"2023-09-23T20:55:55.455525+0000","last_undegraded":"2023-09-23T20:55:55.455525+0000","last_fullsized":"2023-09-23T20:54:45.413689+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:03.418893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.455936+0000","last_change":"2023-09-23T20:55:42.209431+0000","last_active":"2023-09-23T20:55:55.455936+0000","last_peered":"2023-09-23T20:55:55.455936+0000","last_clean":"2023-09-23T20:55:55.455936+0000","last_became_active":"2023-09-23T20:55:42.208997+0000","last_became_peered":"2023-09-23T20:55:42.208997+0000","last_unstale":"2023-09-23T20:55:55.455936+0000","last_undegraded":"2023-09-23T20:55:55.455936+0000","last_fullsized":"2023-09-23T20:55:55.455936+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:55.403635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,2],"avail_no_missing":["6","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"247'5059","reported_seq":5097,"reported_epoch":247,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:55:57.051100+0000","last_change":"2023-09-23T20:54:46.035241+0000","last_active":"2023-09-23T20:55:57.051100+0000","last_peered":"2023-09-23T20:55:57.051100+0000","last_clean":"2023-09-23T20:54:43.441608+0000","last_became_active":"2023-09-23T20:54:45.422503+0000","last_became_peered":"2023-09-23T20:54:45.422503+0000","last_unstale":"2023-09-23T20:55:57.051100+0000","last_undegraded":"2023-09-23T20:54:46.030921+0000","last_fullsized":"2023-09-23T20:54:45.415165+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5059,"log_dups_size":0,"ondisk_log_size":5059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:19.358306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331546624,"num_objects":5059,"num_object_clones":0,"num_object_copies":15177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5059,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5059,"num_write_kb":323776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":5059}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.453653+0000","last_change":"2023-09-23T20:55:42.419152+0000","last_active":"2023-09-23T20:55:55.453653+0000","last_peered":"2023-09-23T20:55:55.453653+0000","last_clean":"2023-09-23T20:55:55.453653+0000","last_became_active":"2023-09-23T20:55:42.418680+0000","last_became_peered":"2023-09-23T20:55:42.418680+0000","last_unstale":"2023-09-23T20:55:55.453653+0000","last_undegraded":"2023-09-23T20:55:55.453653+0000","last_fullsized":"2023-09-23T20:55:55.453653+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:23.601902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":236,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.462813+0000","last_change":"2023-09-23T20:54:45.424055+0000","last_active":"2023-09-23T20:55:55.462813+0000","last_peered":"2023-09-23T20:55:55.462813+0000","last_clean":"2023-09-23T20:54:43.443367+0000","last_became_active":"2023-09-23T20:54:45.424055+0000","last_became_peered":"2023-09-23T20:54:45.424055+0000","last_unstale":"2023-09-23T20:55:55.462813+0000","last_undegraded":"2023-09-23T20:55:55.462813+0000","last_fullsized":"2023-09-23T20:54:45.414076+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:32.457001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"247'5240","reported_seq":5277,"reported_epoch":247,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:56.941718+0000","last_change":"2023-09-23T20:55:31.900894+0000","last_active":"2023-09-23T20:55:56.941718+0000","last_peered":"2023-09-23T20:55:56.941718+0000","last_clean":"2023-09-23T20:55:56.941718+0000","last_became_active":"2023-09-23T20:55:31.896508+0000","last_became_peered":"2023-09-23T20:55:31.896508+0000","last_unstale":"2023-09-23T20:55:56.941718+0000","last_undegraded":"2023-09-23T20:55:56.941718+0000","last_fullsized":"2023-09-23T20:55:56.941718+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5240,"log_dups_size":0,"ondisk_log_size":5240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:18.855957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343408640,"num_objects":5240,"num_object_clones":0,"num_object_copies":15720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5240,"num_objects_unfound":0,"num_objects_dirty":5240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5240,"num_write_kb":335360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":5240}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":219,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.447781+0000","last_change":"2023-09-23T20:55:31.904846+0000","last_active":"2023-09-23T20:55:54.447781+0000","last_peered":"2023-09-23T20:55:54.447781+0000","last_clean":"2023-09-23T20:55:54.447781+0000","last_became_active":"2023-09-23T20:55:31.904421+0000","last_became_peered":"2023-09-23T20:55:31.904421+0000","last_unstale":"2023-09-23T20:55:54.447781+0000","last_undegraded":"2023-09-23T20:55:54.447781+0000","last_fullsized":"2023-09-23T20:55:54.447781+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:28.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":107,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.448443+0000","last_change":"2023-09-23T20:55:42.211277+0000","last_active":"2023-09-23T20:55:54.448443+0000","last_peered":"2023-09-23T20:55:54.448443+0000","last_clean":"2023-09-23T20:55:54.448443+0000","last_became_active":"2023-09-23T20:55:42.210841+0000","last_became_peered":"2023-09-23T20:55:42.210841+0000","last_unstale":"2023-09-23T20:55:54.448443+0000","last_undegraded":"2023-09-23T20:55:54.448443+0000","last_fullsized":"2023-09-23T20:55:54.448443+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:59.690653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,2],"avail_no_missing":["3","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"247'5133","reported_seq":7437,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:55:56.822584+0000","last_change":"2023-09-23T20:55:53.904917+0000","last_active":"2023-09-23T20:55:56.822584+0000","last_peered":"2023-09-23T20:55:56.822584+0000","last_clean":"2023-09-23T20:55:29.867795+0000","last_became_active":"2023-09-23T20:55:32.044783+0000","last_became_peered":"2023-09-23T20:55:32.044783+0000","last_unstale":"2023-09-23T20:55:56.822584+0000","last_undegraded":"2023-09-23T20:55:53.901965+0000","last_fullsized":"2023-09-23T20:55:31.886041+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5133,"log_dups_size":0,"ondisk_log_size":5133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:19.826927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336396288,"num_objects":5133,"num_object_clones":0,"num_object_copies":15399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3460,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5133,"num_write_kb":328512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1131,"num_bytes_recovered":74121216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4","objects":1673},{"shards":"0,4,7","objects":3460}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.448728+0000","last_change":"2023-09-23T20:55:42.222674+0000","last_active":"2023-09-23T20:55:54.448728+0000","last_peered":"2023-09-23T20:55:54.448728+0000","last_clean":"2023-09-23T20:55:54.448728+0000","last_became_active":"2023-09-23T20:55:42.222249+0000","last_became_peered":"2023-09-23T20:55:42.222249+0000","last_unstale":"2023-09-23T20:55:54.448728+0000","last_undegraded":"2023-09-23T20:55:54.448728+0000","last_fullsized":"2023-09-23T20:55:54.448728+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:42.535494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":234,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.457056+0000","last_change":"2023-09-23T20:55:41.230263+0000","last_active":"2023-09-23T20:55:55.457056+0000","last_peered":"2023-09-23T20:55:55.457056+0000","last_clean":"2023-09-23T20:55:55.457056+0000","last_became_active":"2023-09-23T20:55:41.229855+0000","last_became_peered":"2023-09-23T20:55:41.229855+0000","last_unstale":"2023-09-23T20:55:55.457056+0000","last_undegraded":"2023-09-23T20:55:55.457056+0000","last_fullsized":"2023-09-23T20:55:55.457056+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:38.214893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":140,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.454356+0000","last_change":"2023-09-23T20:54:45.425247+0000","last_active":"2023-09-23T20:55:55.454356+0000","last_peered":"2023-09-23T20:55:55.454356+0000","last_clean":"2023-09-23T20:30:22.093910+0000","last_became_active":"2023-09-23T20:54:45.425247+0000","last_became_peered":"2023-09-23T20:54:45.425247+0000","last_unstale":"2023-09-23T20:55:55.454356+0000","last_undegraded":"2023-09-23T20:55:55.454356+0000","last_fullsized":"2023-09-23T20:54:45.411310+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:26.737116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"247'5110","reported_seq":5141,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T20:55:56.958791+0000","last_change":"2023-09-23T20:54:35.777943+0000","last_active":"2023-09-23T20:55:56.958791+0000","last_peered":"2023-09-23T20:55:56.958791+0000","last_clean":"2023-09-23T20:55:56.958791+0000","last_became_active":"2023-09-23T20:54:35.777642+0000","last_became_peered":"2023-09-23T20:54:35.777642+0000","last_unstale":"2023-09-23T20:55:56.958791+0000","last_undegraded":"2023-09-23T20:55:56.958791+0000","last_fullsized":"2023-09-23T20:55:56.958791+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5110,"log_dups_size":0,"ondisk_log_size":5110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:24.764426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334888960,"num_objects":5110,"num_object_clones":0,"num_object_copies":15330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5110,"num_write_kb":327040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":243,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.457275+0000","last_change":"2023-09-23T20:55:55.457047+0000","last_active":"2023-09-23T20:55:55.457275+0000","last_peered":"2023-09-23T20:55:55.457275+0000","last_clean":"2023-09-23T20:54:43.400618+0000","last_became_active":"2023-09-23T20:55:55.457047+0000","last_became_peered":"2023-09-23T20:55:55.457047+0000","last_unstale":"2023-09-23T20:55:55.457275+0000","last_undegraded":"2023-09-23T20:55:55.457275+0000","last_fullsized":"2023-09-23T20:55:55.455361+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:25.720024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":177,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.455171+0000","last_change":"2023-09-23T20:55:05.920213+0000","last_active":"2023-09-23T20:55:55.455171+0000","last_peered":"2023-09-23T20:55:55.455171+0000","last_clean":"2023-09-23T20:55:55.455171+0000","last_became_active":"2023-09-23T20:55:05.917924+0000","last_became_peered":"2023-09-23T20:55:05.917924+0000","last_unstale":"2023-09-23T20:55:55.455171+0000","last_undegraded":"2023-09-23T20:55:55.455171+0000","last_fullsized":"2023-09-23T20:55:55.455171+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:17.149381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":224,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.461319+0000","last_change":"2023-09-23T20:55:55.460569+0000","last_active":"2023-09-23T20:55:55.461319+0000","last_peered":"2023-09-23T20:55:55.461319+0000","last_clean":"2023-09-23T20:55:55.461319+0000","last_became_active":"2023-09-23T20:55:55.460193+0000","last_became_peered":"2023-09-23T20:55:55.460193+0000","last_unstale":"2023-09-23T20:55:55.461319+0000","last_undegraded":"2023-09-23T20:55:55.461319+0000","last_fullsized":"2023-09-23T20:55:55.461319+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:25.084251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.9","version":"244'5158","reported_seq":5817,"reported_epoch":246,"state":"peering","last_fresh":"2023-09-23T20:55:55.477122+0000","last_change":"2023-09-23T20:55:55.477122+0000","last_active":"2023-09-23T20:55:54.472272+0000","last_peered":"2023-09-23T20:55:54.440444+0000","last_clean":"2023-09-23T20:55:29.724749+0000","last_became_active":"2023-09-23T20:55:42.430804+0000","last_became_peered":"2023-09-23T20:55:42.430804+0000","last_unstale":"2023-09-23T20:55:55.477122+0000","last_undegraded":"2023-09-23T20:55:55.477122+0000","last_fullsized":"2023-09-23T20:55:55.477122+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5158,"log_dups_size":0,"ondisk_log_size":5158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:42.182844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338034688,"num_objects":5158,"num_object_clones":0,"num_object_copies":15474,"num_objects_missing_on_primary":0,"num_objects_missing":5128,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5158,"num_write_kb":330112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":306,"num_bytes_recovered":20054016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":112,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.448232+0000","last_change":"2023-09-23T20:55:42.206124+0000","last_active":"2023-09-23T20:55:54.448232+0000","last_peered":"2023-09-23T20:55:54.448232+0000","last_clean":"2023-09-23T20:55:54.448232+0000","last_became_active":"2023-09-23T20:55:42.205742+0000","last_became_peered":"2023-09-23T20:55:42.205742+0000","last_unstale":"2023-09-23T20:55:54.448232+0000","last_undegraded":"2023-09-23T20:55:54.448232+0000","last_fullsized":"2023-09-23T20:55:54.448232+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:54.425439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,2],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":147,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.454074+0000","last_change":"2023-09-23T20:55:42.214181+0000","last_active":"2023-09-23T20:55:55.454074+0000","last_peered":"2023-09-23T20:55:55.454074+0000","last_clean":"2023-09-23T20:55:55.454074+0000","last_became_active":"2023-09-23T20:55:42.213051+0000","last_became_peered":"2023-09-23T20:55:42.213051+0000","last_unstale":"2023-09-23T20:55:55.454074+0000","last_undegraded":"2023-09-23T20:55:55.454074+0000","last_fullsized":"2023-09-23T20:55:55.454074+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:45.267984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,2],"avail_no_missing":["6","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":169,"reported_epoch":247,"state":"active+undersized","last_fresh":"2023-09-23T20:55:56.456379+0000","last_change":"2023-09-23T20:55:55.459619+0000","last_active":"2023-09-23T20:55:56.456379+0000","last_peered":"2023-09-23T20:55:56.456379+0000","last_clean":"2023-09-23T20:40:19.822501+0000","last_became_active":"2023-09-23T20:55:55.459619+0000","last_became_peered":"2023-09-23T20:55:55.459619+0000","last_unstale":"2023-09-23T20:55:56.456379+0000","last_undegraded":"2023-09-23T20:55:56.456379+0000","last_fullsized":"2023-09-23T20:55:55.454589+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:29.786959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"247'5123","reported_seq":5161,"reported_epoch":247,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:55:56.989850+0000","last_change":"2023-09-23T20:54:46.041835+0000","last_active":"2023-09-23T20:55:56.989850+0000","last_peered":"2023-09-23T20:55:56.989850+0000","last_clean":"2023-09-23T20:54:43.441988+0000","last_became_active":"2023-09-23T20:54:45.416641+0000","last_became_peered":"2023-09-23T20:54:45.416641+0000","last_unstale":"2023-09-23T20:55:56.989850+0000","last_undegraded":"2023-09-23T20:54:46.031915+0000","last_fullsized":"2023-09-23T20:54:45.413226+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5123,"log_dups_size":0,"ondisk_log_size":5123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:09.640397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335740928,"num_objects":5123,"num_object_clones":0,"num_object_copies":15369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5123,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5123,"num_write_kb":327872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":5123}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":229,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.460630+0000","last_change":"2023-09-23T20:55:55.459071+0000","last_active":"2023-09-23T20:55:55.460630+0000","last_peered":"2023-09-23T20:55:55.460630+0000","last_clean":"2023-09-23T20:55:55.460630+0000","last_became_active":"2023-09-23T20:55:55.458736+0000","last_became_peered":"2023-09-23T20:55:55.458736+0000","last_unstale":"2023-09-23T20:55:55.460630+0000","last_undegraded":"2023-09-23T20:55:55.460630+0000","last_fullsized":"2023-09-23T20:55:55.460630+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:22.884692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":421,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.451877+0000","last_change":"2023-09-23T20:55:31.901766+0000","last_active":"2023-09-23T20:55:55.451877+0000","last_peered":"2023-09-23T20:55:55.451877+0000","last_clean":"2023-09-23T20:55:55.451877+0000","last_became_active":"2023-09-23T20:55:31.901308+0000","last_became_peered":"2023-09-23T20:55:31.901308+0000","last_unstale":"2023-09-23T20:55:55.451877+0000","last_undegraded":"2023-09-23T20:55:55.451877+0000","last_fullsized":"2023-09-23T20:55:55.451877+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:50.996186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.448057+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:55:54.448057+0000","last_peered":"2023-09-23T20:55:54.448057+0000","last_clean":"2023-09-23T20:55:54.448057+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:55:54.448057+0000","last_undegraded":"2023-09-23T20:55:54.448057+0000","last_fullsized":"2023-09-23T20:55:54.448057+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":89,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.455705+0000","last_change":"2023-09-23T20:55:30.903001+0000","last_active":"2023-09-23T20:55:55.455705+0000","last_peered":"2023-09-23T20:55:55.455705+0000","last_clean":"2023-09-23T20:55:55.455705+0000","last_became_active":"2023-09-23T20:55:30.902659+0000","last_became_peered":"2023-09-23T20:55:30.902659+0000","last_unstale":"2023-09-23T20:55:55.455705+0000","last_undegraded":"2023-09-23T20:55:55.455705+0000","last_fullsized":"2023-09-23T20:55:55.455705+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:04:30.417586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"247'5240","reported_seq":7160,"reported_epoch":247,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:56:03.386125+0000","last_change":"2023-09-23T20:55:53.903813+0000","last_active":"2023-09-23T20:56:03.386125+0000","last_peered":"2023-09-23T20:56:03.386125+0000","last_clean":"2023-09-23T20:55:29.702830+0000","last_became_active":"2023-09-23T20:55:41.360983+0000","last_became_peered":"2023-09-23T20:55:41.360983+0000","last_unstale":"2023-09-23T20:56:03.386125+0000","last_undegraded":"2023-09-23T20:55:41.351558+0000","last_fullsized":"2023-09-23T20:55:41.200912+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5240,"log_dups_size":0,"ondisk_log_size":5240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:11.949016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343408640,"num_objects":5240,"num_object_clones":0,"num_object_copies":15720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":8767,"num_objects_unfound":0,"num_objects_dirty":5240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5240,"num_write_kb":335360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":714,"num_bytes_recovered":46792704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":483},{"shards":"0,2,5","objects":471},{"shards":"0,2,5,7","objects":275},{"shards":"0,2,7","objects":4010}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":251,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.452118+0000","last_change":"2023-09-23T20:55:31.902075+0000","last_active":"2023-09-23T20:55:55.452118+0000","last_peered":"2023-09-23T20:55:55.452118+0000","last_clean":"2023-09-23T20:55:55.452118+0000","last_became_active":"2023-09-23T20:55:31.901716+0000","last_became_peered":"2023-09-23T20:55:31.901716+0000","last_unstale":"2023-09-23T20:55:55.452118+0000","last_undegraded":"2023-09-23T20:55:55.452118+0000","last_fullsized":"2023-09-23T20:55:55.452118+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:42.881725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":198,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.452502+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:55:55.452502+0000","last_peered":"2023-09-23T20:55:55.452502+0000","last_clean":"2023-09-23T20:55:55.452502+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:55:55.452502+0000","last_undegraded":"2023-09-23T20:55:55.452502+0000","last_fullsized":"2023-09-23T20:55:55.452502+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":245,"reported_epoch":247,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:56.457594+0000","last_change":"2023-09-23T20:55:42.208602+0000","last_active":"2023-09-23T20:55:56.457594+0000","last_peered":"2023-09-23T20:55:56.457594+0000","last_clean":"2023-09-23T20:55:56.457594+0000","last_became_active":"2023-09-23T20:55:42.208127+0000","last_became_peered":"2023-09-23T20:55:42.208127+0000","last_unstale":"2023-09-23T20:55:56.457594+0000","last_undegraded":"2023-09-23T20:55:56.457594+0000","last_fullsized":"2023-09-23T20:55:56.457594+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:48.991250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,2],"avail_no_missing":["0","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"247'5052","reported_seq":5083,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T20:55:56.984100+0000","last_change":"2023-09-23T20:54:35.774443+0000","last_active":"2023-09-23T20:55:56.984100+0000","last_peered":"2023-09-23T20:55:56.984100+0000","last_clean":"2023-09-23T20:55:56.984100+0000","last_became_active":"2023-09-23T20:54:35.774079+0000","last_became_peered":"2023-09-23T20:54:35.774079+0000","last_unstale":"2023-09-23T20:55:56.984100+0000","last_undegraded":"2023-09-23T20:55:56.984100+0000","last_fullsized":"2023-09-23T20:55:56.984100+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5052,"log_dups_size":0,"ondisk_log_size":5052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:29.601484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331087872,"num_objects":5052,"num_object_clones":0,"num_object_copies":15156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5052,"num_write_kb":323328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":70,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.457593+0000","last_change":"2023-09-23T20:55:31.897824+0000","last_active":"2023-09-23T20:55:55.457593+0000","last_peered":"2023-09-23T20:55:55.457593+0000","last_clean":"2023-09-23T20:55:55.457593+0000","last_became_active":"2023-09-23T20:55:31.897494+0000","last_became_peered":"2023-09-23T20:55:31.897494+0000","last_unstale":"2023-09-23T20:55:55.457593+0000","last_undegraded":"2023-09-23T20:55:55.457593+0000","last_fullsized":"2023-09-23T20:55:55.457593+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:47.946040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":122,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.453764+0000","last_change":"2023-09-23T20:55:42.222973+0000","last_active":"2023-09-23T20:55:55.453764+0000","last_peered":"2023-09-23T20:55:55.453764+0000","last_clean":"2023-09-23T20:55:55.453764+0000","last_became_active":"2023-09-23T20:55:42.222508+0000","last_became_peered":"2023-09-23T20:55:42.222508+0000","last_unstale":"2023-09-23T20:55:55.453764+0000","last_undegraded":"2023-09-23T20:55:55.453764+0000","last_fullsized":"2023-09-23T20:55:55.453764+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:20.222504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,2],"avail_no_missing":["5","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.454265+0000","last_change":"2023-09-23T20:55:31.896355+0000","last_active":"2023-09-23T20:55:55.454265+0000","last_peered":"2023-09-23T20:55:55.454265+0000","last_clean":"2023-09-23T20:55:55.454265+0000","last_became_active":"2023-09-23T20:55:31.895895+0000","last_became_peered":"2023-09-23T20:55:31.895895+0000","last_unstale":"2023-09-23T20:55:55.454265+0000","last_undegraded":"2023-09-23T20:55:55.454265+0000","last_fullsized":"2023-09-23T20:55:55.454265+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:38.430947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,7],"avail_no_missing":["5","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"247'5109","reported_seq":5150,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:55:56.973026+0000","last_change":"2023-09-23T20:55:42.488677+0000","last_active":"2023-09-23T20:55:56.973026+0000","last_peered":"2023-09-23T20:55:56.973026+0000","last_clean":"2023-09-23T20:55:40.446441+0000","last_became_active":"2023-09-23T20:55:42.474251+0000","last_became_peered":"2023-09-23T20:55:42.474251+0000","last_unstale":"2023-09-23T20:55:56.973026+0000","last_undegraded":"2023-09-23T20:55:42.241706+0000","last_fullsized":"2023-09-23T20:55:42.205059+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5109,"log_dups_size":0,"ondisk_log_size":5109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:44.884851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334823424,"num_objects":5109,"num_object_clones":0,"num_object_copies":15327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5109,"num_write_kb":326976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":5058},{"shards":"4,6","objects":51}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":223,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.448357+0000","last_change":"2023-09-23T20:55:30.901977+0000","last_active":"2023-09-23T20:55:54.448357+0000","last_peered":"2023-09-23T20:55:54.448357+0000","last_clean":"2023-09-23T20:55:54.448357+0000","last_became_active":"2023-09-23T20:55:30.898501+0000","last_became_peered":"2023-09-23T20:55:30.898501+0000","last_unstale":"2023-09-23T20:55:54.448357+0000","last_undegraded":"2023-09-23T20:55:54.448357+0000","last_fullsized":"2023-09-23T20:55:54.448357+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:12.088088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":179,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.462694+0000","last_change":"2023-09-23T20:54:45.423898+0000","last_active":"2023-09-23T20:55:55.462694+0000","last_peered":"2023-09-23T20:55:55.462694+0000","last_clean":"2023-09-23T20:50:51.147256+0000","last_became_active":"2023-09-23T20:54:45.423898+0000","last_became_peered":"2023-09-23T20:54:45.423898+0000","last_unstale":"2023-09-23T20:55:55.462694+0000","last_undegraded":"2023-09-23T20:55:55.462694+0000","last_fullsized":"2023-09-23T20:54:45.413783+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:27.287200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":217,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.447732+0000","last_change":"2023-09-23T20:55:31.896219+0000","last_active":"2023-09-23T20:55:54.447732+0000","last_peered":"2023-09-23T20:55:54.447732+0000","last_clean":"2023-09-23T20:55:54.447732+0000","last_became_active":"2023-09-23T20:55:31.895818+0000","last_became_peered":"2023-09-23T20:55:31.895818+0000","last_unstale":"2023-09-23T20:55:54.447732+0000","last_undegraded":"2023-09-23T20:55:54.447732+0000","last_fullsized":"2023-09-23T20:55:54.447732+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:25.430597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"247'5156","reported_seq":5199,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:55:56.965280+0000","last_change":"2023-09-23T20:55:42.486661+0000","last_active":"2023-09-23T20:55:56.965280+0000","last_peered":"2023-09-23T20:55:56.965280+0000","last_clean":"2023-09-23T20:55:40.403186+0000","last_became_active":"2023-09-23T20:55:42.475024+0000","last_became_peered":"2023-09-23T20:55:42.475024+0000","last_unstale":"2023-09-23T20:55:56.965280+0000","last_undegraded":"2023-09-23T20:55:42.395490+0000","last_fullsized":"2023-09-23T20:55:42.205499+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5156,"log_dups_size":0,"ondisk_log_size":5156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:21.157328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337903616,"num_objects":5156,"num_object_clones":0,"num_object_copies":15468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5156,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":5108},{"shards":"3,6","objects":48}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":101,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.459352+0000","last_change":"2023-09-23T20:55:42.403360+0000","last_active":"2023-09-23T20:55:55.459352+0000","last_peered":"2023-09-23T20:55:55.459352+0000","last_clean":"2023-09-23T20:55:55.459352+0000","last_became_active":"2023-09-23T20:55:42.399880+0000","last_became_peered":"2023-09-23T20:55:42.399880+0000","last_unstale":"2023-09-23T20:55:55.459352+0000","last_undegraded":"2023-09-23T20:55:55.459352+0000","last_fullsized":"2023-09-23T20:55:55.459352+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:05.195834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":223,"reported_epoch":247,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:55:56.457345+0000","last_change":"2023-09-23T20:54:45.424530+0000","last_active":"2023-09-23T20:55:56.457345+0000","last_peered":"2023-09-23T20:55:56.457345+0000","last_clean":"2023-09-23T20:54:43.442024+0000","last_became_active":"2023-09-23T20:54:45.424530+0000","last_became_peered":"2023-09-23T20:54:45.424530+0000","last_unstale":"2023-09-23T20:55:56.457345+0000","last_undegraded":"2023-09-23T20:54:45.413586+0000","last_fullsized":"2023-09-23T20:54:45.413470+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:37.620016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":166,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.454402+0000","last_change":"2023-09-23T20:54:35.777462+0000","last_active":"2023-09-23T20:55:55.454402+0000","last_peered":"2023-09-23T20:55:55.454402+0000","last_clean":"2023-09-23T20:55:55.454402+0000","last_became_active":"2023-09-23T20:54:35.777151+0000","last_became_peered":"2023-09-23T20:54:35.777151+0000","last_unstale":"2023-09-23T20:55:55.454402+0000","last_undegraded":"2023-09-23T20:55:55.454402+0000","last_fullsized":"2023-09-23T20:55:55.454402+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:58.147893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":196,"reported_epoch":246,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:55.452985+0000","last_change":"2023-09-23T20:55:31.896894+0000","last_active":"2023-09-23T20:55:55.452985+0000","last_peered":"2023-09-23T20:55:55.452985+0000","last_clean":"2023-09-23T20:55:55.452985+0000","last_became_active":"2023-09-23T20:55:31.896378+0000","last_became_peered":"2023-09-23T20:55:31.896378+0000","last_unstale":"2023-09-23T20:55:55.452985+0000","last_undegraded":"2023-09-23T20:55:55.452985+0000","last_fullsized":"2023-09-23T20:55:55.452985+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:06.236591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":237,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.449773+0000","last_change":"2023-09-23T20:55:42.223291+0000","last_active":"2023-09-23T20:55:54.449773+0000","last_peered":"2023-09-23T20:55:54.449773+0000","last_clean":"2023-09-23T20:55:54.449773+0000","last_became_active":"2023-09-23T20:55:42.222866+0000","last_became_peered":"2023-09-23T20:55:42.222866+0000","last_unstale":"2023-09-23T20:55:54.449773+0000","last_undegraded":"2023-09-23T20:55:54.449773+0000","last_fullsized":"2023-09-23T20:55:54.449773+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:52.497037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,2],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":233,"reported_epoch":246,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:55:55.453836+0000","last_change":"2023-09-23T20:54:45.424269+0000","last_active":"2023-09-23T20:55:55.453836+0000","last_peered":"2023-09-23T20:55:55.453836+0000","last_clean":"2023-09-23T20:54:43.400415+0000","last_became_active":"2023-09-23T20:54:45.424269+0000","last_became_peered":"2023-09-23T20:54:45.424269+0000","last_unstale":"2023-09-23T20:55:55.453836+0000","last_undegraded":"2023-09-23T20:54:45.412205+0000","last_fullsized":"2023-09-23T20:54:45.412044+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:23.009382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":230,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:55:54.448971+0000","last_change":"2023-09-23T20:54:45.423507+0000","last_active":"2023-09-23T20:55:54.448971+0000","last_peered":"2023-09-23T20:55:54.448971+0000","last_clean":"2023-09-23T20:54:43.398488+0000","last_became_active":"2023-09-23T20:54:45.423507+0000","last_became_peered":"2023-09-23T20:54:45.423507+0000","last_unstale":"2023-09-23T20:55:54.448971+0000","last_undegraded":"2023-09-23T20:55:54.448971+0000","last_fullsized":"2023-09-23T20:54:45.408675+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:03.743561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":152,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.466302+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T20:55:55.466302+0000","last_peered":"2023-09-23T20:55:55.466302+0000","last_clean":"2023-09-23T20:55:55.466302+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T20:55:55.466302+0000","last_undegraded":"2023-09-23T20:55:55.466302+0000","last_fullsized":"2023-09-23T20:55:55.466302+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":149,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.447969+0000","last_change":"2023-09-23T20:54:35.770871+0000","last_active":"2023-09-23T20:55:54.447969+0000","last_peered":"2023-09-23T20:55:54.447969+0000","last_clean":"2023-09-23T20:55:54.447969+0000","last_became_active":"2023-09-23T20:54:35.770577+0000","last_became_peered":"2023-09-23T20:54:35.770577+0000","last_unstale":"2023-09-23T20:55:54.447969+0000","last_undegraded":"2023-09-23T20:55:54.447969+0000","last_fullsized":"2023-09-23T20:55:54.447969+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:23.786293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":232,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:55:54.448030+0000","last_change":"2023-09-23T20:55:42.246662+0000","last_active":"2023-09-23T20:55:54.448030+0000","last_peered":"2023-09-23T20:55:54.448030+0000","last_clean":"2023-09-23T20:55:54.448030+0000","last_became_active":"2023-09-23T20:55:42.246211+0000","last_became_peered":"2023-09-23T20:55:42.246211+0000","last_unstale":"2023-09-23T20:55:54.448030+0000","last_undegraded":"2023-09-23T20:55:54.448030+0000","last_fullsized":"2023-09-23T20:55:54.448030+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:30.333010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,2],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:55:54.448430+0000","last_change":"2023-09-23T20:55:41.229955+0000","last_active":"2023-09-23T20:55:54.448430+0000","last_peered":"2023-09-23T20:55:54.448430+0000","last_clean":"2023-09-23T20:55:54.448430+0000","last_became_active":"2023-09-23T20:55:41.229618+0000","last_became_peered":"2023-09-23T20:55:41.229618+0000","last_unstale":"2023-09-23T20:55:54.448430+0000","last_undegraded":"2023-09-23T20:55:54.448430+0000","last_fullsized":"2023-09-23T20:55:54.448430+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:20.277449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":217,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:55:55.454518+0000","last_change":"2023-09-23T20:55:41.246080+0000","last_active":"2023-09-23T20:55:55.454518+0000","last_peered":"2023-09-23T20:55:55.454518+0000","last_clean":"2023-09-23T20:55:55.454518+0000","last_became_active":"2023-09-23T20:55:41.220693+0000","last_became_peered":"2023-09-23T20:55:41.220693+0000","last_unstale":"2023-09-23T20:55:55.454518+0000","last_undegraded":"2023-09-23T20:55:55.454518+0000","last_fullsized":"2023-09-23T20:55:55.454518+0000","mapping_epoch":241,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":242,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.722007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":250,"reported_epoch":246,"state":"active+undersized","last_fresh":"2023-09-23T20:55:55.451551+0000","last_change":"2023-09-23T20:54:45.422394+0000","last_active":"2023-09-23T20:55:55.451551+0000","last_peered":"2023-09-23T20:55:55.451551+0000","last_clean":"2023-09-23T20:54:43.441483+0000","last_became_active":"2023-09-23T20:54:45.422394+0000","last_became_peered":"2023-09-23T20:54:45.422394+0000","last_unstale":"2023-09-23T20:55:55.451551+0000","last_undegraded":"2023-09-23T20:55:55.451551+0000","last_fullsized":"2023-09-23T20:54:45.418681+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:53.640829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":74,"acting":85,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5398986752,"num_objects":82382,"num_object_clones":0,"num_object_copies":247146,"num_objects_missing_on_primary":0,"num_objects_missing":5128,"num_objects_degraded":34070,"num_objects_misplaced":14007,"num_objects_unfound":0,"num_objects_dirty":82382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82382,"num_write_kb":5272448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2153,"num_bytes_recovered":141099008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15367208960,"data_stored":15367208960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82382,"ondisk_log_size":82382,"up":44,"acting":40,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":14950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":80,"acting":91,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":2,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":17,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201608,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463245,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2623824,"kb_used_data":2281736,"kb_used_omap":0,"kb_used_meta":342080,"kb_avail":91121328,"statfs":{"total":95995035648,"available":93308239872,"internally_reserved":0,"allocated":2336497664,"data_stored":2335826575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":350289920},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":3.3370000000000002,"5min":3.3109999999999999,"15min":2.8340000000000001},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":52.484000000000002,"5min":79.016000000000005,"15min":92.164000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.92000000000000004,"5min":2.4140000000000001,"15min":2.4390000000000001},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.20000000000000001},"max":{"1min":3.0510000000000002,"5min":78.843000000000004,"15min":92.138000000000005},"last":0.33000000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:54:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6400000000000001,"5min":2.6400000000000001,"15min":2.6400000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.509,"5min":13.509,"15min":13.509},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.5899999999999999,"15min":2.5899999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":13.528,"5min":13.528,"15min":13.528},"last":0.433}]},{"osd":2,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2240000000000002,"5min":2.214,"15min":2.1600000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":52.808,"5min":52.808,"15min":52.808},"last":0.318},{"interface":"front","average":{"1min":3.431,"5min":2.3450000000000002,"15min":2.3860000000000001},"min":{"1min":0.40799999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":53.140000000000001,"5min":53.140000000000001,"15min":53.140000000000001},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":6.8780000000000001,"15min":6.0030000000000001},"min":{"1min":0.38200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.4660000000000002,"5min":250.804,"15min":250.804},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.71199999999999997,"5min":6.8700000000000001,"15min":5.9299999999999997},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.853,"5min":250.446,"15min":250.446},"last":0.46100000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.147,"15min":1.0529999999999999},"min":{"1min":0.26300000000000001,"5min":0.248,"15min":0.248},"max":{"1min":3.976,"5min":9.1590000000000007,"15min":9.1590000000000007},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.996,"5min":1.401,"15min":1.347},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.1850000000000001,"5min":8.0839999999999996,"15min":9.8650000000000002},"last":0.378}]},{"osd":5,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":3.2000000000000002,"15min":1.8660000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26000000000000001},"max":{"1min":4.0179999999999998,"5min":105.121,"15min":105.121},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.323,"5min":3.1760000000000002,"15min":1.954},"min":{"1min":0.35299999999999998,"5min":0.34100000000000003,"15min":0.30099999999999999},"max":{"1min":6.6529999999999996,"5min":106.30500000000001,"15min":106.30500000000001},"last":0.29999999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:55:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.7469999999999999,"15min":1.673},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.22600000000000001},"max":{"1min":7.7930000000000001,"5min":101.04600000000001,"15min":101.04600000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.726,"15min":1.724},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.8010000000000002,"5min":101.081,"15min":101.081},"last":0.42999999999999999}]}]},{"osd":1,"up_from":206,"seq":884763262996,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":605540,"kb_used_data":426076,"kb_used_omap":5,"kb_used_meta":179450,"kb_avail":93139612,"statfs":{"total":95995035648,"available":95374962688,"internally_reserved":0,"allocated":436301824,"data_stored":435349991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":183757539},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":2.4319999999999999,"15min":2.4319999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0510000000000002,"5min":6.0510000000000002,"15min":6.0510000000000002},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.8090000000000002,"5min":2.8090000000000002,"15min":2.8090000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.601000000000001,"5min":11.601000000000001,"15min":11.601000000000001},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":4.359,"5min":4.359,"15min":4.359},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":28.492999999999999,"5min":28.492999999999999,"15min":28.492999999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.754,"5min":2.754,"15min":2.754},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":27.742999999999999,"5min":27.742999999999999,"15min":27.742999999999999},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.714,"15min":2.714},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.898,"5min":10.898,"15min":10.898},"last":0.41299999999999998},{"interface":"front","average":{"1min":4.2560000000000002,"5min":4.2560000000000002,"15min":4.2560000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":23.893000000000001,"5min":23.893000000000001,"15min":23.893000000000001},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":2.8940000000000001,"15min":2.8940000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.1099999999999994,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":10.938000000000001,"5min":10.938000000000001,"15min":10.938000000000001},"last":0.57599999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":2.8940000000000001,"15min":2.8940000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":9.5340000000000007,"5min":9.5340000000000007,"15min":9.5340000000000007},"last":0.42699999999999999},{"interface":"front","average":{"1min":3.2050000000000001,"5min":3.2050000000000001,"15min":3.2050000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":12.808999999999999,"5min":12.808999999999999,"15min":12.808999999999999},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":2.8079999999999998,"15min":2.8079999999999998},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.4880000000000004,"5min":7.4880000000000004,"15min":7.4880000000000004},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.036,"5min":3.036,"15min":3.036},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":6.9359999999999999,"5min":6.9359999999999999,"15min":6.9359999999999999},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:54:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9950000000000001,"5min":3.9950000000000001,"15min":3.9950000000000001},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":12.379,"5min":12.379,"15min":12.379},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.1600000000000001,"5min":2.1600000000000001,"15min":2.1600000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.066000000000001,"5min":10.066000000000001,"15min":10.066000000000001},"last":0.54500000000000004}]}]},{"osd":0,"up_from":92,"seq":395136991415,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3864804,"kb_used_data":3307292,"kb_used_omap":5,"kb_used_meta":557498,"kb_avail":89880348,"statfs":{"total":95995035648,"available":92037476352,"internally_reserved":0,"allocated":3386667008,"data_stored":3385628710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":570878586},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:54:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5069999999999997,"5min":5.5069999999999997,"15min":5.5069999999999997},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":89.626000000000005,"5min":89.626000000000005,"15min":89.626000000000005},"last":0.30599999999999999},{"interface":"front","average":{"1min":6.0309999999999997,"5min":6.0309999999999997,"15min":6.0309999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":89.489999999999995,"5min":89.489999999999995,"15min":89.489999999999995},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":2.7320000000000002,"15min":2.2320000000000002},"min":{"1min":0.192,"5min":0.188,"15min":0.188},"max":{"1min":2.839,"5min":114.002,"15min":114.002},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.877,"5min":2.9430000000000001,"15min":3.556},"min":{"1min":0.253,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.0430000000000001,"5min":113.69499999999999,"15min":113.69499999999999},"last":0.27900000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":7.2699999999999996,"15min":4.4939999999999998},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":6.2779999999999996,"5min":94.947999999999993,"15min":94.947999999999993},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.0860000000000001,"5min":7.3129999999999997,"15min":4.6669999999999998},"min":{"1min":0.33000000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.5569999999999999,"5min":94.382999999999996,"15min":94.382999999999996},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.77,"15min":1.823},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.1520000000000001,"5min":10.525,"15min":12.973000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.4370000000000001,"15min":1.706},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.3079999999999998,"5min":12.461,"15min":12.461},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":5.2160000000000002,"15min":3.2570000000000001},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":4.0960000000000001,"5min":154.30600000000001,"15min":154.30600000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.70999999999999996,"5min":4.2539999999999996,"15min":2.7480000000000002},"min":{"1min":0.35699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.0259999999999998,"5min":154.42400000000001,"15min":154.42400000000001},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.585,"15min":1.9059999999999999},"min":{"1min":0.376,"5min":0.317,"15min":0.307},"max":{"1min":2.4649999999999999,"5min":12.510999999999999,"15min":14.037000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.7749999999999999,"15min":1.8149999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.9780000000000002,"5min":10.026999999999999,"15min":20.399999999999999},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:55:21 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":5.6589999999999998,"15min":3.27},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.2300000000000004,"5min":164.26400000000001,"15min":164.26400000000001},"last":0.503},{"interface":"front","average":{"1min":1.2649999999999999,"5min":5.7519999999999998,"15min":3.1920000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.0129999999999999,"5min":164.23099999999999,"15min":164.23099999999999},"last":0.433}]}]},{"osd":2,"up_from":0,"seq":554050781306,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":148,"seq":635655159904,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2079192,"kb_used_data":1757584,"kb_used_omap":2,"kb_used_meta":321597,"kb_avail":91665960,"statfs":{"total":95995035648,"available":93865943040,"internally_reserved":0,"allocated":1799766016,"data_stored":1798729681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":329315381},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":2.7570000000000001,"15min":9.5399999999999991},"min":{"1min":0.19600000000000001,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":1.3939999999999999,"5min":95.507999999999996,"15min":253.68100000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":3.0609999999999999,"15min":8.9670000000000005},"min":{"1min":0.19500000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.1059999999999999,"5min":95.259,"15min":253.46199999999999},"last":0.26300000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.9969999999999999,"5min":7.9969999999999999,"15min":7.9969999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.5049999999999999,"5min":2.5049999999999999,"15min":2.5049999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":13.18,"5min":13.18,"15min":13.18},"last":0.624}]},{"osd":2,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.5249999999999999,"15min":1.1559999999999999},"min":{"1min":0.22,"5min":0.182,"15min":0.182},"max":{"1min":0.60599999999999998,"5min":12.859,"15min":12.859},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":1.798,"15min":1.1899999999999999},"min":{"1min":0.245,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.4370000000000001,"5min":19.050999999999998,"15min":19.050999999999998},"last":0.33100000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.72,"15min":1.6200000000000001},"min":{"1min":0.27900000000000003,"5min":0.222,"15min":0.222},"max":{"1min":2.077,"5min":10.188000000000001,"15min":10.188000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":1.5309999999999999,"15min":1.8759999999999999},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":3.004,"5min":11.276,"15min":16.408999999999999},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":2.7829999999999999,"15min":1.6699999999999999},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.46,"5min":64.528000000000006,"15min":64.528000000000006},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.78300000000000003,"5min":2.3010000000000002,"15min":1.98},"min":{"1min":0.377,"5min":0.249,"15min":0.249},"max":{"1min":2.2690000000000001,"5min":14.054,"15min":14.054},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.3220000000000001,"15min":1.996},"min":{"1min":0.39400000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.093,"5min":19.568000000000001,"15min":19.568000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.123,"5min":2.008,"15min":2.2930000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.6920000000000002,"5min":15.573,"15min":16.451000000000001},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":4.7380000000000004,"15min":2.5550000000000002},"min":{"1min":0.42499999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.9730000000000001,"5min":215.70099999999999,"15min":215.70099999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.4570000000000001,"5min":4.6820000000000004,"15min":3.0099999999999998},"min":{"1min":0.437,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":9.1649999999999991,"5min":222.11699999999999,"15min":222.11699999999999},"last":0.45100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986518,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3151764,"kb_used_data":2721932,"kb_used_omap":0,"kb_used_meta":429824,"kb_avail":90593388,"statfs":{"total":95995035648,"available":92767629312,"internally_reserved":0,"allocated":2787258368,"data_stored":2786582657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":440139776},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:15 2023","interfaces":[{"interface":"back","average":{"1min":6.085,"5min":5.9450000000000003,"15min":3.2839999999999998},"min":{"1min":0.252,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":102.358,"5min":146.25299999999999,"15min":146.25299999999999},"last":0.432},{"interface":"front","average":{"1min":6.3869999999999996,"5min":6.0720000000000001,"15min":3.0870000000000002},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.22600000000000001},"max":{"1min":102.58199999999999,"5min":144.77000000000001,"15min":144.77000000000001},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 20:54:08 2023","interfaces":[{"interface":"back","average":{"1min":6.0999999999999996,"5min":6.0999999999999996,"15min":6.0999999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":75.445999999999998,"5min":75.445999999999998,"15min":75.445999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":6.4199999999999999,"5min":6.4199999999999999,"15min":6.4199999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.734999999999999,"5min":71.734999999999999,"15min":71.734999999999999},"last":0.48299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:55:15 2023","interfaces":[{"interface":"back","average":{"1min":5.3769999999999998,"5min":2.4340000000000002,"15min":1.6890000000000001},"min":{"1min":0.35399999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":99.274000000000001,"5min":99.274000000000001,"15min":99.274000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":5.5140000000000002,"5min":2.7799999999999998,"15min":2.3780000000000001},"min":{"1min":0.29599999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":99.363,"5min":99.363,"15min":99.363},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:55:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":4.1470000000000002,"15min":5.9749999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.1390000000000002,"5min":142.28399999999999,"15min":142.28399999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.78900000000000003,"5min":4.3710000000000004,"15min":5.8079999999999998},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.202,"5min":142.34700000000001,"15min":142.34700000000001},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:55:55 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.929,"15min":1.24},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":7.4720000000000004,"5min":72.304000000000002,"15min":72.304000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.9850000000000001,"15min":1.325},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.2279999999999998,"5min":72.290000000000006,"15min":72.290000000000006},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 20:55:15 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":1.3520000000000001,"15min":1.1819999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4210000000000003,"5min":11.887,"15min":16.271000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":1.0960000000000001,"15min":1.0029999999999999},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":5.1189999999999998,"5min":8.1609999999999996,"15min":11.718},"last":0.36499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.9630000000000001,"15min":1.4330000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.161,"5min":73.268000000000001,"15min":73.268000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.218,"15min":1.4239999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.1740000000000004,"5min":69.760000000000005,"15min":69.760000000000005},"last":0.45700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724882,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2251076,"kb_used_data":1739452,"kb_used_omap":0,"kb_used_meta":511616,"kb_avail":91494076,"statfs":{"total":95995035648,"available":93689933824,"internally_reserved":0,"allocated":1781198848,"data_stored":1780535265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":523894784},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":2.8490000000000002,"15min":1.9930000000000001},"min":{"1min":0.23699999999999999,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":1.8029999999999999,"5min":124.873,"15min":124.873},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.626,"5min":2.9300000000000002,"15min":1.9399999999999999},"min":{"1min":0.254,"5min":0.223,"15min":0.223},"max":{"1min":1.2010000000000001,"5min":123.554,"15min":123.554},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:54:10 2023","interfaces":[{"interface":"back","average":{"1min":3.621,"5min":3.621,"15min":3.621},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.178000000000001,"5min":20.178000000000001,"15min":20.178000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.823,"5min":2.823,"15min":2.823},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.670999999999999,"5min":19.670999999999999,"15min":19.670999999999999},"last":0.95299999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.595,"15min":2.0209999999999999},"min":{"1min":0.254,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":2.04,"5min":10.859,"15min":13.394},"last":0.309},{"interface":"front","average":{"1min":0.78000000000000003,"5min":1.6910000000000001,"15min":1.6319999999999999},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.5390000000000001,"5min":8.5980000000000008,"15min":14.263},"last":0.29999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":5.2919999999999998,"5min":8.0899999999999999,"15min":3.8490000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":98.369,"5min":198.33099999999999,"15min":198.33099999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":5.6230000000000002,"5min":8.1199999999999992,"15min":3.6339999999999999},"min":{"1min":0.32000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":98.683000000000007,"5min":198.47300000000001,"15min":198.47300000000001},"last":0.28100000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:55:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":1.3620000000000001,"15min":0.98299999999999998},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.24099999999999999},"max":{"1min":5.0810000000000004,"5min":10.906000000000001,"15min":10.906000000000001},"last":0.433},{"interface":"front","average":{"1min":1.173,"5min":1.657,"15min":1.2310000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.9910000000000001,"5min":12.539,"15min":12.539},"last":0.26400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.3660000000000001,"15min":1.1459999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":2.0249999999999999,"5min":12.727,"15min":12.727},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.599,"15min":1.161},"min":{"1min":0.313,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":1.7629999999999999,"5min":12.593999999999999,"15min":12.593999999999999},"last":0.38100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:55:47 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":2.1579999999999999,"15min":1.538},"min":{"1min":0.312,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":3.952,"5min":31.091999999999999,"15min":31.091999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":2.0659999999999998,"15min":1.2609999999999999},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.28699999999999998},"max":{"1min":5.0209999999999999,"5min":31.780000000000001,"15min":31.780000000000001},"last":0.36199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3383230464,"data_stored":3383230464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2024800256,"data_stored":2024800256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1796931584,"data_stored":1796931584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2787704832,"data_stored":2787704832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1780940800,"data_stored":1780940800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2333474816,"data_stored":2333474816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1260126208,"data_stored":1260126208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:56:10.125 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-09-23T20:56:10.210 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:10.224 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:10.316 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:10.346 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:10.447 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:10.551 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:10.587 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:10.645 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:10.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:10.726 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:10.726 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:10.749 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:10.780 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:10.947 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:10.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:11.050 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:11.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 83274 83258 61.1972 13 0.989705 0.0162634 2023-09-23T20:56:11.079 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.126 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:11.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:11.232 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:11.262 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.559 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:56:11.570 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:11.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:11.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:11.675 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:11.705 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.755 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:11.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:11.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:11 smithi123 ceph-mon[193903]: pgmap v787: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.1 MiB/s wr, 33 op/s; 48880/248223 objects degraded (19.692%); 13559/248223 objects misplaced (5.462%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:11.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:56:11.865 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:11.920 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:11.973 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:56:11.999 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:12.045 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:12.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:12.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 83497 83481 60.6478 13.9375 0.00368736 0.0164151 2023-09-23T20:56:12.152 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:11 smithi121 ceph-mon[211526]: pgmap v787: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.1 MiB/s wr, 33 op/s; 48880/248223 objects degraded (19.692%); 13559/248223 objects misplaced (5.462%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:11 smithi121 ceph-mon[219709]: pgmap v787: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.1 MiB/s wr, 33 op/s; 48880/248223 objects degraded (19.692%); 13559/248223 objects misplaced (5.462%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:12.185 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:12.243 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:12.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:12.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:12.347 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:12.377 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:12.590 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:12.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:12.695 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:12.722 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:12.764 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:12.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:12.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:12.867 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:12.911 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:13.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 83805 83789 60.1721 19.25 0.875083 0.016554 2023-09-23T20:56:13.289 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:13.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:13.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:13.396 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:13.420 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:56:13.448 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:13.559 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:13.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:13.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:13.562 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:13.562 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:13.563 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[211526]: pgmap v788: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 3.0 MiB/s wr, 48 op/s; 48879/248667 objects degraded (19.656%); 13559/248667 objects misplaced (5.453%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:13.563 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3001050490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:13.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:13.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:13.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[219709]: pgmap v788: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 3.0 MiB/s wr, 48 op/s; 48879/248667 objects degraded (19.656%); 13559/248667 objects misplaced (5.453%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:13.563 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3001050490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:13.663 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:13.686 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:13.686 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:13.696 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:13.711 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:13.711 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:13.711 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:13 smithi123 ceph-mon[193903]: pgmap v788: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 3.0 MiB/s wr, 48 op/s; 48879/248667 objects degraded (19.656%); 13559/248667 objects misplaced (5.453%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:56:13.712 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3001050490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:13.754 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:13.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:13.859 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:13.899 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:13.969 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:13.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:14.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 84165 84149 59.7441 22.5 0.0111796 0.0166686 2023-09-23T20:56:14.073 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:14.102 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:14.153 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:14.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:14.258 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:14.296 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:14.536 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:56:14.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48879/248667 objects degraded (19.656%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:14.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:14.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[219709]: failed to encode map e249 with expected crc 2023-09-23T20:56:14.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[219709]: failed to encode map e249 with expected crc 2023-09-23T20:56:14.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3001050490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:14.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[219709]: osdmap e249: 8 total, 7 up, 6 in 2023-09-23T20:56:14.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48879/248667 objects degraded (19.656%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:14.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:14.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[211526]: failed to encode map e249 with expected crc 2023-09-23T20:56:14.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[211526]: failed to encode map e249 with expected crc 2023-09-23T20:56:14.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3001050490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:14.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:14 smithi121 ceph-mon[211526]: osdmap e249: 8 total, 7 up, 6 in 2023-09-23T20:56:14.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:14 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48879/248667 objects degraded (19.656%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:14.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:14 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:14.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:14 smithi123 ceph-mon[193903]: failed to encode map e249 with expected crc 2023-09-23T20:56:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:14 smithi123 ceph-mon[193903]: failed to encode map e249 with expected crc 2023-09-23T20:56:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3001050490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:14 smithi123 ceph-mon[193903]: osdmap e249: 8 total, 7 up, 6 in 2023-09-23T20:56:15.040 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:15.044 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:56:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:56:14] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-09-23T20:56:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:15.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:15.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 84492 84476 59.3025 20.4375 0.00543581 0.016794 2023-09-23T20:56:15.139 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:15.146 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:15.173 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:15.225 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:15.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:15.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:15.274 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:15.334 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:15.366 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:15.415 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:15.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:15 smithi123 ceph-mon[193903]: pgmap v790: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 10 MiB/s wr, 160 op/s; 49090/252006 objects degraded (19.480%); 13242/252006 objects misplaced (5.255%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:56:15.522 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:15.550 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:15.613 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:15.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:15.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:15.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:15 smithi121 ceph-mon[211526]: pgmap v790: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 10 MiB/s wr, 160 op/s; 49090/252006 objects degraded (19.480%); 13242/252006 objects misplaced (5.255%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:56:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:15 smithi121 ceph-mon[219709]: pgmap v790: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 10 MiB/s wr, 160 op/s; 49090/252006 objects degraded (19.480%); 13242/252006 objects misplaced (5.255%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:56:15.720 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:15.749 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:15.800 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:15.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:15.907 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:15.933 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:15.986 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:15.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:15.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:16.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 90 15 84742 84727 58.818 15.6875 0.967823 0.0169239 2023-09-23T20:56:16.094 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:16.122 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:16.163 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:16.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:16.269 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:16.301 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:16.449 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 1 pgp_num to 1 2023-09-23T20:56:16.471 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:16.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:16.576 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:16.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3310547421' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:56:16.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:16 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:56:16.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3310547421' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:56:16.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:16 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:56:16.605 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3310547421' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:56:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:16 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:56:17.057 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.7653 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 84742 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 58.3524 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 57.0944 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 216.062 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 933 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 913.53 2023-09-23T20:56:17.058 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 3457 2023-09-23T20:56:17.059 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T20:56:17.059 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0170613 2023-09-23T20:56:17.059 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.233461 2023-09-23T20:56:17.059 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 13.5629 2023-09-23T20:56:17.059 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00106683 2023-09-23T20:56:17.061 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:56:17.172 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:56:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:56:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[211526]: osdmap e250: 8 total, 7 up, 6 in 2023-09-23T20:56:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[211526]: failed to encode map e250 with expected crc 2023-09-23T20:56:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[211526]: pgmap v792: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 11 MiB/s wr, 171 op/s; 49090/252006 objects degraded (19.480%); 13242/252006 objects misplaced (5.255%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:56:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:56:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[219709]: osdmap e250: 8 total, 7 up, 6 in 2023-09-23T20:56:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[219709]: failed to encode map e250 with expected crc 2023-09-23T20:56:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:17 smithi121 ceph-mon[219709]: pgmap v792: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 11 MiB/s wr, 171 op/s; 49090/252006 objects degraded (19.480%); 13242/252006 objects misplaced (5.255%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:56:17.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:17 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:56:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:17 smithi123 ceph-mon[193903]: osdmap e250: 8 total, 7 up, 6 in 2023-09-23T20:56:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:17 smithi123 ceph-mon[193903]: failed to encode map e250 with expected crc 2023-09-23T20:56:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:17 smithi123 ceph-mon[193903]: pgmap v792: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 11 MiB/s wr, 171 op/s; 49090/252006 objects degraded (19.480%); 13242/252006 objects misplaced (5.255%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:56:18.541 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:18.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:18.644 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:18.671 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:18.715 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:18.720 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:18 smithi123 ceph-mon[193903]: failed to encode map e250 with expected crc 2023-09-23T20:56:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:18.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:18.821 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:18.847 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:18 smithi121 ceph-mon[211526]: failed to encode map e250 with expected crc 2023-09-23T20:56:18.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:18 smithi121 ceph-mon[219709]: failed to encode map e250 with expected crc 2023-09-23T20:56:18.850 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.399 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:19.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:19.504 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:19.531 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.574 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:19 smithi121 ceph-mon[211526]: pgmap v793: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 262 op/s; 49167/253002 objects degraded (19.433%); 12812/253002 objects misplaced (5.064%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:56:19.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:19 smithi121 ceph-mon[219709]: pgmap v793: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 262 op/s; 49167/253002 objects degraded (19.433%); 12812/253002 objects misplaced (5.064%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:56:19.679 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:19.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:19 smithi123 ceph-mon[193903]: pgmap v793: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 262 op/s; 49167/253002 objects degraded (19.433%); 12812/253002 objects misplaced (5.064%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:56:19.705 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.749 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:19.766 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:19.855 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:19.881 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:19.927 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:19.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:19.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:19.945 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:56:19.975 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:56:20.012 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:56:20.036 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:20.048 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:56:20.063 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:20.075 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:56:20.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:56:20.116 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:20.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:20.137 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:56:20.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:56:20.137 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:56:20.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:56:20.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:56:20.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:56:20.138 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:56:20.138 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:56:20.138 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:56:20.145 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:56:20.145 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:56:20.145 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T20:56:20.145 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:56:20.223 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:20.250 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:20.295 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:20.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:20.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:20.403 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:20.430 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:20.478 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:20.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:20.584 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:20.621 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:20 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 49167/253002 objects degraded (19.433%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2243387879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:20 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 49167/253002 objects degraded (19.433%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2243387879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:20 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 49167/253002 objects degraded (19.433%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2243387879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:21.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:21 smithi123 ceph-mon[193903]: pgmap v794: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 127 B/s rd, 15 MiB/s wr, 286 op/s; 49167/253071 objects degraded (19.428%); 12821/253071 objects misplaced (5.066%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:56:21.794 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:21.798 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:21 smithi121 ceph-mon[211526]: pgmap v794: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 127 B/s rd, 15 MiB/s wr, 286 op/s; 49167/253071 objects degraded (19.428%); 12821/253071 objects misplaced (5.066%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:56:21.798 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:21 smithi121 ceph-mon[219709]: pgmap v794: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 127 B/s rd, 15 MiB/s wr, 286 op/s; 49167/253071 objects degraded (19.428%); 12821/253071 objects misplaced (5.066%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:56:21.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:21.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:21.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:21.932 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:22.610 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:22.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:22.714 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:22.741 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:23.658 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:56:23.659 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:56:23.660 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:56:23.899 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:23.901 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:23 smithi121 ceph-mon[211526]: pgmap v795: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 109 B/s rd, 13 MiB/s wr, 252 op/s; 49076/252825 objects degraded (19.411%); 12821/252825 objects misplaced (5.071%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:56:23.902 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:23 smithi121 ceph-mon[219709]: pgmap v795: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 109 B/s rd, 13 MiB/s wr, 252 op/s; 49076/252825 objects degraded (19.411%); 12821/252825 objects misplaced (5.071%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:56:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:23.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:24.004 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:24.040 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.101 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:24.208 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:23 smithi123 ceph-mon[193903]: pgmap v795: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 109 B/s rd, 13 MiB/s wr, 252 op/s; 49076/252825 objects degraded (19.411%); 12821/252825 objects misplaced (5.071%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:56:24.233 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.245 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:24.276 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:24.383 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:24.410 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.462 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:24.521 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6] out_osds: [7, 2] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:56:24.521 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:24.521 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:56:24.521 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.614388 2023-09-23T20:56:24.521 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0.6143884433733597 2023-09-23T20:56:24.569 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:24.606 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.782 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:24.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:24.889 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:24.899 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:56:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:56:24] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-09-23T20:56:24.899 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:24 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48993/250695 objects degraded (19.543%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:24.915 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:24.959 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:24.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:24.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:25.067 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:25.092 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:25.134 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:25.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:25.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:24 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48993/250695 objects degraded (19.543%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:25.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:24 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48993/250695 objects degraded (19.543%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:25.241 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:25.267 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:25.311 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:25.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:25.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:25.416 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:25.444 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:25.501 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:25.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:25.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:25.609 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:25.643 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:25.661 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:56:25.795 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:25.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:25.899 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:25.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:25 smithi121 ceph-mon[211526]: pgmap v796: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 196 op/s; 48993/250695 objects degraded (19.543%); 12266/250695 objects misplaced (4.893%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:25.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2971272548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:25.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:25 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:25.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:25 smithi121 ceph-mon[219709]: pgmap v796: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 196 op/s; 48993/250695 objects degraded (19.543%); 12266/250695 objects misplaced (4.893%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:25.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2971272548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:25.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:25 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:25.922 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:56:25.945 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:25 smithi123 ceph-mon[193903]: pgmap v796: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 196 op/s; 48993/250695 objects degraded (19.543%); 12266/250695 objects misplaced (4.893%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2971272548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:25 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:56:26.517 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:26.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:26.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:26.621 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:26.646 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:26.687 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:26.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:26.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:26.793 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:26.818 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:26.863 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:26.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:26.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:26.967 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:26.993 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.037 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:27.146 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[211526]: failed to encode map e251 with expected crc 2023-09-23T20:56:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[211526]: failed to encode map e251 with expected crc 2023-09-23T20:56:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[211526]: osdmap e251: 8 total, 7 up, 6 in 2023-09-23T20:56:27.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[211526]: pgmap v798: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 196 op/s; 48993/250695 objects degraded (19.543%); 12266/250695 objects misplaced (4.893%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:27.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:27.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[219709]: failed to encode map e251 with expected crc 2023-09-23T20:56:27.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[219709]: failed to encode map e251 with expected crc 2023-09-23T20:56:27.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:27.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[219709]: osdmap e251: 8 total, 7 up, 6 in 2023-09-23T20:56:27.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:26 smithi121 ceph-mon[219709]: pgmap v798: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 196 op/s; 48993/250695 objects degraded (19.543%); 12266/250695 objects misplaced (4.893%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:27.181 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:26 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:26 smithi123 ceph-mon[193903]: failed to encode map e251 with expected crc 2023-09-23T20:56:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:26 smithi123 ceph-mon[193903]: failed to encode map e251 with expected crc 2023-09-23T20:56:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:56:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:26 smithi123 ceph-mon[193903]: osdmap e251: 8 total, 7 up, 6 in 2023-09-23T20:56:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:26 smithi123 ceph-mon[193903]: pgmap v798: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 4.8 MiB/s wr, 196 op/s; 48993/250695 objects degraded (19.543%); 12266/250695 objects misplaced (4.893%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:27.257 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:27.274 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:27.362 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:27.404 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.543 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:27.563 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:27.668 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:27.697 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.747 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:27.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:27.805 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:56:27.805 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:56:27.805 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T20:56:27.805 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T20:56:27.805 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:56:27.854 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:27.881 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:27.932 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:27.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:28.039 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:28.071 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.123 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:28.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:28.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:28.227 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:28.254 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.297 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:28.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:28.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:28.441 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:28.468 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:28.468 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:28.472 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:29.201 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:56:29.474 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:29 smithi121 ceph-mon[211526]: pgmap v799: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 1.1 MiB/s wr, 158 op/s; 48815/249300 objects degraded (19.581%); 11762/249300 objects misplaced (4.718%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:56:29.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:29 smithi121 ceph-mon[219709]: pgmap v799: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 1.1 MiB/s wr, 158 op/s; 48815/249300 objects degraded (19.581%); 11762/249300 objects misplaced (4.718%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:56:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:29 smithi123 ceph-mon[193903]: pgmap v799: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 1.1 MiB/s wr, 158 op/s; 48815/249300 objects degraded (19.581%); 11762/249300 objects misplaced (4.718%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:56:29.530 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:29.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:29.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:29.640 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:29.669 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:29.712 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:29.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:29.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:29.845 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:29.893 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:29.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:29.998 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:30.028 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:30.364 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:56:30.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:30.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:30.704 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:30.731 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:30.774 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:30.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:30.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:30.881 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:30.929 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:30.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:30 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48815/248694 objects degraded (19.629%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:31.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:30 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48815/248694 objects degraded (19.629%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:30 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48815/248694 objects degraded (19.629%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:31.600 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:31.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:31.704 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:31.731 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:31.747 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:31.774 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:31.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:31.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:31.916 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:32.030 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:32.155 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:32.158 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:31 smithi121 ceph-mon[219709]: pgmap v800: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 48815/248694 objects degraded (19.629%); 11723/248694 objects misplaced (4.714%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:56:32.158 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:31 smithi121 ceph-mon[211526]: pgmap v800: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 48815/248694 objects degraded (19.629%); 11723/248694 objects misplaced (4.714%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:56:32.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:32.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:32.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:31 smithi123 ceph-mon[193903]: pgmap v800: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 48815/248694 objects degraded (19.629%); 11723/248694 objects misplaced (4.714%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:56:32.260 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:32.300 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:32.726 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:32.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:32.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:32.766 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:56:32.804 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:56:32.834 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:32.844 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:56:32.864 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:32.878 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:56:32.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3387217233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:32.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3387217233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:32.912 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:56:32.915 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:32.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:32.938 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:56:32.966 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:56:32.966 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:56:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:56:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:56:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:56:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:56:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:56:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:56:32.967 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:56:33.020 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:33.049 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:33.095 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:33.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:33.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:33.200 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:33.227 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3387217233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:33.228 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:33.281 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:33.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:33.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:33.418 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:33.922 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.1 primary-affinity to 0.614388 (8402642) 2023-09-23T20:56:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:33 smithi121 ceph-mon[211526]: pgmap v801: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 48717/248394 objects degraded (19.613%); 11723/248394 objects misplaced (4.720%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:56:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3251820821' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]: dispatch 2023-09-23T20:56:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:33 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]: dispatch 2023-09-23T20:56:34.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:33 smithi121 ceph-mon[219709]: pgmap v801: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 48717/248394 objects degraded (19.613%); 11723/248394 objects misplaced (4.720%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:56:34.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3251820821' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]: dispatch 2023-09-23T20:56:34.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:33 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]: dispatch 2023-09-23T20:56:34.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:33 smithi123 ceph-mon[193903]: pgmap v801: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 48717/248394 objects degraded (19.613%); 11723/248394 objects misplaced (4.720%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T20:56:34.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3251820821' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]: dispatch 2023-09-23T20:56:34.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:33 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]: dispatch 2023-09-23T20:56:35.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:56:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:56:34] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-09-23T20:56:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]': finished 2023-09-23T20:56:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: osdmap e252: 8 total, 7 up, 6 in 2023-09-23T20:56:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[219709]: pgmap v803: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 48624/246354 objects degraded (19.737%); 11132/246354 objects misplaced (4.519%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]': finished 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: osdmap e252: 8 total, 7 up, 6 in 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:34 smithi121 ceph-mon[211526]: pgmap v803: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 48624/246354 objects degraded (19.737%); 11132/246354 objects misplaced (4.519%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:56:35.183 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:35.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6143884433733597}]': finished 2023-09-23T20:56:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: osdmap e252: 8 total, 7 up, 6 in 2023-09-23T20:56:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: failed to encode map e252 with expected crc 2023-09-23T20:56:35.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:34 smithi123 ceph-mon[193903]: pgmap v803: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 48624/246354 objects degraded (19.737%); 11132/246354 objects misplaced (4.519%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:56:35.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:35.314 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:35.367 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:35.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:35.472 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:35.502 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:35.544 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:35.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:35.627 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:56:35.628 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:56:35.651 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:35.685 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48624/246354 objects degraded (19.737%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48624/246354 objects degraded (19.737%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48624/246354 objects degraded (19.737%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:37.194 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:37.302 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:37.329 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.373 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:37 smithi121 ceph-mon[211526]: pgmap v804: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 144 op/s; 48624/246354 objects degraded (19.737%); 11132/246354 objects misplaced (4.519%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:56:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:37 smithi121 ceph-mon[219709]: pgmap v804: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 144 op/s; 48624/246354 objects degraded (19.737%); 11132/246354 objects misplaced (4.519%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:56:37.478 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:37.539 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.629 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:56:37.703 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:37.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:37 smithi123 ceph-mon[193903]: pgmap v804: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 144 op/s; 48624/246354 objects degraded (19.737%); 11132/246354 objects misplaced (4.519%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:56:37.809 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:37.845 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:37.900 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:37.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:38.006 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:38.043 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.055 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:38.122 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:38.134 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:38.134 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:38.227 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:38.254 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.299 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:38.410 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:38.442 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.483 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:38.533 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:56:38.591 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:38.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/643093160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:38.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:38 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:38.618 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.662 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:38.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:38.772 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:38.798 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:38.841 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:38.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:38.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/643093160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:38.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:38 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:38.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/643093160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:56:38.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:38 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:56:38.949 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:38.982 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:39.339 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:39.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:39.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:39.443 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:39.470 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[219709]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[219709]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/643093160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[219709]: osdmap e253: 8 total, 7 up, 6 in 2023-09-23T20:56:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[219709]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[219709]: pgmap v806: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 48430/244836 objects degraded (19.781%); 10540/244836 objects misplaced (4.305%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:56:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[219709]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[211526]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[211526]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/643093160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[211526]: osdmap e253: 8 total, 7 up, 6 in 2023-09-23T20:56:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[211526]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[211526]: pgmap v806: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 48430/244836 objects degraded (19.781%); 10540/244836 objects misplaced (4.305%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:56:39.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:39 smithi121 ceph-mon[211526]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:39 smithi123 ceph-mon[193903]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:39 smithi123 ceph-mon[193903]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/643093160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:56:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:39 smithi123 ceph-mon[193903]: osdmap e253: 8 total, 7 up, 6 in 2023-09-23T20:56:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:39 smithi123 ceph-mon[193903]: failed to encode map e253 with expected crc 2023-09-23T20:56:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:39 smithi123 ceph-mon[193903]: pgmap v806: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 48430/244836 objects degraded (19.781%); 10540/244836 objects misplaced (4.305%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:56:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:39 smithi123 ceph-mon[193903]: failed to encode map e253 with expected crc 2023-09-23T20:56:40.078 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:40.184 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:40.203 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:56:40.211 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.260 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:40.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:40.367 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:40.408 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.616 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:40.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:40.721 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:40.746 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.787 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:40.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:40.893 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:40.919 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:40.961 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:40.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:41.067 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:41.100 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.468 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:41.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:41.573 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:41.599 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.643 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:41.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:41.748 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:41.778 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.824 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:41.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:41.929 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:41.955 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:41.998 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:42.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:42.103 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:42.127 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:41 smithi123 ceph-mon[193903]: pgmap v807: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 48430/244167 objects degraded (19.835%); 10498/244167 objects misplaced (4.300%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:56:42.127 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48430/244167 objects degraded (19.835%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:42.130 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.133 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:41 smithi121 ceph-mon[211526]: pgmap v807: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 48430/244167 objects degraded (19.835%); 10498/244167 objects misplaced (4.300%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:56:42.133 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48430/244167 objects degraded (19.835%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:42.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:41 smithi121 ceph-mon[219709]: pgmap v807: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 48430/244167 objects degraded (19.835%); 10498/244167 objects misplaced (4.300%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T20:56:42.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48430/244167 objects degraded (19.835%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:42.174 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:42.217 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6] out_osds: [7, 2] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:56:42.217 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:42.217 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T20:56:42.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T20:56:42.279 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:42.311 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.397 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:42.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:42.443 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:42.506 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:42.554 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.731 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:42.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:42.836 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:42.864 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:42.907 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:42.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:43.015 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:43.054 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:43.120 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:43.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:43.130 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:56:43.150 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:56:43.170 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:56:43.193 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:56:43.216 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:56:43.226 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:43.226 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:43.241 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:56:43.252 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:56:43.276 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:56:43.294 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:43.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:43.400 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:43.426 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:43.473 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:43.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:43.579 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:43.602 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:43.602 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:43.602 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:43 smithi123 ceph-mon[193903]: pgmap v808: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 48341/243903 objects degraded (19.820%); 10498/243903 objects misplaced (4.304%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:43.603 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/811549036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:43.605 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:43.647 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:43.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:43.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[211526]: pgmap v808: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 48341/243903 objects degraded (19.820%); 10498/243903 objects misplaced (4.304%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/811549036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[219709]: pgmap v808: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 48341/243903 objects degraded (19.820%); 10498/243903 objects misplaced (4.304%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:56:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/811549036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:43.753 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:43.786 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:44.135 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:56:44.681 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:44.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:44.785 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:44.814 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:45.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:56:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:56:44] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-09-23T20:56:45.540 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:45.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:56:45.594 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:56:45.648 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:45.681 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:45.745 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:45.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:45.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:45.848 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:45.877 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:45.922 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:45.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:45.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:45.986 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:45.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:45 smithi121 ceph-mon[211526]: pgmap v809: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 48239/242106 objects degraded (19.925%); 9822/242106 objects misplaced (4.057%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:56:45.997 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:45 smithi121 ceph-mon[219709]: pgmap v809: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 48239/242106 objects degraded (19.925%); 9822/242106 objects misplaced (4.057%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:56:46.027 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:46.053 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:46.097 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:46.102 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:45 smithi123 ceph-mon[193903]: pgmap v809: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 48239/242106 objects degraded (19.925%); 9822/242106 objects misplaced (4.057%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:56:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:46.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:46.204 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:46.232 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:46.290 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:46.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:46.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:46.395 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:46.426 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:46.628 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:46.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:46.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:46.732 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:46.762 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:46.808 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:46.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:46.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:46.914 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:46.940 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48239/242106 objects degraded (19.925%), 12 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:46.941 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48239/242106 objects degraded (19.925%), 12 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:46.943 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:46.994 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:46.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:46.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:47.099 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:47.129 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48239/242106 objects degraded (19.925%), 12 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:47.534 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:47.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:47.582 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPushReply(10.8 253/241 [PushReplyOp(10:1e12743b:::benchmark_data_smithi121_358375_object47885:head)])", 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:47.577308+0000", 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0040254619999999996, 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040492050000000002, 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:47.577308+0000", 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:47.577308+0000", 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:56:47.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:47.577307+0000", 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:47.577314+0000", 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6549999999999998e-06 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:47.577326+0000", 2023-09-23T20:56:47.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2532e-05 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:47.577335+0000", 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3899999999999993e-06 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:47.578961+0000", 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016256879999999999 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:56:47.586 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:56:47.587 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:56:47.587 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:56:47.587 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:56:47.587 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 1 2023-09-23T20:56:47.587 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:56:47.595 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:56:47.639 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:47.665 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.709 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:47.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:47.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:47.814 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:47.853 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:47.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:47 smithi123 ceph-mon[193903]: pgmap v810: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 48239/242106 objects degraded (19.925%); 9822/242106 objects misplaced (4.057%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:56:48.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:47 smithi121 ceph-mon[211526]: pgmap v810: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 48239/242106 objects degraded (19.925%); 9822/242106 objects misplaced (4.057%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:56:48.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:47 smithi121 ceph-mon[219709]: pgmap v810: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 48239/242106 objects degraded (19.925%); 9822/242106 objects misplaced (4.057%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:56:49.224 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:49.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:49.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:49.329 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:49.362 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:49.405 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:49.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:49.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:49.511 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:49.537 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:49.579 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:49.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:49.685 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:49.715 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:49.766 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:49.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:49.841 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:56:49.871 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:49.914 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:49.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:49 smithi123 ceph-mon[193903]: pgmap v811: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 48050/240765 objects degraded (19.957%); 9108/240765 objects misplaced (3.783%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:56:50.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:49 smithi121 ceph-mon[211526]: pgmap v811: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 48050/240765 objects degraded (19.957%); 9108/240765 objects misplaced (3.783%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:56:50.154 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:49 smithi121 ceph-mon[219709]: pgmap v811: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 48050/240765 objects degraded (19.957%); 9108/240765 objects misplaced (3.783%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:56:50.184 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:50.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:50.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:50.290 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:50.317 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:50.375 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:50.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:50.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:50.393 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:50.483 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:50.511 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:50.553 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:50.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:50.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:50.658 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:50.686 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:50.737 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:50.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:50.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:50.849 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:50.914 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.091 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:51.096 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:51.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:51 smithi121 conmon[211495]: 2023-09-23T20:56:51.117+0000 7fdc50c66700 -1 mon.a@0(leader).osd e253 definitely_dead 0 2023-09-23T20:56:51.201 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:51.229 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.281 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:51.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:51.387 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:51.412 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.457 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:51.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:51.563 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:51.589 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.634 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:51.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:51.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:51.768 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.822 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:51.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:51.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:51.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:51 smithi123 ceph-mon[193903]: pgmap v812: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 48050/240114 objects degraded (20.011%); 9063/240114 objects misplaced (3.774%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:56:51.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3278489304' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:56:51.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:56:51.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48050/240114 objects degraded (20.011%), 12 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:51.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3153123411' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:51.883 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:51.894 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.2. 2023-09-23T20:56:51.895 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:56:51.931 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:51.957 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:51.998 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:52.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:52.107 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:56:52.132 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[219709]: pgmap v812: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 48050/240114 objects degraded (20.011%); 9063/240114 objects misplaced (3.774%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:56:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3278489304' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:56:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:56:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48050/240114 objects degraded (20.011%), 12 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3153123411' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[211526]: pgmap v812: 89 pgs: 2 active+recovery_wait+degraded, 27 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 34 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 48050/240114 objects degraded (20.011%); 9063/240114 objects misplaced (3.774%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:56:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3278489304' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:56:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:56:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48050/240114 objects degraded (20.011%), 12 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3153123411' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:56:52.173 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:56:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:56:52.278 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:52.303 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.344 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:52.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:52.449 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:52.475 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.516 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:52.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:52.623 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:52.649 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.691 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:52.799 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:52.825 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:52.867 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:52.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:52.972 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:53.005 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:53.049 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:53.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:53.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:53.153 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: osdmap e254: 8 total, 7 up, 7 in 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[211526]: osdmap e255: 8 total, 7 up, 7 in 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: osdmap e254: 8 total, 7 up, 7 in 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:52 smithi121 ceph-mon[219709]: osdmap e255: 8 total, 7 up, 7 in 2023-09-23T20:56:53.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:56:53.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:56:53.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:56:53.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: osdmap e254: 8 total, 7 up, 7 in 2023-09-23T20:56:53.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.178 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.179 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.179 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.179 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: failed to encode map e254 with expected crc 2023-09-23T20:56:53.179 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:52 smithi123 ceph-mon[193903]: osdmap e255: 8 total, 7 up, 7 in 2023-09-23T20:56:53.180 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:53.226 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:53.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:53.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:53.332 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:53.358 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:53.402 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:53.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:53.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:53.508 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:53.545 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: pgmap v814: 89 pgs: 2 peering, 5 remapped+peering, 2 active+recovery_wait+degraded, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 32 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 48050/240114 objects degraded (20.011%); 9063/240114 objects misplaced (3.774%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[211526]: osdmap e256: 8 total, 7 up, 7 in 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: pgmap v814: 89 pgs: 2 peering, 5 remapped+peering, 2 active+recovery_wait+degraded, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 32 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 48050/240114 objects degraded (20.011%); 9063/240114 objects misplaced (3.774%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:53 smithi121 ceph-mon[219709]: osdmap e256: 8 total, 7 up, 7 in 2023-09-23T20:56:54.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: pgmap v814: 89 pgs: 2 peering, 5 remapped+peering, 2 active+recovery_wait+degraded, 22 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 32 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 48050/240114 objects degraded (20.011%); 9063/240114 objects misplaced (3.774%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:56:54.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: failed to encode map e255 with expected crc 2023-09-23T20:56:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:53 smithi123 ceph-mon[193903]: osdmap e256: 8 total, 7 up, 7 in 2023-09-23T20:56:54.729 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:54.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:54.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:54.807 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T20:56:54.835 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:54.874 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:54.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[211526]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[211526]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[211526]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[211526]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[211526]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.998 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:56:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:56:54] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-09-23T20:56:54.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[219709]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[219709]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[219709]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[219709]: failed to encode map e256 with expected crc 2023-09-23T20:56:54.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:54 smithi121 ceph-mon[219709]: failed to encode map e256 with expected crc 2023-09-23T20:56:55.039 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:55.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:55.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:55.144 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:56:55.174 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:55.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:54 smithi123 ceph-mon[193903]: failed to encode map e256 with expected crc 2023-09-23T20:56:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:54 smithi123 ceph-mon[193903]: failed to encode map e256 with expected crc 2023-09-23T20:56:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:54 smithi123 ceph-mon[193903]: failed to encode map e256 with expected crc 2023-09-23T20:56:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:54 smithi123 ceph-mon[193903]: failed to encode map e256 with expected crc 2023-09-23T20:56:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:54 smithi123 ceph-mon[193903]: failed to encode map e256 with expected crc 2023-09-23T20:56:55.564 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:56:55.789 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:56:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:55.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:56:55.893 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:56:55.954 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[211526]: pgmap v817: 89 pgs: 4 peering, 9 remapped+peering, 2 active+recovery_wait+degraded, 17 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 31 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 33380/238800 objects degraded (13.978%); 6312/238800 objects misplaced (2.643%); 6.7 MiB/s, 108 objects/s recovering 2023-09-23T20:56:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:56:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[211526]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[211526]: osdmap e257: 8 total, 7 up, 7 in 2023-09-23T20:56:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[211526]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[211526]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[219709]: pgmap v817: 89 pgs: 4 peering, 9 remapped+peering, 2 active+recovery_wait+degraded, 17 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 31 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 33380/238800 objects degraded (13.978%); 6312/238800 objects misplaced (2.643%); 6.7 MiB/s, 108 objects/s recovering 2023-09-23T20:56:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:56:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[219709]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[219709]: osdmap e257: 8 total, 7 up, 7 in 2023-09-23T20:56:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[219709]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:55 smithi121 ceph-mon[219709]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:55 smithi123 ceph-mon[193903]: pgmap v817: 89 pgs: 4 peering, 9 remapped+peering, 2 active+recovery_wait+degraded, 17 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 31 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 33380/238800 objects degraded (13.978%); 6312/238800 objects misplaced (2.643%); 6.7 MiB/s, 108 objects/s recovering 2023-09-23T20:56:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:55 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:56:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:55 smithi123 ceph-mon[193903]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:55 smithi123 ceph-mon[193903]: osdmap e257: 8 total, 7 up, 7 in 2023-09-23T20:56:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:55 smithi123 ceph-mon[193903]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:55 smithi123 ceph-mon[193903]: failed to encode map e257 with expected crc 2023-09-23T20:56:56.331 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:56:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:56.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:56:56.435 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:56.485 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:56.598 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:56:56.627 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:56:56.651 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:56.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:56.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:56.658 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:56:56.684 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:56:56.710 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:56:56.716 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:56:56.716 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:56:56.732 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:56:56.756 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:56:56.756 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:56:56.776 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:56:56.776 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:56:56.777 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:56:56.777 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:56:56.777 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:56:56.777 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:56:56.777 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:56:56.777 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:56:56.777 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:56:56.925 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:56.967 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:56:56.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:56.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:56:57.074 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:56:57.102 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:57.173 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:56:57.176 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:56 smithi123 ceph-mon[193903]: failed to encode map e258 with expected crc 2023-09-23T20:56:57.176 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:56 smithi123 ceph-mon[193903]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T20:56:57.176 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:56 smithi123 ceph-mon[193903]: failed to encode map e258 with expected crc 2023-09-23T20:56:57.176 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 33380/238800 objects degraded (13.978%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:57.176 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:56 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3671021070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:57.176 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:56 smithi123 ceph-mon[193903]: pgmap v820: 89 pgs: 4 peering, 9 remapped+peering, 2 active+recovery_wait+degraded, 17 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 31 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 33380/238800 objects degraded (13.978%); 6312/238800 objects misplaced (2.643%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:56:57.176 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:56:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:57.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:56:57.278 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:56:57.304 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:57.356 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:56:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:57.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:56:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[219709]: failed to encode map e258 with expected crc 2023-09-23T20:56:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[219709]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T20:56:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[219709]: failed to encode map e258 with expected crc 2023-09-23T20:56:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 33380/238800 objects degraded (13.978%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3671021070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[219709]: pgmap v820: 89 pgs: 4 peering, 9 remapped+peering, 2 active+recovery_wait+degraded, 17 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 31 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 33380/238800 objects degraded (13.978%); 6312/238800 objects misplaced (2.643%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:56:57.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:56:57.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[211526]: failed to encode map e258 with expected crc 2023-09-23T20:56:57.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[211526]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T20:56:57.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[211526]: failed to encode map e258 with expected crc 2023-09-23T20:56:57.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 33380/238800 objects degraded (13.978%), 11 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T20:56:57.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3671021070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:56:57.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[211526]: pgmap v820: 89 pgs: 4 peering, 9 remapped+peering, 2 active+recovery_wait+degraded, 17 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 31 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 33380/238800 objects degraded (13.978%); 6312/238800 objects misplaced (2.643%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:56:57.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:56:57.465 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:57.496 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:58.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:56:59.604 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:56:59.662 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:56:59.663 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:56:59.665 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:59.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:56:59.770 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:56:59.808 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2] out_osds: [7] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:56:59.808 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:56:59.808 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T20:56:59.808 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:56:59.808 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:56:59.809 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T20:56:59.813 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:56:59.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:59 smithi121 ceph-mon[211526]: pgmap v821: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 79 op/s; 33365/238704 objects degraded (13.978%); 5276/238704 objects misplaced (2.210%); 18 MiB/s, 339 objects/s recovering 2023-09-23T20:56:59.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:59.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:56:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:56:59.939 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:56:59.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:56:59.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:00.044 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:00.073 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:00.119 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:00.123 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:59 smithi123 ceph-mon[193903]: pgmap v821: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 79 op/s; 33365/238704 objects degraded (13.978%); 5276/238704 objects misplaced (2.210%); 18 MiB/s, 339 objects/s recovering 2023-09-23T20:57:00.123 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:00.123 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:56:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:00.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:00.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:59 smithi121 ceph-mon[219709]: pgmap v821: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 79 op/s; 33365/238704 objects degraded (13.978%); 5276/238704 objects misplaced (2.210%); 18 MiB/s, 339 objects/s recovering 2023-09-23T20:57:00.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:00.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:56:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:00.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:00.254 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:00.297 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:00.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:00.401 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:00.426 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:00.467 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:00.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:00.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:00.518 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:00.576 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:00.607 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:01.265 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:01.370 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:01.398 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: pgmap v823: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 30014/238647 objects degraded (12.577%); 8262/238647 objects misplaced (3.462%); 15 MiB/s, 287 objects/s recovering 2023-09-23T20:57:01.584 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 33365/238704 objects degraded (13.978%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:01.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: pgmap v823: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 30014/238647 objects degraded (12.577%); 8262/238647 objects misplaced (3.462%); 15 MiB/s, 287 objects/s recovering 2023-09-23T20:57:01.586 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:57:01.586 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 33365/238704 objects degraded (13.978%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:01.641 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:01.642 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: failed to encode map e259 with expected crc 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: pgmap v823: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 30014/238647 objects degraded (12.577%); 8262/238647 objects misplaced (3.462%); 15 MiB/s, 287 objects/s recovering 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:57:01.643 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 33365/238704 objects degraded (13.978%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:01.664 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:57:01.745 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:01.772 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:01.819 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:01.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:01.925 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:01.952 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.098 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:02.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:02.202 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:02.229 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.273 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:02.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:02.313 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:57:02.314 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "" 2023-09-23T20:57:02.314 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:57:02.378 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:02.405 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:02 smithi123 ceph-mon[193903]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:02 smithi123 ceph-mon[193903]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:02 smithi123 ceph-mon[193903]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T20:57:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:02 smithi123 ceph-mon[193903]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:02 smithi123 ceph-mon[193903]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.548 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:02.643 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[219709]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[219709]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[219709]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[219709]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[219709]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[211526]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[211526]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[211526]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[211526]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:02 smithi121 ceph-mon[211526]: failed to encode map e260 with expected crc 2023-09-23T20:57:02.652 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:02.678 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:02.930 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:02.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:02.976 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:03.034 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:03.060 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.101 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:03.207 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:03.220 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:03.232 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.273 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:03.382 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:03.419 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.704 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:57:03.739 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:03.843 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:03.870 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:03.912 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:03.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:03 smithi123 ceph-mon[193903]: failed to encode map e260 with expected crc 2023-09-23T20:57:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:03 smithi123 ceph-mon[193903]: failed to encode map e261 with expected crc 2023-09-23T20:57:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:03 smithi123 ceph-mon[193903]: failed to encode map e261 with expected crc 2023-09-23T20:57:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:03 smithi123 ceph-mon[193903]: osdmap e261: 8 total, 7 up, 7 in 2023-09-23T20:57:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:03 smithi123 ceph-mon[193903]: pgmap v826: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 30014/238647 objects degraded (12.577%); 8262/238647 objects misplaced (3.462%); 15 MiB/s, 287 objects/s recovering 2023-09-23T20:57:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2332418182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:04.019 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:04.045 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:04.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:04.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:04.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[211526]: failed to encode map e260 with expected crc 2023-09-23T20:57:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[211526]: failed to encode map e261 with expected crc 2023-09-23T20:57:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[211526]: failed to encode map e261 with expected crc 2023-09-23T20:57:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[211526]: osdmap e261: 8 total, 7 up, 7 in 2023-09-23T20:57:04.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[211526]: pgmap v826: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 30014/238647 objects degraded (12.577%); 8262/238647 objects misplaced (3.462%); 15 MiB/s, 287 objects/s recovering 2023-09-23T20:57:04.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2332418182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:04.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[219709]: failed to encode map e260 with expected crc 2023-09-23T20:57:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[219709]: failed to encode map e261 with expected crc 2023-09-23T20:57:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[219709]: failed to encode map e261 with expected crc 2023-09-23T20:57:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[219709]: osdmap e261: 8 total, 7 up, 7 in 2023-09-23T20:57:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[219709]: pgmap v826: 89 pgs: 1 activating, 2 peering, 2 remapped+peering, 2 active+recovery_wait+degraded, 24 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 33 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 30014/238647 objects degraded (12.577%); 8262/238647 objects misplaced (3.462%); 15 MiB/s, 287 objects/s recovering 2023-09-23T20:57:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2332418182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:04.193 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:04.225 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:04.953 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:57:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:57:04] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2023-09-23T20:57:04.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:04.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[211526]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[211526]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[211526]: osdmap e262: 8 total, 7 up, 7 in 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[211526]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[211526]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[219709]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[219709]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[219709]: osdmap e262: 8 total, 7 up, 7 in 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[219709]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:04 smithi121 ceph-mon[219709]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:04 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:04 smithi123 ceph-mon[193903]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:04 smithi123 ceph-mon[193903]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:04 smithi123 ceph-mon[193903]: osdmap e262: 8 total, 7 up, 7 in 2023-09-23T20:57:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:04 smithi123 ceph-mon[193903]: failed to encode map e262 with expected crc 2023-09-23T20:57:04.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:04 smithi123 ceph-mon[193903]: failed to encode map e262 with expected crc 2023-09-23T20:57:05.569 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:05.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:05.673 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:05.710 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:05.771 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:05.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:05.876 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:05.909 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:05.959 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:05.964 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:05 smithi123 ceph-mon[193903]: pgmap v828: 89 pgs: 1 activating, 3 peering, 2 remapped+peering, 1 active+recovery_wait+degraded, 24 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 25053/238524 objects degraded (10.503%); 7550/238524 objects misplaced (3.165%); 13 MiB/s, 279 objects/s recovering 2023-09-23T20:57:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:05.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:06.016 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:05 smithi121 ceph-mon[211526]: pgmap v828: 89 pgs: 1 activating, 3 peering, 2 remapped+peering, 1 active+recovery_wait+degraded, 24 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 25053/238524 objects degraded (10.503%); 7550/238524 objects misplaced (3.165%); 13 MiB/s, 279 objects/s recovering 2023-09-23T20:57:06.016 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:05 smithi121 ceph-mon[219709]: pgmap v828: 89 pgs: 1 activating, 3 peering, 2 remapped+peering, 1 active+recovery_wait+degraded, 24 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 34 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 25053/238524 objects degraded (10.503%); 7550/238524 objects misplaced (3.165%); 13 MiB/s, 279 objects/s recovering 2023-09-23T20:57:06.066 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:06.093 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:06.137 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:06.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:06.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:06.243 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:06.282 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:06.324 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:06.324 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 25053/238524 objects degraded (10.503%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 25053/238524 objects degraded (10.503%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 25053/238524 objects degraded (10.503%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:07.286 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:07.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:07.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:07.391 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:07.421 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:07.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:57:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:07 smithi123 ceph-mon[193903]: pgmap v829: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 36 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 20723/238401 objects degraded (8.692%); 7550/238401 objects misplaced (3.167%); 10 MiB/s, 219 objects/s recovering 2023-09-23T20:57:07.987 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2] out_osds: [7] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:57:07.987 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:57:07.987 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T20:57:07.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T20:57:08.017 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:07 smithi121 ceph-mon[211526]: pgmap v829: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 36 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 20723/238401 objects degraded (8.692%); 7550/238401 objects misplaced (3.167%); 10 MiB/s, 219 objects/s recovering 2023-09-23T20:57:08.017 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:07 smithi121 ceph-mon[219709]: pgmap v829: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 36 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 20723/238401 objects degraded (8.692%); 7550/238401 objects misplaced (3.167%); 10 MiB/s, 219 objects/s recovering 2023-09-23T20:57:08.165 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:08.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:08.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:08.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:57:08.276 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:08.309 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:08.358 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:08.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:08.466 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:08.506 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:08.667 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:57:08.674 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:08.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:08.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:57:08.740 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:57:08.773 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:57:08.777 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:08.787 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:08.802 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:57:08.819 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:08.830 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:08.834 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:57:08.861 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:57:08.861 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:57:08.861 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:57:08.861 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:57:08.861 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:57:08.862 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:57:08.862 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:57:08.862 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:57:08.862 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:57:08.867 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:08.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:08.976 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1004061478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:09.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1004061478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:09.004 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1004061478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:09.007 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:09.159 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:09.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:09.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:09.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:09.296 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:09.344 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:09.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:09.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:09.451 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:09.482 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:09 smithi121 ceph-mon[211526]: pgmap v830: 89 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 20723/238401 objects degraded (8.692%); 7550/238401 objects misplaced (3.167%); 8.2 MiB/s, 174 objects/s recovering 2023-09-23T20:57:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:09 smithi121 ceph-mon[219709]: pgmap v830: 89 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 20723/238401 objects degraded (8.692%); 7550/238401 objects misplaced (3.167%); 8.2 MiB/s, 174 objects/s recovering 2023-09-23T20:57:09.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:09 smithi123 ceph-mon[193903]: pgmap v830: 89 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 20723/238401 objects degraded (8.692%); 7550/238401 objects misplaced (3.167%); 8.2 MiB/s, 174 objects/s recovering 2023-09-23T20:57:10.528 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:10.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:10.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:10.633 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:10.666 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:10.717 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:10.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:10.822 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:10.850 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:10.898 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:10.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:11.005 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:11.039 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:11.058 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:57:11.090 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:57:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:11 smithi121 ceph-mon[211526]: pgmap v831: 89 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 20723/238401 objects degraded (8.692%); 6975/238401 objects misplaced (2.926%); 12 MiB/s, 232 objects/s recovering 2023-09-23T20:57:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 20723/238401 objects degraded (8.692%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:11 smithi121 ceph-mon[219709]: pgmap v831: 89 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 20723/238401 objects degraded (8.692%); 6975/238401 objects misplaced (2.926%); 12 MiB/s, 232 objects/s recovering 2023-09-23T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 20723/238401 objects degraded (8.692%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:11 smithi123 ceph-mon[193903]: pgmap v831: 89 pgs: 1 activating, 1 activating+undersized+degraded+remapped, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 20723/238401 objects degraded (8.692%); 6975/238401 objects misplaced (2.926%); 12 MiB/s, 232 objects/s recovering 2023-09-23T20:57:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 20723/238401 objects degraded (8.692%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:12.480 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:12.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:12.586 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:12.612 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:12.655 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:12.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:12.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:12.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:12.794 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:12.994 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:12.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:12.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:13.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:57:13.099 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:13.118 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:13.118 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:13.146 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:13.246 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:13.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:13.353 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:13.384 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:13.887 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:13.888 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:57:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:13.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:14.000 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:14.027 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:14.074 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:14.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[211526]: pgmap v832: 89 pgs: 1 activating, 26 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 15858/238380 objects degraded (6.652%); 11523/238380 objects misplaced (4.834%); 9.8 MiB/s, 189 objects/s recovering 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[211526]: osdmap e263: 8 total, 7 up, 7 in 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[211526]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[211526]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[211526]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4052993145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 conmon[211495]: 2023-09-23T20:57:14.051+0000 7fdc50c66700 -1 mon.a@0(leader).osd e264 definitely_dead 0 2023-09-23T20:57:14.140 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[219709]: pgmap v832: 89 pgs: 1 activating, 26 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 15858/238380 objects degraded (6.652%); 11523/238380 objects misplaced (4.834%); 9.8 MiB/s, 189 objects/s recovering 2023-09-23T20:57:14.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[219709]: osdmap e263: 8 total, 7 up, 7 in 2023-09-23T20:57:14.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[219709]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[219709]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[219709]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4052993145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:14.141 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:13 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:14.178 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:14.215 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:13 smithi123 ceph-mon[193903]: pgmap v832: 89 pgs: 1 activating, 26 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 37 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 15858/238380 objects degraded (6.652%); 11523/238380 objects misplaced (4.834%); 9.8 MiB/s, 189 objects/s recovering 2023-09-23T20:57:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:13 smithi123 ceph-mon[193903]: osdmap e263: 8 total, 7 up, 7 in 2023-09-23T20:57:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:13 smithi123 ceph-mon[193903]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:13 smithi123 ceph-mon[193903]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:13 smithi123 ceph-mon[193903]: failed to encode map e263 with expected crc 2023-09-23T20:57:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4052993145' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:13 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:14.637 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:14.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:14.741 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:14.772 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:14.882 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:14.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:14.902 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: osdmap e264: 8 total, 7 up, 7 in 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/937678994' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:57:14.903 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:57:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:57:14] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: osdmap e264: 8 total, 7 up, 7 in 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: failed to encode map e264 with expected crc 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/937678994' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:57:14.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:14 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:57:14.914 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.7. 2023-09-23T20:57:14.998 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:15.024 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:15.065 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:15.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:15.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:15.170 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:15.201 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:15.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: failed to encode map e264 with expected crc 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: failed to encode map e264 with expected crc 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: osdmap e264: 8 total, 7 up, 7 in 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: failed to encode map e264 with expected crc 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: failed to encode map e264 with expected crc 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/937678994' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:57:15.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:14 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T20:57:15.725 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90505 10.9 10:99aea1ee:::benchmark_data_smithi121_358375_object5760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.3174349190000001, 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3174611869999999, 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.726 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90505 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:57:15.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268395+0000", 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268402+0000", 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2389999999999996e-06 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268415+0000", 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2850999999999999e-05 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268422+0000", 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.644e-06 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.520421+0000", 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.251998897 2023-09-23T20:57:15.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.526858+0000", 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064371389999999997 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.557595+0000", 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.030736329 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.563877+0000", 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006281989 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998305+0000", 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43442851399999999 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998317+0000", 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1959e-05 2023-09-23T20:57:15.731 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90379 10.9 10:951c4b03:::benchmark_data_smithi121_358375_object5634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.6569253690000001, 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6570134570000001, 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.732 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90379 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928904+0000", 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928913+0000", 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928927+0000", 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4329e-05 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928935+0000", 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9729999999999997e-06 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.954600+0000", 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025665337 2023-09-23T20:57:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.964941+0000", 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010340223000000001 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.088603+0000", 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12366287400000001 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.095790+0000", 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071866090000000001 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998534+0000", 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.90274379599999999 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998541+0000", 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1199999999999996e-06 2023-09-23T20:57:15.737 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90448 10.9 10:9dd18145:::benchmark_data_smithi121_358375_object5703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.5583318830000001, 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5584849670000001, 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.738 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90448 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027497+0000", 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027505+0000", 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0879999999999997e-06 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027521+0000", 2023-09-23T20:57:15.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6795000000000001e-05 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027531+0000", 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3649999999999993e-06 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.359164+0000", 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.331633185 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.365457+0000", 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006293267 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.418540+0000", 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.053082826999999999 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.429706+0000", 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011166376 2023-09-23T20:57:15.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998075+0000", 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.56836851300000002 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998082+0000", 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5100000000000001e-06 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.743 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90304 10.9 10:95d08781:::benchmark_data_smithi121_358375_object5559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 2.5850418509999997, 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.585249278, 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90304 2023-09-23T20:57:15.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000787+0000", 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000797+0000", 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6229999999999992e-06 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000815+0000", 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8391999999999999e-05 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000827+0000", 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1934e-05 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.076647+0000", 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.075820121000000004 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083118+0000", 2023-09-23T20:57:15.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064707690000000003 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.125639+0000", 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.042520998999999997 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.133609+0000", 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079701110000000002 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998466+0000", 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8648562869999998 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998472+0000", 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8070000000000002e-06 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90319 10.9 10:9c23793e:::benchmark_data_smithi121_358375_object5574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:57:15.749 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 2.5136070749999999, 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5138664849999999, 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90319 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:57:15.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072222+0000", 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072231+0000", 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2539999999999992e-06 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072248+0000", 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7422e-05 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072260+0000", 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2139e-05 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083953+0000", 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011692759 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.090352+0000", 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063994910000000002 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.175493+0000", 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.085140748000000002 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.187526+0000", 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01203284 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998112+0000", 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8105862510000001 2023-09-23T20:57:15.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998120+0000", 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.418e-06 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90350 10.9 10:9252fbfb:::benchmark_data_smithi121_358375_object5605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:57:15.755 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.9033631180000001, 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9036757529999999, 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90350 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682466+0000", 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682473+0000", 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7800000000000003e-06 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682487+0000", 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.402e-05 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682496+0000", 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7860000000000002e-06 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.827509+0000", 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.145013317 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.834217+0000", 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067071600000000002 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.884874+0000", 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050657455999999997 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.902020+0000", 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017146446999999999 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998607+0000", 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.096586142 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998613+0000", 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7830000000000001e-06 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90514 10.9 10:96e73e59:::benchmark_data_smithi121_358375_object5769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.92427182500000005, 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92463589000000002, 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.761 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90514 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661558+0000", 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661567+0000", 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.183e-06 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661583+0000", 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6585000000000001e-05 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661593+0000", 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6190000000000006e-06 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.825075+0000", 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16348148200000001 2023-09-23T20:57:15.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.835611+0000", 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010536495 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.936853+0000", 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10124217200000001 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.943598+0000", 2023-09-23T20:57:15.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067444640000000004 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998433+0000", 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.054835452 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998439+0000", 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6470000000000003e-06 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90471 10.9 10:9fd1a800:::benchmark_data_smithi121_358375_object5726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.479431669, 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.479849153, 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90471 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106398+0000", 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:15.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106407+0000", 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9089999999999995e-06 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106423+0000", 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5962999999999999e-05 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106433+0000", 2023-09-23T20:57:15.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1589999999999999e-06 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.431330+0000", 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32489747699999999 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.444379+0000", 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013049432999999999 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.479473+0000", 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.035093725999999999 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.485924+0000", 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064507230000000002 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997976+0000", 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.51205186199999997 2023-09-23T20:57:15.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997987+0000", 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1669e-05 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90301 10.9 10:95826f21:::benchmark_data_smithi121_358375_object5556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.0797766979999999, 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.080256302, 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.772 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90301 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506053+0000", 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506062+0000", 2023-09-23T20:57:15.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7669999999999998e-06 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506077+0000", 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4562999999999999e-05 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506088+0000", 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0779e-05 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.665171+0000", 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15908349199999999 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.672174+0000", 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070033109999999999 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.759608+0000", 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087433832000000003 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.770158+0000", 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010549433 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998499+0000", 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2283415890000002 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998505+0000", 2023-09-23T20:57:15.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6810000000000003e-06 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90424 10.9 10:980e8aaf:::benchmark_data_smithi121_358375_object5679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.60428839, 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.604823954, 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.778 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90424 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:57:15.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981541+0000", 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981548+0000", 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1489999999999997e-06 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981561+0000", 2023-09-23T20:57:15.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2448e-05 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981569+0000", 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1499999999999999e-06 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.224643+0000", 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243074701 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.237465+0000", 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012821737 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.280151+0000", 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.04268632 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.286562+0000", 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064109060000000001 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998360+0000", 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71179806400000001 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998368+0000", 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5050000000000001e-06 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90523 10.9 10.f32176e9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.783 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.743128+0000", 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.84270306399999995, 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.84329194699999999, 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90523 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743128+0000", 2023-09-23T20:57:15.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743128+0000", 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743126+0000", 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743133+0000", 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0520000000000004e-06 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743149+0000", 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5480000000000001e-05 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743156+0000", 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3000000000000004e-06 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998680+0000", 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25552361299999998 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998687+0000", 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1949999999999997e-06 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.787 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90429 10.9 10:9b165b02:::benchmark_data_smithi121_358375_object5684:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.5971918349999998, 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5978196929999999, 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90429 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988638+0000", 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988645+0000", 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2840000000000001e-06 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988661+0000", 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6053999999999999e-05 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988669+0000", 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9419999999999995e-06 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.238554+0000", 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24988542499999999 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.246423+0000", 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0078688479999999995 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.311852+0000", 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.065429004999999998 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.318196+0000", 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063438970000000003 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998162+0000", 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.67996552200000004 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998172+0000", 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0208e-05 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90489 10.9 10:97f30eb8:::benchmark_data_smithi121_358375_object5744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.375077809, 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.375757801, 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.793 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90489 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210752+0000", 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210761+0000", 2023-09-23T20:57:15.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9970000000000004e-06 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210778+0000", 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7181999999999999e-05 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210789+0000", 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0818e-05 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.446681+0000", 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23589284299999999 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.453972+0000", 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072908900000000004 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.511615+0000", 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057642634999999998 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.518011+0000", 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006395815 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998399+0000", 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.480388125 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998406+0000", 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4789999999999999e-06 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90434 10.9 10:90493f14:::benchmark_data_smithi121_358375_object5689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.5917954569999999, 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5925268749999999, 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.799 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90434 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:57:15.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994034+0000", 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994041+0000", 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4219999999999997e-06 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994052+0000", 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994059+0000", 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4060000000000002e-06 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.247296+0000", 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25323742700000001 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.254030+0000", 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067339619999999996 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.341839+0000", 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087808943 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.348061+0000", 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006222216 2023-09-23T20:57:15.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998641+0000", 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65057928700000001 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998645+0000", 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8470000000000003e-06 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90436 10.9 10:925ee57b:::benchmark_data_smithi121_358375_object5691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.5901396779999999, 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5909225120000001, 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90436 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.805 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:15.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995690+0000", 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995697+0000", 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5919999999999997e-06 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995714+0000", 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7144999999999999e-05 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995724+0000", 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0030000000000001e-05 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.350440+0000", 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.354715958 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.356739+0000", 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062994560000000002 2023-09-23T20:57:15.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.388748+0000", 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032009263000000003 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.395007+0000", 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006258726 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:15.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998567+0000", 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.60355961700000005 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998573+0000", 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8619999999999999e-06 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 15 2023-09-23T20:57:15.810 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:57:15.811 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:16.001 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:16.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:16.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:16.105 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:16.131 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: pgmap v835: 89 pgs: 1 peering, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 38 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 33 op/s; 16999/237585 objects degraded (7.155%); 9566/237585 objects misplaced (4.026%); 7.8 MiB/s, 165 objects/s recovering 2023-09-23T20:57:16.131 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:57:16.131 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.131 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.131 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T20:57:16.131 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: osdmap e265: 8 total, 7 up, 8 in 2023-09-23T20:57:16.131 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.132 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:15 smithi123 ceph-mon[193903]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.134 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:16.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: pgmap v835: 89 pgs: 1 peering, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 38 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 33 op/s; 16999/237585 objects degraded (7.155%); 9566/237585 objects misplaced (4.026%); 7.8 MiB/s, 165 objects/s recovering 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: osdmap e265: 8 total, 7 up, 8 in 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[211526]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: pgmap v835: 89 pgs: 1 peering, 26 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 38 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 33 op/s; 16999/237585 objects degraded (7.155%); 9566/237585 objects misplaced (4.026%); 7.8 MiB/s, 165 objects/s recovering 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T20:57:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T20:57:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: osdmap e265: 8 total, 7 up, 8 in 2023-09-23T20:57:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:15 smithi121 ceph-mon[219709]: failed to encode map e265 with expected crc 2023-09-23T20:57:16.180 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:16.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:16.287 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:16.315 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:16.590 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T20:57:16.790 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: osdmap e266: 8 total, 7 up, 8 in 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: failed to encode map e266 with expected crc 2023-09-23T20:57:16.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16999/237585 objects degraded (7.155%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:17.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:57:17.119 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:17.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:17.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:17.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: osdmap e266: 8 total, 7 up, 8 in 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.147 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16999/237585 objects degraded (7.155%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:17.224 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: osdmap e266: 8 total, 7 up, 8 in 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: failed to encode map e266 with expected crc 2023-09-23T20:57:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16999/237585 objects degraded (7.155%), 8 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:17.258 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:17.386 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:17.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:17.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:17.492 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:17.517 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:17.561 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:17.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:17.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:17.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:17.692 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:17.734 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:17.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:17.839 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:17.865 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:17.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: pgmap v838: 89 pgs: 4 remapped+peering, 2 peering, 22 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 38 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 16992/237513 objects degraded (7.154%); 5018/237513 objects misplaced (2.113%); 6.9 MiB/s, 191 objects/s recovering 2023-09-23T20:57:17.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: pgmap v838: 89 pgs: 4 remapped+peering, 2 peering, 22 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 38 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 16992/237513 objects degraded (7.154%); 5018/237513 objects misplaced (2.113%); 6.9 MiB/s, 191 objects/s recovering 2023-09-23T20:57:17.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: osdmap e267: 8 total, 7 up, 8 in 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: 10.8 starting backfill to osd.7 from (0'0,236'4725] MIN to 264'5752 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: failed to encode map e267 with expected crc 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:17.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:17.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:57:17.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:57:17.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:17.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[219709]: osdmap e268: 8 total, 7 up, 8 in 2023-09-23T20:57:17.918 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:17.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:17.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:18.025 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:18.052 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:18.095 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:18.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:18.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:18.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: osdmap e267: 8 total, 7 up, 8 in 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: 10.8 starting backfill to osd.7 from (0'0,236'4725] MIN to 264'5752 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:18.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:57:18.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:57:18.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:18.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:17 smithi121 ceph-mon[211526]: osdmap e268: 8 total, 7 up, 8 in 2023-09-23T20:57:18.199 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: pgmap v838: 89 pgs: 4 remapped+peering, 2 peering, 22 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 38 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 16992/237513 objects degraded (7.154%); 5018/237513 objects misplaced (2.113%); 6.9 MiB/s, 191 objects/s recovering 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: osdmap e267: 8 total, 7 up, 8 in 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: 10.8 starting backfill to osd.7 from (0'0,236'4725] MIN to 264'5752 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: failed to encode map e267 with expected crc 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:18.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:17 smithi123 ceph-mon[193903]: osdmap e268: 8 total, 7 up, 8 in 2023-09-23T20:57:18.226 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:18.270 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:18.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:18.375 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:18.404 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:18.539 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:18.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:18.589 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:18.643 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:18.688 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:18.749 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:18.833 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:18.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:18.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[211526]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[211526]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[211526]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.937 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[211526]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.938 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[219709]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.938 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[219709]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.938 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[219709]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.938 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:18 smithi121 ceph-mon[219709]: failed to encode map e268 with expected crc 2023-09-23T20:57:18.938 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:18.965 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.009 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:19.115 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:19.150 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:18 smithi123 ceph-mon[193903]: failed to encode map e268 with expected crc 2023-09-23T20:57:19.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:18 smithi123 ceph-mon[193903]: failed to encode map e268 with expected crc 2023-09-23T20:57:19.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:18 smithi123 ceph-mon[193903]: failed to encode map e268 with expected crc 2023-09-23T20:57:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:18 smithi123 ceph-mon[193903]: failed to encode map e268 with expected crc 2023-09-23T20:57:19.349 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:57:19.375 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:57:19.397 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:57:19.399 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:19.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:19.420 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:57:19.440 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:57:19.470 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:57:19.491 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:57:19.491 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:57:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:57:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:57:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:57:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:57:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:57:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:57:19.492 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:57:19.504 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:19.531 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.575 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:19.680 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:19.705 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:19.749 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:19.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:19.853 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:19.883 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:19 smithi121 ceph-mon[211526]: pgmap v841: 89 pgs: 4 remapped+peering, 1 peering, 21 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 46 op/s; 16397/237027 objects degraded (6.918%); 10039/237027 objects misplaced (4.235%); 3.5 MiB/s, 142 objects/s recovering 2023-09-23T20:57:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1232462311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:19 smithi121 ceph-mon[219709]: pgmap v841: 89 pgs: 4 remapped+peering, 1 peering, 21 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 46 op/s; 16397/237027 objects degraded (6.918%); 10039/237027 objects misplaced (4.235%); 3.5 MiB/s, 142 objects/s recovering 2023-09-23T20:57:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1232462311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:20.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:19 smithi123 ceph-mon[193903]: pgmap v841: 89 pgs: 4 remapped+peering, 1 peering, 21 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 16 active+undersized, 40 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 46 op/s; 16397/237027 objects degraded (6.918%); 10039/237027 objects misplaced (4.235%); 3.5 MiB/s, 142 objects/s recovering 2023-09-23T20:57:20.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1232462311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:21.286 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:21.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:21.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:21.392 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:21.417 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:21.459 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:21.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:21.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:21.565 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:21.590 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:21.590 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:57:21.591 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:57:21.591 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T20:57:21.591 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T20:57:21.591 DEBUG:teuthology.orchestra.run.smithi121:osd.1> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T20:57:21.593 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl start ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:57:21.625 INFO:journalctl@ceph.osd.1.smithi121.stdout:-- Logs begin at Sat 2023-09-23 20:08:31 UTC. -- 2023-09-23T20:57:21.632 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:21.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:21.714 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:21 smithi121 systemd[1]: Starting Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T20:57:21.739 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:21.771 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[219709]: pgmap v842: 89 pgs: 4 remapped+peering, 1 peering, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 49 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 41 op/s; 14668/236889 objects degraded (6.192%); 5196/236889 objects misplaced (2.193%); 7.8 MiB/s, 1 keys/s, 181 objects/s recovering 2023-09-23T20:57:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[219709]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[219709]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[219709]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[219709]: osdmap e269: 8 total, 7 up, 8 in 2023-09-23T20:57:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14668/236889 objects degraded (6.192%), 5 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[211526]: pgmap v842: 89 pgs: 4 remapped+peering, 1 peering, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 49 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 41 op/s; 14668/236889 objects degraded (6.192%); 5196/236889 objects misplaced (2.193%); 7.8 MiB/s, 1 keys/s, 181 objects/s recovering 2023-09-23T20:57:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[211526]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[211526]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[211526]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[211526]: osdmap e269: 8 total, 7 up, 8 in 2023-09-23T20:57:22.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14668/236889 objects degraded (6.192%), 5 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:21 smithi123 ceph-mon[193903]: pgmap v842: 89 pgs: 4 remapped+peering, 1 peering, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 49 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 41 op/s; 14668/236889 objects degraded (6.192%); 5196/236889 objects misplaced (2.193%); 7.8 MiB/s, 1 keys/s, 181 objects/s recovering 2023-09-23T20:57:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:21 smithi123 ceph-mon[193903]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:21 smithi123 ceph-mon[193903]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:21 smithi123 ceph-mon[193903]: failed to encode map e269 with expected crc 2023-09-23T20:57:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:21 smithi123 ceph-mon[193903]: osdmap e269: 8 total, 7 up, 8 in 2023-09-23T20:57:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14668/236889 objects degraded (6.192%), 5 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:22.457 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:22.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:22.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:22.561 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:22.587 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:22.628 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:22.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:22.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:22.647 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:57:22.648 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:57:22.733 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:22.759 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:22.801 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:22.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:22.905 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:22.954 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:23.046 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:23.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:23.121 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:23.121 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:23.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[219709]: osdmap e270: 8 total, 7 up, 8 in 2023-09-23T20:57:23.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[219709]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[219709]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[219709]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[219709]: pgmap v845: 89 pgs: 4 remapped+peering, 1 peering, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 49 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 37 op/s; 14668/236889 objects degraded (6.192%); 5196/236889 objects misplaced (2.193%); 7.6 MiB/s, 1 keys/s, 177 objects/s recovering 2023-09-23T20:57:23.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[211526]: osdmap e270: 8 total, 7 up, 8 in 2023-09-23T20:57:23.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[211526]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[211526]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[211526]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:22 smithi121 ceph-mon[211526]: pgmap v845: 89 pgs: 4 remapped+peering, 1 peering, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 49 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 37 op/s; 14668/236889 objects degraded (6.192%); 5196/236889 objects misplaced (2.193%); 7.6 MiB/s, 1 keys/s, 177 objects/s recovering 2023-09-23T20:57:23.151 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:23.184 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:22 smithi123 ceph-mon[193903]: osdmap e270: 8 total, 7 up, 8 in 2023-09-23T20:57:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:22 smithi123 ceph-mon[193903]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:22 smithi123 ceph-mon[193903]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:22 smithi123 ceph-mon[193903]: failed to encode map e270 with expected crc 2023-09-23T20:57:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:22 smithi123 ceph-mon[193903]: pgmap v845: 89 pgs: 4 remapped+peering, 1 peering, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 49 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 37 op/s; 14668/236889 objects degraded (6.192%); 5196/236889 objects misplaced (2.193%); 7.6 MiB/s, 1 keys/s, 177 objects/s recovering 2023-09-23T20:57:23.513 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:23.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:23.617 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:23.643 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:23.685 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:23.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:23.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:23.791 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:23.817 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:23.858 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:23.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:23.943 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:57:23.964 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:23.991 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:24.031 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:24.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:24.139 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:24.170 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:23 smithi123 ceph-mon[193903]: failed to encode map e270 with expected crc 2023-09-23T20:57:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/229938204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:23 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:24.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:23 smithi121 ceph-mon[219709]: failed to encode map e270 with expected crc 2023-09-23T20:57:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/229938204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:23 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:23 smithi121 ceph-mon[211526]: failed to encode map e270 with expected crc 2023-09-23T20:57:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/229938204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:23 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:24.649 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:57:24.931 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:57:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:57:24] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T20:57:24.931 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:57:24.931 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[211526]: failed to encode map e271 with expected crc 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[211526]: osdmap e271: 8 total, 7 up, 8 in 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[211526]: failed to encode map e271 with expected crc 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[211526]: pgmap v847: 89 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 12 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 9 op/s; 14668/236889 objects degraded (6.192%); 4505/236889 objects misplaced (1.902%); 12 MiB/s, 1 keys/s, 246 objects/s recovering 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[219709]: failed to encode map e271 with expected crc 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[219709]: osdmap e271: 8 total, 7 up, 8 in 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[219709]: failed to encode map e271 with expected crc 2023-09-23T20:57:24.932 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:24 smithi121 ceph-mon[219709]: pgmap v847: 89 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 12 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 9 op/s; 14668/236889 objects degraded (6.192%); 4505/236889 objects misplaced (1.902%); 12 MiB/s, 1 keys/s, 246 objects/s recovering 2023-09-23T20:57:25.080 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:25.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:25.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:25.185 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:25.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:24 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:57:25.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:24 smithi123 ceph-mon[193903]: failed to encode map e271 with expected crc 2023-09-23T20:57:25.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:24 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:25.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:24 smithi123 ceph-mon[193903]: osdmap e271: 8 total, 7 up, 8 in 2023-09-23T20:57:25.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:24 smithi123 ceph-mon[193903]: failed to encode map e271 with expected crc 2023-09-23T20:57:25.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:24 smithi123 ceph-mon[193903]: pgmap v847: 89 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 12 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 9 op/s; 14668/236889 objects degraded (6.192%); 4505/236889 objects misplaced (1.902%); 12 MiB/s, 1 keys/s, 246 objects/s recovering 2023-09-23T20:57:25.211 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:25.252 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:25.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:25.359 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:25.385 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:25.427 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:25.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:25.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:25.532 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:25.558 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:25.600 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:25.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:25.707 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:25.732 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:25.774 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:25.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:25.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:25.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:26.077 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:26.833 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:26.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:26.937 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:26.965 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.007 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:27.112 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:27.137 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.179 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:27.285 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:27.311 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.352 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:27.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:27.458 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:27 smithi123 ceph-mon[193903]: pgmap v848: 89 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 12 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14668/236889 objects degraded (6.192%); 4505/236889 objects misplaced (1.902%); 6.5 MiB/s, 162 objects/s recovering 2023-09-23T20:57:27.483 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.523 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:27.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:27.632 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:27.657 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:27 smithi121 ceph-mon[211526]: pgmap v848: 89 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 12 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14668/236889 objects degraded (6.192%); 4505/236889 objects misplaced (1.902%); 6.5 MiB/s, 162 objects/s recovering 2023-09-23T20:57:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:27 smithi121 ceph-mon[219709]: pgmap v848: 89 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 12 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 50 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 14668/236889 objects degraded (6.192%); 4505/236889 objects misplaced (1.902%); 6.5 MiB/s, 162 objects/s recovering 2023-09-23T20:57:27.699 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:27.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:27.806 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:28.238 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:28.354 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:28.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:28.457 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:28.484 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:28.793 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:28.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:28.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:28.813 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:57:28.898 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:28.926 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:28.938 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:28.971 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:28.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:28.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:29.076 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:29.107 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:29.325 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:29.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:57:29.376 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:29 smithi121 bash[391572]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T20:57:29.430 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:29.456 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:29.498 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:29.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:29.605 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:29.630 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:29 smithi121 ceph-mon[211526]: pgmap v849: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 6.9 MiB/s, 154 objects/s recovering 2023-09-23T20:57:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:29 smithi121 ceph-mon[219709]: pgmap v849: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 6.9 MiB/s, 154 objects/s recovering 2023-09-23T20:57:29.671 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:29.678 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:29 smithi123 ceph-mon[193903]: pgmap v849: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 6.9 MiB/s, 154 objects/s recovering 2023-09-23T20:57:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:29.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:29.779 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:29.814 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:30 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16183/236889 objects degraded (6.831%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:30 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16183/236889 objects degraded (6.831%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:30 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16183/236889 objects degraded (6.831%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:30.898 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:57:31.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:31 smithi123 ceph-mon[193903]: pgmap v850: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 6.1 MiB/s, 137 objects/s recovering 2023-09-23T20:57:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:31 smithi121 ceph-mon[211526]: pgmap v850: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 6.1 MiB/s, 137 objects/s recovering 2023-09-23T20:57:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:31 smithi121 ceph-mon[219709]: pgmap v850: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 6.1 MiB/s, 137 objects/s recovering 2023-09-23T20:57:33.580 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:33.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:33.684 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:33.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:33 smithi123 ceph-mon[193903]: osdmap e272: 8 total, 7 up, 8 in 2023-09-23T20:57:33.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:33 smithi123 ceph-mon[193903]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:33 smithi123 ceph-mon[193903]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:33 smithi123 ceph-mon[193903]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:33 smithi123 ceph-mon[193903]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:33 smithi123 ceph-mon[193903]: pgmap v852: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 3.0 MiB/s, 86 objects/s recovering 2023-09-23T20:57:33.717 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:33.765 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:33.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:33.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:33.775 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:33.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[211526]: osdmap e272: 8 total, 7 up, 8 in 2023-09-23T20:57:33.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[211526]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[211526]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[211526]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[211526]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[211526]: pgmap v852: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 3.0 MiB/s, 86 objects/s recovering 2023-09-23T20:57:33.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[219709]: osdmap e272: 8 total, 7 up, 8 in 2023-09-23T20:57:33.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[219709]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[219709]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[219709]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[219709]: failed to encode map e272 with expected crc 2023-09-23T20:57:33.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:33 smithi121 ceph-mon[219709]: pgmap v852: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 51 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 2711/236889 objects misplaced (1.144%); 3.0 MiB/s, 86 objects/s recovering 2023-09-23T20:57:33.874 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:33.905 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:33.940 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:34.181 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:34.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:34.285 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:34.318 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.367 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:34.472 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:34.505 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.555 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:34.596 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:57:34.616 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:57:34.660 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:34.690 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:34 smithi123 ceph-mon[193903]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.690 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:34 smithi123 ceph-mon[193903]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.690 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:34 smithi123 ceph-mon[193903]: osdmap e273: 8 total, 7 up, 8 in 2023-09-23T20:57:34.691 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:34 smithi123 ceph-mon[193903]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.691 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.735 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:34.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:34.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[211526]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[211526]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[211526]: osdmap e273: 8 total, 7 up, 8 in 2023-09-23T20:57:34.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[211526]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[219709]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[219709]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[219709]: osdmap e273: 8 total, 7 up, 8 in 2023-09-23T20:57:34.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:34 smithi121 ceph-mon[219709]: failed to encode map e273 with expected crc 2023-09-23T20:57:34.841 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:34.869 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:34.910 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:34.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:35.015 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:35.041 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:35.078 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:57:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:57:34] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T20:57:35.082 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:35.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:35.110 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:57:35.129 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:57:35.148 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:57:35.167 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:57:35.186 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:57:35.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:57:35.189 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:35.223 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:35.698 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:35 smithi121 ceph-mon[211526]: failed to encode map e273 with expected crc 2023-09-23T20:57:35.698 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1969375055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:35.698 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:35 smithi121 ceph-mon[211526]: pgmap v854: 89 pgs: 1 activating+undersized+degraded+remapped, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 1/236889 objects misplaced (0.000%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:57:35.699 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:35 smithi121 ceph-mon[219709]: failed to encode map e273 with expected crc 2023-09-23T20:57:35.699 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1969375055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:35.699 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:35 smithi121 ceph-mon[219709]: pgmap v854: 89 pgs: 1 activating+undersized+degraded+remapped, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 1/236889 objects misplaced (0.000%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:57:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:35 smithi123 ceph-mon[193903]: failed to encode map e273 with expected crc 2023-09-23T20:57:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1969375055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:35 smithi123 ceph-mon[193903]: pgmap v854: 89 pgs: 1 activating+undersized+degraded+remapped, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 1/236889 objects misplaced (0.000%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T20:57:35.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:35.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:57:36.017 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:57:36.051 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:36.077 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:36.119 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:36.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:36.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:36.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:36.250 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:36.291 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:36.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:36.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:36.400 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:36.438 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89926 10.9 10:9376cfc7:::benchmark_data_smithi121_358375_object5181:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992321+0000", 2023-09-23T20:57:36.490 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 43.493544757000002, 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.078511505000002, 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89926 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.491 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992321+0000", 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992321+0000", 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992320+0000", 2023-09-23T20:57:36.492 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992325+0000", 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9529999999999998e-06 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992333+0000", 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7989999999999994e-06 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.493 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992337+0000", 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5650000000000003e-06 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992578+0000", 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000240194 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.494 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992587+0000", 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4830000000000005e-06 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.953580+0000", 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 17.960992713 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.962103+0000", 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085235780000000004 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.495 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.973237+0000", 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011133848999999999 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.014661+0000", 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.041424157000000003 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.496 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.018893+0000", 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042316200000000002 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.020092+0000", 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0011993469999999999 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.024572+0000", 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0044795290000000003 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.497 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.029970+0000", 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053978680000000001 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.070832+0000", 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.040862480999999999 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.498 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89934 10.9 10:99add1d4:::benchmark_data_smithi121_358375_object5189:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 43.493508038999998, 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.125213574, 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89934 2023-09-23T20:57:36.499 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992360+0000", 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.999e-06 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992365+0000", 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4210000000000003e-06 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992369+0000", 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8870000000000002e-06 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992624+0000", 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00025508600000000002 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992633+0000", 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1339999999999993e-06 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.982585+0000", 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 17.989951529999999 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.503 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.991708+0000", 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091229280000000006 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.001314+0000", 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096056409999999998 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.021104+0000", 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019790161000000001 2023-09-23T20:57:36.504 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.076468+0000", 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.055364728000000002 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.081776+0000", 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053078259999999999 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.505 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.086509+0000", 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047326829999999997 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.093298+0000", 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067891399999999999 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.506 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.117571+0000", 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024272848 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89935 10.9 10:99d64049:::benchmark_data_smithi121_358375_object5190:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 43.49348956, 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.185285084, 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.507 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89935 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.508 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.509 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992377+0000", 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.635e-06 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992381+0000", 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2710000000000001e-06 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992383+0000", 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7920000000000001e-06 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.510 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992670+0000", 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00028668500000000001 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992679+0000", 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9819999999999997e-06 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.118245+0000", 2023-09-23T20:57:36.511 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.125566417000002 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.129062+0000", 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010817108000000001 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.138856+0000", 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097938880000000006 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.155528+0000", 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016671136999999999 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.157350+0000", 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018224879999999999 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.513 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.157942+0000", 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00059154200000000002 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.159783+0000", 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018418969999999999 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.162715+0000", 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029311860000000001 2023-09-23T20:57:36.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.177661+0000", 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014946291 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89981 10.9 10:994f28fe:::benchmark_data_smithi121_358375_object5236:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992398+0000", 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 43.493467727000002, 2023-09-23T20:57:36.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.146739247999999, 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89981 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992398+0000", 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.516 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992398+0000", 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992397+0000", 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.517 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992399+0000", 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5099999999999999e-06 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992402+0000", 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1970000000000001e-06 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992405+0000", 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9790000000000002e-06 2023-09-23T20:57:36.518 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992715+0000", 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00030991800000000001 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992724+0000", 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5080000000000005e-06 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.519 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.147118+0000", 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.154393974000001 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.153688+0000", 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065708340000000002 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.203940+0000", 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050252012999999998 2023-09-23T20:57:36.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.210878+0000", 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069378920000000002 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.987311+0000", 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.77643294500000004 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.992743+0000", 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054319370000000004 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.002286+0000", 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095426230000000001 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.089643+0000", 2023-09-23T20:57:36.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087357120999999996 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.115322+0000", 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025679124000000001 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.118245+0000", 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029228069999999999 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.120479+0000", 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002233938 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.124182+0000", 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037032990000000002 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.139137+0000", 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014954857 2023-09-23T20:57:36.524 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90001 10.9 10:90c824f6:::benchmark_data_smithi121_358375_object5256:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992408+0000", 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 43.493457808999999, 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.173996376000002, 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.525 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90001 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992408+0000", 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992408+0000", 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992407+0000", 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992409+0000", 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.449e-06 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992412+0000", 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.853e-06 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992414+0000", 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6070000000000002e-06 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992748+0000", 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000333247 2023-09-23T20:57:36.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992756+0000", 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8699999999999998e-06 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.211516+0000", 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.218759336000002 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.220416+0000", 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089002730000000002 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.264440+0000", 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044023962 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.271591+0000", 2023-09-23T20:57:36.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071505479999999996 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.038619+0000", 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.76702871699999997 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.043853+0000", 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052333670000000001 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:57:36.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.053317+0000", 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094646399999999999 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.115885+0000", 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.062567431000000007 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.141137+0000", 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025252508 2023-09-23T20:57:36.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.143947+0000", 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028093950000000001 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.146421+0000", 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024744300000000001 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.149991+0000", 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035700340000000001 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.166404+0000", 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016412930999999999 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90057 10.9 10:95b13069:::benchmark_data_smithi121_358375_object5312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:54.167609+0000", 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 42.318256884, 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.023931561000001, 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90057 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:54.167609+0000", 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:54.167609+0000", 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:54.167607+0000", 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:54.167616+0000", 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9670000000000007e-06 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:54.167634+0000", 2023-09-23T20:57:36.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8732999999999998e-05 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:54.167645+0000", 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0231000000000001e-05 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.222536+0000", 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 17.054891298000001 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.229231+0000", 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006695393 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.299256+0000", 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.070024410999999995 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.305728+0000", 2023-09-23T20:57:36.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064727919999999998 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.090352+0000", 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.78462378200000005 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.095730+0000", 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053782300000000003 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.105218+0000", 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094874859999999998 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.141979+0000", 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036761418999999997 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.168258+0000", 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026279019000000001 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.170141+0000", 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001882606 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.171984+0000", 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018428979999999999 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.174962+0000", 2023-09-23T20:57:36.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029777440000000001 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.191540+0000", 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016578407 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90301 10.9 10:95826f21:::benchmark_data_smithi121_358375_object5556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:57:36.549 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 23.979811050999999, 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 23.226904614999999, 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90301 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:57:36.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506053+0000", 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506062+0000", 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7669999999999998e-06 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506077+0000", 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4562999999999999e-05 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506088+0000", 2023-09-23T20:57:36.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0779e-05 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.665171+0000", 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15908349199999999 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.672174+0000", 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070033109999999999 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.759608+0000", 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087433832000000003 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.770158+0000", 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010549433 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998499+0000", 2023-09-23T20:57:36.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2283415890000002 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998505+0000", 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6810000000000003e-06 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892231+0000", 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372554500000001 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892240+0000", 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4369999999999996e-06 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.600926+0000", 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.708686151999999 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.608105+0000", 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071792979999999998 2023-09-23T20:57:36.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.617691+0000", 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095853580000000004 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.654316+0000", 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036625188000000003 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.709183+0000", 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.054866572000000002 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.712311+0000", 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031283980000000001 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.714254+0000", 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001943005 2023-09-23T20:57:36.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.717912+0000", 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036581460000000001 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.732959+0000", 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015046838 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90304 10.9 10:95d08781:::benchmark_data_smithi121_358375_object5559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 23.485076203999999, 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.706967114000001, 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90304 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.560 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000787+0000", 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000797+0000", 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6229999999999992e-06 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000815+0000", 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8391999999999999e-05 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000827+0000", 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1934e-05 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.076647+0000", 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.075820121000000004 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083118+0000", 2023-09-23T20:57:36.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064707690000000003 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.125639+0000", 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.042520998999999997 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.133609+0000", 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079701110000000002 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998466+0000", 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8648562869999998 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998472+0000", 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8070000000000002e-06 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892193+0000", 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372040100000005 2023-09-23T20:57:36.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892202+0000", 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4790000000000002e-06 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.565782+0000", 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.673579972999999 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.572988+0000", 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072056059999999998 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.582601+0000", 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096135609999999996 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.596634+0000", 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014032559999999999 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.598447+0000", 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001812924 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.600285+0000", 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018377759999999999 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.676784+0000", 2023-09-23T20:57:36.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.076499811000000001 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.680024+0000", 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032395729999999999 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.707756+0000", 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027732273000000002 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.569 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90319 10.9 10:9c23793e:::benchmark_data_smithi121_358375_object5574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 23.413641427999998, 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.227369583000002, 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.570 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90319 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072222+0000", 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072231+0000", 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2539999999999992e-06 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072248+0000", 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7422e-05 2023-09-23T20:57:36.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072260+0000", 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2139e-05 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083953+0000", 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011692759 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.090352+0000", 2023-09-23T20:57:36.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063994910000000002 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.175493+0000", 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.085140748000000002 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.187526+0000", 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01203284 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998112+0000", 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8105862510000001 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998120+0000", 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.418e-06 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891970+0000", 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89384994900000003 2023-09-23T20:57:36.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891979+0000", 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6320000000000003e-06 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.208032+0000", 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.316052247000002 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.217965+0000", 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.009933753 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.229822+0000", 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011856366 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.248481+0000", 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018659464000000001 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.278359+0000", 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.029878236999999998 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.280279+0000", 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019199919999999999 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.282082+0000", 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001802367 2023-09-23T20:57:36.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.285240+0000", 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031586299999999999 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.299594+0000", 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014353223999999999 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90350 10.9 10:9252fbfb:::benchmark_data_smithi121_358375_object5605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.803397471, 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.226020969, 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90350 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682466+0000", 2023-09-23T20:57:36.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682473+0000", 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7800000000000003e-06 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682487+0000", 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.402e-05 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682496+0000", 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7860000000000002e-06 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.827509+0000", 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.145013317 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.834217+0000", 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067071600000000002 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.884874+0000", 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050657455999999997 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.902020+0000", 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017146446999999999 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998607+0000", 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.096586142 2023-09-23T20:57:36.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998613+0000", 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7830000000000001e-06 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892350+0000", 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373614700000004 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892359+0000", 2023-09-23T20:57:36.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3430000000000002e-06 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.797647+0000", 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.905288455000001 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.805039+0000", 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0073917929999999998 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.814794+0000", 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097547199999999997 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.846753+0000", 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.031959301000000002 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.848628+0000", 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001875172 2023-09-23T20:57:36.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.850451+0000", 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018229469999999999 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.887256+0000", 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036804591999999997 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.890406+0000", 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003149982 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.908489+0000", 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018083054000000001 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90379 10.9 10:951c4b03:::benchmark_data_smithi121_358375_object5634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:57:36.589 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.556959721999998, 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.839626358, 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90379 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928904+0000", 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928913+0000", 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928927+0000", 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4329e-05 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928935+0000", 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9729999999999997e-06 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.954600+0000", 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025665337 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.964941+0000", 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010340223000000001 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.088603+0000", 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12366287400000001 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.095790+0000", 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071866090000000001 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998534+0000", 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.90274379599999999 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998541+0000", 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1199999999999996e-06 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892274+0000", 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373288500000003 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892284+0000", 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9399999999999997e-06 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.625055+0000", 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.732770945999999 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.633921+0000", 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088660279999999998 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.644084+0000", 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010163669 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.733850+0000", 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089765882000000005 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.736155+0000", 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023041950000000002 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.738094+0000", 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001939276 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.740271+0000", 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021772879999999999 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.744526+0000", 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042552850000000001 2023-09-23T20:57:36.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.768532+0000", 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024005678999999999 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90424 10.9 10:980e8aaf:::benchmark_data_smithi121_358375_object5679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.504322742999999, 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.531567125999999, 2023-09-23T20:57:36.599 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90424 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981541+0000", 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981548+0000", 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1489999999999997e-06 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981561+0000", 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2448e-05 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981569+0000", 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1499999999999999e-06 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.224643+0000", 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243074701 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.237465+0000", 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012821737 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.280151+0000", 2023-09-23T20:57:36.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.04268632 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.286562+0000", 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064109060000000001 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998360+0000", 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71179806400000001 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998368+0000", 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5050000000000001e-06 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892082+0000", 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89371398099999999 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892091+0000", 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1409999999999994e-06 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.424130+0000", 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.532038986 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.435357+0000", 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011226817 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.448022+0000", 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01266514 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.486130+0000", 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038108329000000003 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.487944+0000", 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018138589999999999 2023-09-23T20:57:36.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.489799+0000", 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001854332 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.492508+0000", 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027092589999999999 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.496591+0000", 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040830900000000002 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.513110+0000", 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016518877000000001 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90429 10.9 10:9b165b02:::benchmark_data_smithi121_358375_object5684:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.609 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.497226187999999, 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.409704737999999, 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90429 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988638+0000", 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988645+0000", 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2840000000000001e-06 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988661+0000", 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6053999999999999e-05 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988669+0000", 2023-09-23T20:57:36.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9419999999999995e-06 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.238554+0000", 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24988542499999999 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.246423+0000", 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0078688479999999995 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.311852+0000", 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.065429004999999998 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.318196+0000", 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063438970000000003 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998162+0000", 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.67996552200000004 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998172+0000", 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0208e-05 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892007+0000", 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89383517700000004 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892016+0000", 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4290000000000007e-06 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.301575+0000", 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.409558344000001 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.309095+0000", 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075199719999999998 2023-09-23T20:57:36.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.319256+0000", 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010161106 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.376130+0000", 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.056873817 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.378120+0000", 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019904839999999998 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.378762+0000", 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00064167 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.381391+0000", 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026288940000000001 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.384518+0000", 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031269750000000002 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.398344+0000", 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013826188 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90434 10.9 10:90493f14:::benchmark_data_smithi121_358375_object5689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.491829809999999, 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.989979289000001, 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90434 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994034+0000", 2023-09-23T20:57:36.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994041+0000", 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4219999999999997e-06 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994052+0000", 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994059+0000", 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4060000000000002e-06 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.247296+0000", 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25323742700000001 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.254030+0000", 2023-09-23T20:57:36.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067339619999999996 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.341839+0000", 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087808943 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.348061+0000", 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006222216 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998641+0000", 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65057928700000001 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998645+0000", 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8470000000000003e-06 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892386+0000", 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89374099500000004 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892396+0000", 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5119999999999997e-06 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.851132+0000", 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.958735591 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.858452+0000", 2023-09-23T20:57:36.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0073205420000000002 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.868309+0000", 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098567020000000002 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.884649+0000", 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016340086 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.886658+0000", 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020087680000000002 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.913375+0000", 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026717060000000001 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.962703+0000", 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.049328300999999998 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.965943+0000", 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003240368 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.984015+0000", 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018071474000000001 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90436 10.9 10:925ee57b:::benchmark_data_smithi121_358375_object5691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.490174030999999, 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.84919506, 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90436 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995690+0000", 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995697+0000", 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5919999999999997e-06 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995714+0000", 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7144999999999999e-05 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995724+0000", 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0030000000000001e-05 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.350440+0000", 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.354715958 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.356739+0000", 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062994560000000002 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.388748+0000", 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032009263000000003 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.395007+0000", 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006258726 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998567+0000", 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.60355961700000005 2023-09-23T20:57:36.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998573+0000", 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8619999999999999e-06 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892312+0000", 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373954700000002 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892321+0000", 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9789999999999999e-06 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.769494+0000", 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.877172975000001 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.779991+0000", 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010496959 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.790177+0000", 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010185801 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.822235+0000", 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032058013000000003 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.824316+0000", 2023-09-23T20:57:36.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002081252 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.826167+0000", 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001850862 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.828002+0000", 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018350759999999999 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.831036+0000", 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003033446 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.844886+0000", 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013850843999999999 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.639 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90448 10.9 10:9dd18145:::benchmark_data_smithi121_358375_object5703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.458366236, 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.248909286, 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90448 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027497+0000", 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027505+0000", 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0879999999999997e-06 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027521+0000", 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6795000000000001e-05 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027531+0000", 2023-09-23T20:57:36.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3649999999999993e-06 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.359164+0000", 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.331633185 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.365457+0000", 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006293267 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.418540+0000", 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.053082826999999999 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.429706+0000", 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011166376 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998075+0000", 2023-09-23T20:57:36.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.56836851300000002 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998082+0000", 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5100000000000001e-06 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891929+0000", 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89384610799999997 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891940+0000", 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1442e-05 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.172182+0000", 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.280241611000001 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.179816+0000", 2023-09-23T20:57:36.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076342859999999997 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.193670+0000", 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013853811000000001 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.246965+0000", 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.05329536 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.247710+0000", 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00074513800000000001 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.250763+0000", 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030532549999999999 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.252934+0000", 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002170866 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.256527+0000", 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035923309999999998 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.276408+0000", 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019881827000000001 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90471 10.9 10:9fd1a800:::benchmark_data_smithi121_358375_object5726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.379466021999999, 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.062235608000002, 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90471 2023-09-23T20:57:36.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106398+0000", 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106407+0000", 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9089999999999995e-06 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106423+0000", 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5962999999999999e-05 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106433+0000", 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1589999999999999e-06 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.431330+0000", 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32489747699999999 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.444379+0000", 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013049432999999999 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.479473+0000", 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.035093725999999999 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.485924+0000", 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064507230000000002 2023-09-23T20:57:36.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997976+0000", 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.51205186199999997 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997987+0000", 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1669e-05 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891608+0000", 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89362018099999996 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891623+0000", 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5492e-05 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.072757+0000", 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.181133587000001 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for readable", 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.087146+0000", 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014389206 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.107028+0000", 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019882529999999999 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.114642+0000", 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076134410000000003 2023-09-23T20:57:36.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.124398+0000", 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097560830000000005 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.146747+0000", 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022348922 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.148740+0000", 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019930780000000001 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.149391+0000", 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00065059799999999995 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.151298+0000", 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019071190000000001 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.154367+0000", 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030688310000000002 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.168635+0000", 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014268471 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90489 10.9 10:97f30eb8:::benchmark_data_smithi121_358375_object5744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.275112161999999, 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.329841758000001, 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90489 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210752+0000", 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210761+0000", 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9970000000000004e-06 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210778+0000", 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7181999999999999e-05 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210789+0000", 2023-09-23T20:57:36.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0818e-05 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.446681+0000", 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23589284299999999 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.453972+0000", 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072908900000000004 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.511615+0000", 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057642634999999998 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.518011+0000", 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006395815 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998399+0000", 2023-09-23T20:57:36.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.480388125 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998406+0000", 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4789999999999999e-06 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892119+0000", 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.893712795 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892129+0000", 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3829999999999998e-06 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.457483+0000", 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.565354894999999 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.466620+0000", 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091370259999999995 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.476805+0000", 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010184886000000001 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.491775+0000", 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014969623 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.513722+0000", 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.021947326999999999 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.516436+0000", 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027137620000000002 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.519247+0000", 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028114009999999998 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.523691+0000", 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0044435380000000003 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.540595+0000", 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016903975000000002 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90505 10.9 10:99aea1ee:::benchmark_data_smithi121_358375_object5760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 22.217469271999999, 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.154937368999999, 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90505 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.671 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268395+0000", 2023-09-23T20:57:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268402+0000", 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2389999999999996e-06 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268415+0000", 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2850999999999999e-05 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268422+0000", 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.644e-06 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.520421+0000", 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.251998897 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.526858+0000", 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064371389999999997 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.557595+0000", 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.030736329 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.563877+0000", 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006281989 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998305+0000", 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43442851399999999 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998317+0000", 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1959e-05 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892044+0000", 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372699 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892054+0000", 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3389999999999999e-06 2023-09-23T20:57:36.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.328034+0000", 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.435980598 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.336888+0000", 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088543010000000002 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.347140+0000", 2023-09-23T20:57:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010251873999999999 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.379404+0000", 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032263781999999998 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.400240+0000", 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.020836174999999998 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.402072+0000", 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018316369999999999 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.403897+0000", 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001824579 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.406955+0000", 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030584589999999999 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.423333+0000", 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016378508 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90514 10.9 10:96e73e59:::benchmark_data_smithi121_358375_object5769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 21.824306178000001, 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.014633163999999, 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:57:36.680 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90514 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:57:36.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661558+0000", 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661567+0000", 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.183e-06 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661583+0000", 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6585000000000001e-05 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661593+0000", 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6190000000000006e-06 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.825075+0000", 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16348148200000001 2023-09-23T20:57:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.835611+0000", 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010536495 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.936853+0000", 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10124217200000001 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.943598+0000", 2023-09-23T20:57:36.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067444640000000004 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998433+0000", 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.054835452 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998439+0000", 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6470000000000003e-06 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892156+0000", 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89371742200000004 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892165+0000", 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1449999999999997e-06 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.541432+0000", 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.649266821000001 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.548683+0000", 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072504270000000003 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.558326+0000", 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096435710000000001 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.591157+0000", 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032831143 2023-09-23T20:57:36.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.593006+0000", 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001848948 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.594819+0000", 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001812501 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.654981+0000", 2023-09-23T20:57:36.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.060162001999999999 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.658403+0000", 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034224960000000001 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.676192+0000", 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017789158999999999 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:57:36.690 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:57:36.690 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:57:36.771 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:36 smithi121 ceph-osd[393770]: -- 172.21.15.121:0/3499634525 <== mon.2 v2:172.21.15.121:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5624e82b64e0 con 0x5624e8289400 2023-09-23T20:57:36.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:57:37.163 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:36 smithi121 bash[393576]: 1baf61c043a7b1fbcf0037176efa6395864fc54098a8b1e0bcac4b5ce7a0b0f0 2023-09-23T20:57:37.163 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:36 smithi121 systemd[1]: Started Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T20:57:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:37 smithi123 ceph-mon[193903]: pgmap v855: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 1/236889 objects misplaced (0.000%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:57:37.503 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:37.511 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:37 smithi121 conmon[393696]: 2023-09-23T20:57:37.308+0000 7f378e0a0640 -1 Falling back to public interface 2023-09-23T20:57:37.511 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:37 smithi121 ceph-mon[219709]: pgmap v855: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 1/236889 objects misplaced (0.000%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:57:37.512 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:37 smithi121 ceph-mon[211526]: pgmap v855: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 16183/236889 objects degraded (6.831%); 1/236889 objects misplaced (0.000%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:57:37.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:37.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:37.614 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:37.639 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:37.675 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:57:37.681 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:37.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:37.743 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:37.787 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:37.812 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:37.858 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:37.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:37.963 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:37.992 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:38.018 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:57:38.031 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:38.041 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:38.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:38.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:38.148 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:38.208 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:38.539 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:57:38.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/669174844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:38.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:38 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:38.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/669174844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:38.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:38 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:38.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/669174844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:38.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:38 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:57:39.136 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:39.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:39.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:39.240 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:39.269 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:39.315 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:39.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:39.423 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:39.464 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:39.597 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:39.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:39.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:39.698 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:39.704 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:39.734 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:39.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:39.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: osdmap e274: 8 total, 7 up, 8 in 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[211526]: pgmap v857: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 16082/235899 objects degraded (6.817%); 1/235899 objects misplaced (0.000%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: osdmap e274: 8 total, 7 up, 8 in 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:39 smithi121 ceph-mon[219709]: pgmap v857: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 16082/235899 objects degraded (6.817%); 1/235899 objects misplaced (0.000%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:57:39.783 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: osdmap e274: 8 total, 7 up, 8 in 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: failed to encode map e274 with expected crc 2023-09-23T20:57:39.785 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:39 smithi123 ceph-mon[193903]: pgmap v857: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 15 active+clean+remapped, 5 active+undersized+degraded, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 16082/235899 objects degraded (6.817%); 1/235899 objects misplaced (0.000%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T20:57:39.786 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:57:39.786 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:57:39.786 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T20:57:39.786 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:57:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:39.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:39.887 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:39.915 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.007 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:40.017 INFO:teuthology.orchestra.run.smithi121.stderr:no valid command found; 10 closest matches: 2023-09-23T20:57:40.017 INFO:teuthology.orchestra.run.smithi121.stderr:0 2023-09-23T20:57:40.017 INFO:teuthology.orchestra.run.smithi121.stderr:1 2023-09-23T20:57:40.017 INFO:teuthology.orchestra.run.smithi121.stderr:2 2023-09-23T20:57:40.017 INFO:teuthology.orchestra.run.smithi121.stderr:abort 2023-09-23T20:57:40.018 INFO:teuthology.orchestra.run.smithi121.stderr:assert 2023-09-23T20:57:40.018 INFO:teuthology.orchestra.run.smithi121.stderr:bluefs debug_inject_read_zeros 2023-09-23T20:57:40.018 INFO:teuthology.orchestra.run.smithi121.stderr:bluefs files list 2023-09-23T20:57:40.018 INFO:teuthology.orchestra.run.smithi121.stderr:bluefs stats 2023-09-23T20:57:40.018 INFO:teuthology.orchestra.run.smithi121.stderr:bluestore bluefs device info [] 2023-09-23T20:57:40.018 INFO:teuthology.orchestra.run.smithi121.stderr:config diff 2023-09-23T20:57:40.018 INFO:teuthology.orchestra.run.smithi121.stderr:admin_socket: invalid command 2023-09-23T20:57:40.110 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:40.138 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.181 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:40.284 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:40.310 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.457 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:40.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:57:40.560 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:40.587 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.639 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:40.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:40 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16082/235899 objects degraded (6.817%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:40.746 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:40.774 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:40.818 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:40.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:40.869 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:40 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16082/235899 objects degraded (6.817%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:40.869 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:40 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16082/235899 objects degraded (6.817%), 6 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:57:40.894 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-09-23T20:57:40.922 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:40.948 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:40.962 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:41.071 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:41.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:41.175 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:41.205 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:41.258 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:41.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:41.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:41.361 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:41.392 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:41.538 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:41.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:41.547 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:41 smithi121 conmon[393696]: 2023-09-23T20:57:41.267+0000 7f378e0a0640 -1 osd.1 228 log_to_monitors true 2023-09-23T20:57:41.608 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:57:41.642 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:41.668 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:41.710 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:41.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:41.813 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:41.838 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:41 smithi123 ceph-mon[193903]: pgmap v858: 89 pgs: 15 active+clean+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 14567/235371 objects degraded (6.189%); 848/235371 objects misplaced (0.360%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:57:41.838 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:41 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:57:41.838 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:41 smithi123 ceph-mon[193903]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:57:41.840 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:41.877 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:41 smithi121 ceph-mon[211526]: pgmap v858: 89 pgs: 15 active+clean+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 14567/235371 objects degraded (6.189%); 848/235371 objects misplaced (0.360%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:57:41.877 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:41 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:57:41.877 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:41 smithi121 ceph-mon[211526]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:57:41.877 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:41 smithi121 ceph-mon[219709]: pgmap v858: 89 pgs: 15 active+clean+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 14567/235371 objects degraded (6.189%); 848/235371 objects misplaced (0.360%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:57:41.878 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:41 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:57:41.878 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:41 smithi121 ceph-mon[219709]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:57:42.053 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:42.146 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:42.158 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:42.185 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.203 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:57:42.223 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:57:42.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:42.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:42.257 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:57:42.284 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:57:42.310 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:57:42.328 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:57:42.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:42.355 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:57:42.355 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:57:42.355 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:57:42.355 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:57:42.355 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:57:42.356 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:57:42.356 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:57:42.356 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:57:42.356 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:57:42.365 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.425 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:42.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:42.535 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:42.548 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 20:57:42 smithi121 conmon[393696]: 2023-09-23T20:57:42.286+0000 7f377ed0c700 -1 osd.1 275 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:57:42.550 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/319670928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:42.550 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:57:42.550 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: osdmap e275: 8 total, 7 up, 8 in 2023-09-23T20:57:42.550 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: from='osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:57:42.550 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:57:42.550 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[211526]: failed to encode map e275 with expected crc 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/319670928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: osdmap e275: 8 total, 7 up, 8 in 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: from='osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:42.551 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:42 smithi121 ceph-mon[219709]: failed to encode map e275 with expected crc 2023-09-23T20:57:42.563 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.569 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:42.569 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:42.611 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:42.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:42.716 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:42.742 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/319670928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: osdmap e275: 8 total, 7 up, 8 in 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: from='osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:42.823 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:42 smithi123 ceph-mon[193903]: failed to encode map e275 with expected crc 2023-09-23T20:57:42.887 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:42.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:42.990 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:43.016 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:43.058 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:43.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:43.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:43.161 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:43.193 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:43.395 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:57:43.397 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:43.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:43.500 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:43.528 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: pgmap v860: 89 pgs: 15 active+clean+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 14567/235371 objects degraded (6.189%); 848/235371 objects misplaced (0.360%); 224 KiB/s, 3 objects/s recovering 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: failed to encode map e276 with expected crc 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964] boot 2023-09-23T20:57:43.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[211526]: osdmap e276: 8 total, 8 up, 8 in 2023-09-23T20:57:43.576 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:43.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:43.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:43.684 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:43.711 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: pgmap v860: 89 pgs: 15 active+clean+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 14567/235371 objects degraded (6.189%); 848/235371 objects misplaced (0.360%); 224 KiB/s, 3 objects/s recovering 2023-09-23T20:57:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:57:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:57:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: failed to encode map e276 with expected crc 2023-09-23T20:57:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964] boot 2023-09-23T20:57:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:43 smithi123 ceph-mon[193903]: osdmap e276: 8 total, 8 up, 8 in 2023-09-23T20:57:43.757 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:43.843 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:43.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:43.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: failed to encode map e275 with expected crc 2023-09-23T20:57:43.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: pgmap v860: 89 pgs: 15 active+clean+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 52 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 14567/235371 objects degraded (6.189%); 848/235371 objects misplaced (0.360%); 224 KiB/s, 3 objects/s recovering 2023-09-23T20:57:43.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:57:43.849 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:57:43.849 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: failed to encode map e276 with expected crc 2023-09-23T20:57:43.849 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: osd.1 [v2:172.21.15.121:6810/3255216964,v1:172.21.15.121:6811/3255216964] boot 2023-09-23T20:57:43.849 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:43 smithi121 ceph-mon[219709]: osdmap e276: 8 total, 8 up, 8 in 2023-09-23T20:57:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:57:43.910 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:57:43.949 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:43.977 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:44.024 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:44.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:44.127 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:44.155 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:44.352 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:44.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:44.455 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:44.482 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:44.525 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:44.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:44.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:44.628 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:44.651 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:44 smithi123 ceph-mon[193903]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.652 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:44 smithi123 ceph-mon[193903]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.652 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:44 smithi123 ceph-mon[193903]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.652 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:44 smithi123 ceph-mon[193903]: osdmap e277: 8 total, 8 up, 8 in 2023-09-23T20:57:44.652 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:44 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.652 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:44 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.652 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:44 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.654 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:44.903 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:44.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:44.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:44.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:57:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:57:44] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[211526]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[211526]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[211526]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[211526]: osdmap e277: 8 total, 8 up, 8 in 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[219709]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[219709]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[219709]: failed to encode map e276 with expected crc 2023-09-23T20:57:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[219709]: osdmap e277: 8 total, 8 up, 8 in 2023-09-23T20:57:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:44.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:44 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.008 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:45.036 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:45.133 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:45.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:45.237 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:45.251 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:57:45.274 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:45.329 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:45.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:45.374 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:45.374 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:45.432 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:45.468 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:45.668 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:45.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:45.775 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:45.810 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:45.859 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:45.861 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/364951827' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: pgmap v863: 89 pgs: 9 peering, 12 remapped+peering, 4 active+clean+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 51 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 4863/233562 objects degraded (2.082%); 847/233562 objects misplaced (0.363%); 299 KiB/s, 4 objects/s recovering 2023-09-23T20:57:45.862 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:45.863 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:45.863 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:45.863 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:45 smithi123 ceph-mon[193903]: osdmap e278: 8 total, 8 up, 8 in 2023-09-23T20:57:45.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:45.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:45.895 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:57:45.911 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:57:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/364951827' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: pgmap v863: 89 pgs: 9 peering, 12 remapped+peering, 4 active+clean+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 51 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 4863/233562 objects degraded (2.082%); 847/233562 objects misplaced (0.363%); 299 KiB/s, 4 objects/s recovering 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:45.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[211526]: osdmap e278: 8 total, 8 up, 8 in 2023-09-23T20:57:45.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e277 with expected crc 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/364951827' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: pgmap v863: 89 pgs: 9 peering, 12 remapped+peering, 4 active+clean+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 51 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 4863/233562 objects degraded (2.082%); 847/233562 objects misplaced (0.363%); 299 KiB/s, 4 objects/s recovering 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:57:45.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:45 smithi121 ceph-mon[219709]: osdmap e278: 8 total, 8 up, 8 in 2023-09-23T20:57:45.964 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:46.039 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.160 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:57:46.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:46.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:46.285 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:46.311 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.354 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:46.454 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:46.460 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:46.487 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.510 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:46.532 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:46.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:46.572 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:46.637 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:46.663 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.750 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.751 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4863/233562 objects degraded (2.082%), 2 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:46.756 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:46.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:46.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:46.861 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:46.866 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.866 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.866 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.866 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4863/233562 objects degraded (2.082%), 2 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: failed to encode map e278 with expected crc 2023-09-23T20:57:46.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4863/233562 objects degraded (2.082%), 2 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:57:46.899 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:46.999 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:47.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:47.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:47.108 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:47.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:47 smithi123 ceph-mon[193903]: pgmap v865: 89 pgs: 9 peering, 12 remapped+peering, 1 active+clean+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 58 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 4863/233433 objects degraded (2.083%); 847/233433 objects misplaced (0.363%); 655 KiB/s, 10 objects/s recovering 2023-09-23T20:57:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:47 smithi121 ceph-mon[211526]: pgmap v865: 89 pgs: 9 peering, 12 remapped+peering, 1 active+clean+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 58 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 4863/233433 objects degraded (2.083%); 847/233433 objects misplaced (0.363%); 655 KiB/s, 10 objects/s recovering 2023-09-23T20:57:47.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:47 smithi121 ceph-mon[219709]: pgmap v865: 89 pgs: 9 peering, 12 remapped+peering, 1 active+clean+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 58 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 4863/233433 objects degraded (2.083%); 847/233433 objects misplaced (0.363%); 655 KiB/s, 10 objects/s recovering 2023-09-23T20:57:48.100 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:48.100 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:48.268 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:57:48.645 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:48.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:48.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:48.779 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:48.828 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:57:48.987 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:48.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:48.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:49.093 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:49.127 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:49.193 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:49.484 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:49.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:49.588 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:49.634 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:49.683 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:49.702 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:49.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:49.815 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:49.844 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:49.891 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:49.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:49.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:50.022 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:50.113 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:49 smithi121 ceph-mon[219709]: pgmap v866: 89 pgs: 2 peering, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 15345/232317 objects degraded (6.605%); 847/232317 objects misplaced (0.365%); 501 KiB/s, 7 objects/s recovering 2023-09-23T20:57:50.113 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:49 smithi121 ceph-mon[211526]: pgmap v866: 89 pgs: 2 peering, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 15345/232317 objects degraded (6.605%); 847/232317 objects misplaced (0.365%); 501 KiB/s, 7 objects/s recovering 2023-09-23T20:57:50.196 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:57:50.197 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:57:50.197 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T20:57:50.197 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:57:50.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:49 smithi123 ceph-mon[193903]: pgmap v866: 89 pgs: 2 peering, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 15345/232317 objects degraded (6.605%); 847/232317 objects misplaced (0.365%); 501 KiB/s, 7 objects/s recovering 2023-09-23T20:57:50.262 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:50.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:50.365 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:50.393 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:50.486 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:50.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:50.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:50.590 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:57:50.626 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:50.789 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:57:50.789 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:57:51.121 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:50 smithi121 ceph-mon[211526]: pgmap v867: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 15102/231588 objects degraded (6.521%); 847/231588 objects misplaced (0.366%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:57:51.121 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:50 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15102/231588 objects degraded (6.521%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:51.121 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4101766085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:51.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:50 smithi121 ceph-mon[219709]: pgmap v867: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 15102/231588 objects degraded (6.521%); 847/231588 objects misplaced (0.366%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:57:51.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:50 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15102/231588 objects degraded (6.521%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:51.122 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4101766085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:51.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:50 smithi123 ceph-mon[193903]: pgmap v867: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 15102/231588 objects degraded (6.521%); 847/231588 objects misplaced (0.366%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:57:51.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:50 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15102/231588 objects degraded (6.521%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:51.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4101766085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:57:51.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:57:51.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:57:51.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:57:51.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:57:51.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.252 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:57:51.290 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:57:51.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:51.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:57:51.394 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:51.427 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:51.527 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:57:51.875 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:52.840 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:52.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:52.948 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:52.980 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186212 8.8 8:10804386:::benchmark_data_smithi121_252781_object77037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:57:53.489 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 586.99101352399998, 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1681942240000001, 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186212 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495490+0000", 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495489+0000", 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495493+0000", 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1779999999999999e-06 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495500+0000", 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7600000000000002e-06 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.495504+0000", 2023-09-23T20:57:53.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9720000000000003e-06 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500662+0000", 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051571189999999999 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.500891+0000", 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022905000000000001 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 7", 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.501010+0000", 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119268 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509849+0000", 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088391219999999996 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.509859+0000", 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0032e-05 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513758+0000", 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0038989819999999 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513773+0000", 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5116999999999999e-05 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602445+0000", 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0886714449999999 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602553+0000", 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010843199999999999 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663605+0000", 2023-09-23T20:57:53.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.061052150999999999 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663684+0000", 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9040000000000002e-05 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186226 8.8 8:109cf42b:::benchmark_data_smithi121_252781_object77051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:57:53.497 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 586.95688107399997, 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0872477360000001, 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186226 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529622+0000", 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529620+0000", 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529629+0000", 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8680000000000005e-06 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529649+0000", 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9874e-05 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529657+0000", 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6680000000000001e-06 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529724+0000", 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7138000000000007e-05 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.529737+0000", 2023-09-23T20:57:53.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3083e-05 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513799+0000", 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98406165899999998 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513808+0000", 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8489999999999995e-06 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602617+0000", 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0888096490000001 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602726+0000", 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010896 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:57:53.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.602846+0000", 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011963099999999999 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616554+0000", 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013708016 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616622+0000", 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7794999999999997e-05 2023-09-23T20:57:53.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.505 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.505 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616656+0000", 2023-09-23T20:57:53.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4397999999999998e-05 2023-09-23T20:57:53.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616718+0000", 2023-09-23T20:57:53.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1920000000000003e-05 2023-09-23T20:57:53.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616870+0000", 2023-09-23T20:57:53.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015218 2023-09-23T20:57:53.509 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.509 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.509 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.509 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186232 8.8 8:18ce4fcc:::benchmark_data_smithi121_252781_object77057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T20:57:53.510 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:57:53.510 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 586.95358455400003, 2023-09-23T20:57:53.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1309886809999998, 2023-09-23T20:57:53.510 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.510 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.510 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186232 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532919+0000", 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532918+0000", 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532923+0000", 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7319999999999998e-06 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532936+0000", 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.293e-05 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532942+0000", 2023-09-23T20:57:53.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6980000000000003e-06 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532993+0000", 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1447999999999999e-05 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:06.532999+0000", 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3389999999999998e-06 2023-09-23T20:57:53.514 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513832+0000", 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98083224099999999 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.513840+0000", 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1070000000000005e-06 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607582+0000", 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0937428 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.607747+0000", 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016451500000000001 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.608823+0000", 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001075559 2023-09-23T20:57:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616564+0000", 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077410530000000003 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.616924+0000", 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00036078199999999998 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663714+0000", 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.046789848000000002 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663776+0000", 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2114999999999997e-05 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663908+0000", 2023-09-23T20:57:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013124799999999999 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186215 8.0 8:00f6bf9b:::benchmark_data_smithi121_252781_object77040:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 585.97222665699996, 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.162298345, 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.519 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186215 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.520 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514277+0000", 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514275+0000", 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514284+0000", 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9570000000000008e-06 2023-09-23T20:57:53.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514297+0000", 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2663000000000001e-05 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514305+0000", 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2970000000000004e-06 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514439+0000", 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013414199999999999 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514449+0000", 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3139999999999993e-06 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649698+0000", 2023-09-23T20:57:53.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1352495979999999 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649809+0000", 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110911 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676465+0000", 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026655729999999999 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676575+0000", 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110305 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186222 8.0 8:0eef9874:::benchmark_data_smithi121_252781_object77047:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 585.97216906100005, 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1614129209999999, 2023-09-23T20:57:53.525 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186222 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514334+0000", 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514339+0000", 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1680000000000003e-06 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514348+0000", 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0140000000000004e-06 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514353+0000", 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:57:53.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514474+0000", 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121105 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514482+0000", 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4039999999999998e-06 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649859+0000", 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.135377012 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.649954+0000", 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4914999999999995e-05 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.650033+0000", 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9120000000000001e-05 2023-09-23T20:57:53.530 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663245+0000", 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013211926000000001 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674215+0000", 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010970126 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.531 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675506+0000", 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001290914 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675559+0000", 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2643999999999999e-05 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675747+0000", 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000188815 2023-09-23T20:57:53.532 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186230 8.0 8:041ac878:::benchmark_data_smithi121_252781_object77055:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 585.972143684, 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1616748569999999, 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.533 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186230 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514360+0000", 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.534 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514359+0000", 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514363+0000", 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8360000000000002e-06 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514370+0000", 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4899999999999997e-06 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514374+0000", 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6750000000000001e-06 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514504+0000", 2023-09-23T20:57:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012982599999999999 2023-09-23T20:57:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514512+0000", 2023-09-23T20:57:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5959999999999998e-06 2023-09-23T20:57:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654104+0000", 2023-09-23T20:57:53.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.139592663 2023-09-23T20:57:53.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654250+0000", 2023-09-23T20:57:53.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014595400000000001 2023-09-23T20:57:53.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:57:53.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.654394+0000", 2023-09-23T20:57:53.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143442 2023-09-23T20:57:53.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.540 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.540 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.540 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663683+0000", 2023-09-23T20:57:53.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092891689999999999 2023-09-23T20:57:53.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.541 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674249+0000", 2023-09-23T20:57:53.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010566002 2023-09-23T20:57:53.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675814+0000", 2023-09-23T20:57:53.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015649539999999999 2023-09-23T20:57:53.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.675875+0000", 2023-09-23T20:57:53.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1224999999999997e-05 2023-09-23T20:57:53.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.676035+0000", 2023-09-23T20:57:53.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015948000000000001 2023-09-23T20:57:53.543 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.544 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.544 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.544 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:186231 8.0 8:0bcd4c07:::benchmark_data_smithi121_252781_object77056:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T20:57:53.544 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:57:53.544 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 585.97212304899995, 2023-09-23T20:57:53.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.168650363, 2023-09-23T20:57:53.545 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.545 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.545 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.545 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.545 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.545 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 186231 2023-09-23T20:57:53.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.546 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.546 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:57:53.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:57:53.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514380+0000", 2023-09-23T20:57:53.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514383+0000", 2023-09-23T20:57:53.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:57:53.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514388+0000", 2023-09-23T20:57:53.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5969999999999999e-06 2023-09-23T20:57:53.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.550 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514393+0000", 2023-09-23T20:57:53.550 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.16e-06 2023-09-23T20:57:53.550 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514533+0000", 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140841 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:07.514540+0000", 2023-09-23T20:57:53.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9890000000000003e-06 2023-09-23T20:57:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659037+0000", 2023-09-23T20:57:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.144496156 2023-09-23T20:57:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659183+0000", 2023-09-23T20:57:53.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014606600000000001 2023-09-23T20:57:53.553 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:57:53.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.659334+0000", 2023-09-23T20:57:53.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015159700000000001 2023-09-23T20:57:53.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.554 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.554 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.663874+0000", 2023-09-23T20:57:53.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045394600000000004 2023-09-23T20:57:53.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.674301+0000", 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010427439 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682815+0000", 2023-09-23T20:57:53.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085137149999999998 2023-09-23T20:57:53.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.682865+0000", 2023-09-23T20:57:53.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0510999999999998e-05 2023-09-23T20:57:53.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:08.683031+0000", 2023-09-23T20:57:53.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016547400000000001 2023-09-23T20:57:53.557 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.557 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.557 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.558 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209245 8.8 8:1443b4f6:::benchmark_data_smithi121_252781_object100070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:57:53.558 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:57:53.558 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 552.75812924399997, 2023-09-23T20:57:53.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.08464988, 2023-09-23T20:57:53.558 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209245 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.559 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728374+0000", 2023-09-23T20:57:53.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728373+0000", 2023-09-23T20:57:53.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:53.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728379+0000", 2023-09-23T20:57:53.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5089999999999999e-06 2023-09-23T20:57:53.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728395+0000", 2023-09-23T20:57:53.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5710999999999999e-05 2023-09-23T20:57:53.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728402+0000", 2023-09-23T20:57:53.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7190000000000004e-06 2023-09-23T20:57:53.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728795+0000", 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039319799999999999 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728805+0000", 2023-09-23T20:57:53.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6690000000000001e-06 2023-09-23T20:57:53.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.565 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757335+0000", 2023-09-23T20:57:53.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0285302249999999 2023-09-23T20:57:53.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757427+0000", 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1818999999999996e-05 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:57:53.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.757509+0000", 2023-09-23T20:57:53.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1922000000000006e-05 2023-09-23T20:57:53.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.768732+0000", 2023-09-23T20:57:53.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011223713 2023-09-23T20:57:53.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812940+0000", 2023-09-23T20:57:53.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.044207659000000003 2023-09-23T20:57:53.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812972+0000", 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1633e-05 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813001+0000", 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9300000000000001e-05 2023-09-23T20:57:53.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.570 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.570 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.570 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813024+0000", 2023-09-23T20:57:53.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3244999999999999e-05 2023-09-23T20:57:53.570 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.570 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.570 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.571 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.44585.0:209248 8.8 8:136e166f:::benchmark_data_smithi121_252781_object100073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T20:57:53.571 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:57:53.571 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 552.75809278099996, 2023-09-23T20:57:53.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.084736245, 2023-09-23T20:57:53.571 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.572 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.572 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.572 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.44585", 2023-09-23T20:57:53.572 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3201485503", 2023-09-23T20:57:53.572 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 209248 2023-09-23T20:57:53.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.572 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.573 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.573 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:57:53.573 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.573 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.573 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728411+0000", 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728410+0000", 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728413+0000", 2023-09-23T20:57:53.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.756e-06 2023-09-23T20:57:53.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728419+0000", 2023-09-23T20:57:53.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3940000000000001e-06 2023-09-23T20:57:53.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728423+0000", 2023-09-23T20:57:53.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9400000000000004e-06 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728828+0000", 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00040448300000000002 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:40.728835+0000", 2023-09-23T20:57:53.578 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8469999999999998e-06 2023-09-23T20:57:53.578 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.578 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.578 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761382+0000", 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0325472419999999 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761496+0000", 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113714 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:57:53.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.761583+0000", 2023-09-23T20:57:53.580 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7459000000000004e-05 2023-09-23T20:57:53.580 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.580 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.772112+0000", 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010529221 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.812957+0000", 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040844889000000002 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813061+0000", 2023-09-23T20:57:53.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000104036 2023-09-23T20:57:53.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813109+0000", 2023-09-23T20:57:53.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7704999999999998e-05 2023-09-23T20:57:53.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:48:41.813147+0000", 2023-09-23T20:57:53.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8046000000000001e-05 2023-09-23T20:57:53.584 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.584 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.584 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.584 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136323 9.6 9:6c612c1e:::benchmark_data_smithi121_293770_object33435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:57:53.584 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 323.27183755999999, 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0778323409999999, 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:57:53.585 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136323 2023-09-23T20:57:53.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.586 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:57:53.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214666+0000", 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214665+0000", 2023-09-23T20:57:53.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.588 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.588 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214669+0000", 2023-09-23T20:57:53.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2310000000000001e-06 2023-09-23T20:57:53.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214675+0000", 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6250000000000001e-06 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.589 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214680+0000", 2023-09-23T20:57:53.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.301e-06 2023-09-23T20:57:53.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214759+0000", 2023-09-23T20:57:53.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9574999999999996e-05 2023-09-23T20:57:53.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214869+0000", 2023-09-23T20:57:53.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010993399999999999 2023-09-23T20:57:53.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214978+0000", 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000108773 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224530+0000", 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095521080000000001 2023-09-23T20:57:53.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224538+0000", 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0970000000000006e-06 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.283176+0000", 2023-09-23T20:57:53.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0586380129999999 2023-09-23T20:57:53.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:57:53.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.283513+0000", 2023-09-23T20:57:53.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000336769 2023-09-23T20:57:53.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292448+0000", 2023-09-23T20:57:53.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0089352670000000002 2023-09-23T20:57:53.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.292498+0000", 2023-09-23T20:57:53.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0102000000000002e-05 2023-09-23T20:57:53.596 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.596 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.596 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.597 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74225 10.9 10:9b7c5c1b:::benchmark_data_smithi121_358375_object74224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:57:53.597 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.710286+0000", 2023-09-23T20:57:53.597 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 143.776217153, 2023-09-23T20:57:53.597 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3339274059999999, 2023-09-23T20:57:53.597 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.597 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:57:53.598 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.598 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:57:53.598 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:53.598 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74225 2023-09-23T20:57:53.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.598 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.598 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710286+0000", 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710286+0000", 2023-09-23T20:57:53.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.600 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710282+0000", 2023-09-23T20:57:53.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999957 2023-09-23T20:57:53.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710348+0000", 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6082999999999997e-05 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710359+0000", 2023-09-23T20:57:53.601 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1491000000000001e-05 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710364+0000", 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.206e-06 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710407+0000", 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3637000000000003e-05 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710560+0000", 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015250499999999999 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710732+0000", 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017187699999999999 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.718766+0000", 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080340979999999999 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.727854+0000", 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0090881409999999992 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885871+0000", 2023-09-23T20:57:53.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.15801679499999999 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885883+0000", 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1973999999999999e-05 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899218+0000", 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.013335289 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899229+0000", 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1124000000000001e-05 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.044111+0000", 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1448817710000001 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.044214+0000", 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102935 2023-09-23T20:57:53.606 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74232 10.9 10:9163f21b:::benchmark_data_smithi121_358375_object74231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.716015+0000", 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 143.77048877999999, 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.329761704, 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:53.607 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74232 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716015+0000", 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716015+0000", 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716013+0000", 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716234+0000", 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022076299999999999 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716248+0000", 2023-09-23T20:57:53.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3532e-05 2023-09-23T20:57:53.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716259+0000", 2023-09-23T20:57:53.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1138000000000001e-05 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716302+0000", 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2920000000000002e-05 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716418+0000", 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115862 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716541+0000", 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012331199999999999 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.718820+0000", 2023-09-23T20:57:53.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022795559999999999 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.729911+0000", 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011090645 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885918+0000", 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.15600684300000001 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885925+0000", 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3590000000000002e-06 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899269+0000", 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0133434050000001 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899275+0000", 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0410000000000003e-06 2023-09-23T20:57:53.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045708+0000", 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.146433093 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045776+0000", 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8604999999999998e-05 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74256 10.9 10:9d9325d5:::benchmark_data_smithi121_358375_object74255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.874623+0000", 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 143.611880246, 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1713063940000001, 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74256 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.616 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874623+0000", 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874623+0000", 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.617 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874622+0000", 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874668+0000", 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5856000000000003e-05 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874680+0000", 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2401999999999999e-05 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874685+0000", 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6380000000000003e-06 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874732+0000", 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6709999999999998e-05 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874873+0000", 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014162800000000001 2023-09-23T20:57:53.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.875031+0000", 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015807899999999999 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.877600+0000", 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025683099999999999 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.878245+0000", 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00064534499999999997 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.892611+0000", 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014365562 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.892621+0000", 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0226e-05 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899301+0000", 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0066803090000001 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899307+0000", 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6520000000000003e-06 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045830+0000", 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.146523298 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.622 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045930+0000", 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9457999999999995e-05 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74271 10.9 10:993b4897:::benchmark_data_smithi121_358375_object74270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.909039+0000", 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 143.57746473700001, 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1620222980000001, 2023-09-23T20:57:53.623 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74271 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909039+0000", 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.624 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909039+0000", 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909037+0000", 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909098+0000", 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0850999999999997e-05 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.625 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909118+0000", 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0117999999999999e-05 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909126+0000", 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2110000000000001e-06 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909186+0000", 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0157e-05 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909198+0000", 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1755e-05 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899327+0000", 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.990129386 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.627 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899333+0000", 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6910000000000002e-06 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045959+0000", 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.14662636 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.046111+0000", 2023-09-23T20:57:53.628 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015205500000000001 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.046294+0000", 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000182938 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.056140+0000", 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0098462289999999997 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061791+0000", 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056506589999999997 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.067344+0000", 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055534979999999996 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.067403+0000", 2023-09-23T20:57:53.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9017000000000001e-05 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071061+0000", 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036576600000000001 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74276 10.9 10:916b3091:::benchmark_data_smithi121_358375_object74275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.915396+0000", 2023-09-23T20:57:53.631 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 143.571107109, 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.159243085, 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74276 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915396+0000", 2023-09-23T20:57:53.632 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915396+0000", 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915395+0000", 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.633 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915445+0000", 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9669000000000002e-05 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915456+0000", 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1321000000000001e-05 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915461+0000", 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0590000000000002e-06 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915508+0000", 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7253999999999999e-05 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915517+0000", 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1599999999999998e-06 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899353+0000", 2023-09-23T20:57:53.635 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98383633000000004 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899359+0000", 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3860000000000004e-06 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048395+0000", 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1490360150000001 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.636 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048548+0000", 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015308499999999999 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048724+0000", 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017566700000000001 2023-09-23T20:57:53.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.638 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.638 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061802+0000", 2023-09-23T20:57:53.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013078227 2023-09-23T20:57:53.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061864+0000", 2023-09-23T20:57:53.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2067000000000001e-05 2023-09-23T20:57:53.639 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.639 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071113+0000", 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092486760000000008 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071163+0000", 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9515999999999998e-05 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.640 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.641 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.641 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074639+0000", 2023-09-23T20:57:53.641 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003476884 2023-09-23T20:57:53.641 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.641 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.641 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.641 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74277 10.9 10:902456f8:::benchmark_data_smithi121_358375_object74276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 143.570632999, 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1624961859999998, 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.642 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.643 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:57:53.643 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:53.643 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74277 2023-09-23T20:57:53.643 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.643 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.643 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.643 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.644 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.645 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.645 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.645 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T20:57:53.645 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.645 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.645 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.645 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.646 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915934+0000", 2023-09-23T20:57:53.646 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4109999999999994e-05 2023-09-23T20:57:53.646 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.646 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.646 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.646 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915944+0000", 2023-09-23T20:57:53.646 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7089999999999997e-06 2023-09-23T20:57:53.647 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.647 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.647 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.647 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915948+0000", 2023-09-23T20:57:53.647 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.33e-06 2023-09-23T20:57:53.647 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.647 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915982+0000", 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4347000000000002e-05 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915990+0000", 2023-09-23T20:57:53.648 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8930000000000005e-06 2023-09-23T20:57:53.649 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.649 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.649 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899380+0000", 2023-09-23T20:57:53.649 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98338946299999996 2023-09-23T20:57:53.649 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899385+0000", 2023-09-23T20:57:53.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4439999999999999e-06 2023-09-23T20:57:53.650 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.650 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.050951+0000", 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1515660750000001 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.051065+0000", 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113613 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:57:53.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.051231+0000", 2023-09-23T20:57:53.652 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016582399999999999 2023-09-23T20:57:53.652 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061811+0000", 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010579965 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061902+0000", 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1799000000000007e-05 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074825+0000", 2023-09-23T20:57:53.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012922417 2023-09-23T20:57:53.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074866+0000", 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0949999999999999e-05 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078367+0000", 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035008359999999998 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.655 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.656 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.656 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.656 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74279 10.9 10:9ea30411:::benchmark_data_smithi121_358375_object74278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T20:57:53.656 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.920027+0000", 2023-09-23T20:57:53.656 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 143.566476384, 2023-09-23T20:57:53.656 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.161850388, 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74279 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.657 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920027+0000", 2023-09-23T20:57:53.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.658 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.659 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920027+0000", 2023-09-23T20:57:53.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.659 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.659 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.659 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920026+0000", 2023-09-23T20:57:53.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920073+0000", 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6397000000000003e-05 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920085+0000", 2023-09-23T20:57:53.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.287e-05 2023-09-23T20:57:53.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920091+0000", 2023-09-23T20:57:53.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1490000000000002e-06 2023-09-23T20:57:53.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.662 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.662 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920130+0000", 2023-09-23T20:57:53.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9684999999999997e-05 2023-09-23T20:57:53.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.662 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920138+0000", 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5689999999999999e-06 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899415+0000", 2023-09-23T20:57:53.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97927704699999996 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899421+0000", 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9020000000000004e-06 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053407+0000", 2023-09-23T20:57:53.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1539860749999999 2023-09-23T20:57:53.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.665 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053558+0000", 2023-09-23T20:57:53.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000151216 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053723+0000", 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165048 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:57:53.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061817+0000", 2023-09-23T20:57:53.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080936069999999992 2023-09-23T20:57:53.667 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.667 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.667 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061971+0000", 2023-09-23T20:57:53.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000154582 2023-09-23T20:57:53.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:57:53.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078458+0000", 2023-09-23T20:57:53.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016486923000000001 2023-09-23T20:57:53.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078518+0000", 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0068000000000002e-05 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.081877+0000", 2023-09-23T20:57:53.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033591329999999998 2023-09-23T20:57:53.670 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.670 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.670 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.670 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.670 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:85868 10.8 e250/241 10:190b85f5:::benchmark_data_smithi121_358375_object1123:head v 250'5457, mlcod=230'67)", 2023-09-23T20:57:53.670 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:56:22.694413+0000", 2023-09-23T20:57:53.671 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 90.792090756999997, 2023-09-23T20:57:53.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013925162, 2023-09-23T20:57:53.671 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.671 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.671 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.671 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.694413+0000", 2023-09-23T20:57:53.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.672 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.672 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.694413+0000", 2023-09-23T20:57:53.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.694412+0000", 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.694417+0000", 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5240000000000002e-06 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.694427+0000", 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0576e-05 2023-09-23T20:57:53.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.694431+0000", 2023-09-23T20:57:53.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8330000000000004e-06 2023-09-23T20:57:53.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695458+0000", 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0010272949999999999 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695485+0000", 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6401999999999999e-05 2023-09-23T20:57:53.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.708293+0000", 2023-09-23T20:57:53.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012808043 2023-09-23T20:57:53.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.708338+0000", 2023-09-23T20:57:53.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4929999999999998e-05 2023-09-23T20:57:53.678 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.678 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.678 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:85870 10.8 e250/241 10:13715a99:::benchmark_data_smithi121_358375_object1125:head v 250'5458, mlcod=230'67)", 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:56:22.695444+0000", 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 90.791059310999998, 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0129201, 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695444+0000", 2023-09-23T20:57:53.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695444+0000", 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695444+0000", 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.682 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.682 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695448+0000", 2023-09-23T20:57:53.682 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7879999999999997e-06 2023-09-23T20:57:53.682 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.682 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.682 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.682 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695462+0000", 2023-09-23T20:57:53.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3726999999999999e-05 2023-09-23T20:57:53.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.695466+0000", 2023-09-23T20:57:53.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1819999999999999e-06 2023-09-23T20:57:53.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.700393+0000", 2023-09-23T20:57:53.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004926911 2023-09-23T20:57:53.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.700426+0000", 2023-09-23T20:57:53.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2385000000000002e-05 2023-09-23T20:57:53.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.708342+0000", 2023-09-23T20:57:53.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079164189999999992 2023-09-23T20:57:53.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:22.708364+0000", 2023-09-23T20:57:53.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2163000000000001e-05 2023-09-23T20:57:53.689 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.689 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.689 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.690 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.690 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.690 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:89280 10.5 e253/242 10:abee6021:::benchmark_data_smithi121_358375_object4535:head v 253'5612, mlcod=0'0)", 2023-09-23T20:57:53.690 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:56:46.750286+0000", 2023-09-23T20:57:53.690 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 66.736217456999995, 2023-09-23T20:57:53.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012143664, 2023-09-23T20:57:53.690 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:57:53.691 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:57:53.691 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:57:53.691 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.691 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:57:53.691 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750286+0000", 2023-09-23T20:57:53.691 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:57:53.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750286+0000", 2023-09-23T20:57:53.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:57:53.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:57:53.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750285+0000", 2023-09-23T20:57:53.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:57:53.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:57:53.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750298+0000", 2023-09-23T20:57:53.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2534000000000001e-05 2023-09-23T20:57:53.694 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:57:53.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750314+0000", 2023-09-23T20:57:53.694 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6055000000000001e-05 2023-09-23T20:57:53.694 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750321+0000", 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0439999999999998e-06 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750365+0000", 2023-09-23T20:57:53.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4320000000000003e-05 2023-09-23T20:57:53.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:57:53.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.750389+0000", 2023-09-23T20:57:53.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4375e-05 2023-09-23T20:57:53.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.762383+0000", 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011993091000000001 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:56:46.762430+0000", 2023-09-23T20:57:53.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7135000000000002e-05 2023-09-23T20:57:53.698 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.698 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.698 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.698 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:57:53.698 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:57:53.698 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:57:53.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:53.700 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:53.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:53.801 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:53.829 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:53.874 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:53.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:53.977 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:54.004 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:53 smithi121 ceph-mon[211526]: pgmap v868: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 15102/231588 objects degraded (6.521%); 847/231588 objects misplaced (0.366%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:57:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:53 smithi121 ceph-mon[219709]: pgmap v868: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 15102/231588 objects degraded (6.521%); 847/231588 objects misplaced (0.366%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:57:54.223 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:54.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:53 smithi123 ceph-mon[193903]: pgmap v868: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 15102/231588 objects degraded (6.521%); 847/231588 objects misplaced (0.366%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:57:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:54.279 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:57:54.327 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:57:54.355 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.397 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:57:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:57:54.506 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:54.539 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.591 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:54.643 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:54.694 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:54.722 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:54.787 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:54.789 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:57:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:57:54] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-23T20:57:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:54.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:54.890 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:54.891 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:57:54.920 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:55.075 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/556918235' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:55.075 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:55.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/556918235' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:55.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:55.122 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:55.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:55.227 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/556918235' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:55.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:57:55.261 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:55.340 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:55.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:55.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:55.518 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:55.551 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:55.740 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:57:55.804 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:55.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:55.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:55.914 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:56.063 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: pgmap v869: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 14780/230052 objects degraded (6.425%); 847/230052 objects misplaced (0.368%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:57:56.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:56.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:56.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: osdmap e279: 8 total, 8 up, 8 in 2023-09-23T20:57:56.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[211526]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: pgmap v869: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 14780/230052 objects degraded (6.425%); 847/230052 objects misplaced (0.368%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:57:56.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:56.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:56.065 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: osdmap e279: 8 total, 8 up, 8 in 2023-09-23T20:57:56.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:55 smithi121 ceph-mon[219709]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.066 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:56.083 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: pgmap v869: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 14780/230052 objects degraded (6.425%); 847/230052 objects misplaced (0.368%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:57:56.083 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:57:56.083 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.083 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:57:56.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: osdmap e279: 8 total, 8 up, 8 in 2023-09-23T20:57:56.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:55 smithi123 ceph-mon[193903]: failed to encode map e279 with expected crc 2023-09-23T20:57:56.260 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:57:56.260 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:57:56.260 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T20:57:56.260 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:57:56.481 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:57:56.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:56.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:56.586 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:56.616 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:56.660 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:56.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:56.670 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:56.764 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:56.795 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:56.960 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:57:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:56.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:57.063 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:57.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14780/230052 objects degraded (6.425%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:57.095 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14780/230052 objects degraded (6.425%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:57.099 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:57.128 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:57:57.153 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14780/230052 objects degraded (6.425%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:57:57.302 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:57.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:57.406 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:57.436 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:57.488 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:57:57.490 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:57.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:57.594 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:57:57.623 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:57.678 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:57:57.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:57.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:57:57.782 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:57.810 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:57.882 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:57 smithi121 ceph-mon[211526]: pgmap v871: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 14780/229938 objects degraded (6.428%); 847/229938 objects misplaced (0.368%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:57:57.883 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:57.883 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:57.883 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:57 smithi121 ceph-mon[219709]: pgmap v871: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 14780/229938 objects degraded (6.428%); 847/229938 objects misplaced (0.368%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:57:57.883 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:57.883 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:57 smithi123 ceph-mon[193903]: pgmap v871: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 14780/229938 objects degraded (6.428%); 847/229938 objects misplaced (0.368%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:57:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:57:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:57:58.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:58.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:58.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:57:58.243 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:58.404 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:57:58.404 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:57:58.405 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:57:58.405 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:57:58.411 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:57:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:58.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:57:58.528 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:58.555 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:58.606 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:57:58.651 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:58.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:58.758 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:57:58.787 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:58.935 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:57:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:58.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:57:59.001 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:59.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:57:59.049 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:57:59.072 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:59.081 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:57:59.114 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:57:59.129 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:57:59.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:59.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:57:59.141 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:57:59.147 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:57:59.147 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:57:59.147 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T20:57:59.147 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T20:57:59.147 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:57:59.156 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:57:59.162 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:57:59.187 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:57:59.212 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:57:59.234 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:57:59.236 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:57:59.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.237 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.237 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.237 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.237 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.237 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.237 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.237 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:57:59.270 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:59.488 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:57:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:59.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:57:59.593 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:57:59.619 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:57:59.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:57:59.889 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:57:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:57:59.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:57:59.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:00.023 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.082 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:00.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:00.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:59 smithi121 ceph-mon[219709]: pgmap v872: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 14780/228927 objects degraded (6.456%); 847/228927 objects misplaced (0.370%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:58:00.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:57:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1456267178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:00.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:59 smithi121 ceph-mon[211526]: pgmap v872: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 14780/228927 objects degraded (6.456%); 847/228927 objects misplaced (0.370%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:58:00.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:57:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1456267178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:00.190 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:00.213 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:59 smithi123 ceph-mon[193903]: pgmap v872: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 14780/228927 objects degraded (6.456%); 847/228927 objects misplaced (0.370%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:58:00.214 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:57:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1456267178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:00.216 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.267 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:00.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:00.277 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:00.371 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:00.397 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.492 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:00.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:00.596 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:00.622 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.716 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:00.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:00.820 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:00.858 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:00.912 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:00.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:00.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:01.016 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:01.042 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:01.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:01.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:01.190 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:01.216 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:01.414 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:01.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:01.521 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:01.549 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:01.846 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:01.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:01.889 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:01 smithi121 ceph-mon[219709]: pgmap v873: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 14438/228243 objects degraded (6.326%); 847/228243 objects misplaced (0.371%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:58:01.889 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14438/228243 objects degraded (6.326%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:01.890 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:01 smithi121 ceph-mon[211526]: pgmap v873: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 14438/228243 objects degraded (6.326%); 847/228243 objects misplaced (0.371%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:58:01.890 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14438/228243 objects degraded (6.326%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:01.897 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:01.897 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:01.953 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:01.983 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.130 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:02.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:02.133 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:01 smithi123 ceph-mon[193903]: pgmap v873: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 14438/228243 objects degraded (6.326%); 847/228243 objects misplaced (0.371%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:58:02.133 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14438/228243 objects degraded (6.326%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:02.234 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:02.261 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.302 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:02.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:02.406 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:02.432 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.508 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:02.558 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:58:02.611 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:02.638 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.684 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:02.789 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:02.825 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:02.921 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:02.970 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:02.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:02.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:03.077 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:03.110 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:03.202 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:03.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:03.215 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:58:03.216 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:58:03.310 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:03.340 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:03.394 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:03.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:03.498 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:03.537 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:03.666 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:58:03.846 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:03.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:03.953 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:03.979 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:04.077 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:04.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:04.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:03 smithi121 ceph-mon[219709]: pgmap v874: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 14438/228243 objects degraded (6.326%); 847/228243 objects misplaced (0.371%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:58:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:03 smithi121 ceph-mon[211526]: pgmap v874: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 14438/228243 objects degraded (6.326%); 847/228243 objects misplaced (0.371%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:58:04.185 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:04.211 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:03 smithi123 ceph-mon[193903]: pgmap v874: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 14438/228243 objects degraded (6.326%); 847/228243 objects misplaced (0.371%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:58:04.214 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:04.463 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:04.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:04.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:04.538 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:04.538 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:04.567 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:04.603 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:04.721 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":874,"stamp":"2023-09-23T20:58:02.824672+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4985001912,"num_objects":76081,"num_object_clones":0,"num_object_copies":228243,"num_objects_missing_on_primary":4579,"num_objects_missing":4579,"num_objects_degraded":14438,"num_objects_misplaced":847,"num_objects_unfound":0,"num_objects_dirty":76081,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":93729,"num_write_kb":5425934,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13846,"num_bytes_recovered":797006288,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93645,"ondisk_log_size":93645,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17792312,"kb_used_data":14591352,"kb_used_omap":21,"kb_used_meta":3200810,"kb_avail":732168904,"statfs":{"total":767960285184,"available":749740957696,"internally_reserved":0,"allocated":14941544448,"data_stored":14933553080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3277629923},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3,2,0,6,6,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-73072640,"num_objects":-1115,"num_object_clones":0,"num_object_copies":-3345,"num_objects_missing_on_primary":-70,"num_objects_missing":-70,"num_objects_degraded":-664,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1115,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":733,"num_bytes_recovered":48037888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002496"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":63,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.240501+0000","last_change":"2023-09-23T20:57:45.240062+0000","last_active":"2023-09-23T20:57:45.240501+0000","last_peered":"2023-09-23T20:57:45.240501+0000","last_clean":"2023-09-23T20:57:45.240501+0000","last_became_active":"2023-09-23T20:57:45.239810+0000","last_became_peered":"2023-09-23T20:57:45.239810+0000","last_unstale":"2023-09-23T20:57:45.240501+0000","last_undegraded":"2023-09-23T20:57:45.240501+0000","last_fullsized":"2023-09-23T20:57:45.240501+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:48.028187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":250,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230573+0000","last_change":"2023-09-23T20:57:44.249598+0000","last_active":"2023-09-23T20:57:45.230573+0000","last_peered":"2023-09-23T20:57:45.230573+0000","last_clean":"2023-09-23T20:57:45.230573+0000","last_became_active":"2023-09-23T20:57:44.249342+0000","last_became_peered":"2023-09-23T20:57:44.249342+0000","last_unstale":"2023-09-23T20:57:45.230573+0000","last_undegraded":"2023-09-23T20:57:45.230573+0000","last_fullsized":"2023-09-23T20:57:45.230573+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:53.547436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":143,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.235946+0000","last_change":"2023-09-23T20:57:44.324560+0000","last_active":"2023-09-23T20:57:45.235946+0000","last_peered":"2023-09-23T20:57:45.235946+0000","last_clean":"2023-09-23T20:57:45.235946+0000","last_became_active":"2023-09-23T20:57:44.324308+0000","last_became_peered":"2023-09-23T20:57:44.324308+0000","last_unstale":"2023-09-23T20:57:45.235946+0000","last_undegraded":"2023-09-23T20:57:45.235946+0000","last_fullsized":"2023-09-23T20:57:45.235946+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:32.213839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":252,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.240135+0000","last_change":"2023-09-23T20:57:16.910004+0000","last_active":"2023-09-23T20:57:45.240135+0000","last_peered":"2023-09-23T20:57:45.240135+0000","last_clean":"2023-09-23T20:57:45.240135+0000","last_became_active":"2023-09-23T20:57:16.909719+0000","last_became_peered":"2023-09-23T20:57:16.909719+0000","last_unstale":"2023-09-23T20:57:45.240135+0000","last_undegraded":"2023-09-23T20:57:45.240135+0000","last_fullsized":"2023-09-23T20:57:45.240135+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:39.337052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":269,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.282498+0000","last_change":"2023-09-23T20:57:44.250863+0000","last_active":"2023-09-23T20:57:45.282498+0000","last_peered":"2023-09-23T20:57:45.282498+0000","last_clean":"2023-09-23T20:57:45.282498+0000","last_became_active":"2023-09-23T20:57:44.250209+0000","last_became_peered":"2023-09-23T20:57:44.250209+0000","last_unstale":"2023-09-23T20:57:45.282498+0000","last_undegraded":"2023-09-23T20:57:45.282498+0000","last_fullsized":"2023-09-23T20:57:45.282498+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:30.252886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":145,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.250884+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T20:57:45.250884+0000","last_peered":"2023-09-23T20:57:45.250884+0000","last_clean":"2023-09-23T20:57:45.250884+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T20:57:45.250884+0000","last_undegraded":"2023-09-23T20:57:45.250884+0000","last_fullsized":"2023-09-23T20:57:45.250884+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":230,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:48.923746+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:57:48.923746+0000","last_peered":"2023-09-23T20:57:48.923746+0000","last_clean":"2023-09-23T20:57:48.923746+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:57:48.923746+0000","last_undegraded":"2023-09-23T20:57:48.923746+0000","last_fullsized":"2023-09-23T20:57:48.923746+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":269,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230430+0000","last_change":"2023-09-23T20:57:44.251052+0000","last_active":"2023-09-23T20:57:45.230430+0000","last_peered":"2023-09-23T20:57:45.230430+0000","last_clean":"2023-09-23T20:57:45.230430+0000","last_became_active":"2023-09-23T20:57:44.250576+0000","last_became_peered":"2023-09-23T20:57:44.250576+0000","last_unstale":"2023-09-23T20:57:45.230430+0000","last_undegraded":"2023-09-23T20:57:45.230430+0000","last_fullsized":"2023-09-23T20:57:45.230430+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:09.267827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":299,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.249669+0000","last_change":"2023-09-23T20:57:45.249221+0000","last_active":"2023-09-23T20:57:45.249669+0000","last_peered":"2023-09-23T20:57:45.249669+0000","last_clean":"2023-09-23T20:57:45.249669+0000","last_became_active":"2023-09-23T20:57:45.248960+0000","last_became_peered":"2023-09-23T20:57:45.248960+0000","last_unstale":"2023-09-23T20:57:45.249669+0000","last_undegraded":"2023-09-23T20:57:45.249669+0000","last_fullsized":"2023-09-23T20:57:45.249669+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.370914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":226,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230304+0000","last_change":"2023-09-23T20:57:15.910105+0000","last_active":"2023-09-23T20:57:45.230304+0000","last_peered":"2023-09-23T20:57:45.230304+0000","last_clean":"2023-09-23T20:57:45.230304+0000","last_became_active":"2023-09-23T20:57:15.909785+0000","last_became_peered":"2023-09-23T20:57:15.909785+0000","last_unstale":"2023-09-23T20:57:45.230304+0000","last_undegraded":"2023-09-23T20:57:45.230304+0000","last_fullsized":"2023-09-23T20:57:45.230304+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:05.007111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":29,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248123+0000","last_change":"2023-09-23T20:57:15.952549+0000","last_active":"2023-09-23T20:57:45.248123+0000","last_peered":"2023-09-23T20:57:45.248123+0000","last_clean":"2023-09-23T20:57:45.248123+0000","last_became_active":"2023-09-23T20:57:15.913795+0000","last_became_peered":"2023-09-23T20:57:15.913795+0000","last_unstale":"2023-09-23T20:57:45.248123+0000","last_undegraded":"2023-09-23T20:57:45.248123+0000","last_fullsized":"2023-09-23T20:57:45.248123+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:47.933385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":147,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.244193+0000","last_change":"2023-09-23T20:57:45.243915+0000","last_active":"2023-09-23T20:57:45.244193+0000","last_peered":"2023-09-23T20:57:45.244193+0000","last_clean":"2023-09-23T20:57:45.244193+0000","last_became_active":"2023-09-23T20:57:45.243610+0000","last_became_peered":"2023-09-23T20:57:45.243610+0000","last_unstale":"2023-09-23T20:57:45.244193+0000","last_undegraded":"2023-09-23T20:57:45.244193+0000","last_fullsized":"2023-09-23T20:57:45.244193+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:55.808488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":33,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248587+0000","last_change":"2023-09-23T20:57:15.948301+0000","last_active":"2023-09-23T20:57:45.248587+0000","last_peered":"2023-09-23T20:57:45.248587+0000","last_clean":"2023-09-23T20:57:45.248587+0000","last_became_active":"2023-09-23T20:57:15.908308+0000","last_became_peered":"2023-09-23T20:57:15.908308+0000","last_unstale":"2023-09-23T20:57:45.248587+0000","last_undegraded":"2023-09-23T20:57:45.248587+0000","last_fullsized":"2023-09-23T20:57:45.248587+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:19.866857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":278,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.283563+0000","last_change":"2023-09-23T20:57:44.245493+0000","last_active":"2023-09-23T20:57:45.283563+0000","last_peered":"2023-09-23T20:57:45.283563+0000","last_clean":"2023-09-23T20:57:45.283563+0000","last_became_active":"2023-09-23T20:57:44.245140+0000","last_became_peered":"2023-09-23T20:57:44.245140+0000","last_unstale":"2023-09-23T20:57:45.283563+0000","last_undegraded":"2023-09-23T20:57:45.283563+0000","last_fullsized":"2023-09-23T20:57:45.283563+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:48.316166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":30,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.247812+0000","last_change":"2023-09-23T20:57:15.921091+0000","last_active":"2023-09-23T20:57:45.247812+0000","last_peered":"2023-09-23T20:57:45.247812+0000","last_clean":"2023-09-23T20:57:45.247812+0000","last_became_active":"2023-09-23T20:57:15.908112+0000","last_became_peered":"2023-09-23T20:57:15.908112+0000","last_unstale":"2023-09-23T20:57:45.247812+0000","last_undegraded":"2023-09-23T20:57:45.247812+0000","last_fullsized":"2023-09-23T20:57:45.247812+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:43.853034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":292,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248647+0000","last_change":"2023-09-23T20:57:45.247131+0000","last_active":"2023-09-23T20:57:45.248647+0000","last_peered":"2023-09-23T20:57:45.248647+0000","last_clean":"2023-09-23T20:57:45.248647+0000","last_became_active":"2023-09-23T20:57:45.246850+0000","last_became_peered":"2023-09-23T20:57:45.246850+0000","last_unstale":"2023-09-23T20:57:45.248647+0000","last_undegraded":"2023-09-23T20:57:45.248647+0000","last_fullsized":"2023-09-23T20:57:45.248647+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:15.134941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":126,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248365+0000","last_change":"2023-09-23T20:57:16.911653+0000","last_active":"2023-09-23T20:57:45.248365+0000","last_peered":"2023-09-23T20:57:45.248365+0000","last_clean":"2023-09-23T20:57:45.248365+0000","last_became_active":"2023-09-23T20:57:16.911367+0000","last_became_peered":"2023-09-23T20:57:16.911367+0000","last_unstale":"2023-09-23T20:57:45.248365+0000","last_undegraded":"2023-09-23T20:57:45.248365+0000","last_fullsized":"2023-09-23T20:57:45.248365+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:02.083476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248049+0000","last_change":"2023-09-23T20:57:15.906421+0000","last_active":"2023-09-23T20:57:45.248049+0000","last_peered":"2023-09-23T20:57:45.248049+0000","last_clean":"2023-09-23T20:57:45.248049+0000","last_became_active":"2023-09-23T20:57:15.906073+0000","last_became_peered":"2023-09-23T20:57:15.906073+0000","last_unstale":"2023-09-23T20:57:45.248049+0000","last_undegraded":"2023-09-23T20:57:45.248049+0000","last_fullsized":"2023-09-23T20:57:45.248049+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:23.145995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":23,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.246786+0000","last_change":"2023-09-23T20:57:15.909085+0000","last_active":"2023-09-23T20:57:45.246786+0000","last_peered":"2023-09-23T20:57:45.246786+0000","last_clean":"2023-09-23T20:57:45.246786+0000","last_became_active":"2023-09-23T20:57:15.908703+0000","last_became_peered":"2023-09-23T20:57:15.908703+0000","last_unstale":"2023-09-23T20:57:45.246786+0000","last_undegraded":"2023-09-23T20:57:45.246786+0000","last_fullsized":"2023-09-23T20:57:45.246786+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:23.043924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":267,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:48.923722+0000","last_change":"2023-09-23T20:57:15.906249+0000","last_active":"2023-09-23T20:57:48.923722+0000","last_peered":"2023-09-23T20:57:48.923722+0000","last_clean":"2023-09-23T20:57:48.923722+0000","last_became_active":"2023-09-23T20:57:15.905861+0000","last_became_peered":"2023-09-23T20:57:15.905861+0000","last_unstale":"2023-09-23T20:57:48.923722+0000","last_undegraded":"2023-09-23T20:57:48.923722+0000","last_fullsized":"2023-09-23T20:57:48.923722+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:28.875320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":121,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.323028+0000","last_change":"2023-09-23T20:51:06.038448+0000","last_active":"2023-09-23T20:57:45.323028+0000","last_peered":"2023-09-23T20:57:45.323028+0000","last_clean":"2023-09-23T20:57:45.323028+0000","last_became_active":"2023-09-23T20:51:06.005965+0000","last_became_peered":"2023-09-23T20:51:06.005965+0000","last_unstale":"2023-09-23T20:57:45.323028+0000","last_undegraded":"2023-09-23T20:57:45.323028+0000","last_fullsized":"2023-09-23T20:57:45.323028+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:13.397771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":292,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.338415+0000","last_change":"2023-09-23T20:57:45.249600+0000","last_active":"2023-09-23T20:57:45.338415+0000","last_peered":"2023-09-23T20:57:45.338415+0000","last_clean":"2023-09-23T20:57:45.338415+0000","last_became_active":"2023-09-23T20:57:45.249299+0000","last_became_peered":"2023-09-23T20:57:45.249299+0000","last_unstale":"2023-09-23T20:57:45.338415+0000","last_undegraded":"2023-09-23T20:57:45.338415+0000","last_fullsized":"2023-09-23T20:57:45.338415+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:16.553646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"279'5948","reported_seq":6019,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.812298+0000","last_change":"2023-09-23T20:55:55.476703+0000","last_active":"2023-09-23T20:57:58.812298+0000","last_peered":"2023-09-23T20:57:58.812298+0000","last_clean":"2023-09-23T20:57:58.812298+0000","last_became_active":"2023-09-23T20:55:55.467711+0000","last_became_peered":"2023-09-23T20:55:55.467711+0000","last_unstale":"2023-09-23T20:57:58.812298+0000","last_undegraded":"2023-09-23T20:57:58.812298+0000","last_fullsized":"2023-09-23T20:57:58.812298+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:31.924695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313917440,"num_objects":4790,"num_object_clones":0,"num_object_copies":14370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5948,"num_write_kb":343616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":221,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248406+0000","last_change":"2023-09-23T20:57:16.909660+0000","last_active":"2023-09-23T20:57:45.248406+0000","last_peered":"2023-09-23T20:57:45.248406+0000","last_clean":"2023-09-23T20:57:45.248406+0000","last_became_active":"2023-09-23T20:57:16.909322+0000","last_became_peered":"2023-09-23T20:57:16.909322+0000","last_unstale":"2023-09-23T20:57:45.248406+0000","last_undegraded":"2023-09-23T20:57:45.248406+0000","last_fullsized":"2023-09-23T20:57:45.248406+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:50.829223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":261,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.238022+0000","last_change":"2023-09-23T20:57:15.906260+0000","last_active":"2023-09-23T20:57:45.238022+0000","last_peered":"2023-09-23T20:57:45.238022+0000","last_clean":"2023-09-23T20:57:45.238022+0000","last_became_active":"2023-09-23T20:57:15.905929+0000","last_became_peered":"2023-09-23T20:57:15.905929+0000","last_unstale":"2023-09-23T20:57:45.238022+0000","last_undegraded":"2023-09-23T20:57:45.238022+0000","last_fullsized":"2023-09-23T20:57:45.238022+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:09.183508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"279'5647","reported_seq":5710,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.564126+0000","last_change":"2023-09-23T20:54:35.777545+0000","last_active":"2023-09-23T20:57:58.564126+0000","last_peered":"2023-09-23T20:57:58.564126+0000","last_clean":"2023-09-23T20:57:58.564126+0000","last_became_active":"2023-09-23T20:54:35.776552+0000","last_became_peered":"2023-09-23T20:54:35.776552+0000","last_unstale":"2023-09-23T20:57:58.564126+0000","last_undegraded":"2023-09-23T20:57:58.564126+0000","last_fullsized":"2023-09-23T20:57:58.564126+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:29.974409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304545792,"num_objects":4647,"num_object_clones":0,"num_object_copies":13941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5647,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":258,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.259636+0000","last_change":"2023-09-23T20:57:45.258104+0000","last_active":"2023-09-23T20:57:45.259636+0000","last_peered":"2023-09-23T20:57:45.259636+0000","last_clean":"2023-09-23T20:57:45.259636+0000","last_became_active":"2023-09-23T20:57:45.257745+0000","last_became_peered":"2023-09-23T20:57:45.257745+0000","last_unstale":"2023-09-23T20:57:45.259636+0000","last_undegraded":"2023-09-23T20:57:45.259636+0000","last_fullsized":"2023-09-23T20:57:45.259636+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:56.314260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":217,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.235858+0000","last_change":"2023-09-23T20:57:44.324702+0000","last_active":"2023-09-23T20:57:45.235858+0000","last_peered":"2023-09-23T20:57:45.235858+0000","last_clean":"2023-09-23T20:57:45.235858+0000","last_became_active":"2023-09-23T20:57:44.324367+0000","last_became_peered":"2023-09-23T20:57:44.324367+0000","last_unstale":"2023-09-23T20:57:45.235858+0000","last_undegraded":"2023-09-23T20:57:45.235858+0000","last_fullsized":"2023-09-23T20:57:45.235858+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:55.384013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"279'5864","reported_seq":5929,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.741093+0000","last_change":"2023-09-23T20:54:35.769971+0000","last_active":"2023-09-23T20:57:58.741093+0000","last_peered":"2023-09-23T20:57:58.741093+0000","last_clean":"2023-09-23T20:57:58.741093+0000","last_became_active":"2023-09-23T20:54:35.769661+0000","last_became_peered":"2023-09-23T20:54:35.769661+0000","last_unstale":"2023-09-23T20:57:58.741093+0000","last_undegraded":"2023-09-23T20:57:58.741093+0000","last_fullsized":"2023-09-23T20:57:58.741093+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:55.146768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312868864,"num_objects":4774,"num_object_clones":0,"num_object_copies":14322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5864,"num_write_kb":340416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248198+0000","last_change":"2023-09-23T20:57:15.915430+0000","last_active":"2023-09-23T20:57:45.248198+0000","last_peered":"2023-09-23T20:57:45.248198+0000","last_clean":"2023-09-23T20:57:45.248198+0000","last_became_active":"2023-09-23T20:57:15.906106+0000","last_became_peered":"2023-09-23T20:57:15.906106+0000","last_unstale":"2023-09-23T20:57:45.248198+0000","last_undegraded":"2023-09-23T20:57:45.248198+0000","last_fullsized":"2023-09-23T20:57:45.248198+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:31.990035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":157,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.253367+0000","last_change":"2023-09-23T20:57:45.252971+0000","last_active":"2023-09-23T20:57:45.253367+0000","last_peered":"2023-09-23T20:57:45.253367+0000","last_clean":"2023-09-23T20:57:45.253367+0000","last_became_active":"2023-09-23T20:57:45.252608+0000","last_became_peered":"2023-09-23T20:57:45.252608+0000","last_unstale":"2023-09-23T20:57:45.253367+0000","last_undegraded":"2023-09-23T20:57:45.253367+0000","last_fullsized":"2023-09-23T20:57:45.253367+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:34.867894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"279'5861","reported_seq":5954,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.151784+0000","last_change":"2023-09-23T20:57:17.902794+0000","last_active":"2023-09-23T20:57:58.151784+0000","last_peered":"2023-09-23T20:57:58.151784+0000","last_clean":"2023-09-23T20:57:58.151784+0000","last_became_active":"2023-09-23T20:57:17.892913+0000","last_became_peered":"2023-09-23T20:57:17.892913+0000","last_unstale":"2023-09-23T20:57:58.151784+0000","last_undegraded":"2023-09-23T20:57:58.151784+0000","last_fullsized":"2023-09-23T20:57:58.151784+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:46.016662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312279040,"num_objects":4765,"num_object_clones":0,"num_object_copies":14295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5861,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":550,"num_bytes_recovered":13172736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":194,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230838+0000","last_change":"2023-09-23T20:55:55.460433+0000","last_active":"2023-09-23T20:57:45.230838+0000","last_peered":"2023-09-23T20:57:45.230838+0000","last_clean":"2023-09-23T20:57:45.230838+0000","last_became_active":"2023-09-23T20:55:55.459883+0000","last_became_peered":"2023-09-23T20:55:55.459883+0000","last_unstale":"2023-09-23T20:57:45.230838+0000","last_undegraded":"2023-09-23T20:57:45.230838+0000","last_fullsized":"2023-09-23T20:57:45.230838+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:59.444482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":261,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.236002+0000","last_change":"2023-09-23T20:57:44.250822+0000","last_active":"2023-09-23T20:57:45.236002+0000","last_peered":"2023-09-23T20:57:45.236002+0000","last_clean":"2023-09-23T20:57:45.236002+0000","last_became_active":"2023-09-23T20:57:44.250524+0000","last_became_peered":"2023-09-23T20:57:44.250524+0000","last_unstale":"2023-09-23T20:57:45.236002+0000","last_undegraded":"2023-09-23T20:57:45.236002+0000","last_fullsized":"2023-09-23T20:57:45.236002+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:23.511125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"279'5843","reported_seq":7639,"reported_epoch":279,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:57:58.940814+0000","last_change":"2023-09-23T20:57:45.481809+0000","last_active":"2023-09-23T20:57:58.940814+0000","last_peered":"2023-09-23T20:57:58.940814+0000","last_clean":"2023-09-23T20:54:34.751733+0000","last_became_active":"2023-09-23T20:57:45.435535+0000","last_became_peered":"2023-09-23T20:57:45.435535+0000","last_unstale":"2023-09-23T20:57:58.940814+0000","last_undegraded":"2023-09-23T20:57:45.272202+0000","last_fullsized":"2023-09-23T20:57:45.236715+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5843,"log_dups_size":0,"ondisk_log_size":5843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:33.084037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308346880,"num_objects":4705,"num_object_clones":0,"num_object_copies":14115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3955,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5843,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":863,"num_bytes_recovered":56557568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":750}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":244,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230496+0000","last_change":"2023-09-23T20:48:07.518312+0000","last_active":"2023-09-23T20:57:45.230496+0000","last_peered":"2023-09-23T20:57:45.230496+0000","last_clean":"2023-09-23T20:57:45.230496+0000","last_became_active":"2023-09-23T20:48:07.517976+0000","last_became_peered":"2023-09-23T20:48:07.517976+0000","last_unstale":"2023-09-23T20:57:45.230496+0000","last_undegraded":"2023-09-23T20:57:45.230496+0000","last_fullsized":"2023-09-23T20:57:45.230496+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:33.374104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":215,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230758+0000","last_change":"2023-09-23T20:54:35.776908+0000","last_active":"2023-09-23T20:57:45.230758+0000","last_peered":"2023-09-23T20:57:45.230758+0000","last_clean":"2023-09-23T20:57:45.230758+0000","last_became_active":"2023-09-23T20:54:35.776549+0000","last_became_peered":"2023-09-23T20:54:35.776549+0000","last_unstale":"2023-09-23T20:57:45.230758+0000","last_undegraded":"2023-09-23T20:57:45.230758+0000","last_fullsized":"2023-09-23T20:57:45.230758+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.593595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"279'5997","reported_seq":6738,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.122065+0000","last_change":"2023-09-23T20:57:03.729205+0000","last_active":"2023-09-23T20:57:58.122065+0000","last_peered":"2023-09-23T20:57:58.122065+0000","last_clean":"2023-09-23T20:57:58.122065+0000","last_became_active":"2023-09-23T20:57:03.721552+0000","last_became_peered":"2023-09-23T20:57:03.721552+0000","last_unstale":"2023-09-23T20:57:58.122065+0000","last_undegraded":"2023-09-23T20:57:58.122065+0000","last_fullsized":"2023-09-23T20:57:58.122065+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5997,"log_dups_size":0,"ondisk_log_size":5997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:40.786255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321060864,"num_objects":4899,"num_object_clones":0,"num_object_copies":14697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5997,"num_write_kb":348672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":851,"num_bytes_recovered":34406400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":145,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.245729+0000","last_change":"2023-09-23T20:57:44.249701+0000","last_active":"2023-09-23T20:57:45.245729+0000","last_peered":"2023-09-23T20:57:45.245729+0000","last_clean":"2023-09-23T20:57:45.245729+0000","last_became_active":"2023-09-23T20:57:44.249408+0000","last_became_peered":"2023-09-23T20:57:44.249408+0000","last_unstale":"2023-09-23T20:57:45.245729+0000","last_undegraded":"2023-09-23T20:57:45.245729+0000","last_fullsized":"2023-09-23T20:57:45.245729+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:30.094839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":268,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.241252+0000","last_change":"2023-09-23T20:57:45.240954+0000","last_active":"2023-09-23T20:57:45.241252+0000","last_peered":"2023-09-23T20:57:45.241252+0000","last_clean":"2023-09-23T20:57:45.241252+0000","last_became_active":"2023-09-23T20:57:45.240606+0000","last_became_peered":"2023-09-23T20:57:45.240606+0000","last_unstale":"2023-09-23T20:57:45.241252+0000","last_undegraded":"2023-09-23T20:57:45.241252+0000","last_fullsized":"2023-09-23T20:57:45.241252+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:02.023788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"279'5751","reported_seq":5831,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:57:58.075723+0000","last_change":"2023-09-23T20:57:45.476178+0000","last_active":"2023-09-23T20:57:58.075723+0000","last_peered":"2023-09-23T20:57:58.075723+0000","last_clean":"2023-09-23T20:54:43.441608+0000","last_became_active":"2023-09-23T20:57:45.463536+0000","last_became_peered":"2023-09-23T20:57:45.463536+0000","last_unstale":"2023-09-23T20:57:58.075723+0000","last_undegraded":"2023-09-23T20:57:45.293220+0000","last_fullsized":"2023-09-23T20:57:45.242386+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:34.175256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305201152,"num_objects":4657,"num_object_clones":0,"num_object_copies":13971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5751,"num_write_kb":333056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":-108}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":164,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.232861+0000","last_change":"2023-09-23T20:57:16.904405+0000","last_active":"2023-09-23T20:57:45.232861+0000","last_peered":"2023-09-23T20:57:45.232861+0000","last_clean":"2023-09-23T20:57:45.232861+0000","last_became_active":"2023-09-23T20:57:16.904069+0000","last_became_peered":"2023-09-23T20:57:16.904069+0000","last_unstale":"2023-09-23T20:57:45.232861+0000","last_undegraded":"2023-09-23T20:57:45.232861+0000","last_fullsized":"2023-09-23T20:57:45.232861+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:11.371083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":278,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.250807+0000","last_change":"2023-09-23T20:57:44.250118+0000","last_active":"2023-09-23T20:57:45.250807+0000","last_peered":"2023-09-23T20:57:45.250807+0000","last_clean":"2023-09-23T20:57:45.250807+0000","last_became_active":"2023-09-23T20:57:44.249814+0000","last_became_peered":"2023-09-23T20:57:44.249814+0000","last_unstale":"2023-09-23T20:57:45.250807+0000","last_undegraded":"2023-09-23T20:57:45.250807+0000","last_fullsized":"2023-09-23T20:57:45.250807+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:58.298798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"279'5969","reported_seq":6046,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.861546+0000","last_change":"2023-09-23T20:57:16.914501+0000","last_active":"2023-09-23T20:57:58.861546+0000","last_peered":"2023-09-23T20:57:58.861546+0000","last_clean":"2023-09-23T20:57:58.861546+0000","last_became_active":"2023-09-23T20:57:16.907404+0000","last_became_peered":"2023-09-23T20:57:16.907404+0000","last_unstale":"2023-09-23T20:57:58.861546+0000","last_undegraded":"2023-09-23T20:57:58.861546+0000","last_fullsized":"2023-09-23T20:57:58.861546+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5969,"log_dups_size":0,"ondisk_log_size":5969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:39.583426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315424768,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5969,"num_write_kb":345024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":273,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248589+0000","last_change":"2023-09-23T20:57:45.247092+0000","last_active":"2023-09-23T20:57:45.248589+0000","last_peered":"2023-09-23T20:57:45.248589+0000","last_clean":"2023-09-23T20:57:45.248589+0000","last_became_active":"2023-09-23T20:57:45.246776+0000","last_became_peered":"2023-09-23T20:57:45.246776+0000","last_unstale":"2023-09-23T20:57:45.248589+0000","last_undegraded":"2023-09-23T20:57:45.248589+0000","last_fullsized":"2023-09-23T20:57:45.248589+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:46.691931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:47.243893+0000","last_change":"2023-09-23T20:57:45.245751+0000","last_active":"2023-09-23T20:57:47.243893+0000","last_peered":"2023-09-23T20:57:47.243893+0000","last_clean":"2023-09-23T20:57:47.243893+0000","last_became_active":"2023-09-23T20:57:45.245476+0000","last_became_peered":"2023-09-23T20:57:45.245476+0000","last_unstale":"2023-09-23T20:57:47.243893+0000","last_undegraded":"2023-09-23T20:57:47.243893+0000","last_fullsized":"2023-09-23T20:57:47.243893+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:44.987049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"279'5792","reported_seq":12221,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.774617+0000","last_change":"2023-09-23T20:57:21.928776+0000","last_active":"2023-09-23T20:57:58.774617+0000","last_peered":"2023-09-23T20:57:58.774617+0000","last_clean":"2023-09-23T20:57:58.774617+0000","last_became_active":"2023-09-23T20:57:21.923250+0000","last_became_peered":"2023-09-23T20:57:21.923250+0000","last_unstale":"2023-09-23T20:57:58.774617+0000","last_undegraded":"2023-09-23T20:57:58.774617+0000","last_fullsized":"2023-09-23T20:57:58.774617+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5792,"log_dups_size":0,"ondisk_log_size":5792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:36.965869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310378496,"num_objects":4736,"num_object_clones":0,"num_object_copies":14208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5792,"num_write_kb":336896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3885,"num_bytes_recovered":231669760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":34,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.259611+0000","last_change":"2023-09-23T20:57:45.256956+0000","last_active":"2023-09-23T20:57:45.259611+0000","last_peered":"2023-09-23T20:57:45.259611+0000","last_clean":"2023-09-23T20:57:45.259611+0000","last_became_active":"2023-09-23T20:57:45.256606+0000","last_became_peered":"2023-09-23T20:57:45.256606+0000","last_unstale":"2023-09-23T20:57:45.259611+0000","last_undegraded":"2023-09-23T20:57:45.259611+0000","last_fullsized":"2023-09-23T20:57:45.259611+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:42.787842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":276,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.238235+0000","last_change":"2023-09-23T20:56:52.999112+0000","last_active":"2023-09-23T20:57:45.238235+0000","last_peered":"2023-09-23T20:57:45.238235+0000","last_clean":"2023-09-23T20:57:45.238235+0000","last_became_active":"2023-09-23T20:56:52.998744+0000","last_became_peered":"2023-09-23T20:56:52.998744+0000","last_unstale":"2023-09-23T20:57:45.238235+0000","last_undegraded":"2023-09-23T20:57:45.238235+0000","last_fullsized":"2023-09-23T20:57:45.238235+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:58.451042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":137,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.236653+0000","last_change":"2023-09-23T20:57:44.247431+0000","last_active":"2023-09-23T20:57:45.236653+0000","last_peered":"2023-09-23T20:57:45.236653+0000","last_clean":"2023-09-23T20:57:45.236653+0000","last_became_active":"2023-09-23T20:57:44.247156+0000","last_became_peered":"2023-09-23T20:57:44.247156+0000","last_unstale":"2023-09-23T20:57:45.236653+0000","last_undegraded":"2023-09-23T20:57:45.236653+0000","last_fullsized":"2023-09-23T20:57:45.236653+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:56.902057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"279'5805","reported_seq":5868,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.703887+0000","last_change":"2023-09-23T20:54:35.777943+0000","last_active":"2023-09-23T20:57:58.703887+0000","last_peered":"2023-09-23T20:57:58.703887+0000","last_clean":"2023-09-23T20:57:58.703887+0000","last_became_active":"2023-09-23T20:54:35.777642+0000","last_became_peered":"2023-09-23T20:54:35.777642+0000","last_unstale":"2023-09-23T20:57:58.703887+0000","last_undegraded":"2023-09-23T20:57:58.703887+0000","last_fullsized":"2023-09-23T20:57:58.703887+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:24.764426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310444032,"num_objects":4737,"num_object_clones":0,"num_object_copies":14211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5805,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":215,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:47.243842+0000","last_change":"2023-09-23T20:57:44.246836+0000","last_active":"2023-09-23T20:57:47.243842+0000","last_peered":"2023-09-23T20:57:47.243842+0000","last_clean":"2023-09-23T20:57:47.243842+0000","last_became_active":"2023-09-23T20:57:44.246521+0000","last_became_peered":"2023-09-23T20:57:44.246521+0000","last_unstale":"2023-09-23T20:57:47.243842+0000","last_undegraded":"2023-09-23T20:57:47.243842+0000","last_fullsized":"2023-09-23T20:57:47.243842+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:45.073844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":209,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.283572+0000","last_change":"2023-09-23T20:55:05.920213+0000","last_active":"2023-09-23T20:57:45.283572+0000","last_peered":"2023-09-23T20:57:45.283572+0000","last_clean":"2023-09-23T20:57:45.283572+0000","last_became_active":"2023-09-23T20:55:05.917924+0000","last_became_peered":"2023-09-23T20:55:05.917924+0000","last_unstale":"2023-09-23T20:57:45.283572+0000","last_undegraded":"2023-09-23T20:57:45.283572+0000","last_fullsized":"2023-09-23T20:57:45.283572+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:17.149381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":35,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.233225+0000","last_change":"2023-09-23T20:56:53.000153+0000","last_active":"2023-09-23T20:57:45.233225+0000","last_peered":"2023-09-23T20:57:45.233225+0000","last_clean":"2023-09-23T20:57:45.233225+0000","last_became_active":"2023-09-23T20:56:52.999821+0000","last_became_peered":"2023-09-23T20:56:52.999821+0000","last_unstale":"2023-09-23T20:57:45.233225+0000","last_undegraded":"2023-09-23T20:57:45.233225+0000","last_fullsized":"2023-09-23T20:57:45.233225+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.519799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"279'5856","reported_seq":7673,"reported_epoch":279,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:58:00.485047+0000","last_change":"2023-09-23T20:57:35.384520+0000","last_active":"2023-09-23T20:58:00.485047+0000","last_peered":"2023-09-23T20:58:00.485047+0000","last_clean":"2023-09-23T20:55:29.724749+0000","last_became_active":"2023-09-23T20:57:34.820417+0000","last_became_peered":"2023-09-23T20:57:34.820417+0000","last_unstale":"2023-09-23T20:58:00.485047+0000","last_undegraded":"2023-09-23T20:58:00.485047+0000","last_fullsized":"2023-09-23T20:57:16.887143+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5856,"log_dups_size":0,"ondisk_log_size":5856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:40.546208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313917440,"num_objects":4790,"num_object_clones":0,"num_object_copies":14370,"num_objects_missing_on_primary":4579,"num_objects_missing":4579,"num_objects_degraded":0,"num_objects_misplaced":847,"num_objects_unfound":0,"num_objects_dirty":4790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5856,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":820,"num_bytes_recovered":53542912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[3,7],"avail_no_missing":["5"],"object_location_counts":[{"shards":"2,3,5,7","objects":14},{"shards":"2,4,5","objects":455},{"shards":"2,5,7","objects":4124},{"shards":"3,4,5","objects":196},{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":233,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.232936+0000","last_change":"2023-09-23T20:57:10.019515+0000","last_active":"2023-09-23T20:57:45.232936+0000","last_peered":"2023-09-23T20:57:45.232936+0000","last_clean":"2023-09-23T20:57:45.232936+0000","last_became_active":"2023-09-23T20:57:10.019226+0000","last_became_peered":"2023-09-23T20:57:10.019226+0000","last_unstale":"2023-09-23T20:57:45.232936+0000","last_undegraded":"2023-09-23T20:57:45.232936+0000","last_fullsized":"2023-09-23T20:57:45.232936+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:13.610232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":188,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.324352+0000","last_change":"2023-09-23T20:57:45.323895+0000","last_active":"2023-09-23T20:57:45.324352+0000","last_peered":"2023-09-23T20:57:45.324352+0000","last_clean":"2023-09-23T20:57:45.324352+0000","last_became_active":"2023-09-23T20:57:45.323621+0000","last_became_peered":"2023-09-23T20:57:45.323621+0000","last_unstale":"2023-09-23T20:57:45.324352+0000","last_undegraded":"2023-09-23T20:57:45.324352+0000","last_fullsized":"2023-09-23T20:57:45.324352+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:45.813087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":179,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.237050+0000","last_change":"2023-09-23T20:57:44.252913+0000","last_active":"2023-09-23T20:57:45.237050+0000","last_peered":"2023-09-23T20:57:45.237050+0000","last_clean":"2023-09-23T20:57:45.237050+0000","last_became_active":"2023-09-23T20:57:44.252629+0000","last_became_peered":"2023-09-23T20:57:44.252629+0000","last_unstale":"2023-09-23T20:57:45.237050+0000","last_undegraded":"2023-09-23T20:57:45.237050+0000","last_fullsized":"2023-09-23T20:57:45.237050+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:58.536851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"279'5799","reported_seq":5879,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:57:58.407377+0000","last_change":"2023-09-23T20:57:45.486706+0000","last_active":"2023-09-23T20:57:58.407377+0000","last_peered":"2023-09-23T20:57:58.407377+0000","last_clean":"2023-09-23T20:54:43.441988+0000","last_became_active":"2023-09-23T20:57:45.460741+0000","last_became_peered":"2023-09-23T20:57:45.460741+0000","last_unstale":"2023-09-23T20:57:58.407377+0000","last_undegraded":"2023-09-23T20:57:45.278767+0000","last_fullsized":"2023-09-23T20:57:45.242451+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:47.975740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311885824,"num_objects":4759,"num_object_clones":0,"num_object_copies":14277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5279,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5799,"num_write_kb":337856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-107}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":44,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.233631+0000","last_change":"2023-09-23T20:57:15.904314+0000","last_active":"2023-09-23T20:57:45.233631+0000","last_peered":"2023-09-23T20:57:45.233631+0000","last_clean":"2023-09-23T20:57:45.233631+0000","last_became_active":"2023-09-23T20:57:15.903877+0000","last_became_peered":"2023-09-23T20:57:15.903877+0000","last_unstale":"2023-09-23T20:57:45.233631+0000","last_undegraded":"2023-09-23T20:57:45.233631+0000","last_fullsized":"2023-09-23T20:57:45.233631+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:13.571032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":652,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.246624+0000","last_change":"2023-09-23T20:57:16.910407+0000","last_active":"2023-09-23T20:57:45.246624+0000","last_peered":"2023-09-23T20:57:45.246624+0000","last_clean":"2023-09-23T20:57:45.246624+0000","last_became_active":"2023-09-23T20:57:16.909672+0000","last_became_peered":"2023-09-23T20:57:16.909672+0000","last_unstale":"2023-09-23T20:57:45.246624+0000","last_undegraded":"2023-09-23T20:57:45.246624+0000","last_fullsized":"2023-09-23T20:57:45.246624+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:00.601002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":216,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230407+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:57:45.230407+0000","last_peered":"2023-09-23T20:57:45.230407+0000","last_clean":"2023-09-23T20:57:45.230407+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:57:45.230407+0000","last_undegraded":"2023-09-23T20:57:45.230407+0000","last_fullsized":"2023-09-23T20:57:45.230407+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":131,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.249997+0000","last_change":"2023-09-23T20:57:15.910395+0000","last_active":"2023-09-23T20:57:45.249997+0000","last_peered":"2023-09-23T20:57:45.249997+0000","last_clean":"2023-09-23T20:57:45.249997+0000","last_became_active":"2023-09-23T20:57:15.909812+0000","last_became_peered":"2023-09-23T20:57:15.909812+0000","last_unstale":"2023-09-23T20:57:45.249997+0000","last_undegraded":"2023-09-23T20:57:45.249997+0000","last_fullsized":"2023-09-23T20:57:45.249997+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:57.043984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"279'5929","reported_seq":20694,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.128015+0000","last_change":"2023-09-23T20:57:33.445244+0000","last_active":"2023-09-23T20:57:58.128015+0000","last_peered":"2023-09-23T20:57:58.128015+0000","last_clean":"2023-09-23T20:57:58.128015+0000","last_became_active":"2023-09-23T20:57:33.440125+0000","last_became_peered":"2023-09-23T20:57:33.440125+0000","last_unstale":"2023-09-23T20:57:58.128015+0000","last_undegraded":"2023-09-23T20:57:58.128015+0000","last_fullsized":"2023-09-23T20:57:58.128015+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5929,"log_dups_size":0,"ondisk_log_size":5929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:34.082121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317521920,"num_objects":4845,"num_object_clones":0,"num_object_copies":14535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5929,"num_write_kb":344768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5773,"num_bytes_recovered":378339328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":277,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.256705+0000","last_change":"2023-09-23T20:57:45.253710+0000","last_active":"2023-09-23T20:57:45.256705+0000","last_peered":"2023-09-23T20:57:45.256705+0000","last_clean":"2023-09-23T20:57:45.256705+0000","last_became_active":"2023-09-23T20:57:45.253427+0000","last_became_peered":"2023-09-23T20:57:45.253427+0000","last_unstale":"2023-09-23T20:57:45.256705+0000","last_undegraded":"2023-09-23T20:57:45.256705+0000","last_fullsized":"2023-09-23T20:57:45.256705+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:19.420653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":231,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.236032+0000","last_change":"2023-09-23T20:48:07.522987+0000","last_active":"2023-09-23T20:57:45.236032+0000","last_peered":"2023-09-23T20:57:45.236032+0000","last_clean":"2023-09-23T20:57:45.236032+0000","last_became_active":"2023-09-23T20:48:07.522716+0000","last_became_peered":"2023-09-23T20:48:07.522716+0000","last_unstale":"2023-09-23T20:57:45.236032+0000","last_undegraded":"2023-09-23T20:57:45.236032+0000","last_fullsized":"2023-09-23T20:57:45.236032+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:50.833649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":297,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.246006+0000","last_change":"2023-09-23T20:57:45.245426+0000","last_active":"2023-09-23T20:57:45.246006+0000","last_peered":"2023-09-23T20:57:45.246006+0000","last_clean":"2023-09-23T20:57:45.246006+0000","last_became_active":"2023-09-23T20:57:45.245132+0000","last_became_peered":"2023-09-23T20:57:45.245132+0000","last_unstale":"2023-09-23T20:57:45.246006+0000","last_undegraded":"2023-09-23T20:57:45.246006+0000","last_fullsized":"2023-09-23T20:57:45.246006+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:58.863778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"279'5693","reported_seq":5757,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.122327+0000","last_change":"2023-09-23T20:54:35.774443+0000","last_active":"2023-09-23T20:57:58.122327+0000","last_peered":"2023-09-23T20:57:58.122327+0000","last_clean":"2023-09-23T20:57:58.122327+0000","last_became_active":"2023-09-23T20:54:35.774079+0000","last_became_peered":"2023-09-23T20:54:35.774079+0000","last_unstale":"2023-09-23T20:57:58.122327+0000","last_undegraded":"2023-09-23T20:57:58.122327+0000","last_fullsized":"2023-09-23T20:57:58.122327+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:29.601484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306577432,"num_objects":4679,"num_object_clones":0,"num_object_copies":14037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4679,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5693,"num_write_kb":331841,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":224,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.248274+0000","last_change":"2023-09-23T20:57:16.909343+0000","last_active":"2023-09-23T20:57:45.248274+0000","last_peered":"2023-09-23T20:57:45.248274+0000","last_clean":"2023-09-23T20:57:45.248274+0000","last_became_active":"2023-09-23T20:57:16.906734+0000","last_became_peered":"2023-09-23T20:57:16.906734+0000","last_unstale":"2023-09-23T20:57:45.248274+0000","last_undegraded":"2023-09-23T20:57:45.248274+0000","last_fullsized":"2023-09-23T20:57:45.248274+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:17.625758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":163,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.246735+0000","last_change":"2023-09-23T20:57:45.245119+0000","last_active":"2023-09-23T20:57:45.246735+0000","last_peered":"2023-09-23T20:57:45.246735+0000","last_clean":"2023-09-23T20:57:45.246735+0000","last_became_active":"2023-09-23T20:57:45.244800+0000","last_became_peered":"2023-09-23T20:57:45.244800+0000","last_unstale":"2023-09-23T20:57:45.246735+0000","last_undegraded":"2023-09-23T20:57:45.246735+0000","last_fullsized":"2023-09-23T20:57:45.246735+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:00.494845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":94,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.238112+0000","last_change":"2023-09-23T20:57:15.909593+0000","last_active":"2023-09-23T20:57:45.238112+0000","last_peered":"2023-09-23T20:57:45.238112+0000","last_clean":"2023-09-23T20:57:45.238112+0000","last_became_active":"2023-09-23T20:57:15.909267+0000","last_became_peered":"2023-09-23T20:57:15.909267+0000","last_unstale":"2023-09-23T20:57:45.238112+0000","last_undegraded":"2023-09-23T20:57:45.238112+0000","last_fullsized":"2023-09-23T20:57:45.238112+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:48.909976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"279'5811","reported_seq":6939,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.559994+0000","last_change":"2023-09-23T20:56:55.943104+0000","last_active":"2023-09-23T20:57:58.559994+0000","last_peered":"2023-09-23T20:57:58.559994+0000","last_clean":"2023-09-23T20:57:58.559994+0000","last_became_active":"2023-09-23T20:56:55.937950+0000","last_became_peered":"2023-09-23T20:56:55.937950+0000","last_unstale":"2023-09-23T20:57:58.559994+0000","last_undegraded":"2023-09-23T20:57:58.559994+0000","last_fullsized":"2023-09-23T20:57:58.559994+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:10.203931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306511872,"num_objects":4677,"num_object_clones":0,"num_object_copies":14031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5811,"num_write_kb":335616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":517,"num_bytes_recovered":12189696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":270,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.231155+0000","last_change":"2023-09-23T20:57:15.923214+0000","last_active":"2023-09-23T20:57:45.231155+0000","last_peered":"2023-09-23T20:57:45.231155+0000","last_clean":"2023-09-23T20:57:45.231155+0000","last_became_active":"2023-09-23T20:57:15.909700+0000","last_became_peered":"2023-09-23T20:57:15.909700+0000","last_unstale":"2023-09-23T20:57:45.231155+0000","last_undegraded":"2023-09-23T20:57:45.231155+0000","last_fullsized":"2023-09-23T20:57:45.231155+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:47.582106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":175,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.237244+0000","last_change":"2023-09-23T20:57:44.251246+0000","last_active":"2023-09-23T20:57:45.237244+0000","last_peered":"2023-09-23T20:57:45.237244+0000","last_clean":"2023-09-23T20:57:45.237244+0000","last_became_active":"2023-09-23T20:57:44.250908+0000","last_became_peered":"2023-09-23T20:57:44.250908+0000","last_unstale":"2023-09-23T20:57:45.237244+0000","last_undegraded":"2023-09-23T20:57:45.237244+0000","last_fullsized":"2023-09-23T20:57:45.237244+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:55.004865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":261,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230264+0000","last_change":"2023-09-23T20:57:16.905558+0000","last_active":"2023-09-23T20:57:45.230264+0000","last_peered":"2023-09-23T20:57:45.230264+0000","last_clean":"2023-09-23T20:57:45.230264+0000","last_became_active":"2023-09-23T20:57:16.904903+0000","last_became_peered":"2023-09-23T20:57:16.904903+0000","last_unstale":"2023-09-23T20:57:45.230264+0000","last_undegraded":"2023-09-23T20:57:45.230264+0000","last_fullsized":"2023-09-23T20:57:45.230264+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:37.228753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"279'5864","reported_seq":7039,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T20:57:58.731655+0000","last_change":"2023-09-23T20:57:13.897490+0000","last_active":"2023-09-23T20:57:58.731655+0000","last_peered":"2023-09-23T20:57:58.731655+0000","last_clean":"2023-09-23T20:57:58.731655+0000","last_became_active":"2023-09-23T20:57:13.890198+0000","last_became_peered":"2023-09-23T20:57:13.890198+0000","last_unstale":"2023-09-23T20:57:58.731655+0000","last_undegraded":"2023-09-23T20:57:58.731655+0000","last_fullsized":"2023-09-23T20:57:58.731655+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:01.820773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313524224,"num_objects":4784,"num_object_clones":0,"num_object_copies":14352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5864,"num_write_kb":340736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":539,"num_bytes_recovered":14155776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":155,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.252249+0000","last_change":"2023-09-23T20:57:15.904966+0000","last_active":"2023-09-23T20:57:45.252249+0000","last_peered":"2023-09-23T20:57:45.252249+0000","last_clean":"2023-09-23T20:57:45.252249+0000","last_became_active":"2023-09-23T20:57:15.904651+0000","last_became_peered":"2023-09-23T20:57:15.904651+0000","last_unstale":"2023-09-23T20:57:45.252249+0000","last_undegraded":"2023-09-23T20:57:45.252249+0000","last_fullsized":"2023-09-23T20:57:45.252249+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:01.038015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":264,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.240156+0000","last_change":"2023-09-23T20:57:44.255673+0000","last_active":"2023-09-23T20:57:45.240156+0000","last_peered":"2023-09-23T20:57:45.240156+0000","last_clean":"2023-09-23T20:57:45.240156+0000","last_became_active":"2023-09-23T20:57:44.255288+0000","last_became_peered":"2023-09-23T20:57:44.255288+0000","last_unstale":"2023-09-23T20:57:45.240156+0000","last_undegraded":"2023-09-23T20:57:45.240156+0000","last_fullsized":"2023-09-23T20:57:45.240156+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:07.464751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":198,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.239123+0000","last_change":"2023-09-23T20:54:35.777462+0000","last_active":"2023-09-23T20:57:45.239123+0000","last_peered":"2023-09-23T20:57:45.239123+0000","last_clean":"2023-09-23T20:57:45.239123+0000","last_became_active":"2023-09-23T20:54:35.777151+0000","last_became_peered":"2023-09-23T20:54:35.777151+0000","last_unstale":"2023-09-23T20:57:45.239123+0000","last_undegraded":"2023-09-23T20:57:45.239123+0000","last_fullsized":"2023-09-23T20:57:45.239123+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:58.147893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":237,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.236294+0000","last_change":"2023-09-23T20:57:15.908590+0000","last_active":"2023-09-23T20:57:45.236294+0000","last_peered":"2023-09-23T20:57:45.236294+0000","last_clean":"2023-09-23T20:57:45.236294+0000","last_became_active":"2023-09-23T20:57:15.908096+0000","last_became_peered":"2023-09-23T20:57:15.908096+0000","last_unstale":"2023-09-23T20:57:45.236294+0000","last_undegraded":"2023-09-23T20:57:45.236294+0000","last_fullsized":"2023-09-23T20:57:45.236294+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:27.519281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":280,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.246685+0000","last_change":"2023-09-23T20:57:45.244884+0000","last_active":"2023-09-23T20:57:45.246685+0000","last_peered":"2023-09-23T20:57:45.246685+0000","last_clean":"2023-09-23T20:57:45.246685+0000","last_became_active":"2023-09-23T20:57:45.244605+0000","last_became_peered":"2023-09-23T20:57:45.244605+0000","last_unstale":"2023-09-23T20:57:45.246685+0000","last_undegraded":"2023-09-23T20:57:45.246685+0000","last_fullsized":"2023-09-23T20:57:45.246685+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:40.539527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:47.243865+0000","last_change":"2023-09-23T20:57:44.248071+0000","last_active":"2023-09-23T20:57:47.243865+0000","last_peered":"2023-09-23T20:57:47.243865+0000","last_clean":"2023-09-23T20:57:47.243865+0000","last_became_active":"2023-09-23T20:57:44.247784+0000","last_became_peered":"2023-09-23T20:57:44.247784+0000","last_unstale":"2023-09-23T20:57:47.243865+0000","last_undegraded":"2023-09-23T20:57:47.243865+0000","last_fullsized":"2023-09-23T20:57:47.243865+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:28.185246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":273,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230574+0000","last_change":"2023-09-23T20:57:44.249566+0000","last_active":"2023-09-23T20:57:45.230574+0000","last_peered":"2023-09-23T20:57:45.230574+0000","last_clean":"2023-09-23T20:57:45.230574+0000","last_became_active":"2023-09-23T20:57:44.249284+0000","last_became_peered":"2023-09-23T20:57:44.249284+0000","last_unstale":"2023-09-23T20:57:45.230574+0000","last_undegraded":"2023-09-23T20:57:45.230574+0000","last_fullsized":"2023-09-23T20:57:45.230574+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:15.934995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":184,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.252144+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T20:57:45.252144+0000","last_peered":"2023-09-23T20:57:45.252144+0000","last_clean":"2023-09-23T20:57:45.252144+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T20:57:45.252144+0000","last_undegraded":"2023-09-23T20:57:45.252144+0000","last_fullsized":"2023-09-23T20:57:45.252144+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":183,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.230382+0000","last_change":"2023-09-23T20:54:35.770871+0000","last_active":"2023-09-23T20:57:45.230382+0000","last_peered":"2023-09-23T20:57:45.230382+0000","last_clean":"2023-09-23T20:57:45.230382+0000","last_became_active":"2023-09-23T20:54:35.770577+0000","last_became_peered":"2023-09-23T20:54:35.770577+0000","last_unstale":"2023-09-23T20:57:45.230382+0000","last_undegraded":"2023-09-23T20:57:45.230382+0000","last_fullsized":"2023-09-23T20:57:45.230382+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:23.786293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":275,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.231052+0000","last_change":"2023-09-23T20:56:52.997292+0000","last_active":"2023-09-23T20:57:45.231052+0000","last_peered":"2023-09-23T20:57:45.231052+0000","last_clean":"2023-09-23T20:57:45.231052+0000","last_became_active":"2023-09-23T20:56:52.997018+0000","last_became_peered":"2023-09-23T20:56:52.997018+0000","last_unstale":"2023-09-23T20:57:45.231052+0000","last_undegraded":"2023-09-23T20:57:45.231052+0000","last_fullsized":"2023-09-23T20:57:45.231052+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:50.415205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":262,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:50.430946+0000","last_change":"2023-09-23T20:56:53.060040+0000","last_active":"2023-09-23T20:57:50.430946+0000","last_peered":"2023-09-23T20:57:50.430946+0000","last_clean":"2023-09-23T20:57:50.430946+0000","last_became_active":"2023-09-23T20:56:53.059692+0000","last_became_peered":"2023-09-23T20:56:53.059692+0000","last_unstale":"2023-09-23T20:57:50.430946+0000","last_undegraded":"2023-09-23T20:57:50.430946+0000","last_fullsized":"2023-09-23T20:57:50.430946+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:58.628986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":265,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.238415+0000","last_change":"2023-09-23T20:56:53.075074+0000","last_active":"2023-09-23T20:57:45.238415+0000","last_peered":"2023-09-23T20:57:45.238415+0000","last_clean":"2023-09-23T20:57:45.238415+0000","last_became_active":"2023-09-23T20:56:53.061962+0000","last_became_peered":"2023-09-23T20:56:53.061962+0000","last_unstale":"2023-09-23T20:57:45.238415+0000","last_undegraded":"2023-09-23T20:57:45.238415+0000","last_fullsized":"2023-09-23T20:57:45.238415+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:58.791188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":292,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T20:57:45.237058+0000","last_change":"2023-09-23T20:57:44.251315+0000","last_active":"2023-09-23T20:57:45.237058+0000","last_peered":"2023-09-23T20:57:45.237058+0000","last_clean":"2023-09-23T20:57:45.237058+0000","last_became_active":"2023-09-23T20:57:44.251010+0000","last_became_peered":"2023-09-23T20:57:44.251010+0000","last_unstale":"2023-09-23T20:57:45.237058+0000","last_undegraded":"2023-09-23T20:57:45.237058+0000","last_fullsized":"2023-09-23T20:57:45.237058+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:35.232673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4984406040,"num_objects":76057,"num_object_clones":0,"num_object_copies":228171,"num_objects_missing_on_primary":4579,"num_objects_missing":4579,"num_objects_degraded":14438,"num_objects_misplaced":847,"num_objects_unfound":0,"num_objects_dirty":76057,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":93429,"num_write_kb":5423489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13798,"num_bytes_recovered":794034176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14675357696,"data_stored":14675345480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93429,"ondisk_log_size":93429,"up":48,"acting":44,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":20272,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201631,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1661260,"kb_used_data":1213252,"kb_used_omap":0,"kb_used_meta":448000,"kb_avail":92083892,"statfs":{"total":95995035648,"available":94293905408,"internally_reserved":0,"allocated":1242370048,"data_stored":1241592391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":458752000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.2989999999999999,"15min":2.7069999999999999},"min":{"1min":0.23799999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.56100000000000005,"5min":12.108000000000001,"15min":225.56999999999999},"last":0.505},{"interface":"front","average":{"1min":0.434,"5min":1.786,"15min":2.9860000000000002},"min":{"1min":0.311,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":0.93200000000000005,"5min":14.035,"15min":225.61099999999999},"last":0.45400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.9500000000000002,"5min":4.9500000000000002,"15min":4.9500000000000002},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.5779999999999998,"5min":2.5779999999999998,"15min":2.5779999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.4450000000000003,"5min":9.4450000000000003,"15min":9.4450000000000003},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":4.9269999999999996,"15min":3.7400000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.22900000000000001},"max":{"1min":3.1869999999999998,"5min":147.309,"15min":147.309},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":4.4729999999999999,"15min":3.1709999999999998},"min":{"1min":0.28000000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.1739999999999999,"5min":147.32400000000001,"15min":147.32400000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":3.2530000000000001,"15min":6.2919999999999998},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.51600000000000001,"5min":121.90900000000001,"15min":138.345},"last":0.314},{"interface":"front","average":{"1min":0.625,"5min":3.3340000000000001,"15min":6.0279999999999996},"min":{"1min":0.27200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2519999999999998,"5min":121.748,"15min":133.76400000000001},"last":0.36299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":1.0720000000000001,"15min":1.3160000000000001},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.56200000000000006,"5min":9.5559999999999992,"15min":16.648},"last":0.432},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.85799999999999998,"15min":1.165},"min":{"1min":0.28100000000000003,"5min":0.27000000000000002,"15min":0.247},"max":{"1min":0.54300000000000004,"5min":8.9629999999999992,"15min":10.172000000000001},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 20:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.95799999999999996,"15min":1.6279999999999999},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":0.51400000000000001,"5min":8.8109999999999999,"15min":126.28},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.439,"5min":1.321,"15min":2.0870000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.25600000000000001},"max":{"1min":0.59199999999999997,"5min":9.4930000000000003,"15min":126.465},"last":0.48999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":1.3169999999999999,"15min":1.4610000000000001},"min":{"1min":0.36599999999999999,"5min":0.36399999999999999,"15min":0.30399999999999999},"max":{"1min":0.61199999999999999,"5min":9.532,"15min":16.678000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":1.0209999999999999,"15min":1.167},"min":{"1min":0.35099999999999998,"5min":0.33400000000000002,"15min":0.26400000000000001},"max":{"1min":1.018,"5min":8.7129999999999992,"15min":10.154},"last":0.48099999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463268,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2460864,"kb_used_data":2113080,"kb_used_omap":0,"kb_used_meta":347712,"kb_avail":91284288,"statfs":{"total":95995035648,"available":93475110912,"internally_reserved":0,"allocated":2163793920,"data_stored":2163028222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":356057088},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:28 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":2.6190000000000002,"15min":2.7410000000000001},"min":{"1min":0.23999999999999999,"5min":0.224,"15min":0.20399999999999999},"max":{"1min":18.931999999999999,"5min":79.016000000000005,"15min":92.164000000000001},"last":0.316},{"interface":"front","average":{"1min":1.4510000000000001,"5min":1.9990000000000001,"15min":2.3220000000000001},"min":{"1min":0.307,"5min":0.251,"15min":0.20000000000000001},"max":{"1min":18.946000000000002,"5min":78.843000000000004,"15min":92.138000000000005},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:54:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6400000000000001,"5min":2.6400000000000001,"15min":2.6400000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.509,"5min":13.509,"15min":13.509},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.5899999999999999,"15min":2.5899999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":13.528,"5min":13.528,"15min":13.528},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":1.6279999999999999,"15min":1.952},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":1.2749999999999999,"5min":52.808,"15min":52.808},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":1.9590000000000001,"15min":2.1800000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.637,"5min":53.140000000000001,"15min":53.140000000000001},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:57:28 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":5.9400000000000004,"15min":5.4349999999999996},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":29.824999999999999,"5min":250.804,"15min":250.804},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.105,"5min":5.702,"15min":5.3419999999999996},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":29.777999999999999,"5min":250.446,"15min":250.446},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":1.071,"15min":1.05},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.47699999999999998,"5min":9.1590000000000007,"15min":9.1590000000000007},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.39400000000000002,"5min":1.3220000000000001,"15min":1.3440000000000001},"min":{"1min":0.26300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.63900000000000001,"5min":8.6859999999999999,"15min":9.8650000000000002},"last":0.33900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":1.444,"15min":1.853},"min":{"1min":0.315,"5min":0.29299999999999998,"15min":0.26000000000000001},"max":{"1min":0.55600000000000005,"5min":18.962,"15min":105.121},"last":0.435},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.704,"15min":1.9430000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.60099999999999998,"5min":9.5389999999999997,"15min":106.30500000000001},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":1.278,"15min":1.6870000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.22600000000000001},"max":{"1min":0.66500000000000004,"5min":11.388999999999999,"15min":101.04600000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.379,"15min":1.698},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.89500000000000002,"5min":8.8010000000000002,"15min":101.081},"last":0.36599999999999999}]}]},{"osd":1,"up_from":276,"seq":1185410973701,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":412392,"kb_used_data":287008,"kb_used_omap":5,"kb_used_meta":125370,"kb_avail":93332760,"statfs":{"total":95995035648,"available":95572746240,"internally_reserved":0,"allocated":293896192,"data_stored":292630591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":128379619},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.153}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50800000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39900000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.115}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1819999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]}]},{"osd":0,"up_from":92,"seq":395136991438,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3640296,"kb_used_data":3050784,"kb_used_omap":5,"kb_used_meta":589498,"kb_avail":90104856,"statfs":{"total":95995035648,"available":92267372544,"internally_reserved":0,"allocated":3124002816,"data_stored":3122790867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":603646586},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:54:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5069999999999997,"5min":5.5069999999999997,"15min":5.5069999999999997},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":89.626000000000005,"5min":89.626000000000005,"15min":89.626000000000005},"last":0.51200000000000001},{"interface":"front","average":{"1min":6.0309999999999997,"5min":6.0309999999999997,"15min":6.0309999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":89.489999999999995,"5min":89.489999999999995,"15min":89.489999999999995},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":2.2719999999999998,"15min":2.0990000000000002},"min":{"1min":0.161,"5min":0.161,"15min":0.161},"max":{"1min":4.4429999999999996,"5min":114.002,"15min":114.002},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":2.5089999999999999,"15min":3.0779999999999998},"min":{"1min":0.249,"5min":0.214,"15min":0.214},"max":{"1min":4.4009999999999998,"5min":113.69499999999999,"15min":113.69499999999999},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":4.2530000000000001,"15min":4.1559999999999997},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.53800000000000003,"5min":93.578999999999994,"15min":94.947999999999993},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":4.3529999999999998,"15min":4.2960000000000003},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.1680000000000001,"5min":93.680999999999997,"15min":94.382999999999996},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.55,"15min":1.821},"min":{"1min":0.34899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":6.4649999999999999,"5min":10.512,"15min":12.973000000000001},"last":1.365},{"interface":"front","average":{"1min":0.434,"5min":1.121,"15min":1.514},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":0.61599999999999999,"5min":11.493,"15min":12.461},"last":0.627}]},{"osd":5,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":3.431,"15min":3.2029999999999998},"min":{"1min":0.30199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.516,"5min":154.30600000000001,"15min":154.30600000000001},"last":1.4079999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":2.629,"15min":2.5259999999999998},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.59099999999999997,"5min":154.42400000000001,"15min":154.42400000000001},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.29,"15min":1.6990000000000001},"min":{"1min":0.32200000000000001,"5min":0.31,"15min":0.307},"max":{"1min":0.57999999999999996,"5min":9.6660000000000004,"15min":14.037000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.66900000000000004,"5min":1.5449999999999999,"15min":1.7549999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.2599999999999998,"5min":10.026999999999999,"15min":20.399999999999999},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":4.5750000000000002,"15min":3.0489999999999999},"min":{"1min":0.40300000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.999,"5min":164.26400000000001,"15min":164.26400000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.71199999999999997,"5min":4.6020000000000003,"15min":3.1219999999999999},"min":{"1min":0.371,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.2370000000000001,"5min":164.23099999999999,"15min":164.23099999999999},"last":0.52800000000000002}]}]},{"osd":2,"up_from":129,"seq":554050781329,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2201800,"kb_used_data":1852672,"kb_used_omap":7,"kb_used_meta":349112,"kb_avail":91543352,"statfs":{"total":95995035648,"available":93740392448,"internally_reserved":0,"allocated":1897136128,"data_stored":1895952710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":357490769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:19 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":1.9530000000000001,"15min":2.4809999999999999},"min":{"1min":0.19800000000000001,"5min":0.19,"15min":0.17599999999999999},"max":{"1min":26.356000000000002,"5min":26.356000000000002,"15min":89.644999999999996},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.27,"5min":2.0369999999999999,"15min":2.3809999999999998},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.186},"max":{"1min":26.341000000000001,"5min":26.341000000000001,"15min":88.599999999999994},"last":0.25}]},{"osd":1,"last update":"Sat Sep 23 20:54:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1059999999999999,"5min":3.1059999999999999,"15min":3.1059999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":14.976000000000001,"5min":14.976000000000001,"15min":14.976000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.3079999999999998,"15min":2.3079999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":11.778,"5min":11.778,"15min":11.778},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 20:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":2.6680000000000001,"15min":3.0249999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.83599999999999997,"5min":73.766999999999996,"15min":151.43799999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.34999999999999998,"5min":3.0680000000000001,"15min":2.9169999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.54000000000000004,"5min":73.867000000000004,"15min":152.20699999999999},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.4059999999999999,"15min":1.5900000000000001},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":0.79100000000000004,"5min":9.9209999999999994,"15min":9.9209999999999994},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.4139999999999999,"15min":1.825},"min":{"1min":0.30499999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.71299999999999997,"5min":8.5350000000000001,"15min":8.5350000000000001},"last":0.40000000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":3.4500000000000002,"15min":3.2869999999999999},"min":{"1min":0.313,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.76700000000000002,"5min":173.34,"15min":173.34},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":3.1899999999999999,"15min":3.1709999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61199999999999999,"5min":173.15000000000001,"15min":173.15000000000001},"last":0.53800000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.413,"15min":2.0649999999999999},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.95599999999999996,"5min":13.657,"15min":13.657},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":1.696,"15min":2.1930000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32300000000000001},"max":{"1min":0.82399999999999995,"5min":9.8930000000000007,"15min":12.595000000000001},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:57:19 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":3.7839999999999998,"15min":3.266},"min":{"1min":0.35199999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.0110000000000001,"5min":177.30799999999999,"15min":177.30799999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":3.649,"15min":3.2970000000000002},"min":{"1min":0.38300000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.169,"5min":177.40899999999999,"15min":177.40899999999999},"last":0.437}]}]},{"osd":3,"up_from":148,"seq":635655159924,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2061392,"kb_used_data":1673352,"kb_used_omap":2,"kb_used_meta":388029,"kb_avail":91683760,"statfs":{"total":95995035648,"available":93884170240,"internally_reserved":0,"allocated":1713512448,"data_stored":1712265051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":397341749},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3,2,0,6,6,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.28599999999999998,"5min":2.5569999999999999,"15min":7.8520000000000003},"min":{"1min":0.184,"5min":0.184,"15min":0.16700000000000001},"max":{"1min":0.57299999999999995,"5min":95.507999999999996,"15min":253.68100000000001},"last":0.223},{"interface":"front","average":{"1min":0.66800000000000004,"5min":2.8820000000000001,"15min":7.4569999999999999},"min":{"1min":0.222,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.5819999999999999,"5min":95.259,"15min":253.46199999999999},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:54:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.9969999999999999,"5min":7.9969999999999999,"15min":7.9969999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.5049999999999999,"5min":2.5049999999999999,"15min":2.5049999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":13.18,"5min":13.18,"15min":13.18},"last":0.38}]},{"osd":2,"last update":"Sat Sep 23 20:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":1.032,"15min":1.125},"min":{"1min":0.20999999999999999,"5min":0.182,"15min":0.182},"max":{"1min":2.4769999999999999,"5min":12.859,"15min":12.859},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.411,"15min":1.216},"min":{"1min":0.215,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.9849999999999999,"5min":19.050999999999998,"15min":19.050999999999998},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":1.29,"15min":1.5089999999999999},"min":{"1min":0.31,"5min":0.222,"15min":0.222},"max":{"1min":5.851,"5min":10.188000000000001,"15min":10.188000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":1.1120000000000001,"15min":1.696},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.252},"max":{"1min":1.0029999999999999,"5min":10.090999999999999,"15min":16.408999999999999},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.1019999999999999,"15min":1.651},"min":{"1min":0.313,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.9020000000000001,"5min":64.528000000000006,"15min":64.528000000000006},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.89300000000000002,"5min":1.5820000000000001,"15min":1.839},"min":{"1min":0.29299999999999998,"5min":0.249,"15min":0.249},"max":{"1min":5.75,"5min":13.710000000000001,"15min":14.054},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.728,"15min":1.8540000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.7990000000000004,"5min":19.568000000000001,"15min":19.568000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.69199999999999995,"5min":1.667,"15min":2.101},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.5590000000000002,"5min":15.573,"15min":16.451000000000001},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 20:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":4.1470000000000002,"15min":2.504},"min":{"1min":0.35699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.9529999999999998,"5min":215.70099999999999,"15min":215.70099999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.87,"5min":4.2389999999999999,"15min":2.8210000000000002},"min":{"1min":0.41099999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":5.8239999999999998,"5min":222.11699999999999,"15min":222.11699999999999},"last":0.63100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986541,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3035276,"kb_used_data":2596548,"kb_used_omap":0,"kb_used_meta":438720,"kb_avail":90709876,"statfs":{"total":95995035648,"available":92886913024,"internally_reserved":0,"allocated":2658865152,"data_stored":2658087228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":449249280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:16 2023","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":5.3099999999999996,"15min":3.4289999999999998},"min":{"1min":0.25800000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.48299999999999998,"5min":146.25299999999999,"15min":146.25299999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.35999999999999999,"5min":5.3840000000000003,"15min":3.3359999999999999},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23100000000000001},"max":{"1min":0.49299999999999999,"5min":144.77000000000001,"15min":144.77000000000001},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:54:08 2023","interfaces":[{"interface":"back","average":{"1min":6.0999999999999996,"5min":6.0999999999999996,"15min":6.0999999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":75.445999999999998,"5min":75.445999999999998,"15min":75.445999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":6.4199999999999999,"5min":6.4199999999999999,"15min":6.4199999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":71.734999999999999,"5min":71.734999999999999,"15min":71.734999999999999},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:57:16 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":2.867,"15min":1.843},"min":{"1min":0.246,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.60699999999999998,"5min":99.274000000000001,"15min":99.274000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.67700000000000005,"5min":3.202,"15min":2.3959999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":7.242,"5min":99.363,"15min":99.363},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:57:36 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":3.1709999999999998,"15min":5.0499999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.248,"5min":142.28399999999999,"15min":142.28399999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":3.2679999999999998,"15min":4.9500000000000002},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.3119999999999998,"5min":142.34700000000001,"15min":142.34700000000001},"last":1.4079999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":1.8879999999999999,"15min":1.2290000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":2.0819999999999999,"5min":72.304000000000002,"15min":72.304000000000002},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.929,"15min":1.329},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.0800000000000001,"5min":72.290000000000006,"15min":72.290000000000006},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:57:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":1.2549999999999999,"15min":1.202},"min":{"1min":0.30199999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.1549999999999998,"5min":11.887,"15min":16.271000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.435,"5min":1.0209999999999999,"15min":0.995},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67900000000000005,"5min":8.1609999999999996,"15min":11.718},"last":0.31}]},{"osd":7,"last update":"Sat Sep 23 20:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.2330000000000001,"15min":1.4350000000000001},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.5139999999999998,"5min":10.872,"15min":73.268000000000001},"last":0.439},{"interface":"front","average":{"1min":0.46800000000000003,"5min":1.395,"15min":1.4199999999999999},"min":{"1min":0.39400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.63700000000000001,"5min":11.916,"15min":69.760000000000005},"last":0.35499999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724905,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2319032,"kb_used_data":1804656,"kb_used_omap":0,"kb_used_meta":514368,"kb_avail":91426120,"statfs":{"total":95995035648,"available":93620346880,"internally_reserved":0,"allocated":1847967744,"data_stored":1847206020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":526712832},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:57:24 2023","interfaces":[{"interface":"back","average":{"1min":4.0359999999999996,"5min":2.407,"15min":2.3119999999999998},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.19400000000000001},"max":{"1min":75.159999999999997,"5min":80.105999999999995,"15min":124.873},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.984,"5min":2.5430000000000001,"15min":2.3519999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.223},"max":{"1min":75.203999999999994,"5min":80.152000000000001,"15min":123.554},"last":0.39300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:54:10 2023","interfaces":[{"interface":"back","average":{"1min":3.621,"5min":3.621,"15min":3.621},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":20.178000000000001,"5min":20.178000000000001,"15min":20.178000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.823,"5min":2.823,"15min":2.823},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.670999999999999,"5min":19.670999999999999,"15min":19.670999999999999},"last":0.51200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:57:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.075,"15min":1.7869999999999999},"min":{"1min":0.28699999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.5640000000000001,"5min":5.1219999999999999,"15min":13.394},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.3160000000000001,"15min":1.512},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.929,"5min":8.5980000000000008,"15min":14.263},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:57:24 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":3.9489999999999998,"15min":3.6960000000000002},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.5069999999999999,"5min":153.60300000000001,"15min":198.33099999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.77000000000000002,"5min":3.8580000000000001,"15min":3.5590000000000002},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.4800000000000004,"5min":153.774,"15min":198.47300000000001},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.85999999999999999,"15min":0.94499999999999995},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.24099999999999999},"max":{"1min":0.66000000000000003,"5min":5.3230000000000004,"15min":10.906000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.38700000000000001,"5min":1.0740000000000001,"15min":1.1910000000000001},"min":{"1min":0.28599999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.55200000000000005,"5min":12.539,"15min":12.539},"last":0.30599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:57:24 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":1.173,"15min":1.151},"min":{"1min":0.31,"5min":0.28199999999999997,"15min":0.28000000000000003},"max":{"1min":0.67000000000000004,"5min":12.727,"15min":12.727},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.4199999999999999,"15min":1.1739999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":0.64700000000000002,"5min":12.593999999999999,"15min":12.593999999999999},"last":0.35499999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":1.5609999999999999,"15min":1.506},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":0.627,"5min":31.091999999999999,"15min":31.091999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.528,"15min":1.2250000000000001},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":31.780000000000001,"15min":31.780000000000001},"last":0.63900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3119058944,"data_stored":3119054872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":64749568,"data_stored":64749568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1893728256,"data_stored":1893728256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1706033152,"data_stored":1706033152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2653556736,"data_stored":2653552664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1841692672,"data_stored":1841692672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2158628864,"data_stored":2158624792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1237909504,"data_stored":1237909504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:58:04.728 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-09-23T20:58:04.812 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:04.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:04.896 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:58:04] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-09-23T20:58:04.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:04 smithi121 ceph-mon[211526]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:04.919 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:04.948 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:04.970 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:05.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:04 smithi121 ceph-mon[219709]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:05.182 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:05.184 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:04 smithi123 ceph-mon[193903]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:58:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:05.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:05.242 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:58:05.286 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:05.323 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:05.587 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:05.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:05.597 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:05.691 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:05.722 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:05.913 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:58:05.965 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:05.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:05.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:06.069 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:06.095 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:05 smithi123 ceph-mon[193903]: pgmap v875: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14060/226641 objects degraded (6.204%); 847/226641 objects misplaced (0.374%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:06.096 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/105816720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:06.096 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:06.099 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:05 smithi121 ceph-mon[211526]: pgmap v875: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14060/226641 objects degraded (6.204%); 847/226641 objects misplaced (0.374%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/105816720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:05 smithi121 ceph-mon[219709]: pgmap v875: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14060/226641 objects degraded (6.204%); 847/226641 objects misplaced (0.374%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/105816720' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:06.194 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:06.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:06.298 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:06.332 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:06.592 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:06.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:06.696 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:06.725 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:06.772 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:06.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:06.879 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:06.906 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:06 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:06 smithi123 ceph-mon[193903]: failed to encode map e280 with expected crc 2023-09-23T20:58:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:06 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:06 smithi123 ceph-mon[193903]: osdmap e280: 8 total, 8 up, 8 in 2023-09-23T20:58:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:06 smithi123 ceph-mon[193903]: failed to encode map e280 with expected crc 2023-09-23T20:58:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:06 smithi123 ceph-mon[193903]: failed to encode map e280 with expected crc 2023-09-23T20:58:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14060/226641 objects degraded (6.204%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:06.999 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:07.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:07.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:07.103 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:07.133 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:07.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[219709]: failed to encode map e280 with expected crc 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[219709]: osdmap e280: 8 total, 8 up, 8 in 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[219709]: failed to encode map e280 with expected crc 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[219709]: failed to encode map e280 with expected crc 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14060/226641 objects degraded (6.204%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[211526]: failed to encode map e280 with expected crc 2023-09-23T20:58:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[211526]: osdmap e280: 8 total, 8 up, 8 in 2023-09-23T20:58:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[211526]: failed to encode map e280 with expected crc 2023-09-23T20:58:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[211526]: failed to encode map e280 with expected crc 2023-09-23T20:58:07.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14060/226641 objects degraded (6.204%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:07.178 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:07.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:07.227 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:07.227 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:07.283 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:07.317 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:07.435 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:07.544 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:07.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:07.650 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:07.678 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:07.941 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:07 smithi123 ceph-mon[193903]: pgmap v877: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14060/226533 objects degraded (6.207%); 847/226533 objects misplaced (0.374%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:07.961 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:58:07.976 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:07.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:07.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:08.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:07 smithi121 ceph-mon[219709]: pgmap v877: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14060/226533 objects degraded (6.207%); 847/226533 objects misplaced (0.374%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:08.006 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:07 smithi121 ceph-mon[211526]: pgmap v877: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14060/226533 objects degraded (6.207%); 847/226533 objects misplaced (0.374%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:08.080 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:08.109 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:08.153 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:08.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:08.260 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:08.271 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:58:08.294 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:58:08.296 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:08.320 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:58:08.332 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:58:08.347 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:58:08.380 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:58:08.402 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:08.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:08.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:08.409 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:58:08.433 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:58:08.455 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:58:08.456 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.456 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.457 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.457 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.457 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.457 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.457 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.457 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:58:08.505 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:08.538 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:08.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:08.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:08.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:08.698 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:08.725 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:08.970 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:08.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:09.074 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:09.101 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:09.145 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:09.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:09.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/477218214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/477218214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/477218214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:09.250 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:09.277 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:09.429 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:09.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:09.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:09.534 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:58:09.564 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:09.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:09 smithi121 ceph-mon[211526]: pgmap v878: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 14060/225807 objects degraded (6.227%); 847/225807 objects misplaced (0.375%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:09.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:09 smithi121 ceph-mon[219709]: pgmap v878: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 14060/225807 objects degraded (6.227%); 847/225807 objects misplaced (0.375%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136320 9.d 9:b4eaf8c8:::benchmark_data_smithi121_293770_object33432:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214752+0000", 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.74036518299999, 2023-09-23T20:58:09.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.101782247, 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136320 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214752+0000", 2023-09-23T20:58:09.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214752+0000", 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214751+0000", 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214758+0000", 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4969999999999998e-06 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214772+0000", 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.379e-05 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:09.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214778+0000", 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3770000000000001e-06 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214866+0000", 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7578999999999996e-05 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214992+0000", 2023-09-23T20:58:09.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012608199999999999 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215115+0000", 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012328299999999999 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231351+0000", 2023-09-23T20:58:09.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016235991000000002 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231358+0000", 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2620000000000001e-06 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296318+0000", 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0649602810000001 2023-09-23T20:58:09.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296421+0000", 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010260900000000001 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316433+0000", 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020011547000000001 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316535+0000", 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010212700000000001 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136321 9.3 9:cbaf7339:::benchmark_data_smithi121_293770_object33433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214796+0000", 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.74032142599998, 2023-09-23T20:58:09.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.113576339, 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136321 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214796+0000", 2023-09-23T20:58:09.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214796+0000", 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214796+0000", 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214799+0000", 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0259999999999999e-06 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214809+0000", 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0531e-05 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214814+0000", 2023-09-23T20:58:09.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4909999999999999e-06 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214940+0000", 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012664600000000001 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215036+0000", 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5371999999999994e-05 2023-09-23T20:58:09.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215132+0000", 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6465000000000006e-05 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227009+0000", 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011876695 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237869+0000", 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010860071000000001 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237876+0000", 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.063e-06 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.315925+0000", 2023-09-23T20:58:09.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0780493440000001 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316065+0000", 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013960900000000001 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.328296+0000", 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012231103 2023-09-23T20:58:09.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.328373+0000", 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6440999999999995e-05 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136322 9.3 9:c9fc89fe:::benchmark_data_smithi121_293770_object33434:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:58:09.974 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214824+0000", 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.74029366899998, 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.125673253, 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136322 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:09.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214824+0000", 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214824+0000", 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214824+0000", 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214826+0000", 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.728e-06 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214835+0000", 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0850000000000003e-06 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:09.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214841+0000", 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9220000000000002e-06 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.222782+0000", 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079405770000000007 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.222913+0000", 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013101399999999999 2023-09-23T20:58:09.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.223041+0000", 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128493 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237906+0000", 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014864308999999999 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:58:09.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237911+0000", 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3820000000000003e-06 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316294+0000", 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0783833570000001 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316394+0000", 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9423e-05 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340398+0000", 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024004376000000001 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340497+0000", 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8981000000000007e-05 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:09.981 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136324 9.3 9:c6c56bec:::benchmark_data_smithi121_293770_object33436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214846+0000", 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.740272027, 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1260450399999999, 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:09.982 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136324 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214846+0000", 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214846+0000", 2023-09-23T20:58:09.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214845+0000", 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214848+0000", 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3439999999999999e-06 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214853+0000", 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3499999999999996e-06 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214856+0000", 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.568e-06 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227032+0000", 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012175333 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227151+0000", 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119363 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227272+0000", 2023-09-23T20:58:09.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120783 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237950+0000", 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010677657 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237954+0000", 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9249999999999997e-06 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316421+0000", 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0784679429999999 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316514+0000", 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.221e-05 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340813+0000", 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024299313999999999 2023-09-23T20:58:09.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340891+0000", 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7650000000000004e-05 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136326 9.3 9:c6acc5bf:::benchmark_data_smithi121_293770_object33438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214859+0000", 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.74025829499999, 2023-09-23T20:58:09.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.125181303, 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136326 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214859+0000", 2023-09-23T20:58:09.990 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214859+0000", 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214859+0000", 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214861+0000", 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.846e-06 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214864+0000", 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9560000000000002e-06 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214866+0000", 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.17e-06 2023-09-23T20:58:09.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237967+0000", 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023100961999999999 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237970+0000", 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.909e-06 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:09.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316540+0000", 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.078569551 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316636+0000", 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6572999999999998e-05 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3", 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316711+0000", 2023-09-23T20:58:09.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5104000000000005e-05 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.339845+0000", 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023133529 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.339951+0000", 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000106549 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.339984+0000", 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2450999999999998e-05 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340041+0000", 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6918e-05 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136327 9.e 9:7476b4e6:::benchmark_data_smithi121_293770_object33439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214868+0000", 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.74024926700002, 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.094948848, 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136327 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:09.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214868+0000", 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214868+0000", 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214868+0000", 2023-09-23T20:58:09.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214870+0000", 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.699e-06 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214873+0000", 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7690000000000001e-06 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:09.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214875+0000", 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2680000000000001e-06 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214909+0000", 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3696999999999999e-05 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215019+0000", 2023-09-23T20:58:10.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010990800000000001 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215131+0000", 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000112131 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224915+0000", 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0097839450000000005 2023-09-23T20:58:10.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231113+0000", 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.006198856 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231122+0000", 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2479999999999996e-06 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.294619+0000", 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.063497175 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.294749+0000", 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000130407 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.309762+0000", 2023-09-23T20:58:10.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015012942 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.309817+0000", 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5009000000000001e-05 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136329 9.d 9:bf8aaa61:::benchmark_data_smithi121_293770_object33441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214884+0000", 2023-09-23T20:58:10.004 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.74023388699999, 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1040498940000001, 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136329 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214884+0000", 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214884+0000", 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214883+0000", 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214886+0000", 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.17e-06 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214891+0000", 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5779999999999998e-06 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214895+0000", 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9199999999999997e-06 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.219285+0000", 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004389736 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.219408+0000", 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122967 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.219520+0000", 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000111798 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231384+0000", 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011864263 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231389+0000", 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.412e-06 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296450+0000", 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.065061152 2023-09-23T20:58:10.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296538+0000", 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7361000000000006e-05 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.318855+0000", 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022317212999999999 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.318934+0000", 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8712000000000001e-05 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:192458 9.d 9:bca59bae:::benchmark_data_smithi121_293770_object89570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:53.444440+0000", 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 256.510678111, 2023-09-23T20:58:10.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23285765999999999, 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 192458 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444440+0000", 2023-09-23T20:58:10.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444440+0000", 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444439+0000", 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:58:10.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444444+0000", 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0989999999999998e-06 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444455+0000", 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0838e-05 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444460+0000", 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8139999999999997e-06 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444501+0000", 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0816999999999999e-05 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444620+0000", 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011914899999999999 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.444745+0000", 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000124969 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.448515+0000", 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003769938 2023-09-23T20:58:10.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.454510+0000", 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059946610000000001 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.463169+0000", 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0086589900000000001 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.463205+0000", 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5515999999999997e-05 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.677297+0000", 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.21409267200000001 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.019 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:192472 9.d 9:bec7ff72:::benchmark_data_smithi121_293770_object89584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:53.468656+0000", 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 256.48646176900002, 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23253281100000001, 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 192472 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.020 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.468656+0000", 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.468656+0000", 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.021 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.468655+0000", 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.468661+0000", 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1129999999999997e-06 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.677297+0000", 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.208636194 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.677309+0000", 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1588e-05 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.677362+0000", 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3579000000000003e-05 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.677568+0000", 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000206298 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:58:10.024 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.677728+0000", 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015991399999999999 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.696536+0000", 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018808102 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.700847+0000", 2023-09-23T20:58:10.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043103380000000004 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.700952+0000", 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000104986 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.701005+0000", 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.363e-05 2023-09-23T20:58:10.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:53.701189+0000", 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00018329899999999999 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:196328 9.e 9:7ccfe7d0:::benchmark_data_smithi121_293770_object93440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:59.620517+0000", 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 250.33460050299999, 2023-09-23T20:58:10.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23872052199999999, 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 196328 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.620517+0000", 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.620517+0000", 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.620517+0000", 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.620519+0000", 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2560000000000001e-06 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.620524+0000", 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9629999999999997e-06 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.620527+0000", 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.145e-06 2023-09-23T20:58:10.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.633506+0000", 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012979144999999999 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.633632+0000", 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000125656 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:58:10.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.633763+0000", 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013134299999999999 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.638141+0000", 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043781319999999999 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.638460+0000", 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00031842099999999999 2023-09-23T20:58:10.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.858910+0000", 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.22044999300000001 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.858961+0000", 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1366000000000003e-05 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.859238+0000", 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00027646499999999999 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:196331 9.5 e215/164 9:a9740c3c:::benchmark_data_smithi121_293770_object93443:head v 215'12043, mlcod=215'12039)", 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:59.630866+0000", 2023-09-23T20:58:10.034 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 250.32425157500001, 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.22798775499999999, 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.630866+0000", 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.630866+0000", 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.630865+0000", 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.630871+0000", 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0279999999999997e-06 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.630882+0000", 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0464e-05 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.037 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.630887+0000", 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5539999999999997e-06 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.636463+0000", 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055756820000000002 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.636483+0000", 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9553000000000001e-05 2023-09-23T20:58:10.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.858811+0000", 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.22232822599999999 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.858854+0000", 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2911000000000003e-05 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.039 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45101.0:196334 9.a e215/207 9:509b9120:::benchmark_data_smithi121_293770_object93446:head v 215'12223, mlcod=198'8523)", 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:53:59.639395+0000", 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 250.31572315599999, 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.233593523, 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639395+0000", 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639395+0000", 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639394+0000", 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639399+0000", 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6349999999999997e-06 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.042 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639407+0000", 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9729999999999997e-06 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639411+0000", 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.138e-06 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639451+0000", 2023-09-23T20:58:10.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0380000000000003e-05 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.639474+0000", 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2708e-05 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.872955+0000", 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.233481357 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:53:59.872988+0000", 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2895999999999998e-05 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54855 10.1 10:8aabcc1d:::benchmark_data_smithi121_358375_object54854:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.603179+0000", 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 176.35193870200001, 2023-09-23T20:58:10.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25036673199999998, 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54855 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603179+0000", 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603179+0000", 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603178+0000", 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.047 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603235+0000", 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6938999999999998e-05 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603257+0000", 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2316000000000001e-05 2023-09-23T20:58:10.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603266+0000", 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6880000000000005e-06 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603310+0000", 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4462000000000002e-05 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603429+0000", 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118477 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603589+0000", 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016056499999999999 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610291+0000", 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0067014869999999999 2023-09-23T20:58:10.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612288+0000", 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019972969999999999 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853436+0000", 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24114797099999999 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853501+0000", 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4787000000000002e-05 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853546+0000", 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4966999999999998e-05 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.053 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54859 10.c 10:3d9b601b:::benchmark_data_smithi121_358375_object54858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.606114+0000", 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 176.349003552, 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25144838899999999, 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54859 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606114+0000", 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606114+0000", 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606113+0000", 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606167+0000", 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4460999999999999e-05 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606180+0000", 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2628e-05 2023-09-23T20:58:10.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606186+0000", 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2820000000000002e-06 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607593+0000", 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001406869 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607722+0000", 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012907700000000001 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607861+0000", 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013883799999999999 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.611809+0000", 2023-09-23T20:58:10.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039483060000000004 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.615227+0000", 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034176660000000002 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857476+0000", 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24224914 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857525+0000", 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8582999999999999e-05 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857562+0000", 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7622e-05 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54860 10.1 10:891119e1:::benchmark_data_smithi121_358375_object54859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.607403+0000", 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 176.347714454, 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24781665, 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54860 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607403+0000", 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607403+0000", 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607402+0000", 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607448+0000", 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5973000000000002e-05 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607465+0000", 2023-09-23T20:58:10.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6058000000000001e-05 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607472+0000", 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7719999999999994e-06 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607519+0000", 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6366000000000003e-05 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607648+0000", 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000129519 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607791+0000", 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014309699999999999 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610348+0000", 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002556483 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612310+0000", 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019622979999999999 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855142+0000", 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24283174599999999 2023-09-23T20:58:10.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855179+0000", 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7183e-05 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855220+0000", 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0883999999999997e-05 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54864 10.c 10:3c18fb56:::benchmark_data_smithi121_358375_object54863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.607487+0000", 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 176.34763095900001, 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25261208400000001, 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54864 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607487+0000", 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607487+0000", 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607486+0000", 2023-09-23T20:58:10.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607521+0000", 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4876999999999998e-05 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607530+0000", 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9400000000000008e-06 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607536+0000", 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5260000000000001e-06 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609744+0000", 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022083789999999999 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609868+0000", 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000124067 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609971+0000", 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103428 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612926+0000", 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029545740000000002 2023-09-23T20:58:10.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.615266+0000", 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023401440000000002 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860009+0000", 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24474301600000001 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860061+0000", 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1554999999999997e-05 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860099+0000", 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8111999999999997e-05 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.074 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54869 10.1 10:8843e98c:::benchmark_data_smithi121_358375_object54868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.612342+0000", 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 176.342775668, 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24464660699999999, 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.075 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54869 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612342+0000", 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612342+0000", 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612341+0000", 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612398+0000", 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6696000000000001e-05 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.077 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612409+0000", 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1216000000000001e-05 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612413+0000", 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.997e-06 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612982+0000", 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000569375 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613126+0000", 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014301900000000001 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613267+0000", 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000141099 2023-09-23T20:58:10.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.616777+0000", 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035100750000000001 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.617226+0000", 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00044881000000000001 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856896+0000", 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23967075099999999 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856951+0000", 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5024e-05 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856989+0000", 2023-09-23T20:58:10.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7323999999999997e-05 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54872 10.c 10:3107dab5:::benchmark_data_smithi121_358375_object54871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.613034+0000", 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 176.342083739, 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24716744900000001, 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.082 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54872 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613034+0000", 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613034+0000", 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613033+0000", 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613105+0000", 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1774999999999996e-05 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613119+0000", 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4201e-05 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613126+0000", 2023-09-23T20:58:10.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.849e-06 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613171+0000", 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4917999999999999e-05 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613294+0000", 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122571 2023-09-23T20:58:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.613418+0000", 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000124826 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.615521+0000", 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021021270000000001 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.617427+0000", 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019068690000000001 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860133+0000", 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.242705332 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860169+0000", 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6205999999999999e-05 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860201+0000", 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2456999999999998e-05 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.089 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57396 10.1 10:8b26d4cb:::benchmark_data_smithi121_358375_object57395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.694080+0000", 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 174.26103769400001, 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23803892700000001, 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57396 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.090 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694080+0000", 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694080+0000", 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694078+0000", 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694178+0000", 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9501999999999997e-05 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694196+0000", 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7838000000000001e-05 2023-09-23T20:58:10.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694203+0000", 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3980000000000002e-06 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694249+0000", 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6183000000000002e-05 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694410+0000", 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000161056 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694541+0000", 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000130548 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.696782+0000", 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022408760000000002 2023-09-23T20:58:10.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.697093+0000", 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00031133000000000002 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.932027+0000", 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.234933792 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.932079+0000", 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2262999999999998e-05 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.932119+0000", 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9619999999999997e-05 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.096 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:58239 10.1 10:8c649688:::benchmark_data_smithi121_358375_object58238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:16.519453+0000", 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 173.43566445499999, 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.22636798899999999, 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:10.097 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 58239 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519453+0000", 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519453+0000", 2023-09-23T20:58:10.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519452+0000", 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519504+0000", 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2021999999999998e-05 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519524+0000", 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0443000000000001e-05 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519534+0000", 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0336000000000001e-05 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519597+0000", 2023-09-23T20:58:10.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2955999999999998e-05 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519780+0000", 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00018253599999999999 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.519914+0000", 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013410599999999999 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.521060+0000", 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001146406 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.524557+0000", 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034961010000000002 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.745754+0000", 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.22119736300000001 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.745792+0000", 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8359000000000003e-05 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.745821+0000", 2023-09-23T20:58:10.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8954000000000001e-05 2023-09-23T20:58:10.104 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.104 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.104 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.104 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:58:10.104 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:58:10.104 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:58:10.105 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:09 smithi123 ceph-mon[193903]: pgmap v878: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 14060/225807 objects degraded (6.227%); 847/225807 objects misplaced (0.375%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:58:10.445 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:10.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:10.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:10.550 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:10.581 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:10.708 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:58:10.724 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:10.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:10.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:10.832 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:10.859 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:10.876 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:10.902 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:10.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:10.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:11.007 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:11.041 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:11.158 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:11.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:11.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:11.263 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:11.294 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:11.401 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:11.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:11.506 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:11.532 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:11.574 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:11.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:11.679 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:11.710 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:11.761 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:11.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:11.865 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:11.893 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:11.919 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 1 pgp_num to 1 2023-09-23T20:58:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:11.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:11.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:12.045 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:12.073 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[219709]: pgmap v879: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 13649/225288 objects degraded (6.058%); 847/225288 objects misplaced (0.376%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T20:58:12.074 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3142182346' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:58:12.074 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13649/225288 objects degraded (6.058%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:12.074 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:58:12.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[211526]: pgmap v879: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 13649/225288 objects degraded (6.058%); 847/225288 objects misplaced (0.376%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T20:58:12.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3142182346' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:58:12.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13649/225288 objects degraded (6.058%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:12.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:58:12.076 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:12.137 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:12.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:12.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:11 smithi123 ceph-mon[193903]: pgmap v879: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 13649/225288 objects degraded (6.058%); 847/225288 objects misplaced (0.376%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T20:58:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3142182346' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:58:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13649/225288 objects degraded (6.058%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T20:58:12.241 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:12.266 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:12.308 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:12.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:12.415 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:12.448 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:12.555 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: osdmap e281: 8 total, 8 up, 8 in 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[211526]: pgmap v881: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 13649/225288 objects degraded (6.058%); 847/225288 objects misplaced (0.376%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: osdmap e281: 8 total, 8 up, 8 in 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:12 smithi121 ceph-mon[219709]: pgmap v881: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 13649/225288 objects degraded (6.058%); 847/225288 objects misplaced (0.376%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:58:13.206 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T20:58:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: osdmap e281: 8 total, 8 up, 8 in 2023-09-23T20:58:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: failed to encode map e281 with expected crc 2023-09-23T20:58:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:12 smithi123 ceph-mon[193903]: pgmap v881: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 13649/225288 objects degraded (6.058%); 847/225288 objects misplaced (0.376%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T20:58:13.309 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:13.336 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:13.377 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:13.483 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:13.510 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:13.558 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:13.663 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:13.697 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:13.712 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:13.745 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:13.937 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:13.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:13.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:14.042 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:14.068 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:14.110 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:14.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:14.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:14.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:14.249 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:14.431 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:14.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:14.536 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:14.574 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:14.670 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:58:14.840 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3273892909' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:14.840 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:14 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:14.840 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:14 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:14.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3273892909' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:14.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:14 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:14.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:14 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:14.841 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:58:14] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-09-23T20:58:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3273892909' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:14 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:14.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:14 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:14.996 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:15.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:15.101 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:15.130 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:15.636 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:15.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:15.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:15.739 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:15.773 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[211526]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[211526]: osdmap e282: 8 total, 8 up, 8 in 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[211526]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[211526]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[211526]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[211526]: pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 13212/224043 objects degraded (5.897%); 847/224043 objects misplaced (0.378%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:58:15.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[219709]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[219709]: osdmap e282: 8 total, 8 up, 8 in 2023-09-23T20:58:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[219709]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[219709]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[219709]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:15 smithi121 ceph-mon[219709]: pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 13212/224043 objects degraded (5.897%); 847/224043 objects misplaced (0.378%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:58:15.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:15 smithi123 ceph-mon[193903]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:15 smithi123 ceph-mon[193903]: osdmap e282: 8 total, 8 up, 8 in 2023-09-23T20:58:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:15 smithi123 ceph-mon[193903]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:15 smithi123 ceph-mon[193903]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:15 smithi123 ceph-mon[193903]: failed to encode map e282 with expected crc 2023-09-23T20:58:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:15 smithi123 ceph-mon[193903]: pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 13212/224043 objects degraded (5.897%); 847/224043 objects misplaced (0.378%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:58:16.143 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:16.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:16.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:16.247 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:16.276 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:16.734 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:16.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:16.838 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:16.868 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:16.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13212/224043 objects degraded (5.897%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:16.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13212/224043 objects degraded (5.897%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:16.916 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:16.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:16.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:16.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13212/224043 objects degraded (5.897%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:58:17.023 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:17.053 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.441 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:17.546 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:17.571 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.614 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:17.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:17.721 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:17.748 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.798 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:17.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:17.903 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:17.918 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:58:17.945 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:17 smithi121 ceph-mon[219709]: pgmap v884: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 13212/223941 objects degraded (5.900%); 847/223941 objects misplaced (0.378%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T20:58:17.945 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:17 smithi121 ceph-mon[211526]: pgmap v884: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 13212/223941 objects degraded (5.900%); 847/223941 objects misplaced (0.378%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T20:58:17.945 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:58:17.950 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:17.975 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:58:17.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:17 smithi123 ceph-mon[193903]: pgmap v884: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 13212/223941 objects degraded (5.900%); 847/223941 objects misplaced (0.378%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T20:58:18.017 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:58:18.039 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:18.044 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:58:18.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:18.049 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:18.049 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:18.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:58:18.097 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:58:18.118 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.122 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:58:18.146 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:18.178 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.285 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:18.389 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:18.415 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.461 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:18.564 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:18.592 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.637 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:58:18.637 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:58:18.637 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:58:18.637 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T20:58:18.645 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:18.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:18.751 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:18.780 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:18 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:58:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/686935738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:18.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:18 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:58:18.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/686935738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:18.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:18 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:58:18.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/686935738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:19.449 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:58:19.473 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:19.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:19.577 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:19.604 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:19.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:19 smithi121 ceph-mon[219709]: pgmap v885: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 89 op/s; 13212/223146 objects degraded (5.921%); 847/223146 objects misplaced (0.380%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T20:58:19.974 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:19 smithi121 ceph-mon[211526]: pgmap v885: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 89 op/s; 13212/223146 objects degraded (5.921%); 847/223146 objects misplaced (0.380%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T20:58:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:19 smithi123 ceph-mon[193903]: pgmap v885: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 89 op/s; 13212/223146 objects degraded (5.921%); 847/223146 objects misplaced (0.380%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T20:58:20.003 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:58:20.057 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:20.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:20.122 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:20.165 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:20.194 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:20.215 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:20.240 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:20.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:20.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:20.343 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:20.383 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:20.440 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:20.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:20.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:20.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:20.591 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:21.258 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:21.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:21.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:21.364 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:21.404 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:21.669 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:21.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:21.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:21.772 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:21.804 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:22.064 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:58:22.099 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[211526]: pgmap v886: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 12746/222729 objects degraded (5.723%); 847/222729 objects misplaced (0.380%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:58:22.099 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:22.099 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:22.099 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12746/222729 objects degraded (5.723%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:22.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[219709]: pgmap v886: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 12746/222729 objects degraded (5.723%); 847/222729 objects misplaced (0.380%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:58:22.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:22.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:22.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12746/222729 objects degraded (5.723%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:21 smithi123 ceph-mon[193903]: pgmap v886: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 12746/222729 objects degraded (5.723%); 847/222729 objects misplaced (0.380%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:58:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12746/222729 objects degraded (5.723%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:22.677 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:22.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:22.781 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:22.810 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.041 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:23.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:23.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:23.175 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.226 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:23.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:23.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:23.276 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:23.336 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:23.362 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.406 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:23.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:23.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:23.513 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:23.552 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.664 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:23.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:23.769 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:23.807 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:23.854 INFO:teuthology.orchestra.run.smithi121.stdout:moved 3 / 267 (1.1236%) 2023-09-23T20:58:23.854 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 5.56636 -> 4.89738 (expected baseline 5.40399) 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.2 with 23 -> 24 pgs (0.689139 -> 0.719101 * mean) 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.4 with 41 -> 39 pgs (1.22846 -> 1.16854 * mean) 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0231 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0254 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 1.0000 -> 0.9500 2023-09-23T20:58:23.855 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 1.0000 -> 0.9500 2023-09-23T20:58:23.864 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T20:58:23.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:23 smithi121 ceph-mon[219709]: pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 12746/222627 objects degraded (5.725%); 847/222627 objects misplaced (0.380%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:58:23.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:23 smithi121 conmon[198376]: 2023-09-23T20:58:23.850+0000 7f479dcb1700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T20:58:23.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:23 smithi121 ceph-mon[211526]: pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 12746/222627 objects degraded (5.725%); 847/222627 objects misplaced (0.380%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:58:24.105 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:24.159 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:24.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:23 smithi123 ceph-mon[193903]: pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 12746/222627 objects degraded (5.725%); 847/222627 objects misplaced (0.380%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:58:24.266 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:24.316 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:24.897 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:58:25.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:58:24] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-09-23T20:58:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:24 smithi121 ceph-mon[211526]: from='client.35757 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/22229103' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:25.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:24 smithi121 ceph-mon[219709]: from='client.35757 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:25.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/22229103' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:25.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:24 smithi123 ceph-mon[193903]: from='client.35757 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:25.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/22229103' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:25.473 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:25.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:25.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:25.578 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:25.606 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:25.647 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:25.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:25.754 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:25.780 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:25.827 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:25.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:25.895 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: pgmap v888: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 12219/220710 objects degraded (5.536%); 847/220710 objects misplaced (0.384%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:58:25.895 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:25.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/22229103' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:25.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: osdmap e283: 8 total, 8 up, 8 in 2023-09-23T20:58:25.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: failed to encode map e283 with expected crc 2023-09-23T20:58:25.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: failed to encode map e283 with expected crc 2023-09-23T20:58:25.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: failed to encode map e283 with expected crc 2023-09-23T20:58:25.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[211526]: failed to encode map e283 with expected crc 2023-09-23T20:58:25.933 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:25.969 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: pgmap v888: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 12219/220710 objects degraded (5.536%); 847/220710 objects misplaced (0.384%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/22229103' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: osdmap e283: 8 total, 8 up, 8 in 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: failed to encode map e283 with expected crc 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: failed to encode map e283 with expected crc 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: failed to encode map e283 with expected crc 2023-09-23T20:58:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:25 smithi121 ceph-mon[219709]: failed to encode map e283 with expected crc 2023-09-23T20:58:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: pgmap v888: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 12219/220710 objects degraded (5.536%); 847/220710 objects misplaced (0.384%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:58:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/22229103' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: osdmap e283: 8 total, 8 up, 8 in 2023-09-23T20:58:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: failed to encode map e283 with expected crc 2023-09-23T20:58:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: failed to encode map e283 with expected crc 2023-09-23T20:58:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: failed to encode map e283 with expected crc 2023-09-23T20:58:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:25 smithi123 ceph-mon[193903]: failed to encode map e283 with expected crc 2023-09-23T20:58:27.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12219/220710 objects degraded (5.536%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:27.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12219/220710 objects degraded (5.536%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:27.191 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T20:58:27.191 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T20:58:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12219/220710 objects degraded (5.536%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:27.359 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:27.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:27.468 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:27.494 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:27.536 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:27.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:27.641 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:27.668 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:27.710 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:27.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:27.815 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:27.840 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:27.881 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:27.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:27.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:27.985 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:28.013 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:28.059 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:28.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:27 smithi121 ceph-mon[211526]: pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 12219/220608 objects degraded (5.539%); 847/220608 objects misplaced (0.384%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:58:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:28.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:27 smithi121 ceph-mon[219709]: pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 12219/220608 objects degraded (5.539%); 847/220608 objects misplaced (0.384%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:58:28.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:28.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:28.165 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:27 smithi123 ceph-mon[193903]: pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 12219/220608 objects degraded (5.539%); 847/220608 objects misplaced (0.384%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:58:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:28.191 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:28.232 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:28.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:28.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:28.337 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:28.363 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:28.406 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:28.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:28.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:28.514 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:28.549 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:28.712 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:28.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:28.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:28.845 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:28.889 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:28.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:28.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:28.996 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:29.032 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:29.071 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:29.077 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:29.077 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:29.209 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:29.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:29.316 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:29.357 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:29.397 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:58:29.548 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:58:29.576 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:58:29.597 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:58:29.625 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:58:29.646 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:58:29.667 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:58:29.691 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:58:29.724 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:58:29.731 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.731 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.732 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.732 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.732 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.732 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.732 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.732 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:58:29.971 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:29.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:30.076 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:30.103 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:30.148 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:30.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:29 smithi123 ceph-mon[193903]: pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 12219/219594 objects degraded (5.564%); 847/219594 objects misplaced (0.386%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:58:30.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1443598929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:30.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:30.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:30.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:29 smithi121 ceph-mon[211526]: pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 12219/219594 objects degraded (5.564%); 847/219594 objects misplaced (0.386%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:58:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1443598929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:29 smithi121 ceph-mon[219709]: pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 12219/219594 objects degraded (5.564%); 847/219594 objects misplaced (0.386%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T20:58:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1443598929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:30.256 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:30.289 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:30.958 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:30.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:31.063 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:31.089 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:31.108 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:58:31.130 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:31.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:31.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:31.236 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:31.280 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:31 smithi121 ceph-mon[211526]: pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 11544/219078 objects degraded (5.269%); 847/219078 objects misplaced (0.387%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:58:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11544/219078 objects degraded (5.269%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:32.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:31 smithi121 ceph-mon[219709]: pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 11544/219078 objects degraded (5.269%); 847/219078 objects misplaced (0.387%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:58:32.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11544/219078 objects degraded (5.269%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:32.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:31 smithi123 ceph-mon[193903]: pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 11544/219078 objects degraded (5.269%); 847/219078 objects misplaced (0.387%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:58:32.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11544/219078 objects degraded (5.269%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:32.253 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:32.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:32.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:32.358 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:32.387 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:32.432 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:32.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:32.539 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:32.566 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:32.611 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:32.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:32.716 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:32.747 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.303 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:33.407 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:33.433 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.475 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:33.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:33.579 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:33.605 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.646 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:33.754 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:33.791 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:33.822 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:33.828 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:33.970 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:33.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:33.982 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:33 smithi121 ceph-mon[211526]: pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 11544/218967 objects degraded (5.272%); 847/218967 objects misplaced (0.387%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T20:58:33.982 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:33 smithi121 ceph-mon[219709]: pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 11544/218967 objects degraded (5.272%); 847/218967 objects misplaced (0.387%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T20:58:34.083 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:34.108 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:34.150 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:34.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:33 smithi123 ceph-mon[193903]: pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 11544/218967 objects degraded (5.272%); 847/218967 objects misplaced (0.387%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T20:58:34.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:34.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:34.257 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:34.284 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:34.324 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:34.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:34.429 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:34.463 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:34.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:58:34] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-09-23T20:58:35.255 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:58:35.320 INFO:teuthology.orchestra.run.smithi121.stdout:moved 24 / 267 (8.98876%) 2023-09-23T20:58:35.320 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T20:58:35.320 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 5.56636 -> 6.89089 (expected baseline 5.40399) 2023-09-23T20:58:35.320 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.2 with 23 -> 28 pgs (0.689139 -> 0.838951 * mean) 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.4 with 41 -> 33 pgs (1.22846 -> 0.988764 * mean) 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0227 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0250 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 1.0000 -> 0.6054 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 1.0000 -> 0.7390 2023-09-23T20:58:35.321 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 1.0000 -> 0.8927 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='client.55798 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]: dispatch 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]: dispatch 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 10689/217491 objects degraded (4.915%); 847/217491 objects misplaced (0.389%); 10 MiB/s, 159 objects/s recovering 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:58:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='client.55798 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]: dispatch 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]: dispatch 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 10689/217491 objects degraded (4.915%); 847/217491 objects misplaced (0.389%); 10 MiB/s, 159 objects/s recovering 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:58:35.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='client.55798 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]: dispatch 2023-09-23T20:58:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]: dispatch 2023-09-23T20:58:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 10689/217491 objects degraded (4.915%); 847/217491 objects misplaced (0.389%); 10 MiB/s, 159 objects/s recovering 2023-09-23T20:58:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:58:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:58:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:35.889 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:35.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:35.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:35.993 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:36.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:36.063 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:36.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:36.169 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:36.195 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:36.239 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:36.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:36.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:36.350 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:36.375 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:36.417 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:36.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:36.522 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:36.552 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]': finished 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: osdmap e284: 8 total, 8 up, 8 in 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10689/217491 objects degraded (4.915%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[211526]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]': finished 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: osdmap e284: 8 total, 8 up, 8 in 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10689/217491 objects degraded (4.915%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:36 smithi121 ceph-mon[219709]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T20:58:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'39674','4':'48431','6':'58506'}"}]': finished 2023-09-23T20:58:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: osdmap e284: 8 total, 8 up, 8 in 2023-09-23T20:58:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: failed to encode map e284 with expected crc 2023-09-23T20:58:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10689/217491 objects degraded (4.915%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:36 smithi123 ceph-mon[193903]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T20:58:37.061 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:37.169 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:37.199 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:37.418 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:37.425 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:58:37.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T20:58:37.524 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:37.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[219709]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[219709]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[219709]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[219709]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[219709]: pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 10689/217374 objects degraded (4.917%); 847/217374 objects misplaced (0.390%); 13 MiB/s, 200 objects/s recovering 2023-09-23T20:58:37.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[219709]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T20:58:37.529 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[211526]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.529 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[211526]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.529 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[211526]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.530 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[211526]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.530 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[211526]: pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 10689/217374 objects degraded (4.917%); 847/217374 objects misplaced (0.390%); 13 MiB/s, 200 objects/s recovering 2023-09-23T20:58:37.530 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:37 smithi121 ceph-mon[211526]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T20:58:37.554 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:37 smithi123 ceph-mon[193903]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:37 smithi123 ceph-mon[193903]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:37 smithi123 ceph-mon[193903]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:37 smithi123 ceph-mon[193903]: failed to encode map e285 with expected crc 2023-09-23T20:58:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:37 smithi123 ceph-mon[193903]: pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 10689/217374 objects degraded (4.917%); 847/217374 objects misplaced (0.390%); 13 MiB/s, 200 objects/s recovering 2023-09-23T20:58:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:37 smithi123 ceph-mon[193903]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T20:58:37.882 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:37.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:37.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:37.995 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:38.012 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:38.073 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:38.191 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:38.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:38.294 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:91726 10.2 10.67783df2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:43.511418+0000", 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 54.780883162000002, 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73011041099999996, 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:38.295 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 91726 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511418+0000", 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511418+0000", 2023-09-23T20:58:38.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511416+0000", 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511422+0000", 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9599999999999997e-06 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511432+0000", 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8719999999999997e-06 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511441+0000", 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2909999999999996e-06 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511493+0000", 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1400000000000003e-05 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511502+0000", 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.38e-06 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241515+0000", 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73001329000000004 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241528+0000", 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2694000000000001e-05 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:91739 10.2 10.99674552 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:43.571729+0000", 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 54.720571995, 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.66985531499999995, 2023-09-23T20:58:38.300 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 91739 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571729+0000", 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571729+0000", 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571727+0000", 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571737+0000", 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6770000000000007e-06 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571753+0000", 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6886999999999999e-05 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571762+0000", 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1710000000000002e-06 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571815+0000", 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3412999999999998e-05 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.571825+0000", 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9389999999999992e-06 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241572+0000", 2023-09-23T20:58:38.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.66974718899999997 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241584+0000", 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1958e-05 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.305 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:91743 10.2 10.7495c2a2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:43.772198+0000", 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 54.520103094, 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.46941829600000001, 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 91743 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.306 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772198+0000", 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772198+0000", 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772196+0000", 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772206+0000", 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0117e-05 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772228+0000", 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2305000000000001e-05 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772242+0000", 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3517e-05 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772311+0000", 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8971000000000001e-05 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.772320+0000", 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7710000000000001e-06 2023-09-23T20:58:38.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241606+0000", 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.469285642 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241616+0000", 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.91e-06 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.310 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:91746 10.2 10.e284b7a2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:43.801540+0000", 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 54.490760928, 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.44010749199999999, 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:38.311 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 91746 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801540+0000", 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801540+0000", 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.312 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801538+0000", 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801547+0000", 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.276e-06 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.313 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801564+0000", 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7722000000000001e-05 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801572+0000", 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9810000000000003e-06 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801628+0000", 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6100999999999997e-05 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.801639+0000", 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241637+0000", 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43999849499999999 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.315 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241647+0000", 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0042e-05 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91757 10.b e278/277 10:d19b9748:::benchmark_data_smithi121_358375_object7012:head v 278'5695, mlcod=0'0)", 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.478329+0000", 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.813972227000001, 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036032109, 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.316 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.478329+0000", 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.478329+0000", 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.478328+0000", 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.478339+0000", 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1175000000000001e-05 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.478347+0000", 2023-09-23T20:58:38.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1740000000000003e-06 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.478352+0000", 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2240000000000003e-06 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484565+0000", 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062131119999999998 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484590+0000", 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4660000000000001e-05 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514320+0000", 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.029730543000000002 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514361+0000", 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.053e-05 2023-09-23T20:58:38.320 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91772 10.b e278/277 10:dd8ac2c0:::benchmark_data_smithi121_358375_object7027:head v 278'5696, mlcod=0'0)", 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.479140+0000", 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.813161172000001, 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.035240753, 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.479140+0000", 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.479140+0000", 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.479139+0000", 2023-09-23T20:58:38.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.479145+0000", 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9259999999999997e-06 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.479153+0000", 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0339999999999997e-06 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.479160+0000", 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3560000000000004e-06 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.490589+0000", 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011428535 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.490611+0000", 2023-09-23T20:58:38.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2685e-05 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514364+0000", 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.023752115000000001 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514380+0000", 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6727000000000001e-05 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.325 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91783 10.b e278/277 10:d4ea2e7c:::benchmark_data_smithi121_358375_object7038:head v 278'5697, mlcod=0'0)", 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.481697+0000", 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.810604241, 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032708596, 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481697+0000", 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481697+0000", 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481695+0000", 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:38.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481707+0000", 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1199999999999999e-05 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481723+0000", 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6053999999999999e-05 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481732+0000", 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3009999999999995e-06 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.496343+0000", 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01461081 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.496371+0000", 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8237000000000001e-05 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514382+0000", 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018011479 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514405+0000", 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2648999999999999e-05 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.330 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91755 10.0 e278/277 10:092d1f91:::benchmark_data_smithi121_358375_object7010:head v 278'5646, mlcod=0'0)", 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.481935+0000", 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.810366144, 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.048884190000000001, 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481935+0000", 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.331 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481935+0000", 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481934+0000", 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481939+0000", 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4900000000000002e-06 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481948+0000", 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0569999999999999e-06 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.481953+0000", 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7369999999999997e-06 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.489621+0000", 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076684099999999996 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.489643+0000", 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1977e-05 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530776+0000", 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.041133518000000001 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530819+0000", 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2385000000000002e-05 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91761 10.0 e278/277 10:024e028c:::benchmark_data_smithi121_358375_object7016:head v 278'5647, mlcod=0'0)", 2023-09-23T20:58:38.335 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.484133+0000", 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.808167365000003, 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.046707033000000002, 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484133+0000", 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484133+0000", 2023-09-23T20:58:38.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484132+0000", 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484140+0000", 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8439999999999998e-06 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.337 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484150+0000", 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6390000000000004e-06 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.484159+0000", 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9490000000000002e-06 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.495221+0000", 2023-09-23T20:58:38.338 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011061843 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.495245+0000", 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4593e-05 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530822+0000", 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.035576767000000002 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530840+0000", 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8369999999999999e-05 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91804 10.b e278/277 10:d0307335:::benchmark_data_smithi121_358375_object7059:head v 278'5698, mlcod=0'0)", 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.486260+0000", 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.806040643000003, 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.028162513, 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.340 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486260+0000", 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486260+0000", 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486259+0000", 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486287+0000", 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7789000000000001e-05 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486302+0000", 2023-09-23T20:58:38.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4684999999999999e-05 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486310+0000", 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7839999999999992e-06 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.502532+0000", 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016221124999999999 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.502557+0000", 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5357000000000001e-05 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514407+0000", 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011850368 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.514423+0000", 2023-09-23T20:58:38.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5347999999999999e-05 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91778 10.0 e278/277 10:0036b122:::benchmark_data_smithi121_358375_object7033:head v 278'5648, mlcod=0'0)", 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.486601+0000", 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.805699992000001, 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044257089999999999, 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.345 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486601+0000", 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486601+0000", 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.346 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486600+0000", 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486605+0000", 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2689999999999999e-06 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486615+0000", 2023-09-23T20:58:38.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8239999999999995e-06 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.486623+0000", 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3390000000000004e-06 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503081+0000", 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016458675999999998 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503105+0000", 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.353e-05 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530842+0000", 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027737355000000002 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530858+0000", 2023-09-23T20:58:38.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5770999999999999e-05 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91816 10.0 e278/277 10:00ca8f7d:::benchmark_data_smithi121_358375_object7071:head v 278'5649, mlcod=230'439)", 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.503096+0000", 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.789204251999998, 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027780371000000002, 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503096+0000", 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503096+0000", 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503095+0000", 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503123+0000", 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7932e-05 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503136+0000", 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3448999999999999e-05 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.503149+0000", 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3271000000000001e-05 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.513591+0000", 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010441160999999999 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.353 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.513622+0000", 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1411000000000001e-05 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530862+0000", 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017239701999999999 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530877+0000", 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5145e-05 2023-09-23T20:58:38.354 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91822 10.0 e278/277 10:0cb88a64:::benchmark_data_smithi121_358375_object7077:head v 278'5650, mlcod=230'439)", 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.508289+0000", 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.784011431000003, 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022605648999999998, 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.508289+0000", 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.508289+0000", 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.508288+0000", 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:38.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.508296+0000", 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7619999999999995e-06 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.508309+0000", 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2446000000000001e-05 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.508317+0000", 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6680000000000007e-06 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.519095+0000", 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010777999 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.519130+0000", 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4638999999999998e-05 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530880+0000", 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011749707999999999 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.530895+0000", 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5254e-05 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.359 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:91921 10.b e278/277 10:dfc67eea:::benchmark_data_smithi121_358375_object7176:head v 278'5711, mlcod=230'413)", 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:45.655091+0000", 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.637210179999997, 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022740514, 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.360 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655091+0000", 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655091+0000", 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655090+0000", 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:38.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655100+0000", 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0947e-05 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655117+0000", 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6560000000000001e-05 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655128+0000", 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0753e-05 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655174+0000", 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5896000000000002e-05 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.655209+0000", 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5756999999999997e-05 2023-09-23T20:58:38.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.677798+0000", 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022588794999999998 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:45.677831+0000", 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2840999999999999e-05 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.364 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(10.2 283/277 [PushOp(10:48c5589d:::benchmark_data_smithi121_358375_object70098:head, version: 236'4424, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:48c5589d:::benchmark_data_smithi121_358375_object70098:head@236'4424, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:30.473843+0000", 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 7.8184581069999997, 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024895473000000001, 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473843+0000", 2023-09-23T20:58:38.365 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473843+0000", 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473841+0000", 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:38.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473880+0000", 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8698000000000001e-05 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473905+0000", 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5369999999999999e-05 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.367 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473923+0000", 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7059999999999999e-05 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.474026+0000", 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010315999999999999 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.474040+0000", 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3844e-05 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.498738+0000", 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024698563999999999 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(10.2 283/277 [PushOp(10:4041dab0:::benchmark_data_smithi121_358375_object73782:head, version: 236'4651, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:4041dab0:::benchmark_data_smithi121_358375_object73782:head@236'4651, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:31.813976+0000", 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 6.4783251789999996, 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.051674484, 2023-09-23T20:58:38.369 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.813976+0000", 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.813976+0000", 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.370 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.813974+0000", 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.814007+0000", 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3179000000000002e-05 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.814027+0000", 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9590999999999999e-05 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.814043+0000", 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5877000000000001e-05 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.863396+0000", 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.049352683000000001 2023-09-23T20:58:38.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.863424+0000", 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8506000000000001e-05 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:31.865650+0000", 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022259300000000001 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGRecoveryDelete(10.2 e284,277 10:499e3bf8:::benchmark_data_smithi121_358375_object2759:head,252'5452)", 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:35.401326+0000", 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 2.8909744289999999, 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15688563799999999, 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.374 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401326+0000", 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401326+0000", 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401326+0000", 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401330+0000", 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5590000000000001e-06 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401338+0000", 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3930000000000008e-06 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401347+0000", 2023-09-23T20:58:38.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0599999999999997e-06 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.558152+0000", 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15680481399999999 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.558162+0000", 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7510000000000003e-06 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.558212+0000", 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0253999999999997e-05 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:97172 10.b e284/277 10:d101171c:::benchmark_data_smithi121_358375_object12427:head v 284'6024, mlcod=230'413)", 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:35.401436+0000", 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 2.8908648690000001, 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16232443099999999, 2023-09-23T20:58:38.378 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401436+0000", 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401436+0000", 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401436+0000", 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401441+0000", 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9529999999999998e-06 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401447+0000", 2023-09-23T20:58:38.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.737e-06 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401454+0000", 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7460000000000003e-06 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.401960+0000", 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050598600000000004 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.557021+0000", 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.155060701 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.563710+0000", 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0066896079999999997 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:35.563760+0000", 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0009999999999997e-05 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:97235 10.4 e286/285 10:2b4bd522:::benchmark_data_smithi121_358375_object12490:head v 286'6132, mlcod=0'0)", 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:37.483009+0000", 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.80929157399999996, 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.030890129999999998, 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.383 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.483009+0000", 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.483009+0000", 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.483008+0000", 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.483016+0000", 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5120000000000002e-06 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.483032+0000", 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6467999999999999e-05 2023-09-23T20:58:38.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.483041+0000", 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5299999999999996e-06 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.485339+0000", 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022987429999999998 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.485370+0000", 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0405000000000001e-05 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.513841+0000", 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.028471462999999999 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.513899+0000", 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7985999999999997e-05 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.387 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:97236 10.4 e286/285 10:22a0d6a9:::benchmark_data_smithi121_358375_object12491:head v 286'6133, mlcod=231'811)", 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:37.489711+0000", 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.80259012399999996, 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024216330000000001, 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:38.388 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.489711+0000", 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.489711+0000", 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.489709+0000", 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:38.389 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.489720+0000", 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0859000000000001e-05 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.489735+0000", 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4567e-05 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.390 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.489743+0000", 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1170000000000004e-06 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.491961+0000", 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022185099999999999 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.492000+0000", 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8957e-05 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.391 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.513906+0000", 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.021905974000000002 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:37.513927+0000", 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0692000000000001e-05 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:38.392 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:38.393 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:58:38.393 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:38.427 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[211526]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[211526]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[211526]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[219709]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[219709]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[219709]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:38 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:58:38.530 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:38.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:38.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:38.633 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:38.664 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:38 smithi123 ceph-mon[193903]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:38 smithi123 ceph-mon[193903]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:38 smithi123 ceph-mon[193903]: failed to encode map e286 with expected crc 2023-09-23T20:58:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:38 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:58:38.809 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:38.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:38.917 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:38.945 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:38.988 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:38.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:38.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:39.083 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:39.094 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:39.120 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:39.177 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:39.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:39.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:39.282 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:39.321 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:39.360 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:58:39.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3771224898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:39.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:39.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:39 smithi121 ceph-mon[211526]: pgmap v899: 89 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 26308/216420 objects degraded (12.156%); 847/216420 objects misplaced (0.391%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T20:58:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3771224898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:39 smithi121 ceph-mon[219709]: pgmap v899: 89 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 26308/216420 objects degraded (12.156%); 847/216420 objects misplaced (0.391%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T20:58:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3771224898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:58:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:39 smithi123 ceph-mon[193903]: pgmap v899: 89 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 26308/216420 objects degraded (12.156%); 847/216420 objects misplaced (0.391%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T20:58:40.047 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:40.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:40.135 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:58:40.151 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:40.205 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[219709]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:40 smithi121 ceph-mon[211526]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.516 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:40.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:40.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:40.621 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:40.657 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:40.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:40.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:58:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T20:58:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: failed to encode map e287 with expected crc 2023-09-23T20:58:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:40 smithi123 ceph-mon[193903]: failed to encode map e287 with expected crc 2023-09-23T20:58:41.423 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:41.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:41.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:41.527 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:41.553 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:41.594 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:41.598 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:41 smithi123 ceph-mon[193903]: pgmap v901: 89 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 26219/215979 objects degraded (12.140%); 847/215979 objects misplaced (0.392%); 3.0 MiB/s, 110 objects/s recovering 2023-09-23T20:58:41.598 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 26308/216420 objects degraded (12.156%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:41.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:41 smithi121 ceph-mon[219709]: pgmap v901: 89 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 26219/215979 objects degraded (12.140%); 847/215979 objects misplaced (0.392%); 3.0 MiB/s, 110 objects/s recovering 2023-09-23T20:58:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 26308/216420 objects degraded (12.156%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:41 smithi121 ceph-mon[211526]: pgmap v901: 89 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 unknown, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 26219/215979 objects degraded (12.140%); 847/215979 objects misplaced (0.392%); 3.0 MiB/s, 110 objects/s recovering 2023-09-23T20:58:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 26308/216420 objects degraded (12.156%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:41.700 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:41.725 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:41.735 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:41.765 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:58:41.767 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:41.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:41.785 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:58:41.805 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:58:41.826 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:58:41.850 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:58:41.871 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:58:41.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:41.890 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:58:41.901 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:41.913 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.914 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:58:41.943 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:41.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:42.050 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:42.084 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:42.294 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:42.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 267 (0%) 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.89089 -> 6.89089 (expected baseline 5.40399) 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 21 -> 21 pgs (0.629214 -> 0.629214 * mean) 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 44 -> 44 pgs (1.31835 -> 1.31835 * mean) 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T20:58:42.325 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T20:58:42.326 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T20:58:42.326 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0225 2023-09-23T20:58:42.326 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.2248 2023-09-23T20:58:42.330 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T20:58:42.402 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:42.445 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:42.589 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:58:42.599 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:42.604 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1723383840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:42.604 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1723383840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:42.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:42.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:42.706 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1723383840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:42.734 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:43.408 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[219709]: from='client.45758 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:43.408 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:43.408 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:43.408 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[219709]: pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 84 op/s; 31539/215694 objects degraded (14.622%); 847/215694 objects misplaced (0.393%); 2.5 MiB/s, 92 objects/s recovering 2023-09-23T20:58:43.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[211526]: from='client.45758 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:43 smithi121 ceph-mon[211526]: pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 84 op/s; 31539/215694 objects degraded (14.622%); 847/215694 objects misplaced (0.393%); 2.5 MiB/s, 92 objects/s recovering 2023-09-23T20:58:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:43 smithi123 ceph-mon[193903]: from='client.45758 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:43 smithi123 ceph-mon[193903]: pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 84 op/s; 31539/215694 objects degraded (14.622%); 847/215694 objects misplaced (0.393%); 2.5 MiB/s, 92 objects/s recovering 2023-09-23T20:58:43.949 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:43.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:43.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:43.955 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:58:43.955 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T20:58:44.054 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:44.086 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.195 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:44.226 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:44.303 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:44.338 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.540 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:44.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:44.645 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:44.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:44 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:58:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:44 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:58:44.664 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:44.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:44 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:58:44.671 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.699 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:58:44.699 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:58:44.712 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:44.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:44.817 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:44.844 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:44.887 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:44.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:44.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:45.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:45.025 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:58:44] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.43.0" 2023-09-23T20:58:45.061 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:45.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:45.168 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:45.205 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:45.663 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:58:45.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:45 smithi121 ceph-mon[211526]: pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 31461/213660 objects degraded (14.725%); 847/213660 objects misplaced (0.396%); 2.3 MiB/s, 80 objects/s recovering 2023-09-23T20:58:45.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:45 smithi121 ceph-mon[219709]: pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 31461/213660 objects degraded (14.725%); 847/213660 objects misplaced (0.396%); 2.3 MiB/s, 80 objects/s recovering 2023-09-23T20:58:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:45 smithi123 ceph-mon[193903]: pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 31461/213660 objects degraded (14.725%); 847/213660 objects misplaced (0.396%); 2.3 MiB/s, 80 objects/s recovering 2023-09-23T20:58:45.821 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:45.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:45.844 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:58:45.926 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:45.951 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:45.993 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:45.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:46.100 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:46.126 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:46.169 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:46.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:46.276 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:46.326 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:46.386 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:46.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:46.490 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:46.522 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:46.551 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31461/213660 objects degraded (14.725%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:46.562 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31461/213660 objects degraded (14.725%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:46.622 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:46.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:46.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:46.728 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:46.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31461/213660 objects degraded (14.725%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:58:46.755 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:46.801 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:46.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:58:46.908 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:46.938 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:47.211 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:47.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:47.316 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:47.335 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:47.352 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:47.356 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:47.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:47 smithi121 ceph-mon[211526]: pgmap v904: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 31461/213408 objects degraded (14.742%); 847/213408 objects misplaced (0.397%); 2.2 MiB/s, 71 objects/s recovering 2023-09-23T20:58:47.465 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:47 smithi121 ceph-mon[219709]: pgmap v904: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 31461/213408 objects degraded (14.742%); 847/213408 objects misplaced (0.397%); 2.2 MiB/s, 71 objects/s recovering 2023-09-23T20:58:47.474 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:47.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:47.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:47.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:47.613 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:47 smithi123 ceph-mon[193903]: pgmap v904: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 31461/213408 objects degraded (14.742%); 847/213408 objects misplaced (0.397%); 2.2 MiB/s, 71 objects/s recovering 2023-09-23T20:58:47.971 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:47.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:47.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:48.075 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:48.110 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:48.161 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:48.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:48.265 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:48.293 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:48.342 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:48.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:48.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:48.449 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:48.463 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:58:48.478 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:48.526 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:48.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:48.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:moved 19 / 267 (7.11611%) 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.89089 -> 11.2909 (expected baseline 5.40399) 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 21 -> 11 pgs (0.629214 -> 0.329588 * mean) 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 44 -> 52 pgs (1.31835 -> 1.55805 * mean) 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T20:58:48.628 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0223 2023-09-23T20:58:48.629 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0245 2023-09-23T20:58:48.629 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.6054 -> 0.3664 2023-09-23T20:58:48.629 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.7390 -> 0.5499 2023-09-23T20:58:48.629 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8927 -> 0.8020 2023-09-23T20:58:48.631 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T20:58:48.634 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:48.654 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:48 smithi121 conmon[198376]: 2023-09-23T20:58:48.627+0000 7f479dcb1700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T20:58:48.654 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2906279322' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:48.654 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:48.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2906279322' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:48.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:48.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2906279322' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:48.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:58:48.659 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:48.706 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:48.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:48.814 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:48.839 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:48.880 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:48.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:48.986 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:49.027 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.228 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:49.333 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:49.359 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.416 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:49.525 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:49.553 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.595 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:49.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:49.703 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:49.726 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:49.726 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.726 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:49.726 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T20:58:49.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: from='client.45770 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:49.727 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:49 smithi123 ceph-mon[193903]: pgmap v906: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 31353/212004 objects degraded (14.789%); 847/212004 objects misplaced (0.400%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:58:49.728 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:49.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:49.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: from='client.45770 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[219709]: pgmap v906: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 31353/212004 objects degraded (14.789%); 847/212004 objects misplaced (0.400%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T20:58:49.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.734 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.734 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.734 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: failed to encode map e288 with expected crc 2023-09-23T20:58:49.734 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: from='client.45770 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:49.734 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:49 smithi121 ceph-mon[211526]: pgmap v906: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 31353/212004 objects degraded (14.789%); 847/212004 objects misplaced (0.400%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:58:49.770 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:49.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:49.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:49.901 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:49.942 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:49.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:50.048 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:50.077 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:50.223 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T20:58:50.224 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T20:58:50.551 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:50.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:50.660 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:50.686 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:50.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:50.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:50.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:50.834 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:58:50.860 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:50.900 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:50.900 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:58:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:50.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:58:51.000 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:58:51.006 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:51.027 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:58:51.041 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:51.052 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:58:51.078 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:58:51.097 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:58:51.116 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:58:51.150 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:58:51.172 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:58:51.175 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.175 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.176 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.176 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.176 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.176 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.176 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.176 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:58:51.177 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:51.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:51.280 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:58:51.280 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T20:58:51.280 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T20:58:51.280 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T20:58:51.281 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:58:51.281 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:51.307 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:51.349 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:51.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:51.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:51.457 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:51.487 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:51 smithi121 ceph-mon[211526]: pgmap v907: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 31353/211422 objects degraded (14.830%); 847/211422 objects misplaced (0.401%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T20:58:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/766297843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31353/211422 objects degraded (14.830%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:51 smithi121 ceph-mon[219709]: pgmap v907: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 31353/211422 objects degraded (14.830%); 847/211422 objects misplaced (0.401%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T20:58:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/766297843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31353/211422 objects degraded (14.830%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:51 smithi123 ceph-mon[193903]: pgmap v907: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 31353/211422 objects degraded (14.830%); 847/211422 objects misplaced (0.401%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T20:58:52.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/766297843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:58:52.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31353/211422 objects degraded (14.830%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:52.428 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:52.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:52.488 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:58:52.506 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:58:52.531 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:52.562 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:52.776 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:52.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:52.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:52.911 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:53.188 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:53.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:53.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:53.292 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:53.321 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:53.372 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:53.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:53.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:53.404 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:53.480 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:53.522 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:53.540 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:53.639 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:53.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:53.746 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:53.783 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:53.897 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:53.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:53.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:54.002 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:54.028 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:54.035 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:53 smithi121 ceph-mon[219709]: pgmap v908: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 31353/211194 objects degraded (14.846%); 847/211194 objects misplaced (0.401%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:58:54.035 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:53 smithi121 ceph-mon[211526]: pgmap v908: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 31353/211194 objects degraded (14.846%); 847/211194 objects misplaced (0.401%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:58:54.073 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:54.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:54.177 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:58:54.200 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:53 smithi123 ceph-mon[193903]: pgmap v908: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 31353/211194 objects degraded (14.846%); 847/211194 objects misplaced (0.401%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:58:54.203 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:54.244 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:58:54.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:54.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:58:54.351 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:54.381 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:54.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:58:54.889 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:54.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:54.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:54.906 INFO:teuthology.orchestra.run.smithi121.stdout:moved 2 / 267 (0.749064%) 2023-09-23T20:58:54.906 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T20:58:54.906 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.89089 -> 7.53222 (expected baseline 5.40399) 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 21 -> 20 pgs (0.629214 -> 0.599251 * mean) 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 44 -> 46 pgs (1.31835 -> 1.37828 * mean) 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0221 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0243 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.6054 -> 0.5554 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.7390 -> 0.6890 2023-09-23T20:58:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8927 -> 0.8427 2023-09-23T20:58:54.994 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:55.023 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:58:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:58:54] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.43.0" 2023-09-23T20:58:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:54 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]: dispatch 2023-09-23T20:58:55.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:54 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]: dispatch 2023-09-23T20:58:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:54 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]: dispatch 2023-09-23T20:58:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:54 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]: dispatch 2023-09-23T20:58:55.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:54 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]: dispatch 2023-09-23T20:58:55.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:54 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]: dispatch 2023-09-23T20:58:55.582 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:55.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:55.690 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:55.731 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:55.882 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:55.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:55.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:55.990 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:56.017 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: from='client.35817 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: pgmap v909: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 31247/209418 objects degraded (14.921%); 847/209418 objects misplaced (0.404%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T20:58:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]': finished 2023-09-23T20:58:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T20:58:56.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[211526]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: from='client.35817 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: pgmap v909: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 31247/209418 objects degraded (14.921%); 847/209418 objects misplaced (0.404%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T20:58:56.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]': finished 2023-09-23T20:58:56.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T20:58:56.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:55 smithi121 ceph-mon[219709]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: from='client.35817 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:58:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: pgmap v909: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 31247/209418 objects degraded (14.921%); 847/209418 objects misplaced (0.404%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T20:58:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'36398','4':'45155','6':'55230'}"}]': finished 2023-09-23T20:58:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T20:58:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:55 smithi123 ceph-mon[193903]: failed to encode map e289 with expected crc 2023-09-23T20:58:56.592 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:56.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:56.596 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:56.667 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:58:56.667 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:58:56.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:58:56.667 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57325 9.b 9:d45aa29e:::benchmark_data_smithi121_293770_object57324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.096490+0000", 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 489.56787034299998, 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1300021230000001, 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57325 2023-09-23T20:58:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096490+0000", 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096490+0000", 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096488+0000", 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096544+0000", 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6341999999999997e-05 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096578+0000", 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3216999999999997e-05 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096589+0000", 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1024e-05 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096651+0000", 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2706999999999994e-05 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096803+0000", 2023-09-23T20:58:56.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015159799999999999 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.096949+0000", 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000146267 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.099329+0000", 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023796609999999999 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.101731+0000", 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024025470000000001 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.110981+0000", 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092501100000000006 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.110997+0000", 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5291999999999999e-05 2023-09-23T20:58:56.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106431+0000", 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99543410600000004 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106445+0000", 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3692e-05 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.226393+0000", 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1199482549999999 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.226492+0000", 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9518999999999994e-05 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.675 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57369 9.b 9:da8768ce:::benchmark_data_smithi121_293770_object57368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.166112+0000", 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 489.49824849800001, 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.080740686, 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.676 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57369 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166112+0000", 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166112+0000", 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166111+0000", 2023-09-23T20:58:56.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:56.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166157+0000", 2023-09-23T20:58:56.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5760000000000002e-05 2023-09-23T20:58:56.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166167+0000", 2023-09-23T20:58:56.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0291e-05 2023-09-23T20:58:56.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166174+0000", 2023-09-23T20:58:56.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7240000000000003e-06 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166228+0000", 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4268999999999998e-05 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.166242+0000", 2023-09-23T20:58:56.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4525999999999999e-05 2023-09-23T20:58:56.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106469+0000", 2023-09-23T20:58:56.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.94022618300000005 2023-09-23T20:58:56.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106477+0000", 2023-09-23T20:58:56.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0709999999999995e-06 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.229385+0000", 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1229085350000001 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.229579+0000", 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019396100000000001 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.229909+0000", 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00033003399999999999 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234210+0000", 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0043013569999999996 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241345+0000", 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071342230000000003 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243943+0000", 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025985019999999999 2023-09-23T20:58:56.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.244003+0000", 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0109000000000003e-05 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.246853+0000", 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028494240000000001 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.688 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57386 9.b 9:d32d2c48:::benchmark_data_smithi121_293770_object57385:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.180559+0000", 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 489.48380125099999, 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0718573739999999, 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57386 2023-09-23T20:58:56.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180559+0000", 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180559+0000", 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180557+0000", 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180583+0000", 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5701999999999999e-05 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180602+0000", 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8366e-05 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180611+0000", 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9350000000000006e-06 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180673+0000", 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1908000000000004e-05 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.180688+0000", 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4799e-05 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106499+0000", 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92581111500000002 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106507+0000", 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.43e-06 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.231970+0000", 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.125463221 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.232077+0000", 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106759 2023-09-23T20:58:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.232212+0000", 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013490799999999999 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234246+0000", 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002034826 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.246919+0000", 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012672951 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249731+0000", 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028111719999999998 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.249795+0000", 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4807000000000005e-05 2023-09-23T20:58:56.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252417+0000", 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026213289999999999 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57391 9.b 9:da3371e4:::benchmark_data_smithi121_293770_object57390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.183791+0000", 2023-09-23T20:58:56.697 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 489.480569489, 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0859722039999999, 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57391 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183791+0000", 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183791+0000", 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183790+0000", 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183838+0000", 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8195999999999997e-05 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183858+0000", 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9692999999999999e-05 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183867+0000", 2023-09-23T20:58:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7189999999999996e-06 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183941+0000", 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4167999999999999e-05 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.183958+0000", 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7071e-05 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106686+0000", 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92272719299999995 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.106694+0000", 2023-09-23T20:58:56.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6060000000000003e-06 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234279+0000", 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1275844180000001 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234430+0000", 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015113799999999999 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.234598+0000", 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168276 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243796+0000", 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091980459999999997 2023-09-23T20:58:56.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252432+0000", 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086362450000000007 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.267140+0000", 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01470714 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.267194+0000", 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4255000000000001e-05 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.269763+0000", 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025694730000000001 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57396 9.b 9:d278708d:::benchmark_data_smithi121_293770_object57395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.189786+0000", 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 489.47457486100001, 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0825362539999999, 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57396 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.708 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189786+0000", 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189786+0000", 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189784+0000", 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189825+0000", 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0500999999999997e-05 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189839+0000", 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3815000000000001e-05 2023-09-23T20:58:56.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189848+0000", 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5889999999999992e-06 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189915+0000", 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7034999999999998e-05 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.189929+0000", 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4034e-05 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107460+0000", 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.91753040399999997 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107470+0000", 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0519999999999999e-05 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236631+0000", 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1291604989999999 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236763+0000", 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013231000000000001 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.236906+0000", 2023-09-23T20:58:56.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014250299999999999 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243834+0000", 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069287050000000003 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252439+0000", 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086044020000000006 2023-09-23T20:58:56.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.269825+0000", 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017385838000000001 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.269875+0000", 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0822999999999998e-05 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.272322+0000", 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024465889999999999 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57397 9.b 9:d3fe843c:::benchmark_data_smithi121_293770_object57396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 489.47336430799999, 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.084046802, 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.716 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57397 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.190996+0000", 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191048+0000", 2023-09-23T20:58:56.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2011000000000001e-05 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191057+0000", 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0659999999999993e-06 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191064+0000", 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8260000000000004e-06 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191098+0000", 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4050000000000001e-05 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191104+0000", 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9869999999999996e-06 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107494+0000", 2023-09-23T20:58:56.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.91639063600000004 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107502+0000", 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.238908+0000", 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.131405518 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239063+0000", 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015518300000000001 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.239231+0000", 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168252 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243871+0000", 2023-09-23T20:58:56.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046402170000000003 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252445+0000", 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085733660000000007 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.272373+0000", 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019928461000000001 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.272431+0000", 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7723000000000003e-05 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.275043+0000", 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026121289999999999 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:57398 9.b 9:de1ab5d1:::benchmark_data_smithi121_293770_object57397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 489.47319220100002, 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.089526835, 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 57398 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191168+0000", 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191210+0000", 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2212000000000002e-05 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191221+0000", 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1318000000000001e-05 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191228+0000", 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5339999999999996e-06 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191253+0000", 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5191e-05 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:47.191257+0000", 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2169999999999996e-06 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107526+0000", 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.91626845800000001 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:48.107535+0000", 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7690000000000008e-06 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241373+0000", 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.133838047 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241524+0000", 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015076400000000001 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.241701+0000", 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017661599999999999 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.243909+0000", 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002208485 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.252451+0000", 2023-09-23T20:58:56.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085418309999999997 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.278101+0000", 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025650435999999999 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.278155+0000", 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3356999999999998e-05 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:50:49.280695+0000", 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025403700000000001 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175048 9.9 9:98e06086:::benchmark_data_smithi121_293770_object72160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 332.45449848700002, 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43337474100000001, 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.733 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175048 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209865+0000", 2023-09-23T20:58:56.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9799999999999998e-06 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209882+0000", 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7142e-05 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209888+0000", 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.691e-06 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226284+0000", 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01639606 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226451+0000", 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000167028 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226596+0000", 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014432099999999999 2023-09-23T20:58:56.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232724+0000", 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061281840000000001 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.235331+0000", 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026065340000000002 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643145+0000", 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.40781475099999998 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643197+0000", 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1427000000000001e-05 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643237+0000", 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0119e-05 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.739 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175062 9.5 9:af0bbe63:::benchmark_data_smithi121_293770_object72174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.223421+0000", 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 332.44093954200002, 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41800432799999998, 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:58:56.740 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175062 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223421+0000", 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223421+0000", 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223420+0000", 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223426+0000", 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4750000000000001e-06 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223440+0000", 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3971000000000001e-05 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223447+0000", 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5379999999999997e-06 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223496+0000", 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8513000000000001e-05 2023-09-23T20:58:56.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223640+0000", 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014410900000000001 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223777+0000", 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000137279 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232561+0000", 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0087839670000000002 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232568+0000", 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9580000000000001e-06 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641331+0000", 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.4087635 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641388+0000", 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6258000000000002e-05 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641425+0000", 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7688999999999998e-05 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74269 10.c 10:34f5f527:::benchmark_data_smithi121_358375_object74268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e238)", 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:30.911332+0000", 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 205.75302835799999, 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.170509689, 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74269 2023-09-23T20:58:56.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911332+0000", 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911332+0000", 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911330+0000", 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911357+0000", 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6041999999999999e-05 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911376+0000", 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9082000000000001e-05 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911384+0000", 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6349999999999995e-06 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911437+0000", 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2290999999999998e-05 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911445+0000", 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8030000000000003e-06 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052751+0000", 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1413060900000001 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.054736+0000", 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019844519999999998 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:58:56.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.054936+0000", 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000200127 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.062180+0000", 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072435889999999999 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.062196+0000", 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.626e-05 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078622+0000", 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016426151999999999 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078687+0000", 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5153000000000005e-05 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.081842+0000", 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031547390000000002 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82131 10.6 10:6abfd253:::benchmark_data_smithi121_358375_object82130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.473119+0000", 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 182.19124133099999, 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0092071929999999, 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.755 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82131 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473119+0000", 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473119+0000", 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473118+0000", 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473164+0000", 2023-09-23T20:58:56.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6202999999999998e-05 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473179+0000", 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5135e-05 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473190+0000", 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0943e-05 2023-09-23T20:58:56.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473323+0000", 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013271499999999999 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473334+0000", 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1313e-05 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.472285+0000", 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.998951276 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.474038+0000", 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001752408 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.474147+0000", 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000109298 2023-09-23T20:58:56.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482179+0000", 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080320080000000002 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482192+0000", 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2642e-05 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482220+0000", 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8218e-05 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482285+0000", 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4950999999999995e-05 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482326+0000", 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1459999999999999e-05 2023-09-23T20:58:56.762 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82133 10.6 10:63e8454a:::benchmark_data_smithi121_358375_object82132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.473206+0000", 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 182.19115413899999, 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.01243694, 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.763 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82133 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T20:58:56.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473255+0000", 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9543999999999999e-05 2023-09-23T20:58:56.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473263+0000", 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.106e-06 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473278+0000", 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4651000000000001e-05 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473362+0000", 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4215999999999995e-05 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473371+0000", 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9430000000000006e-06 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.476809+0000", 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0034375790000001 2023-09-23T20:58:56.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.476935+0000", 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012644800000000001 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.477029+0000", 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3652000000000004e-05 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482363+0000", 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053339640000000001 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482387+0000", 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.421e-05 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485541+0000", 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031537000000000002 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485602+0000", 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0899000000000001e-05 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485643+0000", 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1616000000000003e-05 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82167 10.6 10:628536db:::benchmark_data_smithi121_358375_object82166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.616057+0000", 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 182.048303111, 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.87047633700000004, 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82167 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616057+0000", 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616057+0000", 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616056+0000", 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616105+0000", 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.867e-05 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616120+0000", 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5238e-05 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616128+0000", 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5939999999999996e-06 2023-09-23T20:58:56.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616192+0000", 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3879000000000003e-05 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616202+0000", 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0848e-05 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479337+0000", 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86313474099999998 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479507+0000", 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016978800000000001 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479677+0000", 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000169586 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485520+0000", 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0058433160000000003 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485651+0000", 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000131278 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486447+0000", 2023-09-23T20:58:56.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00079564700000000002 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486498+0000", 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1010000000000001e-05 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486534+0000", 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5933999999999998e-05 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.777 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82175 10.6 10:6b1c2f60:::benchmark_data_smithi121_358375_object82174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.626165+0000", 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 182.03819539899999, 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86051544000000002, 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.778 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82175 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626165+0000", 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626165+0000", 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626164+0000", 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626188+0000", 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3451999999999998e-05 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626197+0000", 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7440000000000002e-06 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626204+0000", 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.781e-06 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626249+0000", 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4608999999999999e-05 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626256+0000", 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5560000000000002e-06 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482405+0000", 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.85614919499999997 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482547+0000", 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000141285 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482686+0000", 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013936499999999999 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485656+0000", 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029699399999999999 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485728+0000", 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2163999999999996e-05 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486608+0000", 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00087970500000000005 2023-09-23T20:58:56.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486650+0000", 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1770999999999997e-05 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486681+0000", 2023-09-23T20:58:56.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0961999999999999e-05 2023-09-23T20:58:56.786 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.786 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.786 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.787 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:88960 10.1 10:81f0a708:::benchmark_data_smithi121_358375_object4215:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2023-09-23T20:58:56.787 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:44.289422+0000", 2023-09-23T20:58:56.787 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 132.37493854799999, 2023-09-23T20:58:56.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.13222426100000001, 2023-09-23T20:58:56.787 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.787 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.788 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.788 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.788 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.788 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88960 2023-09-23T20:58:56.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.788 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.289422+0000", 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.289422+0000", 2023-09-23T20:58:56.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.289421+0000", 2023-09-23T20:58:56.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:56.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.289426+0000", 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2970000000000003e-06 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.289436+0000", 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8830000000000001e-06 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.289443+0000", 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.861e-06 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.416398+0000", 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12695445599999999 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.416500+0000", 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000102175 2023-09-23T20:58:56.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.416583+0000", 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3479000000000005e-05 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.419067+0000", 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024833300000000002 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.419216+0000", 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014925600000000001 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.421550+0000", 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023336350000000001 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.421580+0000", 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9989e-05 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:44.421646+0000", 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6759000000000002e-05 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90547 10.c 10:38f673f1:::benchmark_data_smithi121_358375_object5802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.927985+0000", 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 98.736375183000007, 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24451520700000001, 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90547 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927985+0000", 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927985+0000", 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927984+0000", 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927989+0000", 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3669999999999996e-06 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927998+0000", 2023-09-23T20:58:56.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8799999999999997e-06 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928004+0000", 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7620000000000001e-06 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928042+0000", 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7648999999999998e-05 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928130+0000", 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8039000000000002e-05 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928220+0000", 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0078000000000001e-05 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.930554+0000", 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023342699999999998 2023-09-23T20:58:56.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.930699+0000", 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014521900000000001 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932382+0000", 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016824559999999999 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932413+0000", 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0815999999999998e-05 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172501+0000", 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24008792700000001 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.802 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.45470.0:90547 10.c e268/265 ondisk, result = 0)", 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.931963+0000", 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 98.732397777000003, 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24055167899999999, 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931963+0000", 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931963+0000", 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931962+0000", 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931965+0000", 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9299999999999999e-06 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931970+0000", 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8409999999999999e-06 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931974+0000", 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2890000000000002e-06 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932374+0000", 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000400739 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932378+0000", 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1690000000000002e-06 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172514+0000", 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24013602100000001 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90548 10.4 10:202f5be9:::benchmark_data_smithi121_358375_object5803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.932521+0000", 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 98.731839292999993, 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243222459, 2023-09-23T20:58:56.807 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90548 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932525+0000", 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9750000000000001e-06 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172121+0000", 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23959696899999999 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172147+0000", 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5612000000000001e-05 2023-09-23T20:58:56.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172546+0000", 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039850399999999999 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172644+0000", 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8318000000000003e-05 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.811 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172716+0000", 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1630000000000004e-05 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175320+0000", 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026049089999999999 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175444+0000", 2023-09-23T20:58:56.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012345800000000001 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175635+0000", 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000191528 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175695+0000", 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9590999999999999e-05 2023-09-23T20:58:56.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.814 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175744+0000", 2023-09-23T20:58:56.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8677000000000002e-05 2023-09-23T20:58:56.814 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.814 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.814 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:96620 10.5 e283/267 10:ae711801:::benchmark_data_smithi121_358375_object11875:head v 283'6061, mlcod=283'6059)", 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:30.354302+0000", 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 26.310058850000001, 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12226716, 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.354302+0000", 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.354302+0000", 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.354301+0000", 2023-09-23T20:58:56.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.354309+0000", 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1680000000000004e-06 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.472817+0000", 2023-09-23T20:58:56.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.11850788600000001 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.472832+0000", 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4856e-05 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473390+0000", 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00055860199999999995 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.473416+0000", 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5511999999999999e-05 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.476516+0000", 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003100701 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.819 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:30.476569+0000", 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2395999999999998e-05 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:100110 10.6 10:677d361a:::benchmark_data_smithi121_358375_object15365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:58:55.929873+0000", 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.73448736599999997, 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.111479066, 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:58:56.820 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 100110 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.929873+0000", 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.929873+0000", 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.929871+0000", 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.929881+0000", 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4499999999999993e-06 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.929896+0000", 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5358000000000001e-05 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.929904+0000", 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1969999999999996e-06 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.929950+0000", 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5639000000000001e-05 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.930041+0000", 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0748999999999995e-05 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:55.930114+0000", 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2779000000000002e-05 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:56.041180+0000", 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.11106632800000001 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:56.041230+0000", 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9926000000000003e-05 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:56.041242+0000", 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2061e-05 2023-09-23T20:58:56.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:56.041308+0000", 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6475000000000003e-05 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:58:56.041352+0000", 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.384e-05 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.827 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.828 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:58:56.828 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:58:56.828 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:58:56.828 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:56.861 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:56.915 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:56.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:57.021 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31247/209418 objects degraded (14.921%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31247/209418 objects degraded (14.921%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:57.047 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:57.097 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:58:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:57.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:58:57.203 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:57.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T20:58:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: failed to encode map e290 with expected crc 2023-09-23T20:58:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31247/209418 objects degraded (14.921%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:58:57.232 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:57.624 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:57.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:57.727 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:58:57.776 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:57.799 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:58:57.838 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:58:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:57.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:58:57.945 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:58:57.982 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:58.141 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:58:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:58.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:58:58.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[219709]: pgmap v912: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 31247/209148 objects degraded (14.940%); 847/209148 objects misplaced (0.405%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3853798094' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[211526]: pgmap v912: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 31247/209148 objects degraded (14.940%); 847/209148 objects misplaced (0.405%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3853798094' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:57 smithi123 ceph-mon[193903]: pgmap v912: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 31247/209148 objects degraded (14.940%); 847/209148 objects misplaced (0.405%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:58:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3853798094' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:57 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:58:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:58:58.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:58:58.246 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:58.251 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:58:58.275 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:58.575 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:58.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:58.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:58.679 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:58:58.712 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:59.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:58 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:59.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:59.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:58 smithi123 ceph-mon[193903]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T20:58:59.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:58 smithi123 ceph-mon[193903]: failed to encode map e291 with expected crc 2023-09-23T20:58:59.241 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:59.241 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:59.241 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[211526]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T20:58:59.241 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[211526]: failed to encode map e291 with expected crc 2023-09-23T20:58:59.241 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:58:59.241 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:58:59.242 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[219709]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T20:58:59.242 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:58 smithi121 ceph-mon[219709]: failed to encode map e291 with expected crc 2023-09-23T20:58:59.301 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:58:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:59.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:58:59.405 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:59.439 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:59.530 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:58:59.544 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:59.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:59.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:59.634 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:58:59.650 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:58:59.653 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:58:59.676 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:58:59.681 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:59.697 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:58:59.719 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:58:59.737 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:58:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:58:59.789 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:58:59.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:59.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:58:59.796 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:58:59.800 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:58:59.895 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:58:59.925 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:58:59.970 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:58:59.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:58:59.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:00.075 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:00.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[211526]: failed to encode map e291 with expected crc 2023-09-23T20:59:00.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[211526]: failed to encode map e291 with expected crc 2023-09-23T20:59:00.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[211526]: pgmap v914: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 31163/207810 objects degraded (14.996%); 847/207810 objects misplaced (0.408%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:00.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/884225902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:00.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[219709]: failed to encode map e291 with expected crc 2023-09-23T20:59:00.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[219709]: failed to encode map e291 with expected crc 2023-09-23T20:59:00.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[219709]: pgmap v914: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 31163/207810 objects degraded (14.996%); 847/207810 objects misplaced (0.408%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:00.102 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:58:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/884225902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:00.105 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:00.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:59 smithi123 ceph-mon[193903]: failed to encode map e291 with expected crc 2023-09-23T20:59:00.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:59 smithi123 ceph-mon[193903]: failed to encode map e291 with expected crc 2023-09-23T20:59:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:59 smithi123 ceph-mon[193903]: pgmap v914: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 31163/207810 objects degraded (14.996%); 847/207810 objects misplaced (0.408%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:59:00.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:58:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/884225902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:01.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:59:01.309 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:01.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:01.419 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:01.452 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:01.499 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:01.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:01.509 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:59:01.509 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:59:01.509 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T20:59:01.510 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:59:01.510 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:59:01.510 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T20:59:01.603 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:01.648 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:01.751 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:01.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:01.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:01.834 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:01.835 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:01.837 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:01.856 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:01.883 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:01.931 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:01.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:02.028 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:01 smithi121 ceph-mon[219709]: pgmap v915: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 31163/207246 objects degraded (15.037%); 847/207246 objects misplaced (0.409%); 885 KiB/s, 13 objects/s recovering 2023-09-23T20:59:02.028 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31163/207246 objects degraded (15.037%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:02.028 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:01 smithi121 ceph-mon[211526]: pgmap v915: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 31163/207246 objects degraded (15.037%); 847/207246 objects misplaced (0.409%); 885 KiB/s, 13 objects/s recovering 2023-09-23T20:59:02.028 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31163/207246 objects degraded (15.037%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:02.032 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:02.065 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:01 smithi123 ceph-mon[193903]: pgmap v915: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 31163/207246 objects degraded (15.037%); 847/207246 objects misplaced (0.409%); 885 KiB/s, 13 objects/s recovering 2023-09-23T20:59:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31163/207246 objects degraded (15.037%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:02.479 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:02.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:02.582 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:02.592 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:02.608 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:02.629 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:59:02.652 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:02.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:02.758 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:02.800 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:02.914 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:02.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:03.019 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:03.045 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:03.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:03.088 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:03.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:03.193 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:03.224 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:03.898 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:03.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:03.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:03.987 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:03 smithi121 ceph-mon[219709]: pgmap v916: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 31163/206931 objects degraded (15.060%); 847/206931 objects misplaced (0.409%); 1007 KiB/s, 15 objects/s recovering 2023-09-23T20:59:03.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:03 smithi121 ceph-mon[211526]: pgmap v916: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 31163/206931 objects degraded (15.060%); 847/206931 objects misplaced (0.409%); 1007 KiB/s, 15 objects/s recovering 2023-09-23T20:59:04.002 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:04.032 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:03 smithi123 ceph-mon[193903]: pgmap v916: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 31163/206931 objects degraded (15.060%); 847/206931 objects misplaced (0.409%); 1007 KiB/s, 15 objects/s recovering 2023-09-23T20:59:04.580 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:04.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:04.604 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:04.685 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:04.736 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:04.738 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:04.898 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:59:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:59:04] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-09-23T20:59:04.922 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:04.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:04.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:05.026 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:05.056 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:05.360 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:05.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:05.465 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:05.495 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:05.866 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:59:05.867 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:59:05.867 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:59:05.937 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:59:05.993 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:05.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:05.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:06.097 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:06.124 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:05 smithi121 ceph-mon[219709]: pgmap v917: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 31095/205251 objects degraded (15.150%); 847/205251 objects misplaced (0.413%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3687921282' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:05 smithi121 ceph-mon[211526]: pgmap v917: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 31095/205251 objects degraded (15.150%); 847/205251 objects misplaced (0.413%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3687921282' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:06.168 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:06.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:05 smithi123 ceph-mon[193903]: pgmap v917: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 31095/205251 objects degraded (15.150%); 847/205251 objects misplaced (0.413%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3687921282' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:06.273 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:06.298 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.339 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:06.448 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:06.483 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.525 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:06.630 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:06.657 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.700 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:06.806 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:06.831 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:06.873 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:06.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:06.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:06.980 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[219709]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3687921282' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[219709]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[219709]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[219709]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31095/205251 objects degraded (15.150%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:07.006 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[211526]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.007 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3687921282' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:07.007 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[211526]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T20:59:07.007 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[211526]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.007 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[211526]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.007 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31095/205251 objects degraded (15.150%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:07.009 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:06 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:06 smithi123 ceph-mon[193903]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3687921282' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:06 smithi123 ceph-mon[193903]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T20:59:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:06 smithi123 ceph-mon[193903]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:06 smithi123 ceph-mon[193903]: failed to encode map e292 with expected crc 2023-09-23T20:59:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31095/205251 objects degraded (15.150%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:07.326 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:07.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:07.430 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:07.456 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:07.498 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:07.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:07.606 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:07.637 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.016 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:08.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:08.119 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:08.145 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.178 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:07 smithi121 ceph-mon[211526]: pgmap v919: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 31095/205044 objects degraded (15.165%); 847/205044 objects misplaced (0.413%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T20:59:08.178 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:07 smithi121 ceph-mon[219709]: pgmap v919: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 31095/205044 objects degraded (15.165%); 847/205044 objects misplaced (0.413%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T20:59:08.190 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:08.194 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:07 smithi123 ceph-mon[193903]: pgmap v919: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 31095/205044 objects degraded (15.165%); 847/205044 objects misplaced (0.413%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T20:59:08.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:08.295 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:08.321 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.362 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:08.366 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:08.366 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:08.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:08.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:08.424 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:08.451 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:08.469 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:08.474 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:08.493 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:08.495 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.528 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:08.538 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:08.547 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:08.567 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:08.568 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.568 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.568 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.568 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.568 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.568 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.569 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.569 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:08.642 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:08.668 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:08.711 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:08.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:08.818 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:08.851 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:09.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/609787360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:09.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/609787360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/609787360' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:09.407 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:09.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:09.445 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:59:09.511 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:09.537 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:09.585 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:09.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:09.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:09.691 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:09.717 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:09.758 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:09.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:09.866 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:09.878 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:09.892 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:09.936 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:09.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:09.952 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:59:10.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:10.070 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:10.113 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:10.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:10.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:10.195 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:09 smithi121 ceph-mon[219709]: pgmap v920: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 31035/204165 objects degraded (15.201%); 847/204165 objects misplaced (0.415%); 953 KiB/s, 14 objects/s recovering 2023-09-23T20:59:10.195 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:09 smithi121 ceph-mon[211526]: pgmap v920: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 31035/204165 objects degraded (15.201%); 847/204165 objects misplaced (0.415%); 953 KiB/s, 14 objects/s recovering 2023-09-23T20:59:10.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:09 smithi123 ceph-mon[193903]: pgmap v920: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 31035/204165 objects degraded (15.201%); 847/204165 objects misplaced (0.415%); 953 KiB/s, 14 objects/s recovering 2023-09-23T20:59:10.252 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:10.303 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:10.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:10.407 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:10.437 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:10.534 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:10.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:10.640 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:10.666 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:10.707 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:10.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:10.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:10.812 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:10.840 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:11.072 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:11.086 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:11.089 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:10 smithi121 ceph-mon[211526]: pgmap v921: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 31035/203799 objects degraded (15.228%); 847/203799 objects misplaced (0.416%); 953 KiB/s, 14 objects/s recovering 2023-09-23T20:59:11.089 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:10 smithi121 ceph-mon[219709]: pgmap v921: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 31035/203799 objects degraded (15.228%); 847/203799 objects misplaced (0.416%); 953 KiB/s, 14 objects/s recovering 2023-09-23T20:59:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:11.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:11.191 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:11.221 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:10 smithi123 ceph-mon[193903]: pgmap v921: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 31035/203799 objects degraded (15.228%); 847/203799 objects misplaced (0.416%); 953 KiB/s, 14 objects/s recovering 2023-09-23T20:59:11.379 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:11.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:11.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:11.484 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:11.518 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:11.732 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:59:11.764 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:59:11.764 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:59:11.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74228 10.8 10:11677ef2:::benchmark_data_smithi121_358375_object74227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.710247+0000", 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 222.05046511800001, 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3347776470000001, 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.765 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74228 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710247+0000", 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710247+0000", 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710246+0000", 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:11.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710282+0000", 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5995999999999999e-05 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710293+0000", 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0908e-05 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880012+0000", 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16971977099999999 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880067+0000", 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4975000000000001e-05 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880189+0000", 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012128700000000001 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880330+0000", 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014141699999999999 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.885871+0000", 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005541308 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899625+0000", 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01375358 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899644+0000", 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8882e-05 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881429+0000", 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98178518699999995 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881444+0000", 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5021000000000001e-05 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.044904+0000", 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1634600930000001 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.045025+0000", 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012044099999999999 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74235 10.8 10:185a920d:::benchmark_data_smithi121_358375_object74234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880053+0000", 2023-09-23T20:59:11.774 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 221.88065879800001, 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1684344960000002, 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74235 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880053+0000", 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880053+0000", 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880052+0000", 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880083+0000", 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1236999999999999e-05 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880095+0000", 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1749999999999999e-05 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880102+0000", 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4189999999999996e-06 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.882848+0000", 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027460869999999999 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.882982+0000", 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000133292 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.883144+0000", 2023-09-23T20:59:11.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016197600000000001 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899953+0000", 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016809233 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899965+0000", 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2680000000000001e-05 2023-09-23T20:59:11.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881491+0000", 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98152574000000004 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881501+0000", 2023-09-23T20:59:11.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6569999999999992e-06 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048339+0000", 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.166837954 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048488+0000", 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014888999999999999 2023-09-23T20:59:11.782 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74237 10.8 10:14c3954e:::benchmark_data_smithi121_358375_object74236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880108+0000", 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 221.88060404800001, 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1685115160000001, 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.783 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74237 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880108+0000", 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880108+0000", 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880107+0000", 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880132+0000", 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4752999999999998e-05 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880138+0000", 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2709999999999998e-06 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880143+0000", 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.045e-06 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.885893+0000", 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057495690000000004 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.886029+0000", 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013593500000000001 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.886174+0000", 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014497800000000001 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900009+0000", 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013835439999999999 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900020+0000", 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0975000000000001e-05 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881537+0000", 2023-09-23T20:59:11.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151694 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881547+0000", 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5840000000000001e-06 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048531+0000", 2023-09-23T20:59:11.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.166983981 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048619+0000", 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8630999999999999e-05 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.791 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74240 10.8 10:168ffde2:::benchmark_data_smithi121_358375_object74239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880148+0000", 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 221.880563962, 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.168593628, 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74240 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.792 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880148+0000", 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880148+0000", 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880148+0000", 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880170+0000", 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2884999999999999e-05 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880177+0000", 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.015e-06 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880182+0000", 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4190000000000001e-06 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.888741+0000", 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085586689999999997 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.888897+0000", 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015591699999999999 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:11.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.889065+0000", 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168944 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900058+0000", 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010992388000000001 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900067+0000", 2023-09-23T20:59:11.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4390000000000006e-06 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881583+0000", 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151589500000003 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881592+0000", 2023-09-23T20:59:11.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2969999999999992e-06 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048657+0000", 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1670643629999999 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048742+0000", 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4847999999999998e-05 2023-09-23T20:59:11.799 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74248 10.8 10:13ca07d3:::benchmark_data_smithi121_358375_object74247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880233+0000", 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 221.88047912900001, 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.168624393, 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.800 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74248 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880233+0000", 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880233+0000", 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880232+0000", 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880261+0000", 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8649999999999998e-05 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880274+0000", 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2418e-05 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880280+0000", 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6159999999999998e-06 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.895079+0000", 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014798412 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.895189+0000", 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110474 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.895291+0000", 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000101677 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900103+0000", 2023-09-23T20:59:11.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048123050000000002 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900112+0000", 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3470000000000005e-06 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881624+0000", 2023-09-23T20:59:11.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151193800000003 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881634+0000", 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2459999999999992e-06 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048773+0000", 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1671391209999999 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048857+0000", 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4610000000000005e-05 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74251 10.8 10:1df39893:::benchmark_data_smithi121_358375_object74250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880285+0000", 2023-09-23T20:59:11.808 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 221.88042720000001, 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1923718449999998, 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74251 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880285+0000", 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880285+0000", 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880284+0000", 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880308+0000", 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4032999999999998e-05 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.811 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880314+0000", 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0329999999999997e-06 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880319+0000", 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8620000000000002e-06 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900140+0000", 2023-09-23T20:59:11.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019820676999999998 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900147+0000", 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.306e-06 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881661+0000", 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151351099999995 2023-09-23T20:59:11.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881669+0000", 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9899999999999997e-06 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048884+0000", 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1672154159999999 2023-09-23T20:59:11.814 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.049012+0000", 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012822300000000001 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.049189+0000", 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017644399999999999 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.057969+0000", 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0087801760000000006 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.057984+0000", 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4987000000000001e-05 2023-09-23T20:59:11.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.069380+0000", 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011396007 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.069449+0000", 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9227000000000007e-05 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.072657+0000", 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032074130000000001 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74255 10.8 10:1bd97acb:::benchmark_data_smithi121_358375_object74254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.818 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880323+0000", 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 221.88038864699999, 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1955500450000001, 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74255 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.819 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880323+0000", 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880323+0000", 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.820 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880323+0000", 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880348+0000", 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5286000000000002e-05 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880354+0000", 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.711e-06 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880358+0000", 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4410000000000003e-06 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900178+0000", 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019819973000000001 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900186+0000", 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4220000000000003e-06 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881695+0000", 2023-09-23T20:59:11.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98150895500000002 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881703+0000", 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1429999999999998e-06 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052047+0000", 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1703437619999999 2023-09-23T20:59:11.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052194+0000", 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000146883 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052399+0000", 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020511699999999999 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.058109+0000", 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057098679999999999 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.059092+0000", 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0009838780000000001 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.072722+0000", 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013629743999999999 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.072795+0000", 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2441999999999995e-05 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.075873+0000", 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030788339999999999 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74261 10.8 10:11b87711:::benchmark_data_smithi121_358375_object74260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.887012+0000", 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 221.873699926, 2023-09-23T20:59:11.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1945222019999999, 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74261 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.829 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887012+0000", 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887012+0000", 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.830 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887010+0000", 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887046+0000", 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5676000000000002e-05 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887061+0000", 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4713e-05 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887069+0000", 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3310000000000004e-06 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900305+0000", 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013235716 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900313+0000", 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9240000000000007e-06 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.833 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881728+0000", 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98141572399999999 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881736+0000", 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6279999999999996e-06 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.054985+0000", 2023-09-23T20:59:11.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1732493509999999 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.055149+0000", 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000164117 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.055369+0000", 2023-09-23T20:59:11.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021944 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.058192+0000", 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028229190000000001 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.061206+0000", 2023-09-23T20:59:11.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030145749999999998 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078474+0000", 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017268002000000001 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078539+0000", 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4863999999999994e-05 2023-09-23T20:59:11.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.081534+0000", 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002994994 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89730 10.9 10.996162f9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.838 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885131+0000", 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.875581092, 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1030085839999999, 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89730 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885131+0000", 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885131+0000", 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885129+0000", 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885138+0000", 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5590000000000006e-06 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885155+0000", 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.77e-05 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885167+0000", 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1764e-05 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885280+0000", 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011285600000000001 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885293+0000", 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2933e-05 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988127+0000", 2023-09-23T20:59:11.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1028341589999999 2023-09-23T20:59:11.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988140+0000", 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2449000000000001e-05 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89747 10.9 10.85e9dae9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885181+0000", 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.87553090399999, 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102989947, 2023-09-23T20:59:11.846 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89747 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.847 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885181+0000", 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885181+0000", 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885181+0000", 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885184+0000", 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.422e-06 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885191+0000", 2023-09-23T20:59:11.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8449999999999997e-06 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885196+0000", 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5160000000000002e-06 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885337+0000", 2023-09-23T20:59:11.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014073699999999999 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885344+0000", 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0820000000000001e-06 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988161+0000", 2023-09-23T20:59:11.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102816861 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988171+0000", 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9760000000000007e-06 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89811 10.9 10.259a2b39 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885200+0000", 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.87551167800001, 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1030167200000001, 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89811 2023-09-23T20:59:11.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885200+0000", 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885200+0000", 2023-09-23T20:59:11.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885200+0000", 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885202+0000", 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8670000000000001e-06 2023-09-23T20:59:11.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885206+0000", 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7519999999999998e-06 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885208+0000", 2023-09-23T20:59:11.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.717e-06 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885376+0000", 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016799100000000001 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885384+0000", 2023-09-23T20:59:11.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1459999999999996e-06 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988206+0000", 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1028212310000001 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988217+0000", 2023-09-23T20:59:11.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1331999999999999e-05 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89822 10.9 10.a18a0aa9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885210+0000", 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.875501523, 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1030388840000001, 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.859 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89822 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885210+0000", 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.860 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885210+0000", 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885210+0000", 2023-09-23T20:59:11.861 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885212+0000", 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4959999999999999e-06 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885217+0000", 2023-09-23T20:59:11.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4999999999999999e-06 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885222+0000", 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6630000000000001e-06 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885417+0000", 2023-09-23T20:59:11.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019500600000000001 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885423+0000", 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4320000000000004e-06 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988240+0000", 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102816367 2023-09-23T20:59:11.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988249+0000", 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6399999999999992e-06 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89838 10.9 10.c26773e9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885286+0000", 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.87542629000001, 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.103001481, 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.866 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89838 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885286+0000", 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885286+0000", 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885285+0000", 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885290+0000", 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2100000000000001e-06 2023-09-23T20:59:11.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885300+0000", 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0438999999999999e-05 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885309+0000", 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0940000000000008e-06 2023-09-23T20:59:11.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885445+0000", 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000136124 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885452+0000", 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2060000000000001e-06 2023-09-23T20:59:11.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988277+0000", 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102825033 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988287+0000", 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0288e-05 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.871 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89860 10.9 10.69271d69 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885332+0000", 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.875380006, 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.10314979, 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.872 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89860 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885332+0000", 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885332+0000", 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885332+0000", 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885334+0000", 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9610000000000001e-06 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885341+0000", 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5030000000000002e-06 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885347+0000", 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7969999999999997e-06 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885481+0000", 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013390799999999999 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885487+0000", 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4320000000000004e-06 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988458+0000", 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029713109999999 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988482+0000", 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3383e-05 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89867 10.9 10.1fecf3c9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885351+0000", 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.87536118700001, 2023-09-23T20:59:11.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1031650850000001, 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89867 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885351+0000", 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885351+0000", 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885351+0000", 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885353+0000", 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2230000000000001e-06 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885358+0000", 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6409999999999999e-06 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885373+0000", 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4381e-05 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885509+0000", 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013643300000000001 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885516+0000", 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988505+0000", 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029893470000001 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988516+0000", 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0743e-05 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.884 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89889 10.9 10.48cc6ac9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885378+0000", 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.87533362100001, 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.103172399, 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.885 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89889 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885378+0000", 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885378+0000", 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885378+0000", 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885381+0000", 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7439999999999999e-06 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885387+0000", 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6900000000000003e-06 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885392+0000", 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.425e-06 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885540+0000", 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000147995 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885546+0000", 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2620000000000004e-06 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988541+0000", 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029948940000001 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988551+0000", 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.842e-06 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89903 10.9 10.32ebf39 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885421+0000", 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.875291172, 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1031602439999999, 2023-09-23T20:59:11.891 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89903 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885421+0000", 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885421+0000", 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885420+0000", 2023-09-23T20:59:11.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885423+0000", 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.932e-06 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885431+0000", 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7179999999999996e-06 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885437+0000", 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6420000000000004e-06 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885794+0000", 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00035752599999999999 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885803+0000", 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1500000000000005e-06 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988572+0000", 2023-09-23T20:59:11.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1027685359999999 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988581+0000", 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2560000000000008e-06 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89915 10.9 10.e3da3b99 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885441+0000", 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.875271166, 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1031719099999999, 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.898 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89915 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885441+0000", 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885441+0000", 2023-09-23T20:59:11.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885440+0000", 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885443+0000", 2023-09-23T20:59:11.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.413e-06 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885449+0000", 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9800000000000003e-06 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885454+0000", 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7709999999999997e-06 2023-09-23T20:59:11.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885829+0000", 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00037520899999999999 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885837+0000", 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8730000000000007e-06 2023-09-23T20:59:11.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988603+0000", 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1027659190000001 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988613+0000", 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0141e-05 2023-09-23T20:59:11.903 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90001 10.9 10.6f241309 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885922+0000", 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 139.874790195, 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029280859999999, 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T20:59:11.904 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90001 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885922+0000", 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885922+0000", 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885922+0000", 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:11.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885923+0000", 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.903e-06 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885929+0000", 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2270000000000001e-06 2023-09-23T20:59:11.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885933+0000", 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.279e-06 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885993+0000", 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0291999999999998e-05 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.886003+0000", 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3840000000000003e-06 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988840+0000", 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1028373460000001 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988850+0000", 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.927e-06 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90540 10.8 10:16053130:::benchmark_data_smithi121_358375_object5795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.937140+0000", 2023-09-23T20:59:11.910 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 116.823572149, 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0512755149999999, 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90540 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937140+0000", 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937140+0000", 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:11.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937139+0000", 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937144+0000", 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3750000000000002e-06 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937154+0000", 2023-09-23T20:59:11.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8230000000000006e-06 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937159+0000", 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.242e-06 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937213+0000", 2023-09-23T20:59:11.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3680000000000001e-05 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937224+0000", 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1433999999999999e-05 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.889907+0000", 2023-09-23T20:59:11.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.952682696 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.889921+0000", 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3973e-05 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.983157+0000", 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0932360480000001 2023-09-23T20:59:11.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.983290+0000", 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013333799999999999 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.983423+0000", 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000132583 2023-09-23T20:59:11.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.986986+0000", 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003562548 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.987037+0000", 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1159000000000003e-05 2023-09-23T20:59:11.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.987300+0000", 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00026364500000000001 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.988288+0000", 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00098737 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.988325+0000", 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7636999999999999e-05 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.988415+0000", 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9988999999999995e-05 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.920 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.921 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.921 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:11.921 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:11.921 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:59:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:11.923 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:11.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:12.024 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:12.049 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31035/203799 objects degraded (15.228%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:12.049 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31035/203799 objects degraded (15.228%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:12.053 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31035/203799 objects degraded (15.228%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:12.293 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:59:12.293 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:59:12.293 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T20:59:12.293 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:59:12.294 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:59:12.294 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T20:59:12.665 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:12.973 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:12.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:12.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:12.992 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:59:13.077 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:13.080 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:13.106 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:13.253 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:13.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:13.357 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:13.384 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:13.428 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:13.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:13.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:13.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:13.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:13 smithi123 ceph-mon[193903]: pgmap v922: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 31035/203559 objects degraded (15.246%); 847/203559 objects misplaced (0.416%); 973 KiB/s, 15 objects/s recovering 2023-09-23T20:59:13.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:13.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:13.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:13 smithi121 ceph-mon[211526]: pgmap v922: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 31035/203559 objects degraded (15.246%); 847/203559 objects misplaced (0.416%); 973 KiB/s, 15 objects/s recovering 2023-09-23T20:59:13.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:13.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:13.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:13 smithi121 ceph-mon[219709]: pgmap v922: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 31035/203559 objects degraded (15.246%); 847/203559 objects misplaced (0.416%); 973 KiB/s, 15 objects/s recovering 2023-09-23T20:59:13.532 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:13.559 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:13.882 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:13.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:13.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:13.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:14.025 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:14.272 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:14.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:14.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:14.310 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:14.344 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:59:14.344 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:14.378 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:14.409 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:14.431 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:59:14.560 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:14.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:14.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:14.577 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4231196516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:14.577 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:14 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:14.578 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4231196516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:14.578 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:14 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:14.664 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:14.688 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4231196516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:14.688 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:14 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:14.692 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:14.737 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:14.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:14.840 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:14.870 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:14.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:59:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:59:14] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-09-23T20:59:15.177 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:15.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:15.280 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:15.308 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:15.353 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:15.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:15.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:15.462 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T20:59:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:15 smithi123 ceph-mon[193903]: pgmap v924: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30951/202077 objects degraded (15.316%); 847/202077 objects misplaced (0.419%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:15.488 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:15.532 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:15.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:15.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:15.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:15.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[219709]: pgmap v924: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30951/202077 objects degraded (15.316%); 847/202077 objects misplaced (0.419%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:15.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T20:59:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: failed to encode map e293 with expected crc 2023-09-23T20:59:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:15 smithi121 ceph-mon[211526]: pgmap v924: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30951/202077 objects degraded (15.316%); 847/202077 objects misplaced (0.419%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:15.673 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:15.897 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:15.897 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:15.914 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:15.938 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:15.959 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:15.981 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:16.002 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:16.024 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.026 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:59:16.036 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:16.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:16.141 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:16.169 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:16.213 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:16.321 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:16.346 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:16.388 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:16.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:16.492 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:16.523 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:16.714 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:16.716 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/985451621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:16.716 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30951/202077 objects degraded (15.316%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:16.718 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/985451621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:16.718 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30951/202077 objects degraded (15.316%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/985451621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30951/202077 objects degraded (15.316%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:16.819 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:16.852 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:16.957 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:16.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:17.054 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:17.054 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:17.062 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:17.092 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:17.198 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:17.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:17.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:17.304 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:17.332 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:17.381 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:17.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:17.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:17.486 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:17.517 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:17.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:17 smithi123 ceph-mon[193903]: pgmap v925: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 30951/201804 objects degraded (15.337%); 847/201804 objects misplaced (0.420%); 1017 KiB/s, 15 objects/s recovering 2023-09-23T20:59:17.762 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:59:17.788 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:17 smithi121 ceph-mon[211526]: pgmap v925: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 30951/201804 objects degraded (15.337%); 847/201804 objects misplaced (0.420%); 1017 KiB/s, 15 objects/s recovering 2023-09-23T20:59:17.788 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:17 smithi121 ceph-mon[219709]: pgmap v925: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 30951/201804 objects degraded (15.337%); 847/201804 objects misplaced (0.420%); 1017 KiB/s, 15 objects/s recovering 2023-09-23T20:59:17.809 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:17.812 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:59:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:17.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:17.917 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:17.944 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:17.988 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:17.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:18.096 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:18.121 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:18.162 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:18.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:18.192 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:18.268 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:18.295 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:18.312 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:18.397 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:18.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:18.503 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:18.534 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:18.584 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:18.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:18.693 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:18.725 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:18.755 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:18.773 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:18.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:18.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:18.878 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:18.906 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:19.796 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:19.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:19.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:19 smithi121 ceph-mon[219709]: pgmap v926: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 30845/199968 objects degraded (15.425%); 847/199968 objects misplaced (0.424%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:19.901 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:19.941 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.137 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:20.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:19 smithi121 ceph-mon[211526]: pgmap v926: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 30845/199968 objects degraded (15.425%); 847/199968 objects misplaced (0.424%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:20.228 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:59:20.228 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:59:20.228 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T20:59:20.228 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:59:20.228 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:59:20.229 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T20:59:20.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:19 smithi123 ceph-mon[193903]: pgmap v926: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 30845/199968 objects degraded (15.425%); 847/199968 objects misplaced (0.424%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:20.243 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:20.279 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.388 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:20.540 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:20.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:20.583 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:20.645 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:20.673 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.770 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:20.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:20.816 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:20.816 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:20.877 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:20.910 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:20.913 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:59:20.966 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2411878974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:20.966 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:21.106 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:21.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:21.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2411878974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:21.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:21.131 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2411878974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:21.131 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:21.209 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:21.236 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:21.342 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:21.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:21.445 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:21.472 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:21.520 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:21.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:21.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:21.624 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:21.653 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:21.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[211526]: pgmap v927: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30845/199215 objects degraded (15.483%); 847/199215 objects misplaced (0.425%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:21.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[211526]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[211526]: failed to encode map e294 with expected crc 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[211526]: failed to encode map e294 with expected crc 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30845/199215 objects degraded (15.483%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[219709]: pgmap v927: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30845/199215 objects degraded (15.483%); 847/199215 objects misplaced (0.425%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:22.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[219709]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T20:59:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[219709]: failed to encode map e294 with expected crc 2023-09-23T20:59:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[219709]: failed to encode map e294 with expected crc 2023-09-23T20:59:22.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30845/199215 objects degraded (15.483%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:22.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:21 smithi123 ceph-mon[193903]: pgmap v927: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30845/199215 objects degraded (15.483%); 847/199215 objects misplaced (0.425%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:22.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:21 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:22.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:22.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:21 smithi123 ceph-mon[193903]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T20:59:22.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:21 smithi123 ceph-mon[193903]: failed to encode map e294 with expected crc 2023-09-23T20:59:22.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:21 smithi123 ceph-mon[193903]: failed to encode map e294 with expected crc 2023-09-23T20:59:22.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30845/199215 objects degraded (15.483%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:22.439 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:22.439 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "" 2023-09-23T20:59:22.439 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:22.808 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:59:22.841 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:22.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:22.944 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:22.974 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.093 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:22 smithi121 ceph-mon[211526]: failed to encode map e294 with expected crc 2023-09-23T20:59:23.093 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:22 smithi121 ceph-mon[219709]: failed to encode map e294 with expected crc 2023-09-23T20:59:23.177 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:23.181 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:22 smithi123 ceph-mon[193903]: failed to encode map e294 with expected crc 2023-09-23T20:59:23.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:23.283 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:23.315 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:23.319 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.348 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:23.373 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:23.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:23.394 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:23.416 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:23.436 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:23.455 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:23.478 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:23.484 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:23.504 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:23.506 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.506 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.506 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.507 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.507 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.507 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.507 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.507 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:23.508 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.552 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:23.658 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:23.683 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:23.725 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:23.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:23.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:23.831 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:23.859 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:23 smithi123 ceph-mon[193903]: pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30845/198909 objects degraded (15.507%); 847/198909 objects misplaced (0.426%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:59:24.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4087767665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:24.328 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:23 smithi121 ceph-mon[211526]: pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30845/198909 objects degraded (15.507%); 847/198909 objects misplaced (0.426%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:59:24.329 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4087767665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:24.329 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:23 smithi121 ceph-mon[219709]: pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30845/198909 objects degraded (15.507%); 847/198909 objects misplaced (0.426%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:59:24.329 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4087767665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:24.516 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:59:24.557 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:24.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:24.661 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:24.689 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:24.734 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:24.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:24.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:24.840 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:24.872 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:24.985 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:24.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:24.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:25.086 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:25.098 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:25.126 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:25.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:59:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:59:24] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-09-23T20:59:25.174 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:25.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:25.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:25.281 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:25.309 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:25.352 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:25.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:25.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:25.457 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:25.491 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89934 10.9 10:99add1d4:::benchmark_data_smithi121_358375_object5189:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.536316973, 2023-09-23T20:59:25.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.125213574, 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89934 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:59:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992360+0000", 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.999e-06 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992365+0000", 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4210000000000003e-06 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992369+0000", 2023-09-23T20:59:25.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8870000000000002e-06 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992624+0000", 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00025508600000000002 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992633+0000", 2023-09-23T20:59:25.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1339999999999993e-06 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.982585+0000", 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 17.989951529999999 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.991708+0000", 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091229280000000006 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.001314+0000", 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096056409999999998 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.021104+0000", 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019790161000000001 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.076468+0000", 2023-09-23T20:59:25.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.055364728000000002 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.081776+0000", 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053078259999999999 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.086509+0000", 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047326829999999997 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.541 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.093298+0000", 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067891399999999999 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.117571+0000", 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024272848 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.542 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89935 10.9 10:99d64049:::benchmark_data_smithi121_358375_object5190:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:59:25.543 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:59:25.543 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.53629849399999, 2023-09-23T20:59:25.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.185285084, 2023-09-23T20:59:25.543 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89935 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:59:25.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992377+0000", 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.635e-06 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992381+0000", 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2710000000000001e-06 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992383+0000", 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7920000000000001e-06 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992670+0000", 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00028668500000000001 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992679+0000", 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9819999999999997e-06 2023-09-23T20:59:25.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.118245+0000", 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.125566417000002 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.129062+0000", 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010817108000000001 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.138856+0000", 2023-09-23T20:59:25.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097938880000000006 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.155528+0000", 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016671136999999999 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.157350+0000", 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018224879999999999 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.157942+0000", 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00059154200000000002 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.159783+0000", 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018418969999999999 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.162715+0000", 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029311860000000001 2023-09-23T20:59:25.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.177661+0000", 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014946291 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89981 10.9 10:994f28fe:::benchmark_data_smithi121_358375_object5236:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992398+0000", 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.53627666099999, 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.146739247999999, 2023-09-23T20:59:25.552 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89981 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992398+0000", 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992398+0000", 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992397+0000", 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992399+0000", 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5099999999999999e-06 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992402+0000", 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1970000000000001e-06 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992405+0000", 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9790000000000002e-06 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992715+0000", 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00030991800000000001 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992724+0000", 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5080000000000005e-06 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.147118+0000", 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.154393974000001 2023-09-23T20:59:25.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.153688+0000", 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065708340000000002 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.203940+0000", 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050252012999999998 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.210878+0000", 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069378920000000002 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.987311+0000", 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.77643294500000004 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.992743+0000", 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054319370000000004 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.002286+0000", 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095426230000000001 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.089643+0000", 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087357120999999996 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.115322+0000", 2023-09-23T20:59:25.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025679124000000001 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.118245+0000", 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029228069999999999 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.120479+0000", 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002233938 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.124182+0000", 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037032990000000002 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.139137+0000", 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014954857 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90001 10.9 10:90c824f6:::benchmark_data_smithi121_358375_object5256:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:59:25.561 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992408+0000", 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 152.536266743, 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.173996376000002, 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90001 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992408+0000", 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992408+0000", 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992407+0000", 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T20:59:25.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992409+0000", 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.449e-06 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992412+0000", 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.853e-06 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992414+0000", 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6070000000000002e-06 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992748+0000", 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000333247 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992756+0000", 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8699999999999998e-06 2023-09-23T20:59:25.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.211516+0000", 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.218759336000002 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.220416+0000", 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089002730000000002 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.264440+0000", 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044023962 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.271591+0000", 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071505479999999996 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.038619+0000", 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.76702871699999997 2023-09-23T20:59:25.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.043853+0000", 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052333670000000001 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.053317+0000", 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094646399999999999 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.115885+0000", 2023-09-23T20:59:25.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.062567431000000007 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.141137+0000", 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025252508 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.143947+0000", 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028093950000000001 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.146421+0000", 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024744300000000001 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.149991+0000", 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035700340000000001 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.166404+0000", 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016412930999999999 2023-09-23T20:59:25.570 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90301 10.9 10:95826f21:::benchmark_data_smithi121_358375_object5556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 133.02261998500001, 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 23.226904614999999, 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.571 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90301 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506053+0000", 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506062+0000", 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7669999999999998e-06 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506077+0000", 2023-09-23T20:59:25.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4562999999999999e-05 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506088+0000", 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0779e-05 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.665171+0000", 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15908349199999999 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.672174+0000", 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070033109999999999 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.759608+0000", 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087433832000000003 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.770158+0000", 2023-09-23T20:59:25.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010549433 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998499+0000", 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2283415890000002 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998505+0000", 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6810000000000003e-06 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892231+0000", 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372554500000001 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892240+0000", 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4369999999999996e-06 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.600926+0000", 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.708686151999999 2023-09-23T20:59:25.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.608105+0000", 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071792979999999998 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.617691+0000", 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095853580000000004 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.654316+0000", 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036625188000000003 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.709183+0000", 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.054866572000000002 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.712311+0000", 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031283980000000001 2023-09-23T20:59:25.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.714254+0000", 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001943005 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.717912+0000", 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036581460000000001 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.732959+0000", 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015046838 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90304 10.9 10:95d08781:::benchmark_data_smithi121_358375_object5559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 132.52788513799999, 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.706967114000001, 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.581 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90304 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000787+0000", 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000797+0000", 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6229999999999992e-06 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000815+0000", 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8391999999999999e-05 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000827+0000", 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1934e-05 2023-09-23T20:59:25.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.076647+0000", 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.075820121000000004 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083118+0000", 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064707690000000003 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.125639+0000", 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.042520998999999997 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.133609+0000", 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079701110000000002 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998466+0000", 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8648562869999998 2023-09-23T20:59:25.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998472+0000", 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8070000000000002e-06 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892193+0000", 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372040100000005 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892202+0000", 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4790000000000002e-06 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.565782+0000", 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.673579972999999 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.572988+0000", 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072056059999999998 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.582601+0000", 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096135609999999996 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.596634+0000", 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014032559999999999 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.598447+0000", 2023-09-23T20:59:25.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001812924 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.600285+0000", 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018377759999999999 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.676784+0000", 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.076499811000000001 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.680024+0000", 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032395729999999999 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.707756+0000", 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027732273000000002 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90319 10.9 10:9c23793e:::benchmark_data_smithi121_358375_object5574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 132.456450362, 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.227369583000002, 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90319 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072222+0000", 2023-09-23T20:59:25.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072231+0000", 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2539999999999992e-06 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072248+0000", 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7422e-05 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072260+0000", 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2139e-05 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083953+0000", 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011692759 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.090352+0000", 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063994910000000002 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.175493+0000", 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.085140748000000002 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.187526+0000", 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01203284 2023-09-23T20:59:25.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998112+0000", 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8105862510000001 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998120+0000", 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.418e-06 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891970+0000", 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89384994900000003 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891979+0000", 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6320000000000003e-06 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.208032+0000", 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.316052247000002 2023-09-23T20:59:25.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.217965+0000", 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.009933753 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.229822+0000", 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011856366 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.248481+0000", 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018659464000000001 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.278359+0000", 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.029878236999999998 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.280279+0000", 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019199919999999999 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.282082+0000", 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001802367 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.285240+0000", 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031586299999999999 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.299594+0000", 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014353223999999999 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90350 10.9 10:9252fbfb:::benchmark_data_smithi121_358375_object5605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.846206405, 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.226020969, 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.602 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90350 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682466+0000", 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682473+0000", 2023-09-23T20:59:25.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7800000000000003e-06 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682487+0000", 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.402e-05 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682496+0000", 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7860000000000002e-06 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.827509+0000", 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.145013317 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.834217+0000", 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067071600000000002 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.884874+0000", 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050657455999999997 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.902020+0000", 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017146446999999999 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998607+0000", 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.096586142 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998613+0000", 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7830000000000001e-06 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892350+0000", 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373614700000004 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892359+0000", 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3430000000000002e-06 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.797647+0000", 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.905288455000001 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.805039+0000", 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0073917929999999998 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.814794+0000", 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097547199999999997 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.846753+0000", 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.031959301000000002 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.848628+0000", 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001875172 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.850451+0000", 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018229469999999999 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.887256+0000", 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036804591999999997 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.890406+0000", 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003149982 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.908489+0000", 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018083054000000001 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90379 10.9 10:951c4b03:::benchmark_data_smithi121_358375_object5634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.59976865600001, 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.839626358, 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90379 2023-09-23T20:59:25.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928904+0000", 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928913+0000", 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928927+0000", 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4329e-05 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928935+0000", 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9729999999999997e-06 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.954600+0000", 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025665337 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.964941+0000", 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010340223000000001 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.088603+0000", 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12366287400000001 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.095790+0000", 2023-09-23T20:59:25.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071866090000000001 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998534+0000", 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.90274379599999999 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998541+0000", 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1199999999999996e-06 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892274+0000", 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373288500000003 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892284+0000", 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9399999999999997e-06 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.625055+0000", 2023-09-23T20:59:25.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.732770945999999 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.633921+0000", 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088660279999999998 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.644084+0000", 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010163669 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.733850+0000", 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089765882000000005 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.736155+0000", 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023041950000000002 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.738094+0000", 2023-09-23T20:59:25.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001939276 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.740271+0000", 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021772879999999999 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.744526+0000", 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042552850000000001 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.768532+0000", 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024005678999999999 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90424 10.9 10:980e8aaf:::benchmark_data_smithi121_358375_object5679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.54713167700001, 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.531567125999999, 2023-09-23T20:59:25.623 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90424 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981541+0000", 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981548+0000", 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1489999999999997e-06 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981561+0000", 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2448e-05 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981569+0000", 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1499999999999999e-06 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.224643+0000", 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243074701 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.237465+0000", 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012821737 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.280151+0000", 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.04268632 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.286562+0000", 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064109060000000001 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998360+0000", 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71179806400000001 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998368+0000", 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5050000000000001e-06 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892082+0000", 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89371398099999999 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892091+0000", 2023-09-23T20:59:25.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1409999999999994e-06 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.424130+0000", 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.532038986 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.435357+0000", 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011226817 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.448022+0000", 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01266514 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.486130+0000", 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038108329000000003 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.487944+0000", 2023-09-23T20:59:25.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018138589999999999 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.489799+0000", 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001854332 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.492508+0000", 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027092589999999999 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.496591+0000", 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040830900000000002 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.513110+0000", 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016518877000000001 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.633 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90429 10.9 10:9b165b02:::benchmark_data_smithi121_358375_object5684:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.54003512200001, 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.409704737999999, 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90429 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988638+0000", 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988645+0000", 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2840000000000001e-06 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988661+0000", 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6053999999999999e-05 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988669+0000", 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9419999999999995e-06 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.238554+0000", 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24988542499999999 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.246423+0000", 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0078688479999999995 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.311852+0000", 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.065429004999999998 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.318196+0000", 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063438970000000003 2023-09-23T20:59:25.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998162+0000", 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.67996552200000004 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998172+0000", 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0208e-05 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892007+0000", 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89383517700000004 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892016+0000", 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4290000000000007e-06 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.301575+0000", 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.409558344000001 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.309095+0000", 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075199719999999998 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.319256+0000", 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010161106 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.376130+0000", 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.056873817 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.378120+0000", 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019904839999999998 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.378762+0000", 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00064167 2023-09-23T20:59:25.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.381391+0000", 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026288940000000001 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.384518+0000", 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031269750000000002 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.398344+0000", 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013826188 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90434 10.9 10:90493f14:::benchmark_data_smithi121_358375_object5689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.53463874400001, 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.989979289000001, 2023-09-23T20:59:25.644 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90434 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994034+0000", 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994041+0000", 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4219999999999997e-06 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994052+0000", 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994059+0000", 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4060000000000002e-06 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.247296+0000", 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25323742700000001 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.254030+0000", 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067339619999999996 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.341839+0000", 2023-09-23T20:59:25.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087808943 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.348061+0000", 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006222216 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998641+0000", 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65057928700000001 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998645+0000", 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8470000000000003e-06 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892386+0000", 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89374099500000004 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892396+0000", 2023-09-23T20:59:25.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5119999999999997e-06 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.851132+0000", 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.958735591 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.858452+0000", 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0073205420000000002 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.868309+0000", 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098567020000000002 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.884649+0000", 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016340086 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.886658+0000", 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020087680000000002 2023-09-23T20:59:25.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.913375+0000", 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026717060000000001 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.962703+0000", 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.049328300999999998 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.965943+0000", 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003240368 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.984015+0000", 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018071474000000001 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90436 10.9 10:925ee57b:::benchmark_data_smithi121_358375_object5691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.532982965, 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.84919506, 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90436 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995690+0000", 2023-09-23T20:59:25.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995697+0000", 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5919999999999997e-06 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995714+0000", 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7144999999999999e-05 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995724+0000", 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0030000000000001e-05 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.350440+0000", 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.354715958 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.356739+0000", 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062994560000000002 2023-09-23T20:59:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.388748+0000", 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032009263000000003 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.395007+0000", 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006258726 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998567+0000", 2023-09-23T20:59:25.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.60355961700000005 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998573+0000", 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8619999999999999e-06 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892312+0000", 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373954700000002 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892321+0000", 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9789999999999999e-06 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.769494+0000", 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.877172975000001 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.779991+0000", 2023-09-23T20:59:25.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010496959 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.790177+0000", 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010185801 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.822235+0000", 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032058013000000003 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.824316+0000", 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002081252 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.826167+0000", 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001850862 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.828002+0000", 2023-09-23T20:59:25.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018350759999999999 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.831036+0000", 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003033446 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.844886+0000", 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013850843999999999 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.664 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90448 10.9 10:9dd18145:::benchmark_data_smithi121_358375_object5703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.50117517000001, 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.248909286, 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.665 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90448 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027497+0000", 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027505+0000", 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0879999999999997e-06 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027521+0000", 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6795000000000001e-05 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027531+0000", 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3649999999999993e-06 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.359164+0000", 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.331633185 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.365457+0000", 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006293267 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.418540+0000", 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.053082826999999999 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.429706+0000", 2023-09-23T20:59:25.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011166376 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998075+0000", 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.56836851300000002 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998082+0000", 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5100000000000001e-06 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891929+0000", 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89384610799999997 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891940+0000", 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1442e-05 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.172182+0000", 2023-09-23T20:59:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.280241611000001 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.179816+0000", 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076342859999999997 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.193670+0000", 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013853811000000001 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.246965+0000", 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.05329536 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.247710+0000", 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00074513800000000001 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.250763+0000", 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030532549999999999 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.252934+0000", 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002170866 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.256527+0000", 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035923309999999998 2023-09-23T20:59:25.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.276408+0000", 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019881827000000001 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90471 10.9 10:9fd1a800:::benchmark_data_smithi121_358375_object5726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.422274956, 2023-09-23T20:59:25.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.062235608000002, 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90471 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106398+0000", 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106407+0000", 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9089999999999995e-06 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106423+0000", 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5962999999999999e-05 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106433+0000", 2023-09-23T20:59:25.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1589999999999999e-06 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.431330+0000", 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32489747699999999 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.444379+0000", 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013049432999999999 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.479473+0000", 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.035093725999999999 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.485924+0000", 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064507230000000002 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997976+0000", 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.51205186199999997 2023-09-23T20:59:25.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997987+0000", 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1669e-05 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891608+0000", 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89362018099999996 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891623+0000", 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5492e-05 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.072757+0000", 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.181133587000001 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for readable", 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.087146+0000", 2023-09-23T20:59:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014389206 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.107028+0000", 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019882529999999999 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.114642+0000", 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076134410000000003 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.124398+0000", 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097560830000000005 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.146747+0000", 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022348922 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.148740+0000", 2023-09-23T20:59:25.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019930780000000001 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.149391+0000", 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00065059799999999995 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.151298+0000", 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019071190000000001 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.154367+0000", 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030688310000000002 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.168635+0000", 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014268471 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90489 10.9 10:97f30eb8:::benchmark_data_smithi121_358375_object5744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.31792109599999, 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.329841758000001, 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90489 2023-09-23T20:59:25.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210752+0000", 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210761+0000", 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9970000000000004e-06 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210778+0000", 2023-09-23T20:59:25.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7181999999999999e-05 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210789+0000", 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0818e-05 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.446681+0000", 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23589284299999999 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.453972+0000", 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072908900000000004 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.511615+0000", 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057642634999999998 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.518011+0000", 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006395815 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998399+0000", 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.480388125 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998406+0000", 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4789999999999999e-06 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892119+0000", 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.893712795 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892129+0000", 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3829999999999998e-06 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.457483+0000", 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.565354894999999 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.466620+0000", 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091370259999999995 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.476805+0000", 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010184886000000001 2023-09-23T20:59:25.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.491775+0000", 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014969623 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.513722+0000", 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.021947326999999999 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.516436+0000", 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027137620000000002 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.519247+0000", 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028114009999999998 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.523691+0000", 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0044435380000000003 2023-09-23T20:59:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.540595+0000", 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016903975000000002 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90505 10.9 10:99aea1ee:::benchmark_data_smithi121_358375_object5760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:59:25.697 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 131.26027820600001, 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.154937368999999, 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90505 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:59:25.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268395+0000", 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268402+0000", 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2389999999999996e-06 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268415+0000", 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2850999999999999e-05 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268422+0000", 2023-09-23T20:59:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.644e-06 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.520421+0000", 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.251998897 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.526858+0000", 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064371389999999997 2023-09-23T20:59:25.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.557595+0000", 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.030736329 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.563877+0000", 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006281989 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998305+0000", 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43442851399999999 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998317+0000", 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1959e-05 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892044+0000", 2023-09-23T20:59:25.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372699 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892054+0000", 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3389999999999999e-06 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.328034+0000", 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.435980598 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.336888+0000", 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088543010000000002 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.347140+0000", 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010251873999999999 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.379404+0000", 2023-09-23T20:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032263781999999998 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.400240+0000", 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.020836174999999998 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.402072+0000", 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018316369999999999 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.403897+0000", 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001824579 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.406955+0000", 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030584589999999999 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.423333+0000", 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016378508 2023-09-23T20:59:25.707 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90514 10.9 10:96e73e59:::benchmark_data_smithi121_358375_object5769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 130.86711511199999, 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.014633163999999, 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.708 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90514 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661558+0000", 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661567+0000", 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.183e-06 2023-09-23T20:59:25.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661583+0000", 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6585000000000001e-05 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661593+0000", 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6190000000000006e-06 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.825075+0000", 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16348148200000001 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.835611+0000", 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010536495 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.936853+0000", 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10124217200000001 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.943598+0000", 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067444640000000004 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998433+0000", 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.054835452 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998439+0000", 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6470000000000003e-06 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892156+0000", 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89371742200000004 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892165+0000", 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1449999999999997e-06 2023-09-23T20:59:25.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.541432+0000", 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.649266821000001 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.548683+0000", 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072504270000000003 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.558326+0000", 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096435710000000001 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.591157+0000", 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032831143 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.593006+0000", 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001848948 2023-09-23T20:59:25.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.594819+0000", 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001812501 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.654981+0000", 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.060162001999999999 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.658403+0000", 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034224960000000001 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.676192+0000", 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017789158999999999 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90523 10.9 10:976e84cf:::benchmark_data_smithi121_358375_object5778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.743128+0000", 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 130.78554635099999, 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.046086119999998, 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90523 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743128+0000", 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743128+0000", 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743126+0000", 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:25.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743133+0000", 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0520000000000004e-06 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743149+0000", 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5480000000000001e-05 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743156+0000", 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3000000000000004e-06 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998680+0000", 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25552361299999998 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998687+0000", 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1949999999999997e-06 2023-09-23T20:59:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892421+0000", 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373391700000004 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892428+0000", 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5700000000000004e-06 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.875874+0000", 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.983445199999998 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.882670+0000", 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006796814 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.985057+0000", 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.102386677 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.993718+0000", 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086608669999999992 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.710912+0000", 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71719427999999996 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.716497+0000", 2023-09-23T20:59:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0055847630000000004 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.726083+0000", 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095864479999999992 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.763944+0000", 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.037860317999999997 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.766094+0000", 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002150683 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.766698+0000", 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00060379499999999996 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.769891+0000", 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031930330000000001 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.773125+0000", 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032341840000000002 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.789214+0000", 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016088683999999999 2023-09-23T20:59:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90564 10.9 10:9972dda2:::benchmark_data_smithi121_358375_object5819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:18.227558+0000", 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 127.301116576, 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.678191501000001, 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T20:59:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90564 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227558+0000", 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227558+0000", 2023-09-23T20:59:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227556+0000", 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227562+0000", 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2580000000000001e-06 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227573+0000", 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0593e-05 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227580+0000", 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4280000000000001e-06 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.994895+0000", 2023-09-23T20:59:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 17.767315675999999 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.010431+0000", 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015535731000000001 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.049810+0000", 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.039379499999999998 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.056645+0000", 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006834311 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.815438+0000", 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.758792841 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T20:59:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.821101+0000", 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005663413 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.830907+0000", 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098062710000000001 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.882525+0000", 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.051617730000000001 2023-09-23T20:59:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.883121+0000", 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00059645399999999995 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.885048+0000", 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001926456 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.886943+0000", 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018950320000000001 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.890040+0000", 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030973580000000001 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.905749+0000", 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015708976999999999 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T20:59:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T20:59:25.738 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T20:59:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:25.739 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:25.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:25.840 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:25.869 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:26.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:25 smithi121 ceph-mon[211526]: pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 30773/197205 objects degraded (15.605%); 847/197205 objects misplaced (0.430%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:25 smithi121 ceph-mon[219709]: pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 30773/197205 objects degraded (15.605%); 847/197205 objects misplaced (0.430%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:25 smithi123 ceph-mon[193903]: pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 30773/197205 objects degraded (15.605%); 847/197205 objects misplaced (0.430%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:26.350 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:59:26.437 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:59:26.490 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:26.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:26.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:26.595 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:26.624 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:26.680 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:26.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:26.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:26.790 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:26.824 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:26.852 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:26.879 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:26.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:26.990 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:27.007 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:27.016 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:27.073 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30773/197205 objects degraded (15.605%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:27.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:27.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:27.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:27.209 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30773/197205 objects degraded (15.605%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:27.210 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30773/197205 objects degraded (15.605%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:27.215 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:27.243 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:27.491 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:27.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:27.594 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:27.621 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:27.663 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:27.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:27.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:27.766 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:27.800 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:27.847 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:27.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:27.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:27.950 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:27.978 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:28.027 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:28.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:28.130 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:28.155 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[219709]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 30773/196965 objects degraded (15.624%); 847/196965 objects misplaced (0.430%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:28.155 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2437486453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[211526]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 30773/196965 objects degraded (15.624%); 847/196965 objects misplaced (0.430%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2437486453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:28.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:28.158 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:28.163 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:59:28.163 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:59:28.164 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T20:59:28.164 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:59:28.164 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:59:28.164 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T20:59:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:27 smithi123 ceph-mon[193903]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 30773/196965 objects degraded (15.624%); 847/196965 objects misplaced (0.430%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2437486453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:28.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:28.360 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:28.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:28.418 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:59:28.465 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:28.497 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:28.646 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:28.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:28.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:28.707 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:28.709 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:28.709 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:59:28.709 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T20:59:28.709 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:28.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:28.779 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:29.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:29.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:29.182 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[219709]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T20:59:29.183 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[219709]: failed to encode map e295 with expected crc 2023-09-23T20:59:29.183 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:29.183 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:29.183 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[211526]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T20:59:29.183 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:28 smithi121 ceph-mon[211526]: failed to encode map e295 with expected crc 2023-09-23T20:59:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:28 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:28 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:28 smithi123 ceph-mon[193903]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T20:59:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:28 smithi123 ceph-mon[193903]: failed to encode map e295 with expected crc 2023-09-23T20:59:29.418 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:29.438 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:29.457 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:29.477 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:29.496 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:29.498 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:29.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:29.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:29.516 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:29.523 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:59:29.535 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:29.554 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:29.555 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.555 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.555 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.555 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.555 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.555 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.556 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:59:29.602 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:29.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:29.796 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:29.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:29.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:29.905 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:29.935 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:30.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:29 smithi123 ceph-mon[193903]: failed to encode map e295 with expected crc 2023-09-23T20:59:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:29 smithi123 ceph-mon[193903]: failed to encode map e295 with expected crc 2023-09-23T20:59:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:29 smithi123 ceph-mon[193903]: pgmap v933: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 30675/195585 objects degraded (15.684%); 847/195585 objects misplaced (0.433%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:30.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3007954495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:30.260 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:30.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:30.281 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[211526]: failed to encode map e295 with expected crc 2023-09-23T20:59:30.281 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[211526]: failed to encode map e295 with expected crc 2023-09-23T20:59:30.281 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[211526]: pgmap v933: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 30675/195585 objects degraded (15.684%); 847/195585 objects misplaced (0.433%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:30.281 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3007954495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:30.281 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[219709]: failed to encode map e295 with expected crc 2023-09-23T20:59:30.281 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[219709]: failed to encode map e295 with expected crc 2023-09-23T20:59:30.281 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[219709]: pgmap v933: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 30675/195585 objects degraded (15.684%); 847/195585 objects misplaced (0.433%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:30.282 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3007954495' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:30.364 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:30.397 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:30.446 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:30.450 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:30.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:30.556 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:30.585 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:30.632 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:30.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:30.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:30.766 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:30.864 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:30.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:30.967 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:31.002 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:31.053 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:31.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:31.159 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:31.188 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:31.216 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:59:31.238 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:31.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:31.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:31.344 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:31.370 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:31.421 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:31.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:31.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:31.528 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:31.556 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:31.604 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:31.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:31.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:31.637 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:31.637 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:59:31.637 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:31.672 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:31.708 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:31.737 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:31.796 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:31.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:31.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:31.937 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:31.985 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:31.987 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:31 smithi121 ceph-mon[211526]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30675/195111 objects degraded (15.722%); 847/195111 objects misplaced (0.434%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:31.987 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30675/195111 objects degraded (15.722%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:31.988 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:31 smithi121 ceph-mon[219709]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30675/195111 objects degraded (15.722%); 847/195111 objects misplaced (0.434%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:31.988 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30675/195111 objects degraded (15.722%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:31.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:31.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:32.089 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:32.116 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:31 smithi123 ceph-mon[193903]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30675/195111 objects degraded (15.722%); 847/195111 objects misplaced (0.434%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:32.116 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30675/195111 objects degraded (15.722%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:32.119 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:32.198 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:32.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:32.300 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:32.301 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:32.329 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:32.434 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:32.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:32.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:32.538 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:32.567 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:32.598 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:32.598 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:32.884 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:32.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:32.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:32.988 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:33.017 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.052 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:33.168 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:33.221 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:59:33.271 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:33.306 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.431 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:33.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:33.538 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:33.566 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.597 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:33.612 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:33.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:33.717 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:33.747 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.795 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:33.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:33.898 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:33.925 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:33.940 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T20:59:33.975 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:33.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:34.079 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:34.104 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:33 smithi121 ceph-mon[219709]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 30675/194823 objects degraded (15.745%); 847/194823 objects misplaced (0.435%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:34.104 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1351735248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:34.104 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:33 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:34.104 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:33 smithi121 ceph-mon[211526]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 30675/194823 objects degraded (15.745%); 847/194823 objects misplaced (0.435%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:34.104 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1351735248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:34.104 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:33 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:34.107 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:34.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:33 smithi123 ceph-mon[193903]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 30675/194823 objects degraded (15.745%); 847/194823 objects misplaced (0.435%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:34.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1351735248' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:34.155 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:33 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:34.809 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:34.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:34.918 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:34.948 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:34.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:34 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:34 smithi123 ceph-mon[193903]: failed to encode map e296 with expected crc 2023-09-23T20:59:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:34 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:34 smithi123 ceph-mon[193903]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T20:59:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:34 smithi123 ceph-mon[193903]: failed to encode map e296 with expected crc 2023-09-23T20:59:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:34 smithi123 ceph-mon[193903]: failed to encode map e296 with expected crc 2023-09-23T20:59:34.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:34 smithi123 ceph-mon[193903]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 30593/193005 objects degraded (15.851%); 847/193005 objects misplaced (0.439%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T20:59:35.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:35.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[211526]: failed to encode map e296 with expected crc 2023-09-23T20:59:35.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:35.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[211526]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T20:59:35.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[211526]: failed to encode map e296 with expected crc 2023-09-23T20:59:35.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[211526]: failed to encode map e296 with expected crc 2023-09-23T20:59:35.023 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[211526]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 30593/193005 objects degraded (15.851%); 847/193005 objects misplaced (0.439%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T20:59:35.023 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:59:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:59:34] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T20:59:35.023 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:35.023 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[219709]: failed to encode map e296 with expected crc 2023-09-23T20:59:35.024 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:35.024 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[219709]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T20:59:35.024 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[219709]: failed to encode map e296 with expected crc 2023-09-23T20:59:35.024 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[219709]: failed to encode map e296 with expected crc 2023-09-23T20:59:35.024 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:34 smithi121 ceph-mon[219709]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 30593/193005 objects degraded (15.851%); 847/193005 objects misplaced (0.439%); 1.1 MiB/s, 16 objects/s recovering 2023-09-23T20:59:35.051 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:35.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:35.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:35.155 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:35.195 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:35.249 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:35.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:35.256 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:35.256 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:35.270 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:35.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:35.326 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:35.354 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:35.354 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:35.377 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:35.381 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:35.405 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:35.426 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:35.459 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.460 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:35.528 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:35.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:35.632 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:35.666 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:36.033 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:59:36.135 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:36.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:36.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:36.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:59:36.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2909615443' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:36.238 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:36.275 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:36.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:59:36.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2909615443' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:36.325 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:59:36.325 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2909615443' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:36.374 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:36.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:36.479 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:36.506 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:36.609 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:36.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:36.635 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:36.713 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:36.742 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:36.837 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:59:36.850 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:36.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:36.954 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:37.008 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:37.078 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30593/193005 objects degraded (15.851%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:36 smithi121 ceph-mon[211526]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 30593/192780 objects degraded (15.869%); 847/192780 objects misplaced (0.439%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30593/193005 objects degraded (15.851%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:36 smithi121 ceph-mon[219709]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 30593/192780 objects degraded (15.869%); 847/192780 objects misplaced (0.439%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:37.188 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:37.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30593/193005 objects degraded (15.851%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:37.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:36 smithi123 ceph-mon[193903]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 30593/192780 objects degraded (15.869%); 847/192780 objects misplaced (0.439%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:37.221 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:37.292 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:59:37.292 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:59:37.292 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:59:37.292 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-09-23T20:59:37.293 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T20:59:37.557 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:37.974 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:37.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:37.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:38.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:38.108 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:38.110 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:38.322 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:38.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:38.346 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:59:38.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:38.461 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:38.594 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:38.594 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:59:38.594 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T20:59:38.594 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:38.790 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:38.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:38.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:38.894 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:38.925 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:39.449 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:39.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:39.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:59:39.555 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:39.593 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:39.593 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:39.711 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:39.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:39.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:39.816 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:39.852 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:39.920 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:39.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:39.947 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:40.023 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:40.061 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:39 smithi121 ceph-mon[211526]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 30505/191394 objects degraded (15.938%); 847/191394 objects misplaced (0.443%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:39 smithi121 ceph-mon[219709]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 30505/191394 objects degraded (15.938%); 847/191394 objects misplaced (0.443%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:40.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:39 smithi123 ceph-mon[193903]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 30505/191394 objects degraded (15.938%); 847/191394 objects misplaced (0.443%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:40.943 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T20:59:41.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1039714368' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:41.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:41.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1039714368' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:41.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:41.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1039714368' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:41.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:59:41.326 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:41.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:41.431 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:41.462 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:42.030 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:42.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:42.081 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:42.081 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:59:42.081 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T20:59:42.081 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T20:59:42.081 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:42.105 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30505/190851 objects degraded (15.984%); 847/190851 objects misplaced (0.444%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.106 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30505/190851 objects degraded (15.984%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:42.135 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:42.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30505/190851 objects degraded (15.984%); 847/190851 objects misplaced (0.444%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30505/190851 objects degraded (15.984%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30505/190851 objects degraded (15.984%); 847/190851 objects misplaced (0.444%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: failed to encode map e297 with expected crc 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:42.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30505/190851 objects degraded (15.984%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:42.231 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:42.586 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:42.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:42.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:42.693 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:42.722 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:42.772 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:42.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:42.830 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:59:42.876 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:42.911 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:42.962 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:42.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:42.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:43.066 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:43.069 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:43.088 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:43.110 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.112 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:43.140 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:43.170 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:43.174 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:43.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:43.195 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:43.217 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:43.237 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:43.237 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:43.238 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.239 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.239 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.239 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.239 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.239 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.239 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.239 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:59:43.278 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:43.310 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.461 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:43.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:43.568 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T20:59:43.606 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:43.658 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:43.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:43.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:43.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:59:43.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:59:43.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30505/190545 objects degraded (16.009%); 847/190545 objects misplaced (0.445%); 946 KiB/s, 14 objects/s recovering 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2398674176' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/284312717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:43.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30505/190545 objects degraded (16.009%); 847/190545 objects misplaced (0.445%); 946 KiB/s, 14 objects/s recovering 2023-09-23T20:59:43.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2398674176' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:59:43.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/284312717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:43.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30505/190545 objects degraded (16.009%); 847/190545 objects misplaced (0.445%); 946 KiB/s, 14 objects/s recovering 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2398674176' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:59:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/284312717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:43.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:43.792 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.044 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:44.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:44.177 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.344 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:44.386 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:59:44.454 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:44.481 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.524 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:44.628 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:44.658 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[211526]: failed to encode map e298 with expected crc 2023-09-23T20:59:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[211526]: failed to encode map e298 with expected crc 2023-09-23T20:59:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2398674176' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:59:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[211526]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T20:59:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[219709]: failed to encode map e298 with expected crc 2023-09-23T20:59:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[219709]: failed to encode map e298 with expected crc 2023-09-23T20:59:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2398674176' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:59:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:44 smithi121 ceph-mon[219709]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T20:59:44.755 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:44.758 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:44 smithi123 ceph-mon[193903]: failed to encode map e298 with expected crc 2023-09-23T20:59:44.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:44 smithi123 ceph-mon[193903]: failed to encode map e298 with expected crc 2023-09-23T20:59:44.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2398674176' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:59:44.759 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:44 smithi123 ceph-mon[193903]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T20:59:44.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:44.810 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:44.822 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:44.822 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:44.860 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:44.896 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:44.968 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:44.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:45.069 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:59:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:59:44] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T20:59:45.072 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:45.105 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:45.152 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:45.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:45.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:45.256 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:45.286 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:45.490 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:59:45.491 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:45.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:45.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:45.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:45.624 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:45.667 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:45.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:45.772 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[211526]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[211526]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[211526]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[211526]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[211526]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[211526]: pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 30389/188247 objects degraded (16.143%); 847/188247 objects misplaced (0.450%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[219709]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[219709]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T20:59:45.800 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[219709]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.801 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[219709]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.801 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[219709]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.801 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:45 smithi121 ceph-mon[219709]: pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 30389/188247 objects degraded (16.143%); 847/188247 objects misplaced (0.450%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:59:45.802 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:45.919 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:45 smithi123 ceph-mon[193903]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:45 smithi123 ceph-mon[193903]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T20:59:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:45 smithi123 ceph-mon[193903]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:45 smithi123 ceph-mon[193903]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:45 smithi123 ceph-mon[193903]: failed to encode map e299 with expected crc 2023-09-23T20:59:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:45 smithi123 ceph-mon[193903]: pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 30389/188247 objects degraded (16.143%); 847/188247 objects misplaced (0.450%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T20:59:46.017 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:46.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:46.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:46.121 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:46.154 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:46.274 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:46.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:46.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:46.380 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:46.411 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:46.521 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:46.591 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:46.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:46.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:46.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30389/188247 objects degraded (16.143%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30389/188247 objects degraded (16.143%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:46.697 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:46.727 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:46.772 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:46.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:46.876 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:46.927 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:46.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30389/188247 objects degraded (16.143%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:46.981 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:46.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:46.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:47.004 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:47.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:47.113 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:47.206 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:47.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:47.314 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:47.345 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:47.591 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T20:59:47.634 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:47.634 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:47.649 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:47.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:47.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:47 smithi123 ceph-mon[193903]: pgmap v946: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 30389/187956 objects degraded (16.168%); 847/187956 objects misplaced (0.451%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2065635506' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:47.753 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:47.783 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:47.832 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:47.834 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:47 smithi121 ceph-mon[211526]: pgmap v946: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 30389/187956 objects degraded (16.168%); 847/187956 objects misplaced (0.451%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:47.834 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2065635506' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:47.834 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:47.835 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:47 smithi121 ceph-mon[219709]: pgmap v946: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 30389/187956 objects degraded (16.168%); 847/187956 objects misplaced (0.451%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T20:59:47.835 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2065635506' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:47.835 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:59:47.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:47.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:47.936 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:47.964 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.012 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:48.117 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:48.144 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.336 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:48.391 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:59:48.439 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:48.467 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.630 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:48 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:48 smithi123 ceph-mon[193903]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:48 smithi123 ceph-mon[193903]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:48 smithi123 ceph-mon[193903]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:48 smithi123 ceph-mon[193903]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T20:59:48.738 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:48.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:48.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[211526]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[211526]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[211526]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:48.775 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[211526]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T20:59:48.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:59:48.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[219709]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[219709]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[219709]: failed to encode map e300 with expected crc 2023-09-23T20:59:48.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:59:48.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:48 smithi121 ceph-mon[219709]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T20:59:48.781 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:48.784 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:48.843 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:48.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:48.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:48.946 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:48.977 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:49.076 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:49.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:49.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:49.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:49.174 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:49.180 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:49.194 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:49.210 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:49.218 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:49.239 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:49.259 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:49.284 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:49.307 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.309 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:59:49.369 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:49.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:49.472 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:49.499 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:49.541 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:49.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:49.577 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:59:49.577 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:59:49.578 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-09-23T20:59:49.578 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:59:49.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:49 smithi121 ceph-mon[211526]: failed to encode map e300 with expected crc 2023-09-23T20:59:49.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:49 smithi121 ceph-mon[211526]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 30277/186378 objects degraded (16.245%); 847/186378 objects misplaced (0.454%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:59:49.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3383312462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:49.590 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:49 smithi121 ceph-mon[219709]: failed to encode map e300 with expected crc 2023-09-23T20:59:49.590 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:49 smithi121 ceph-mon[219709]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 30277/186378 objects degraded (16.245%); 847/186378 objects misplaced (0.454%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:59:49.590 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3383312462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:49.646 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:49.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:49 smithi123 ceph-mon[193903]: failed to encode map e300 with expected crc 2023-09-23T20:59:49.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:49 smithi123 ceph-mon[193903]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 30277/186378 objects degraded (16.245%); 847/186378 objects misplaced (0.454%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T20:59:49.669 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3383312462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:49.672 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:49.935 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:49.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:50.040 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:50.069 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:50.168 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:50.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:50.191 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:50.273 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:50.302 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:50.391 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:59:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T20:59:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T20:59:50.392 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:50.449 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:50.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:50.555 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:50.585 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:50.629 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:50.711 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:59:50.732 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:50.786 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:50.897 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:50.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:51.005 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:51.034 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.081 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:51.095 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:59:51.187 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:51.213 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.261 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:51.367 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:51.395 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.450 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:51.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:51.490 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:51.506 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:51.556 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:51.583 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.629 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:51.733 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:51.761 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.808 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:51.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:51.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:51.912 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:51.945 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:51.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:51 smithi123 ceph-mon[193903]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 30277/185766 objects degraded (16.298%); 847/185766 objects misplaced (0.456%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:51.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30277/185766 objects degraded (16.298%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:52.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:51 smithi121 ceph-mon[211526]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 30277/185766 objects degraded (16.298%); 847/185766 objects misplaced (0.456%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:52.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30277/185766 objects degraded (16.298%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:52.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:51 smithi121 ceph-mon[219709]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 30277/185766 objects degraded (16.298%); 847/185766 objects misplaced (0.456%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T20:59:52.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30277/185766 objects degraded (16.298%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:52.042 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:52.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:52.110 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:59:52.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:52.174 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:52.214 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:52.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:52.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:52.221 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T20:59:52.222 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T20:59:52.318 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:52.343 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:52.537 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:52.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:52.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:52.641 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:52.674 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:52.744 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:52.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:52.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:52.848 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:52.881 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:53.041 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:53.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:53.111 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136312 9.0 9:087a316d:::benchmark_data_smithi121_293770_object33424:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.87964745199997, 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.026362005, 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.112 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136312 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228794+0000", 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228801+0000", 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6000000000000003e-06 2023-09-23T20:59:53.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228812+0000", 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1517e-05 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228821+0000", 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.089e-06 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230287+0000", 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014660960000000001 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230299+0000", 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255083+0000", 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247833159999999 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255158+0000", 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5184000000000004e-05 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136315 9.0 9:07b4e2b9:::benchmark_data_smithi121_293770_object33427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.87763647600002, 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0248562560000001, 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.117 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136315 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230806+0000", 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230811+0000", 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.879e-06 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230823+0000", 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.151e-05 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230830+0000", 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8460000000000002e-06 2023-09-23T20:59:53.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230876+0000", 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6560999999999997e-05 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230884+0000", 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8960000000000003e-06 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255603+0000", 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247184060000001 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255663+0000", 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0364e-05 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.122 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136335 9.0 9:00bf7afd:::benchmark_data_smithi121_293770_object33447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.852007007, 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0152258009999999, 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136335 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256434+0000", 2023-09-23T20:59:53.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256453+0000", 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8349999999999999e-05 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256484+0000", 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0908999999999998e-05 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256494+0000", 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0479000000000001e-05 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256538+0000", 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3940999999999998e-05 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256547+0000", 2023-09-23T20:59:53.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6689999999999995e-06 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255731+0000", 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99918387799999997 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255863+0000", 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000131917 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255979+0000", 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011618899999999999 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.267419+0000", 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011440723999999999 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271524+0000", 2023-09-23T20:59:53.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041050169999999999 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271584+0000", 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9577999999999998e-05 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271662+0000", 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7960999999999994e-05 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.129 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:58207 10.e 10:7c929052:::benchmark_data_smithi121_358375_object58206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 276.61704302300001, 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.26084283200000002, 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.130 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 58207 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:59:53.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491400+0000", 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491437+0000", 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7049999999999999e-05 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491445+0000", 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.909e-06 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.491450+0000", 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.074e-06 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.493204+0000", 2023-09-23T20:59:53.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001754579 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.493354+0000", 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014998499999999999 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.493457+0000", 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102198 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.495328+0000", 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0018714059999999999 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.497890+0000", 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025619940000000002 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752164+0000", 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25427398499999998 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752206+0000", 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2197999999999998e-05 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:16.752243+0000", 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6906999999999997e-05 2023-09-23T20:59:53.136 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90329 10.f 10:f35e32c5:::benchmark_data_smithi121_358375_object5584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.105215+0000", 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 160.00322868699999, 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80166978300000002, 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.137 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90329 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105215+0000", 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105215+0000", 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105214+0000", 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105217+0000", 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3589999999999999e-06 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105228+0000", 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0261999999999999e-05 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105234+0000", 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2149999999999998e-06 2023-09-23T20:59:53.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105290+0000", 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5810000000000003e-05 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105298+0000", 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0020000000000006e-06 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.894981+0000", 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.789683778 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.895080+0000", 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9035000000000003e-05 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.895162+0000", 2023-09-23T20:59:53.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1553999999999999e-05 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900361+0000", 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005198846 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900369+0000", 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.388e-06 2023-09-23T20:59:53.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906827+0000", 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064581009999999999 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906859+0000", 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2100000000000001e-05 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906884+0000", 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4882e-05 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90332 10.f 10:f123f416:::benchmark_data_smithi121_358375_object5587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.300759+0000", 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 159.80768380399999, 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.60945262700000002, 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.145 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90332 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300759+0000", 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300759+0000", 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300758+0000", 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300766+0000", 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9340000000000006e-06 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300782+0000", 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.632e-05 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300791+0000", 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8529999999999998e-06 2023-09-23T20:59:53.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300838+0000", 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6925000000000002e-05 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300849+0000", 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1008e-05 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.897583+0000", 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.59673357400000004 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.897653+0000", 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0284999999999995e-05 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.897718+0000", 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5127000000000002e-05 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906113+0000", 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.008394594 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906163+0000", 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0469000000000001e-05 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910127+0000", 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039636309999999996 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910174+0000", 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6918e-05 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910212+0000", 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8473999999999998e-05 2023-09-23T20:59:53.152 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90333 10.f 10:f4af6242:::benchmark_data_smithi121_358375_object5588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.396755+0000", 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 159.71168844100001, 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.51361582500000003, 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.153 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90333 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396755+0000", 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396755+0000", 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396746+0000", 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999909 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396767+0000", 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1305999999999999e-05 2023-09-23T20:59:53.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396803+0000", 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6508999999999999e-05 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396817+0000", 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.33e-05 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396927+0000", 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110268 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396953+0000", 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6139000000000002e-05 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.898950+0000", 2023-09-23T20:59:53.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.50199691899999999 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.899041+0000", 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.077e-05 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.899104+0000", 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3583000000000004e-05 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906117+0000", 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070128350000000002 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910092+0000", 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039743599999999997 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910283+0000", 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000191053 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910330+0000", 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7914999999999998e-05 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910371+0000", 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0157000000000002e-05 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.160 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90334 10.f 10:f59733bd:::benchmark_data_smithi121_358375_object5589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.505405+0000", 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 159.603038244, 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.407500266, 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.161 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90334 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505405+0000", 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505405+0000", 2023-09-23T20:59:53.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505404+0000", 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505414+0000", 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0674e-05 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505432+0000", 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7810000000000001e-05 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505439+0000", 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.708e-06 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505498+0000", 2023-09-23T20:59:53.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9385999999999997e-05 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505509+0000", 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1235e-05 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900477+0000", 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.39496745700000002 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900562+0000", 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5111999999999994e-05 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900624+0000", 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2447999999999996e-05 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906121+0000", 2023-09-23T20:59:53.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0054971009999999999 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910246+0000", 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004124104 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.912851+0000", 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026054440000000002 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.912878+0000", 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6570000000000001e-05 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.912905+0000", 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7699e-05 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90336 10.f 10:ff0f5e2b:::benchmark_data_smithi121_358375_object5591:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.558079+0000", 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 159.55036436500001, 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.35782227599999999, 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90336 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.169 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558079+0000", 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558079+0000", 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558077+0000", 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558087+0000", 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7599999999999997e-06 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558106+0000", 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9025000000000002e-05 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558113+0000", 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4049999999999997e-06 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558185+0000", 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2113000000000006e-05 2023-09-23T20:59:53.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.558196+0000", 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1677e-05 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.901936+0000", 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34373951400000002 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.902000+0000", 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3768000000000003e-05 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.902060+0000", 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0782000000000002e-05 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906127+0000", 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040670039999999999 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910447+0000", 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043198910000000002 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.915807+0000", 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053596870000000001 2023-09-23T20:59:53.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.915858+0000", 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1242000000000002e-05 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.915901+0000", 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2929000000000002e-05 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.176 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97176 10.d 10:bf52c426:::benchmark_data_smithi121_358375_object12431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.499973+0000", 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 77.608470371999999, 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0090121330000001, 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97176 2023-09-23T20:59:53.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499972+0000", 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499975+0000", 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2090000000000001e-06 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499984+0000", 2023-09-23T20:59:53.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.422e-06 2023-09-23T20:59:53.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499988+0000", 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8739999999999997e-06 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500030+0000", 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2366000000000001e-05 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500039+0000", 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5569999999999996e-06 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242363+0000", 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.74232412699999994 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242370+0000", 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.041e-06 2023-09-23T20:59:53.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482610+0000", 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2402398800000001 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482781+0000", 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000170884 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:59:53.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482955+0000", 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017400099999999999 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490821+0000", 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078662190000000007 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.493967+0000", 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031467309999999998 2023-09-23T20:59:53.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506247+0000", 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012279684000000001 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506289+0000", 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2329000000000001e-05 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.508985+0000", 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026955099999999999 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97184 10.a 10:54b30599:::benchmark_data_smithi121_358375_object12439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.587317+0000", 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 77.521126151000004, 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.974572057, 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.186 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97184 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587316+0000", 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587321+0000", 2023-09-23T20:59:53.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3270000000000002e-06 2023-09-23T20:59:53.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.189 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587333+0000", 2023-09-23T20:59:53.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2383e-05 2023-09-23T20:59:53.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587338+0000", 2023-09-23T20:59:53.190 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9189999999999998e-06 2023-09-23T20:59:53.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587383+0000", 2023-09-23T20:59:53.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5014e-05 2023-09-23T20:59:53.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587389+0000", 2023-09-23T20:59:53.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9660000000000001e-06 2023-09-23T20:59:53.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242357+0000", 2023-09-23T20:59:53.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.65496830699999997 2023-09-23T20:59:53.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242367+0000", 2023-09-23T20:59:53.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0266e-05 2023-09-23T20:59:53.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538285+0000", 2023-09-23T20:59:53.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2959178060000001 2023-09-23T20:59:53.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538406+0000", 2023-09-23T20:59:53.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120308 2023-09-23T20:59:53.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:59:53.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538531+0000", 2023-09-23T20:59:53.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000125761 2023-09-23T20:59:53.197 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.197 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.197 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.197 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.549226+0000", 2023-09-23T20:59:53.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010694789999999999 2023-09-23T20:59:53.197 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.198 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.198 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.198 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551472+0000", 2023-09-23T20:59:53.198 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002246315 2023-09-23T20:59:53.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559794+0000", 2023-09-23T20:59:53.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083212719999999993 2023-09-23T20:59:53.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559841+0000", 2023-09-23T20:59:53.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7197999999999998e-05 2023-09-23T20:59:53.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561889+0000", 2023-09-23T20:59:53.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020482159999999998 2023-09-23T20:59:53.201 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.201 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.201 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.202 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97187 10.0 10:057fb271:::benchmark_data_smithi121_358375_object12442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.202 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.610847+0000", 2023-09-23T20:59:53.202 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 77.497596494999996, 2023-09-23T20:59:53.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8667573179999999, 2023-09-23T20:59:53.203 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.203 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.203 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.203 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.203 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.204 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97187 2023-09-23T20:59:53.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.204 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T20:59:53.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.205 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.205 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.205 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.205 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T20:59:53.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.206 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.206 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610846+0000", 2023-09-23T20:59:53.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T20:59:53.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610850+0000", 2023-09-23T20:59:53.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2509999999999996e-06 2023-09-23T20:59:53.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610863+0000", 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2995e-05 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610869+0000", 2023-09-23T20:59:53.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1480000000000003e-06 2023-09-23T20:59:53.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610945+0000", 2023-09-23T20:59:53.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5284000000000006e-05 2023-09-23T20:59:53.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610953+0000", 2023-09-23T20:59:53.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3750000000000003e-06 2023-09-23T20:59:53.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242784+0000", 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63183069599999997 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242792+0000", 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6510000000000007e-06 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.434568+0000", 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1917759189999999 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.435371+0000", 2023-09-23T20:59:53.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000802339 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451218+0000", 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015847757000000001 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.213 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451292+0000", 2023-09-23T20:59:53.214 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3386000000000005e-05 2023-09-23T20:59:53.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:59:53.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451393+0000", 2023-09-23T20:59:53.214 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101216 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462865+0000", 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011472379 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.466387+0000", 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035211050000000001 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.474961+0000", 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085745549999999993 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.475028+0000", 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7105000000000002e-05 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477604+0000", 2023-09-23T20:59:53.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025758389999999999 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97197 10.d 10:b1ae4ef2:::benchmark_data_smithi121_358375_object12452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.769347+0000", 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 77.339096061000006, 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7457691610000001, 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.217 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97197 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769347+0000", 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769347+0000", 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769346+0000", 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769352+0000", 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4269999999999999e-06 2023-09-23T20:59:53.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769364+0000", 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2561999999999999e-05 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769371+0000", 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4520000000000002e-06 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769417+0000", 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6017999999999999e-05 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769425+0000", 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6619999999999994e-06 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242398+0000", 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.47297264900000002 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242404+0000", 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5640000000000002e-06 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488192+0000", 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.245787612 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488310+0000", 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118377 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488416+0000", 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000105685 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.493980+0000", 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055637339999999999 2023-09-23T20:59:53.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.497354+0000", 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033737319999999999 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.512976+0000", 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015622575 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.513019+0000", 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2478999999999998e-05 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515116+0000", 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020976649999999999 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.225 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97205 10.0 10:0f792b11:::benchmark_data_smithi121_358375_object12460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.899978+0000", 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 77.208465519000001, 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5793486960000001, 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97205 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899978+0000", 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899978+0000", 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899977+0000", 2023-09-23T20:59:53.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899981+0000", 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5700000000000003e-06 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899993+0000", 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1613e-05 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899999+0000", 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5969999999999997e-06 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.900045+0000", 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5772000000000002e-05 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.900053+0000", 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5580000000000003e-06 2023-09-23T20:59:53.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242824+0000", 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34277136600000002 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242832+0000", 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4569999999999999e-06 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.436698+0000", 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1938665129999999 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.437505+0000", 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00080636499999999997 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.456977+0000", 2023-09-23T20:59:53.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019472245999999999 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.457059+0000", 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2544000000000001e-05 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.457160+0000", 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100435 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.466405+0000", 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092448630000000007 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477677+0000", 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011272453999999999 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477732+0000", 2023-09-23T20:59:53.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4910000000000001e-05 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477796+0000", 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4053000000000005e-05 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479326+0000", 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015303929999999999 2023-09-23T20:59:53.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97215 10.d 10:b031210c:::benchmark_data_smithi121_358375_object12470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.106818+0000", 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 77.001624922999994, 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4100548179999999, 2023-09-23T20:59:53.235 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97215 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106818+0000", 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106818+0000", 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106816+0000", 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106830+0000", 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4032000000000001e-05 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106869+0000", 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8349999999999997e-05 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106883+0000", 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4243000000000001e-05 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106982+0000", 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8944000000000007e-05 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.107003+0000", 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1302999999999999e-05 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242425+0000", 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.13542152199999999 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242431+0000", 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1639999999999998e-06 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490848+0000", 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2484173919999999 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490954+0000", 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010638500000000001 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491045+0000", 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0453999999999998e-05 2023-09-23T20:59:53.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.497302+0000", 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062572529999999999 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.500461+0000", 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031586980000000001 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515185+0000", 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014723630999999999 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515224+0000", 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9691999999999999e-05 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.516873+0000", 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001648928 2023-09-23T20:59:53.243 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97221 10.0 10:0777f148:::benchmark_data_smithi121_358375_object12476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.130321+0000", 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.978122225000007, 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3504028450000001, 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.244 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97221 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130321+0000", 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130321+0000", 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130320+0000", 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130327+0000", 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.328e-06 2023-09-23T20:59:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130344+0000", 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6699999999999999e-05 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130352+0000", 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1990000000000006e-06 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130408+0000", 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6060999999999997e-05 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130418+0000", 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0169999999999999e-05 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242919+0000", 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.112500957 2023-09-23T20:59:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242927+0000", 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7819999999999994e-06 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.437568+0000", 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.194641184 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.438303+0000", 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00073454399999999995 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462904+0000", 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024601067000000001 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462984+0000", 2023-09-23T20:59:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9919000000000005e-05 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.463135+0000", 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000151361 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479342+0000", 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016207200000000001 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479403+0000", 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0417000000000001e-05 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479456+0000", 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3798999999999998e-05 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479512+0000", 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5714000000000002e-05 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.480724+0000", 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001211684 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97225 10.a 10:52fbb22c:::benchmark_data_smithi121_358375_object12480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T20:59:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.225436+0000", 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.883007410999994, 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3382990889999999, 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97225 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225436+0000", 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225436+0000", 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225433+0000", 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225446+0000", 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2703e-05 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225471+0000", 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5225e-05 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225481+0000", 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6050000000000004e-06 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225544+0000", 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2649000000000003e-05 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225553+0000", 2023-09-23T20:59:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1549999999999996e-06 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242391+0000", 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016838199000000002 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242398+0000", 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8959999999999997e-06 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541687+0000", 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.299289683 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541850+0000", 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016283399999999999 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541997+0000", 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014649800000000001 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.549235+0000", 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0072378959999999997 2023-09-23T20:59:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551516+0000", 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022813149999999999 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561929+0000", 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010412988999999999 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561972+0000", 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2719000000000002e-05 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.563735+0000", 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001763202 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97254 10.a 10:56bae3f9:::benchmark_data_smithi121_358375_object12509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.794906+0000", 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.313537435000001, 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77559789899999998, 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97254 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794906+0000", 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794906+0000", 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794905+0000", 2023-09-23T20:59:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794911+0000", 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5320000000000003e-06 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794926+0000", 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.447e-05 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794931+0000", 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.451e-06 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.795017+0000", 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5575000000000007e-05 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.795026+0000", 2023-09-23T20:59:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5729999999999997e-06 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.544759+0000", 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.74973301299999995 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.544898+0000", 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000138985 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.545008+0000", 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000109745 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551524+0000", 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065155339999999999 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551553+0000", 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9068999999999999e-05 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.568290+0000", 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016737577 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.568339+0000", 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9119000000000002e-05 2023-09-23T20:59:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570504+0000", 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002164329 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97257 10.a 10:548053b5:::benchmark_data_smithi121_358375_object12512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.920276+0000", 2023-09-23T20:59:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.188166918999997, 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.65229715300000002, 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97257 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T20:59:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920279+0000", 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8709999999999999e-06 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920289+0000", 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4919999999999999e-06 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920293+0000", 2023-09-23T20:59:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1359999999999999e-06 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920345+0000", 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2012999999999998e-05 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920352+0000", 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7309999999999996e-06 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547185+0000", 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.626832944 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547290+0000", 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010505600000000001 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547383+0000", 2023-09-23T20:59:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2948000000000005e-05 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551592+0000", 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042095969999999998 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551600+0000", 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3139999999999998e-06 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570562+0000", 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018962326000000002 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570625+0000", 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3427000000000001e-05 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.572573+0000", 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019480890000000001 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97261 10.0 10:0762f7f9:::benchmark_data_smithi121_358375_object12516:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:37.081781+0000", 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.026662325999993, 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.40037638599999997, 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T20:59:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97261 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081781+0000", 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081781+0000", 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081780+0000", 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081786+0000", 2023-09-23T20:59:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8900000000000004e-06 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081817+0000", 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1344999999999999e-05 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081823+0000", 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8529999999999997e-06 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081881+0000", 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.821e-05 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.081891+0000", 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7289999999999995e-06 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.438341+0000", 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.35645033500000001 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.438977+0000", 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00063621399999999996 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.471339+0000", 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032361981999999997 2023-09-23T20:59:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.471459+0000", 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011931599999999999 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.471651+0000", 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000192695 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479353+0000", 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077012440000000003 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.480795+0000", 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001442486 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.480847+0000", 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1666000000000003e-05 2023-09-23T20:59:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.480901+0000", 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3857000000000003e-05 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482157+0000", 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012566190000000001 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T20:59:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T20:59:53.287 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:53.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:53.317 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:53.412 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:53.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:53.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:53.516 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:53.541 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":949,"stamp":"2023-09-23T20:59:50.860612+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4057077688,"num_objects":61922,"num_object_clones":0,"num_object_copies":185766,"num_objects_missing_on_primary":3737,"num_objects_missing":3737,"num_objects_degraded":30277,"num_objects_misplaced":847,"num_objects_unfound":0,"num_objects_dirty":61922,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":107888,"num_write_kb":5425934,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19641,"num_bytes_recovered":1154045862,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107804,"ondisk_log_size":107804,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":289,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14961172,"kb_used_data":11710932,"kb_used_omap":21,"kb_used_meta":3250154,"kb_avail":735000044,"statfs":{"total":767960285184,"available":752640045056,"internally_reserved":0,"allocated":11991994368,"data_stored":11983260154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3328158179},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-104398848,"num_objects":-1593,"num_object_clones":0,"num_object_copies":-4779,"num_objects_missing_on_primary":-43,"num_objects_missing":-43,"num_objects_degraded":-228,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1593,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":158,"num_bytes_recovered":10354688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.299255"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":22,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.579619+0000","last_change":"2023-09-23T20:59:44.579619+0000","last_active":"2023-09-23T20:59:44.579619+0000","last_peered":"2023-09-23T20:59:44.579619+0000","last_clean":"2023-09-23T20:59:44.579619+0000","last_became_active":"2023-09-23T20:59:44.579313+0000","last_became_peered":"2023-09-23T20:59:44.579313+0000","last_unstale":"2023-09-23T20:59:44.579619+0000","last_undegraded":"2023-09-23T20:59:44.579619+0000","last_fullsized":"2023-09-23T20:59:44.579619+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:35.340190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":271,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.568069+0000","last_change":"2023-09-23T20:57:44.249598+0000","last_active":"2023-09-23T20:59:43.568069+0000","last_peered":"2023-09-23T20:59:43.568069+0000","last_clean":"2023-09-23T20:59:43.568069+0000","last_became_active":"2023-09-23T20:57:44.249342+0000","last_became_peered":"2023-09-23T20:57:44.249342+0000","last_unstale":"2023-09-23T20:59:43.568069+0000","last_undegraded":"2023-09-23T20:59:43.568069+0000","last_fullsized":"2023-09-23T20:59:43.568069+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:53.547436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":172,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567876+0000","last_change":"2023-09-23T20:58:36.390008+0000","last_active":"2023-09-23T20:59:43.567876+0000","last_peered":"2023-09-23T20:59:43.567876+0000","last_clean":"2023-09-23T20:59:43.567876+0000","last_became_active":"2023-09-23T20:58:36.389621+0000","last_became_peered":"2023-09-23T20:58:36.389621+0000","last_unstale":"2023-09-23T20:59:43.567876+0000","last_undegraded":"2023-09-23T20:59:43.567876+0000","last_fullsized":"2023-09-23T20:59:43.567876+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.316905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":274,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.650009+0000","last_change":"2023-09-23T20:57:16.910004+0000","last_active":"2023-09-23T20:59:44.650009+0000","last_peered":"2023-09-23T20:59:44.650009+0000","last_clean":"2023-09-23T20:59:44.650009+0000","last_became_active":"2023-09-23T20:57:16.909719+0000","last_became_peered":"2023-09-23T20:57:16.909719+0000","last_unstale":"2023-09-23T20:59:44.650009+0000","last_undegraded":"2023-09-23T20:59:44.650009+0000","last_fullsized":"2023-09-23T20:59:44.650009+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:39.337052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":298,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567259+0000","last_change":"2023-09-23T20:58:36.250072+0000","last_active":"2023-09-23T20:59:43.567259+0000","last_peered":"2023-09-23T20:59:43.567259+0000","last_clean":"2023-09-23T20:59:43.567259+0000","last_became_active":"2023-09-23T20:58:36.249766+0000","last_became_peered":"2023-09-23T20:58:36.249766+0000","last_unstale":"2023-09-23T20:59:43.567259+0000","last_undegraded":"2023-09-23T20:59:43.567259+0000","last_fullsized":"2023-09-23T20:59:43.567259+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:59.327045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":166,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.575204+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T20:59:44.575204+0000","last_peered":"2023-09-23T20:59:44.575204+0000","last_clean":"2023-09-23T20:59:44.575204+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T20:59:44.575204+0000","last_undegraded":"2023-09-23T20:59:44.575204+0000","last_fullsized":"2023-09-23T20:59:44.575204+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":251,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.650270+0000","last_change":"2023-09-23T20:48:07.523882+0000","last_active":"2023-09-23T20:59:44.650270+0000","last_peered":"2023-09-23T20:59:44.650270+0000","last_clean":"2023-09-23T20:59:44.650270+0000","last_became_active":"2023-09-23T20:48:07.523589+0000","last_became_peered":"2023-09-23T20:48:07.523589+0000","last_unstale":"2023-09-23T20:59:44.650270+0000","last_undegraded":"2023-09-23T20:59:44.650270+0000","last_fullsized":"2023-09-23T20:59:44.650270+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:58.678434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":290,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.568686+0000","last_change":"2023-09-23T20:57:44.251052+0000","last_active":"2023-09-23T20:59:44.568686+0000","last_peered":"2023-09-23T20:59:44.568686+0000","last_clean":"2023-09-23T20:59:44.568686+0000","last_became_active":"2023-09-23T20:57:44.250576+0000","last_became_peered":"2023-09-23T20:57:44.250576+0000","last_unstale":"2023-09-23T20:59:44.568686+0000","last_undegraded":"2023-09-23T20:59:44.568686+0000","last_fullsized":"2023-09-23T20:59:44.568686+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:09.267827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":320,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.597028+0000","last_change":"2023-09-23T20:57:45.249221+0000","last_active":"2023-09-23T20:59:44.597028+0000","last_peered":"2023-09-23T20:59:44.597028+0000","last_clean":"2023-09-23T20:59:44.597028+0000","last_became_active":"2023-09-23T20:57:45.248960+0000","last_became_peered":"2023-09-23T20:57:45.248960+0000","last_unstale":"2023-09-23T20:59:44.597028+0000","last_undegraded":"2023-09-23T20:59:44.597028+0000","last_fullsized":"2023-09-23T20:59:44.597028+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:17.370914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":247,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.576786+0000","last_change":"2023-09-23T20:57:15.910105+0000","last_active":"2023-09-23T20:59:44.576786+0000","last_peered":"2023-09-23T20:59:44.576786+0000","last_clean":"2023-09-23T20:59:44.576786+0000","last_became_active":"2023-09-23T20:57:15.909785+0000","last_became_peered":"2023-09-23T20:57:15.909785+0000","last_unstale":"2023-09-23T20:59:44.576786+0000","last_undegraded":"2023-09-23T20:59:44.576786+0000","last_fullsized":"2023-09-23T20:59:44.576786+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:05.007111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":50,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596398+0000","last_change":"2023-09-23T20:57:15.952549+0000","last_active":"2023-09-23T20:59:44.596398+0000","last_peered":"2023-09-23T20:59:44.596398+0000","last_clean":"2023-09-23T20:59:44.596398+0000","last_became_active":"2023-09-23T20:57:15.913795+0000","last_became_peered":"2023-09-23T20:57:15.913795+0000","last_unstale":"2023-09-23T20:59:44.596398+0000","last_undegraded":"2023-09-23T20:59:44.596398+0000","last_fullsized":"2023-09-23T20:59:44.596398+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:47.933385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":167,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567972+0000","last_change":"2023-09-23T20:57:45.243915+0000","last_active":"2023-09-23T20:59:43.567972+0000","last_peered":"2023-09-23T20:59:43.567972+0000","last_clean":"2023-09-23T20:59:43.567972+0000","last_became_active":"2023-09-23T20:57:45.243610+0000","last_became_peered":"2023-09-23T20:57:45.243610+0000","last_unstale":"2023-09-23T20:59:43.567972+0000","last_undegraded":"2023-09-23T20:59:43.567972+0000","last_fullsized":"2023-09-23T20:59:43.567972+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:55.808488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":54,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.597141+0000","last_change":"2023-09-23T20:57:15.948301+0000","last_active":"2023-09-23T20:59:44.597141+0000","last_peered":"2023-09-23T20:59:44.597141+0000","last_clean":"2023-09-23T20:59:44.597141+0000","last_became_active":"2023-09-23T20:57:15.908308+0000","last_became_peered":"2023-09-23T20:57:15.908308+0000","last_unstale":"2023-09-23T20:59:44.597141+0000","last_undegraded":"2023-09-23T20:59:44.597141+0000","last_fullsized":"2023-09-23T20:59:44.597141+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:19.866857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":299,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.575764+0000","last_change":"2023-09-23T20:57:44.245493+0000","last_active":"2023-09-23T20:59:44.575764+0000","last_peered":"2023-09-23T20:59:44.575764+0000","last_clean":"2023-09-23T20:59:44.575764+0000","last_became_active":"2023-09-23T20:57:44.245140+0000","last_became_peered":"2023-09-23T20:57:44.245140+0000","last_unstale":"2023-09-23T20:59:44.575764+0000","last_undegraded":"2023-09-23T20:59:44.575764+0000","last_fullsized":"2023-09-23T20:59:44.575764+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:48.316166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":51,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.598418+0000","last_change":"2023-09-23T20:57:15.921091+0000","last_active":"2023-09-23T20:59:44.598418+0000","last_peered":"2023-09-23T20:59:44.598418+0000","last_clean":"2023-09-23T20:59:44.598418+0000","last_became_active":"2023-09-23T20:57:15.908112+0000","last_became_peered":"2023-09-23T20:57:15.908112+0000","last_unstale":"2023-09-23T20:59:44.598418+0000","last_undegraded":"2023-09-23T20:59:44.598418+0000","last_fullsized":"2023-09-23T20:59:44.598418+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:43.853034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":312,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.571710+0000","last_change":"2023-09-23T20:57:45.247131+0000","last_active":"2023-09-23T20:59:43.571710+0000","last_peered":"2023-09-23T20:59:43.571710+0000","last_clean":"2023-09-23T20:59:43.571710+0000","last_became_active":"2023-09-23T20:57:45.246850+0000","last_became_peered":"2023-09-23T20:57:45.246850+0000","last_unstale":"2023-09-23T20:59:43.571710+0000","last_undegraded":"2023-09-23T20:59:43.571710+0000","last_fullsized":"2023-09-23T20:59:43.571710+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:15.134941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":157,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596169+0000","last_change":"2023-09-23T20:58:55.938372+0000","last_active":"2023-09-23T20:59:44.596169+0000","last_peered":"2023-09-23T20:59:44.596169+0000","last_clean":"2023-09-23T20:59:44.596169+0000","last_became_active":"2023-09-23T20:58:55.938113+0000","last_became_peered":"2023-09-23T20:58:55.938113+0000","last_unstale":"2023-09-23T20:59:44.596169+0000","last_undegraded":"2023-09-23T20:59:44.596169+0000","last_fullsized":"2023-09-23T20:59:44.596169+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:46.718226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":44,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596310+0000","last_change":"2023-09-23T20:57:15.906421+0000","last_active":"2023-09-23T20:59:44.596310+0000","last_peered":"2023-09-23T20:59:44.596310+0000","last_clean":"2023-09-23T20:59:44.596310+0000","last_became_active":"2023-09-23T20:57:15.906073+0000","last_became_peered":"2023-09-23T20:57:15.906073+0000","last_unstale":"2023-09-23T20:59:44.596310+0000","last_undegraded":"2023-09-23T20:59:44.596310+0000","last_fullsized":"2023-09-23T20:59:44.596310+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:23.145995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596320+0000","last_change":"2023-09-23T20:57:15.909085+0000","last_active":"2023-09-23T20:59:44.596320+0000","last_peered":"2023-09-23T20:59:44.596320+0000","last_clean":"2023-09-23T20:59:44.596320+0000","last_became_active":"2023-09-23T20:57:15.908703+0000","last_became_peered":"2023-09-23T20:57:15.908703+0000","last_unstale":"2023-09-23T20:59:44.596320+0000","last_undegraded":"2023-09-23T20:59:44.596320+0000","last_fullsized":"2023-09-23T20:59:44.596320+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:23.043924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":270,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567409+0000","last_change":"2023-09-23T20:58:36.258695+0000","last_active":"2023-09-23T20:59:43.567409+0000","last_peered":"2023-09-23T20:59:43.567409+0000","last_clean":"2023-09-23T20:59:43.567409+0000","last_became_active":"2023-09-23T20:58:36.258151+0000","last_became_peered":"2023-09-23T20:58:36.258151+0000","last_unstale":"2023-09-23T20:59:43.567409+0000","last_undegraded":"2023-09-23T20:59:43.567409+0000","last_fullsized":"2023-09-23T20:59:43.567409+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:37.235485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":150,"reported_epoch":299,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:59:44.574776+0000","last_change":"2023-09-23T20:58:36.251540+0000","last_active":"2023-09-23T20:59:44.574776+0000","last_peered":"2023-09-23T20:59:44.574776+0000","last_clean":"2023-09-23T20:58:24.953194+0000","last_became_active":"2023-09-23T20:58:36.251226+0000","last_became_peered":"2023-09-23T20:58:36.251226+0000","last_unstale":"2023-09-23T20:59:44.574776+0000","last_undegraded":"2023-09-23T20:58:36.241369+0000","last_fullsized":"2023-09-23T20:59:44.574776+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:46.581009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":312,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567710+0000","last_change":"2023-09-23T20:57:45.249600+0000","last_active":"2023-09-23T20:59:43.567710+0000","last_peered":"2023-09-23T20:59:43.567710+0000","last_clean":"2023-09-23T20:59:43.567710+0000","last_became_active":"2023-09-23T20:57:45.249299+0000","last_became_peered":"2023-09-23T20:57:45.249299+0000","last_unstale":"2023-09-23T20:59:43.567710+0000","last_undegraded":"2023-09-23T20:59:43.567710+0000","last_fullsized":"2023-09-23T20:59:43.567710+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:16.553646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"300'6777","reported_seq":6870,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.749884+0000","last_change":"2023-09-23T20:55:55.476703+0000","last_active":"2023-09-23T20:59:48.749884+0000","last_peered":"2023-09-23T20:59:48.749884+0000","last_clean":"2023-09-23T20:59:48.749884+0000","last_became_active":"2023-09-23T20:55:55.467711+0000","last_became_peered":"2023-09-23T20:55:55.467711+0000","last_unstale":"2023-09-23T20:59:48.749884+0000","last_undegraded":"2023-09-23T20:59:48.749884+0000","last_fullsized":"2023-09-23T20:59:48.749884+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6777,"log_dups_size":0,"ondisk_log_size":6777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:31.924695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259588096,"num_objects":3961,"num_object_clones":0,"num_object_copies":11883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6777,"num_write_kb":343616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":250,"reported_epoch":299,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:59:44.596890+0000","last_change":"2023-09-23T20:58:36.259891+0000","last_active":"2023-09-23T20:59:44.596890+0000","last_peered":"2023-09-23T20:59:44.596890+0000","last_clean":"2023-09-23T20:58:24.951094+0000","last_became_active":"2023-09-23T20:58:36.259657+0000","last_became_peered":"2023-09-23T20:58:36.259657+0000","last_unstale":"2023-09-23T20:59:44.596890+0000","last_undegraded":"2023-09-23T20:58:36.248149+0000","last_fullsized":"2023-09-23T20:59:44.596890+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:18.139037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":282,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596494+0000","last_change":"2023-09-23T20:57:15.906260+0000","last_active":"2023-09-23T20:59:44.596494+0000","last_peered":"2023-09-23T20:59:44.596494+0000","last_clean":"2023-09-23T20:59:44.596494+0000","last_became_active":"2023-09-23T20:57:15.905929+0000","last_became_peered":"2023-09-23T20:57:15.905929+0000","last_unstale":"2023-09-23T20:59:44.596494+0000","last_undegraded":"2023-09-23T20:59:44.596494+0000","last_fullsized":"2023-09-23T20:59:44.596494+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:09.183508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"300'6547","reported_seq":6631,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.737448+0000","last_change":"2023-09-23T20:54:35.777545+0000","last_active":"2023-09-23T20:59:48.737448+0000","last_peered":"2023-09-23T20:59:48.737448+0000","last_clean":"2023-09-23T20:59:48.737448+0000","last_became_active":"2023-09-23T20:54:35.776552+0000","last_became_peered":"2023-09-23T20:54:35.776552+0000","last_unstale":"2023-09-23T20:59:48.737448+0000","last_undegraded":"2023-09-23T20:59:48.737448+0000","last_fullsized":"2023-09-23T20:59:48.737448+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6547,"log_dups_size":0,"ondisk_log_size":6547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:29.974409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245563392,"num_objects":3747,"num_object_clones":0,"num_object_copies":11241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6547,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":287,"reported_epoch":299,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:59:44.595930+0000","last_change":"2023-09-23T20:58:36.260270+0000","last_active":"2023-09-23T20:59:44.595930+0000","last_peered":"2023-09-23T20:59:44.595930+0000","last_clean":"2023-09-23T20:58:24.951147+0000","last_became_active":"2023-09-23T20:58:36.259985+0000","last_became_peered":"2023-09-23T20:58:36.259985+0000","last_unstale":"2023-09-23T20:59:44.595930+0000","last_undegraded":"2023-09-23T20:58:36.246715+0000","last_fullsized":"2023-09-23T20:59:44.595930+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:15.641187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":237,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567974+0000","last_change":"2023-09-23T20:57:44.324702+0000","last_active":"2023-09-23T20:59:43.567974+0000","last_peered":"2023-09-23T20:59:43.567974+0000","last_clean":"2023-09-23T20:59:43.567974+0000","last_became_active":"2023-09-23T20:57:44.324367+0000","last_became_peered":"2023-09-23T20:57:44.324367+0000","last_unstale":"2023-09-23T20:59:43.567974+0000","last_undegraded":"2023-09-23T20:59:43.567974+0000","last_fullsized":"2023-09-23T20:59:43.567974+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:55.384013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"299'6776","reported_seq":6868,"reported_epoch":299,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:46.569697+0000","last_change":"2023-09-23T20:58:37.479225+0000","last_active":"2023-09-23T20:59:46.569697+0000","last_peered":"2023-09-23T20:59:46.569697+0000","last_clean":"2023-09-23T20:58:33.866326+0000","last_became_active":"2023-09-23T20:58:37.454711+0000","last_became_peered":"2023-09-23T20:58:37.454711+0000","last_unstale":"2023-09-23T20:59:46.569697+0000","last_undegraded":"2023-09-23T20:58:37.265426+0000","last_fullsized":"2023-09-23T20:58:37.239656+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6776,"log_dups_size":0,"ondisk_log_size":6776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:38.791081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253100032,"num_objects":3862,"num_object_clones":0,"num_object_copies":11586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5319,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6776,"num_write_kb":340416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":3862},{"shards":"3,4","objects":-646}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":48,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596481+0000","last_change":"2023-09-23T20:57:15.915430+0000","last_active":"2023-09-23T20:59:44.596481+0000","last_peered":"2023-09-23T20:59:44.596481+0000","last_clean":"2023-09-23T20:59:44.596481+0000","last_became_active":"2023-09-23T20:57:15.906106+0000","last_became_peered":"2023-09-23T20:57:15.906106+0000","last_unstale":"2023-09-23T20:59:44.596481+0000","last_undegraded":"2023-09-23T20:59:44.596481+0000","last_fullsized":"2023-09-23T20:59:44.596481+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:22:31.990035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":178,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.576184+0000","last_change":"2023-09-23T20:57:45.252971+0000","last_active":"2023-09-23T20:59:44.576184+0000","last_peered":"2023-09-23T20:59:44.576184+0000","last_clean":"2023-09-23T20:59:44.576184+0000","last_became_active":"2023-09-23T20:57:45.252608+0000","last_became_peered":"2023-09-23T20:57:45.252608+0000","last_unstale":"2023-09-23T20:59:44.576184+0000","last_undegraded":"2023-09-23T20:59:44.576184+0000","last_fullsized":"2023-09-23T20:59:44.576184+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:34.867894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"300'6740","reported_seq":6855,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.495371+0000","last_change":"2023-09-23T20:57:17.902794+0000","last_active":"2023-09-23T20:59:48.495371+0000","last_peered":"2023-09-23T20:59:48.495371+0000","last_clean":"2023-09-23T20:59:48.495371+0000","last_became_active":"2023-09-23T20:57:17.892913+0000","last_became_peered":"2023-09-23T20:57:17.892913+0000","last_unstale":"2023-09-23T20:59:48.495371+0000","last_undegraded":"2023-09-23T20:59:48.495371+0000","last_fullsized":"2023-09-23T20:59:48.495371+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:46.016662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254672896,"num_objects":3886,"num_object_clones":0,"num_object_copies":11658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6740,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":550,"num_bytes_recovered":13172736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":215,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.571567+0000","last_change":"2023-09-23T20:55:55.460433+0000","last_active":"2023-09-23T20:59:43.571567+0000","last_peered":"2023-09-23T20:59:43.571567+0000","last_clean":"2023-09-23T20:59:43.571567+0000","last_became_active":"2023-09-23T20:55:55.459883+0000","last_became_peered":"2023-09-23T20:55:55.459883+0000","last_unstale":"2023-09-23T20:59:43.571567+0000","last_undegraded":"2023-09-23T20:59:43.571567+0000","last_fullsized":"2023-09-23T20:59:43.571567+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:59.444482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":283,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.649590+0000","last_change":"2023-09-23T20:57:44.250822+0000","last_active":"2023-09-23T20:59:44.649590+0000","last_peered":"2023-09-23T20:59:44.649590+0000","last_clean":"2023-09-23T20:59:44.649590+0000","last_became_active":"2023-09-23T20:57:44.250524+0000","last_became_peered":"2023-09-23T20:57:44.250524+0000","last_unstale":"2023-09-23T20:59:44.649590+0000","last_undegraded":"2023-09-23T20:59:44.649590+0000","last_fullsized":"2023-09-23T20:59:44.649590+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:23.511125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"300'6729","reported_seq":17144,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:48.813876+0000","last_change":"2023-09-23T20:58:36.258308+0000","last_active":"2023-09-23T20:59:48.813876+0000","last_peered":"2023-09-23T20:59:48.813876+0000","last_clean":"2023-09-23T20:54:34.751733+0000","last_became_active":"2023-09-23T20:57:45.435535+0000","last_became_peered":"2023-09-23T20:57:45.435535+0000","last_unstale":"2023-09-23T20:59:48.813876+0000","last_undegraded":"2023-09-23T20:57:45.272202+0000","last_fullsized":"2023-09-23T20:57:45.236715+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:33.084037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250281984,"num_objects":3819,"num_object_clones":0,"num_object_copies":11457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":113,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6729,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5161,"num_bytes_recovered":315359232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":3813}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":272,"reported_epoch":298,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:59:43.567333+0000","last_change":"2023-09-23T20:58:36.254342+0000","last_active":"2023-09-23T20:59:43.567333+0000","last_peered":"2023-09-23T20:59:43.567333+0000","last_clean":"2023-09-23T20:58:24.893883+0000","last_became_active":"2023-09-23T20:58:36.254098+0000","last_became_peered":"2023-09-23T20:58:36.254098+0000","last_unstale":"2023-09-23T20:59:43.567333+0000","last_undegraded":"2023-09-23T20:58:36.238561+0000","last_fullsized":"2023-09-23T20:59:43.567333+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:25.061626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":236,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.571470+0000","last_change":"2023-09-23T20:54:35.776908+0000","last_active":"2023-09-23T20:59:43.571470+0000","last_peered":"2023-09-23T20:59:43.571470+0000","last_clean":"2023-09-23T20:59:43.571470+0000","last_became_active":"2023-09-23T20:54:35.776549+0000","last_became_peered":"2023-09-23T20:54:35.776549+0000","last_unstale":"2023-09-23T20:59:43.571470+0000","last_undegraded":"2023-09-23T20:59:43.571470+0000","last_fullsized":"2023-09-23T20:59:43.571470+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.593595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"300'6898","reported_seq":7661,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.411532+0000","last_change":"2023-09-23T20:57:03.729205+0000","last_active":"2023-09-23T20:59:48.411532+0000","last_peered":"2023-09-23T20:59:48.411532+0000","last_clean":"2023-09-23T20:59:48.411532+0000","last_became_active":"2023-09-23T20:57:03.721552+0000","last_became_peered":"2023-09-23T20:57:03.721552+0000","last_unstale":"2023-09-23T20:59:48.411532+0000","last_undegraded":"2023-09-23T20:59:48.411532+0000","last_fullsized":"2023-09-23T20:59:48.411532+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:40.786255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262012928,"num_objects":3998,"num_object_clones":0,"num_object_copies":11994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6898,"num_write_kb":348672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":851,"num_bytes_recovered":34406400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":166,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.576228+0000","last_change":"2023-09-23T20:57:44.249701+0000","last_active":"2023-09-23T20:59:44.576228+0000","last_peered":"2023-09-23T20:59:44.576228+0000","last_clean":"2023-09-23T20:59:44.576228+0000","last_became_active":"2023-09-23T20:57:44.249408+0000","last_became_peered":"2023-09-23T20:57:44.249408+0000","last_unstale":"2023-09-23T20:59:44.576228+0000","last_undegraded":"2023-09-23T20:59:44.576228+0000","last_fullsized":"2023-09-23T20:59:44.576228+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:30.094839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":299,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.578262+0000","last_change":"2023-09-23T20:58:36.260344+0000","last_active":"2023-09-23T20:59:44.578262+0000","last_peered":"2023-09-23T20:59:44.578262+0000","last_clean":"2023-09-23T20:59:44.578262+0000","last_became_active":"2023-09-23T20:58:36.260001+0000","last_became_peered":"2023-09-23T20:58:36.260001+0000","last_unstale":"2023-09-23T20:59:44.578262+0000","last_undegraded":"2023-09-23T20:59:44.578262+0000","last_fullsized":"2023-09-23T20:59:44.578262+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:55.903992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"300'6655","reported_seq":8732,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:48.713766+0000","last_change":"2023-09-23T20:58:37.435471+0000","last_active":"2023-09-23T20:59:48.713766+0000","last_peered":"2023-09-23T20:59:48.713766+0000","last_clean":"2023-09-23T20:54:43.441608+0000","last_became_active":"2023-09-23T20:58:37.424775+0000","last_became_peered":"2023-09-23T20:58:37.424775+0000","last_unstale":"2023-09-23T20:59:48.713766+0000","last_undegraded":"2023-09-23T20:58:37.358473+0000","last_fullsized":"2023-09-23T20:58:37.242618+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6655,"log_dups_size":0,"ondisk_log_size":6655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:34.491158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245956608,"num_objects":3753,"num_object_clones":0,"num_object_copies":11259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9098,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6655,"num_write_kb":333056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":655,"num_bytes_recovered":42926080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":3753}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":195,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.578036+0000","last_change":"2023-09-23T20:58:56.039035+0000","last_active":"2023-09-23T20:59:44.578036+0000","last_peered":"2023-09-23T20:59:44.578036+0000","last_clean":"2023-09-23T20:59:44.578036+0000","last_became_active":"2023-09-23T20:58:56.038610+0000","last_became_peered":"2023-09-23T20:58:56.038610+0000","last_unstale":"2023-09-23T20:59:44.578036+0000","last_undegraded":"2023-09-23T20:59:44.578036+0000","last_fullsized":"2023-09-23T20:59:44.578036+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":290,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:02.235874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":299,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.575846+0000","last_change":"2023-09-23T20:57:44.250118+0000","last_active":"2023-09-23T20:59:44.575846+0000","last_peered":"2023-09-23T20:59:44.575846+0000","last_clean":"2023-09-23T20:59:44.575846+0000","last_became_active":"2023-09-23T20:57:44.249814+0000","last_became_peered":"2023-09-23T20:57:44.249814+0000","last_unstale":"2023-09-23T20:59:44.575846+0000","last_undegraded":"2023-09-23T20:59:44.575846+0000","last_fullsized":"2023-09-23T20:59:44.575846+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:58.298798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"300'6874","reported_seq":6972,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.779692+0000","last_change":"2023-09-23T20:57:16.914501+0000","last_active":"2023-09-23T20:59:48.779692+0000","last_peered":"2023-09-23T20:59:48.779692+0000","last_clean":"2023-09-23T20:59:48.779692+0000","last_became_active":"2023-09-23T20:57:16.907404+0000","last_became_peered":"2023-09-23T20:57:16.907404+0000","last_unstale":"2023-09-23T20:59:48.779692+0000","last_undegraded":"2023-09-23T20:59:48.779692+0000","last_fullsized":"2023-09-23T20:59:48.779692+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:39.583426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256114688,"num_objects":3908,"num_object_clones":0,"num_object_copies":11724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6874,"num_write_kb":345024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":294,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567108+0000","last_change":"2023-09-23T20:57:45.247092+0000","last_active":"2023-09-23T20:59:43.567108+0000","last_peered":"2023-09-23T20:59:43.567108+0000","last_clean":"2023-09-23T20:59:43.567108+0000","last_became_active":"2023-09-23T20:57:45.246776+0000","last_became_peered":"2023-09-23T20:57:45.246776+0000","last_unstale":"2023-09-23T20:59:43.567108+0000","last_undegraded":"2023-09-23T20:59:43.567108+0000","last_fullsized":"2023-09-23T20:59:43.567108+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:46.691931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":32,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567917+0000","last_change":"2023-09-23T20:57:45.245751+0000","last_active":"2023-09-23T20:59:43.567917+0000","last_peered":"2023-09-23T20:59:43.567917+0000","last_clean":"2023-09-23T20:59:43.567917+0000","last_became_active":"2023-09-23T20:57:45.245476+0000","last_became_peered":"2023-09-23T20:57:45.245476+0000","last_unstale":"2023-09-23T20:59:43.567917+0000","last_undegraded":"2023-09-23T20:59:43.567917+0000","last_fullsized":"2023-09-23T20:59:43.567917+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:44.987049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"300'6649","reported_seq":13099,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.739588+0000","last_change":"2023-09-23T20:57:21.928776+0000","last_active":"2023-09-23T20:59:48.739588+0000","last_peered":"2023-09-23T20:59:48.739588+0000","last_clean":"2023-09-23T20:59:48.739588+0000","last_became_active":"2023-09-23T20:57:21.923250+0000","last_became_peered":"2023-09-23T20:57:21.923250+0000","last_unstale":"2023-09-23T20:59:48.739588+0000","last_undegraded":"2023-09-23T20:59:48.739588+0000","last_fullsized":"2023-09-23T20:59:48.739588+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6649,"log_dups_size":0,"ondisk_log_size":6649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:36.965869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254214144,"num_objects":3879,"num_object_clones":0,"num_object_copies":11637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6649,"num_write_kb":336896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3885,"num_bytes_recovered":231669760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":181,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.651249+0000","last_change":"2023-09-23T20:59:44.650895+0000","last_active":"2023-09-23T20:59:44.651249+0000","last_peered":"2023-09-23T20:59:44.651249+0000","last_clean":"2023-09-23T20:59:44.651249+0000","last_became_active":"2023-09-23T20:59:44.650598+0000","last_became_peered":"2023-09-23T20:59:44.650598+0000","last_unstale":"2023-09-23T20:59:44.651249+0000","last_undegraded":"2023-09-23T20:59:44.651249+0000","last_fullsized":"2023-09-23T20:59:44.651249+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:58.620444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":297,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596862+0000","last_change":"2023-09-23T20:56:52.999112+0000","last_active":"2023-09-23T20:59:44.596862+0000","last_peered":"2023-09-23T20:59:44.596862+0000","last_clean":"2023-09-23T20:59:44.596862+0000","last_became_active":"2023-09-23T20:56:52.998744+0000","last_became_peered":"2023-09-23T20:56:52.998744+0000","last_unstale":"2023-09-23T20:59:44.596862+0000","last_undegraded":"2023-09-23T20:59:44.596862+0000","last_fullsized":"2023-09-23T20:59:44.596862+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:58.451042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":167,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.575199+0000","last_change":"2023-09-23T20:58:36.251017+0000","last_active":"2023-09-23T20:59:44.575199+0000","last_peered":"2023-09-23T20:59:44.575199+0000","last_clean":"2023-09-23T20:59:44.575199+0000","last_became_active":"2023-09-23T20:58:36.250690+0000","last_became_peered":"2023-09-23T20:58:36.250690+0000","last_unstale":"2023-09-23T20:59:44.575199+0000","last_undegraded":"2023-09-23T20:59:44.575199+0000","last_fullsized":"2023-09-23T20:59:44.575199+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:22.238510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"300'6677","reported_seq":6771,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:48.727744+0000","last_change":"2023-09-23T20:58:37.549306+0000","last_active":"2023-09-23T20:59:48.727744+0000","last_peered":"2023-09-23T20:59:48.727744+0000","last_clean":"2023-09-23T20:58:34.639005+0000","last_became_active":"2023-09-23T20:58:37.523477+0000","last_became_peered":"2023-09-23T20:58:37.523477+0000","last_unstale":"2023-09-23T20:59:48.727744+0000","last_undegraded":"2023-09-23T20:58:37.289061+0000","last_fullsized":"2023-09-23T20:58:37.245383+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6677,"log_dups_size":0,"ondisk_log_size":6677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:15.757056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253296640,"num_objects":3865,"num_object_clones":0,"num_object_copies":11595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5271,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6677,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":3865},{"shards":"5,6","objects":-655}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":235,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.569202+0000","last_change":"2023-09-23T20:57:44.246836+0000","last_active":"2023-09-23T20:59:43.569202+0000","last_peered":"2023-09-23T20:59:43.569202+0000","last_clean":"2023-09-23T20:59:43.569202+0000","last_became_active":"2023-09-23T20:57:44.246521+0000","last_became_peered":"2023-09-23T20:57:44.246521+0000","last_unstale":"2023-09-23T20:59:43.569202+0000","last_undegraded":"2023-09-23T20:59:43.569202+0000","last_fullsized":"2023-09-23T20:59:43.569202+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:45.073844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":230,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.575697+0000","last_change":"2023-09-23T20:55:05.920213+0000","last_active":"2023-09-23T20:59:44.575697+0000","last_peered":"2023-09-23T20:59:44.575697+0000","last_clean":"2023-09-23T20:59:44.575697+0000","last_became_active":"2023-09-23T20:55:05.917924+0000","last_became_peered":"2023-09-23T20:55:05.917924+0000","last_unstale":"2023-09-23T20:59:44.575697+0000","last_undegraded":"2023-09-23T20:59:44.575697+0000","last_fullsized":"2023-09-23T20:59:44.575697+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:17.149381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":57,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.577950+0000","last_change":"2023-09-23T20:56:53.000153+0000","last_active":"2023-09-23T20:59:44.577950+0000","last_peered":"2023-09-23T20:59:44.577950+0000","last_clean":"2023-09-23T20:59:44.577950+0000","last_became_active":"2023-09-23T20:56:52.999821+0000","last_became_peered":"2023-09-23T20:56:52.999821+0000","last_unstale":"2023-09-23T20:59:44.577950+0000","last_undegraded":"2023-09-23T20:59:44.577950+0000","last_fullsized":"2023-09-23T20:59:44.577950+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.519799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"299'6700","reported_seq":10223,"reported_epoch":299,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:59:46.549758+0000","last_change":"2023-09-23T20:57:35.384520+0000","last_active":"2023-09-23T20:59:46.549758+0000","last_peered":"2023-09-23T20:59:46.549758+0000","last_clean":"2023-09-23T20:55:29.724749+0000","last_became_active":"2023-09-23T20:57:34.820417+0000","last_became_peered":"2023-09-23T20:57:34.820417+0000","last_unstale":"2023-09-23T20:59:46.549758+0000","last_undegraded":"2023-09-23T20:59:46.549758+0000","last_fullsized":"2023-09-23T20:57:16.887143+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:40.546208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258605056,"num_objects":3946,"num_object_clones":0,"num_object_copies":11838,"num_objects_missing_on_primary":3737,"num_objects_missing":3737,"num_objects_degraded":0,"num_objects_misplaced":847,"num_objects_unfound":0,"num_objects_dirty":3946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6700,"num_write_kb":340672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1663,"num_bytes_recovered":108789760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[3,7],"avail_no_missing":["5"],"object_location_counts":[{"shards":"2,3,5,7","objects":13},{"shards":"2,4,5","objects":455},{"shards":"2,5,7","objects":3282},{"shards":"3,4,5","objects":196}],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":255,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.578288+0000","last_change":"2023-09-23T20:57:10.019515+0000","last_active":"2023-09-23T20:59:44.578288+0000","last_peered":"2023-09-23T20:59:44.578288+0000","last_clean":"2023-09-23T20:59:44.578288+0000","last_became_active":"2023-09-23T20:57:10.019226+0000","last_became_peered":"2023-09-23T20:57:10.019226+0000","last_unstale":"2023-09-23T20:59:44.578288+0000","last_undegraded":"2023-09-23T20:59:44.578288+0000","last_fullsized":"2023-09-23T20:59:44.578288+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:13.610232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":190,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.581544+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T20:59:44.581544+0000","last_peered":"2023-09-23T20:59:44.581544+0000","last_clean":"2023-09-23T20:59:44.581544+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T20:59:44.581544+0000","last_undegraded":"2023-09-23T20:59:44.581544+0000","last_fullsized":"2023-09-23T20:59:44.581544+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":208,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567997+0000","last_change":"2023-09-23T20:58:36.259879+0000","last_active":"2023-09-23T20:59:43.567997+0000","last_peered":"2023-09-23T20:59:43.567997+0000","last_clean":"2023-09-23T20:59:43.567997+0000","last_became_active":"2023-09-23T20:58:36.259593+0000","last_became_peered":"2023-09-23T20:58:36.259593+0000","last_unstale":"2023-09-23T20:59:43.567997+0000","last_undegraded":"2023-09-23T20:59:43.567997+0000","last_fullsized":"2023-09-23T20:59:43.567997+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:29.785898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"300'6721","reported_seq":6827,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:48.653227+0000","last_change":"2023-09-23T20:58:36.273414+0000","last_active":"2023-09-23T20:59:48.653227+0000","last_peered":"2023-09-23T20:59:48.653227+0000","last_clean":"2023-09-23T20:54:43.441988+0000","last_became_active":"2023-09-23T20:57:45.460741+0000","last_became_peered":"2023-09-23T20:57:45.460741+0000","last_unstale":"2023-09-23T20:59:48.653227+0000","last_undegraded":"2023-09-23T20:57:45.278767+0000","last_fullsized":"2023-09-23T20:57:45.242451+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:47.975740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251461632,"num_objects":3837,"num_object_clones":0,"num_object_copies":11511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6721,"num_write_kb":337856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":-1028}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":66,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.578498+0000","last_change":"2023-09-23T20:57:15.904314+0000","last_active":"2023-09-23T20:59:44.578498+0000","last_peered":"2023-09-23T20:59:44.578498+0000","last_clean":"2023-09-23T20:59:44.578498+0000","last_became_active":"2023-09-23T20:57:15.903877+0000","last_became_peered":"2023-09-23T20:57:15.903877+0000","last_unstale":"2023-09-23T20:59:44.578498+0000","last_undegraded":"2023-09-23T20:59:44.578498+0000","last_fullsized":"2023-09-23T20:59:44.578498+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:13.571032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":683,"reported_epoch":299,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:44.595915+0000","last_change":"2023-09-23T20:58:37.361635+0000","last_active":"2023-09-23T20:59:44.595915+0000","last_peered":"2023-09-23T20:59:44.595915+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T20:58:37.361300+0000","last_became_peered":"2023-09-23T20:58:37.361300+0000","last_unstale":"2023-09-23T20:59:44.595915+0000","last_undegraded":"2023-09-23T20:58:37.243369+0000","last_fullsized":"2023-09-23T20:58:37.242679+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:32.247925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":237,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567044+0000","last_change":"2023-09-23T20:51:05.996444+0000","last_active":"2023-09-23T20:59:43.567044+0000","last_peered":"2023-09-23T20:59:43.567044+0000","last_clean":"2023-09-23T20:59:43.567044+0000","last_became_active":"2023-09-23T20:51:05.995230+0000","last_became_peered":"2023-09-23T20:51:05.995230+0000","last_unstale":"2023-09-23T20:59:43.567044+0000","last_undegraded":"2023-09-23T20:59:43.567044+0000","last_fullsized":"2023-09-23T20:59:43.567044+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:07.464131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.574971+0000","last_change":"2023-09-23T20:57:15.910395+0000","last_active":"2023-09-23T20:59:44.574971+0000","last_peered":"2023-09-23T20:59:44.574971+0000","last_clean":"2023-09-23T20:59:44.574971+0000","last_became_active":"2023-09-23T20:57:15.909812+0000","last_became_peered":"2023-09-23T20:57:15.909812+0000","last_unstale":"2023-09-23T20:59:44.574971+0000","last_undegraded":"2023-09-23T20:59:44.574971+0000","last_fullsized":"2023-09-23T20:59:44.574971+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:57.043984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"300'6830","reported_seq":21617,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.541055+0000","last_change":"2023-09-23T20:57:33.445244+0000","last_active":"2023-09-23T20:59:48.541055+0000","last_peered":"2023-09-23T20:59:48.541055+0000","last_clean":"2023-09-23T20:59:48.541055+0000","last_became_active":"2023-09-23T20:57:33.440125+0000","last_became_peered":"2023-09-23T20:57:33.440125+0000","last_unstale":"2023-09-23T20:59:48.541055+0000","last_undegraded":"2023-09-23T20:59:48.541055+0000","last_fullsized":"2023-09-23T20:59:48.541055+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:34.082121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258473984,"num_objects":3944,"num_object_clones":0,"num_object_copies":11832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6830,"num_write_kb":344768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5773,"num_bytes_recovered":378339328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":298,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.595809+0000","last_change":"2023-09-23T20:57:45.253710+0000","last_active":"2023-09-23T20:59:44.595809+0000","last_peered":"2023-09-23T20:59:44.595809+0000","last_clean":"2023-09-23T20:59:44.595809+0000","last_became_active":"2023-09-23T20:57:45.253427+0000","last_became_peered":"2023-09-23T20:57:45.253427+0000","last_unstale":"2023-09-23T20:59:44.595809+0000","last_undegraded":"2023-09-23T20:59:44.595809+0000","last_fullsized":"2023-09-23T20:59:44.595809+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:19.420653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":262,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.652079+0000","last_change":"2023-09-23T20:58:36.389979+0000","last_active":"2023-09-23T20:59:44.652079+0000","last_peered":"2023-09-23T20:59:44.652079+0000","last_clean":"2023-09-23T20:59:44.652079+0000","last_became_active":"2023-09-23T20:58:36.389608+0000","last_became_peered":"2023-09-23T20:58:36.389608+0000","last_unstale":"2023-09-23T20:59:44.652079+0000","last_undegraded":"2023-09-23T20:59:44.652079+0000","last_fullsized":"2023-09-23T20:59:44.652079+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:07.950069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":301,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.597922+0000","last_change":"2023-09-23T20:59:44.597643+0000","last_active":"2023-09-23T20:59:44.597922+0000","last_peered":"2023-09-23T20:59:44.597922+0000","last_clean":"2023-09-23T20:59:44.597922+0000","last_became_active":"2023-09-23T20:59:44.597342+0000","last_became_peered":"2023-09-23T20:59:44.597342+0000","last_unstale":"2023-09-23T20:59:44.597922+0000","last_undegraded":"2023-09-23T20:59:44.597922+0000","last_fullsized":"2023-09-23T20:59:44.597922+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:23.604688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"300'6558","reported_seq":6653,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:59:48.707646+0000","last_change":"2023-09-23T20:58:37.485623+0000","last_active":"2023-09-23T20:59:48.707646+0000","last_peered":"2023-09-23T20:59:48.707646+0000","last_clean":"2023-09-23T20:58:34.832734+0000","last_became_active":"2023-09-23T20:58:37.469137+0000","last_became_peered":"2023-09-23T20:58:37.469137+0000","last_unstale":"2023-09-23T20:59:48.707646+0000","last_undegraded":"2023-09-23T20:58:37.281954+0000","last_fullsized":"2023-09-23T20:58:37.242658+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:46.422121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249888792,"num_objects":3814,"num_object_clones":0,"num_object_copies":11442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5186,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3814,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6558,"num_write_kb":331841,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":3814},{"shards":"4,6","objects":-635}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":255,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596555+0000","last_change":"2023-09-23T20:58:36.260188+0000","last_active":"2023-09-23T20:59:44.596555+0000","last_peered":"2023-09-23T20:59:44.596555+0000","last_clean":"2023-09-23T20:59:44.596555+0000","last_became_active":"2023-09-23T20:58:36.259859+0000","last_became_peered":"2023-09-23T20:58:36.259859+0000","last_unstale":"2023-09-23T20:59:44.596555+0000","last_undegraded":"2023-09-23T20:59:44.596555+0000","last_fullsized":"2023-09-23T20:59:44.596555+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:37.749346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":165,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.595956+0000","last_change":"2023-09-23T20:59:44.595679+0000","last_active":"2023-09-23T20:59:44.595956+0000","last_peered":"2023-09-23T20:59:44.595956+0000","last_clean":"2023-09-23T20:59:44.595956+0000","last_became_active":"2023-09-23T20:59:44.595341+0000","last_became_peered":"2023-09-23T20:59:44.595341+0000","last_unstale":"2023-09-23T20:59:44.595956+0000","last_undegraded":"2023-09-23T20:59:44.595956+0000","last_fullsized":"2023-09-23T20:59:44.595956+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:03.685196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":115,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596980+0000","last_change":"2023-09-23T20:57:15.909593+0000","last_active":"2023-09-23T20:59:44.596980+0000","last_peered":"2023-09-23T20:59:44.596980+0000","last_clean":"2023-09-23T20:59:44.596980+0000","last_became_active":"2023-09-23T20:57:15.909267+0000","last_became_peered":"2023-09-23T20:57:15.909267+0000","last_unstale":"2023-09-23T20:59:44.596980+0000","last_undegraded":"2023-09-23T20:59:44.596980+0000","last_fullsized":"2023-09-23T20:59:44.596980+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:48.909976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"300'6689","reported_seq":7838,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.733450+0000","last_change":"2023-09-23T20:56:55.943104+0000","last_active":"2023-09-23T20:59:48.733450+0000","last_peered":"2023-09-23T20:59:48.733450+0000","last_clean":"2023-09-23T20:59:48.733450+0000","last_became_active":"2023-09-23T20:56:55.937950+0000","last_became_peered":"2023-09-23T20:56:55.937950+0000","last_unstale":"2023-09-23T20:59:48.733450+0000","last_undegraded":"2023-09-23T20:59:48.733450+0000","last_fullsized":"2023-09-23T20:59:48.733450+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6689,"log_dups_size":0,"ondisk_log_size":6689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:10.203931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248971264,"num_objects":3799,"num_object_clones":0,"num_object_copies":11397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6689,"num_write_kb":335616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":517,"num_bytes_recovered":12189696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":291,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567796+0000","last_change":"2023-09-23T20:57:15.923214+0000","last_active":"2023-09-23T20:59:43.567796+0000","last_peered":"2023-09-23T20:59:43.567796+0000","last_clean":"2023-09-23T20:59:43.567796+0000","last_became_active":"2023-09-23T20:57:15.909700+0000","last_became_peered":"2023-09-23T20:57:15.909700+0000","last_unstale":"2023-09-23T20:59:43.567796+0000","last_undegraded":"2023-09-23T20:59:43.567796+0000","last_fullsized":"2023-09-23T20:59:43.567796+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:47.582106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":195,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.569294+0000","last_change":"2023-09-23T20:57:44.251246+0000","last_active":"2023-09-23T20:59:43.569294+0000","last_peered":"2023-09-23T20:59:43.569294+0000","last_clean":"2023-09-23T20:59:43.569294+0000","last_became_active":"2023-09-23T20:57:44.250908+0000","last_became_peered":"2023-09-23T20:57:44.250908+0000","last_unstale":"2023-09-23T20:59:43.569294+0000","last_undegraded":"2023-09-23T20:59:43.569294+0000","last_fullsized":"2023-09-23T20:59:43.569294+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:55.004865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":291,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567026+0000","last_change":"2023-09-23T20:58:36.253832+0000","last_active":"2023-09-23T20:59:43.567026+0000","last_peered":"2023-09-23T20:59:43.567026+0000","last_clean":"2023-09-23T20:59:43.567026+0000","last_became_active":"2023-09-23T20:58:36.253552+0000","last_became_peered":"2023-09-23T20:58:36.253552+0000","last_unstale":"2023-09-23T20:59:43.567026+0000","last_undegraded":"2023-09-23T20:59:43.567026+0000","last_fullsized":"2023-09-23T20:59:43.567026+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:59.549604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"300'6768","reported_seq":7964,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-09-23T20:59:48.714420+0000","last_change":"2023-09-23T20:57:13.897490+0000","last_active":"2023-09-23T20:59:48.714420+0000","last_peered":"2023-09-23T20:59:48.714420+0000","last_clean":"2023-09-23T20:59:48.714420+0000","last_became_active":"2023-09-23T20:57:13.890198+0000","last_became_peered":"2023-09-23T20:57:13.890198+0000","last_unstale":"2023-09-23T20:59:48.714420+0000","last_undegraded":"2023-09-23T20:59:48.714420+0000","last_fullsized":"2023-09-23T20:59:48.714420+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":224,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:54:34.751733+0000","last_clean_scrub_stamp":"2023-09-23T20:54:34.751733+0000","objects_scrubbed":0,"log_size":6768,"log_dups_size":0,"ondisk_log_size":6768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:01.820773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254279680,"num_objects":3880,"num_object_clones":0,"num_object_copies":11640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6768,"num_write_kb":340736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":539,"num_bytes_recovered":14155776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":176,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.575050+0000","last_change":"2023-09-23T20:57:15.904966+0000","last_active":"2023-09-23T20:59:44.575050+0000","last_peered":"2023-09-23T20:59:44.575050+0000","last_clean":"2023-09-23T20:59:44.575050+0000","last_became_active":"2023-09-23T20:57:15.904651+0000","last_became_peered":"2023-09-23T20:57:15.904651+0000","last_unstale":"2023-09-23T20:59:44.575050+0000","last_undegraded":"2023-09-23T20:59:44.575050+0000","last_fullsized":"2023-09-23T20:59:44.575050+0000","mapping_epoch":265,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:01.038015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":45,"reported_epoch":298,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:59:43.569458+0000","last_change":"2023-09-23T20:58:36.257022+0000","last_active":"2023-09-23T20:59:43.569458+0000","last_peered":"2023-09-23T20:59:43.569458+0000","last_clean":"2023-09-23T20:30:24.498209+0000","last_became_active":"2023-09-23T20:58:36.256755+0000","last_became_peered":"2023-09-23T20:58:36.256755+0000","last_unstale":"2023-09-23T20:59:43.569458+0000","last_undegraded":"2023-09-23T20:58:36.243043+0000","last_fullsized":"2023-09-23T20:59:43.569458+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:15.525957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":219,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.596799+0000","last_change":"2023-09-23T20:54:35.777462+0000","last_active":"2023-09-23T20:59:44.596799+0000","last_peered":"2023-09-23T20:59:44.596799+0000","last_clean":"2023-09-23T20:59:44.596799+0000","last_became_active":"2023-09-23T20:54:35.777151+0000","last_became_peered":"2023-09-23T20:54:35.777151+0000","last_unstale":"2023-09-23T20:59:44.596799+0000","last_undegraded":"2023-09-23T20:59:44.596799+0000","last_fullsized":"2023-09-23T20:59:44.596799+0000","mapping_epoch":224,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":225,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:58.147893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":28,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.598575+0000","last_change":"2023-09-23T20:58:36.260126+0000","last_active":"2023-09-23T20:59:44.598575+0000","last_peered":"2023-09-23T20:59:44.598575+0000","last_clean":"2023-09-23T20:59:44.598575+0000","last_became_active":"2023-09-23T20:58:36.259767+0000","last_became_peered":"2023-09-23T20:58:36.259767+0000","last_unstale":"2023-09-23T20:59:44.598575+0000","last_undegraded":"2023-09-23T20:59:44.598575+0000","last_fullsized":"2023-09-23T20:59:44.598575+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:19.882219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":302,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.578023+0000","last_change":"2023-09-23T20:57:45.244884+0000","last_active":"2023-09-23T20:59:44.578023+0000","last_peered":"2023-09-23T20:59:44.578023+0000","last_clean":"2023-09-23T20:59:44.578023+0000","last_became_active":"2023-09-23T20:57:45.244605+0000","last_became_peered":"2023-09-23T20:57:45.244605+0000","last_unstale":"2023-09-23T20:59:44.578023+0000","last_undegraded":"2023-09-23T20:59:44.578023+0000","last_fullsized":"2023-09-23T20:59:44.578023+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:40.539527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":47,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.576185+0000","last_change":"2023-09-23T20:57:44.248071+0000","last_active":"2023-09-23T20:59:43.576185+0000","last_peered":"2023-09-23T20:59:43.576185+0000","last_clean":"2023-09-23T20:59:43.576185+0000","last_became_active":"2023-09-23T20:57:44.247784+0000","last_became_peered":"2023-09-23T20:57:44.247784+0000","last_unstale":"2023-09-23T20:59:43.576185+0000","last_undegraded":"2023-09-23T20:59:43.576185+0000","last_fullsized":"2023-09-23T20:59:43.576185+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:28.185246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":303,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567476+0000","last_change":"2023-09-23T20:58:36.258286+0000","last_active":"2023-09-23T20:59:43.567476+0000","last_peered":"2023-09-23T20:59:43.567476+0000","last_clean":"2023-09-23T20:59:43.567476+0000","last_became_active":"2023-09-23T20:58:36.257911+0000","last_became_peered":"2023-09-23T20:58:36.257911+0000","last_unstale":"2023-09-23T20:59:43.567476+0000","last_undegraded":"2023-09-23T20:59:43.567476+0000","last_fullsized":"2023-09-23T20:59:43.567476+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:18.216370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":205,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.575950+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T20:59:44.575950+0000","last_peered":"2023-09-23T20:59:44.575950+0000","last_clean":"2023-09-23T20:59:44.575950+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T20:59:44.575950+0000","last_undegraded":"2023-09-23T20:59:44.575950+0000","last_fullsized":"2023-09-23T20:59:44.575950+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":213,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567920+0000","last_change":"2023-09-23T20:58:36.254795+0000","last_active":"2023-09-23T20:59:43.567920+0000","last_peered":"2023-09-23T20:59:43.567920+0000","last_clean":"2023-09-23T20:59:43.567920+0000","last_became_active":"2023-09-23T20:58:36.254486+0000","last_became_peered":"2023-09-23T20:58:36.254486+0000","last_unstale":"2023-09-23T20:59:43.567920+0000","last_undegraded":"2023-09-23T20:59:43.567920+0000","last_fullsized":"2023-09-23T20:59:43.567920+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:51.059069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":296,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.567330+0000","last_change":"2023-09-23T20:56:52.997292+0000","last_active":"2023-09-23T20:59:43.567330+0000","last_peered":"2023-09-23T20:59:43.567330+0000","last_clean":"2023-09-23T20:59:43.567330+0000","last_became_active":"2023-09-23T20:56:52.997018+0000","last_became_peered":"2023-09-23T20:56:52.997018+0000","last_unstale":"2023-09-23T20:59:43.567330+0000","last_undegraded":"2023-09-23T20:59:43.567330+0000","last_fullsized":"2023-09-23T20:59:43.567330+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:50.415205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":282,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:59:43.662376+0000","last_change":"2023-09-23T20:56:53.060040+0000","last_active":"2023-09-23T20:59:43.662376+0000","last_peered":"2023-09-23T20:59:43.662376+0000","last_clean":"2023-09-23T20:59:43.662376+0000","last_became_active":"2023-09-23T20:56:53.059692+0000","last_became_peered":"2023-09-23T20:56:53.059692+0000","last_unstale":"2023-09-23T20:59:43.662376+0000","last_undegraded":"2023-09-23T20:59:43.662376+0000","last_fullsized":"2023-09-23T20:59:43.662376+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:58.628986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":250,"reported_epoch":299,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:59:44.577985+0000","last_change":"2023-09-23T20:58:36.253368+0000","last_active":"2023-09-23T20:59:44.577985+0000","last_peered":"2023-09-23T20:59:44.577985+0000","last_clean":"2023-09-23T20:56:38.555777+0000","last_became_active":"2023-09-23T20:58:36.252956+0000","last_became_peered":"2023-09-23T20:58:36.252956+0000","last_unstale":"2023-09-23T20:59:44.577985+0000","last_undegraded":"2023-09-23T20:58:36.249817+0000","last_fullsized":"2023-09-23T20:59:44.577985+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:21.466186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":313,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:59:44.574767+0000","last_change":"2023-09-23T20:57:44.251315+0000","last_active":"2023-09-23T20:59:44.574767+0000","last_peered":"2023-09-23T20:59:44.574767+0000","last_clean":"2023-09-23T20:59:44.574767+0000","last_became_active":"2023-09-23T20:57:44.251010+0000","last_became_peered":"2023-09-23T20:57:44.251010+0000","last_unstale":"2023-09-23T20:59:44.574767+0000","last_undegraded":"2023-09-23T20:59:44.574767+0000","last_fullsized":"2023-09-23T20:59:44.574767+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:35.232673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4056481816,"num_objects":61898,"num_object_clones":0,"num_object_copies":185694,"num_objects_missing_on_primary":3737,"num_objects_missing":3737,"num_objects_degraded":30265,"num_objects_misplaced":847,"num_objects_unfound":0,"num_objects_dirty":61898,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":107588,"num_write_kb":5423489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19595,"num_bytes_recovered":1151074304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11949191168,"data_stored":11949178952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107588,"ondisk_log_size":107588,"up":48,"acting":41,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":19718,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201653,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1448244,"kb_used_data":988076,"kb_used_omap":0,"kb_used_meta":460160,"kb_avail":92296908,"statfs":{"total":95995035648,"available":94512033792,"internally_reserved":0,"allocated":1011789824,"data_stored":1010947815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":471203840},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.84899999999999998,"15min":2.5609999999999999},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.71099999999999997,"5min":16.033000000000001,"15min":225.56999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.433,"5min":0.78700000000000003,"15min":2.8239999999999998},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.70199999999999996,"5min":16.052,"15min":225.61099999999999},"last":0.47399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:59:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.58399999999999996,"15min":0.60999999999999999},"min":{"1min":0.28799999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.59999999999999998,"5min":3.3100000000000001,"15min":3.3100000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.45700000000000002,"15min":0.45900000000000002},"min":{"1min":0.29699999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.84599999999999997,"5min":1.778,"15min":1.778},"last":0.45000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.9930000000000001,"15min":3.3839999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.22900000000000001},"max":{"1min":0.63700000000000001,"5min":102.17,"15min":147.309},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.9530000000000001,"15min":2.9249999999999998},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.64600000000000002,"5min":102.152,"15min":147.32400000000001},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5740000000000001,"5min":1.5,"15min":5.1210000000000004},"min":{"1min":0.316,"5min":0.249,"15min":0.249},"max":{"1min":22.274999999999999,"5min":43.707000000000001,"15min":138.345},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.946,"5min":1.4159999999999999,"15min":4.9950000000000001},"min":{"1min":0.33300000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":22.251000000000001,"5min":43.302999999999997,"15min":133.76400000000001},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 20:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.76700000000000002,"15min":1.27},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.59199999999999997,"5min":9.5559999999999992,"15min":16.648},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.53400000000000003,"15min":1.155},"min":{"1min":0.29499999999999998,"5min":0.27000000000000002,"15min":0.247},"max":{"1min":0.55500000000000005,"5min":5.7229999999999999,"15min":10.172000000000001},"last":0.36199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.66700000000000004,"15min":1.623},"min":{"1min":0.29699999999999999,"5min":0.249,"15min":0.23300000000000001},"max":{"1min":0.57899999999999996,"5min":5.7480000000000002,"15min":126.28},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.85299999999999998,"15min":2.0499999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":0.57499999999999996,"5min":9.4930000000000003,"15min":126.465},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.0049999999999999,"15min":1.4279999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30399999999999999},"max":{"1min":0.57999999999999996,"5min":9.532,"15min":16.678000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.65900000000000003,"15min":1.145},"min":{"1min":0.35799999999999998,"5min":0.33400000000000002,"15min":0.26400000000000001},"max":{"1min":0.60099999999999998,"5min":5.649,"15min":10.154},"last":0.46400000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463290,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2063004,"kb_used_data":1717012,"kb_used_omap":0,"kb_used_meta":345984,"kb_avail":91682148,"statfs":{"total":95995035648,"available":93882519552,"internally_reserved":0,"allocated":1758220288,"data_stored":1757394250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":354287616},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":6.1020000000000003,"5min":2.5289999999999999,"15min":2.9140000000000001},"min":{"1min":0.27000000000000002,"5min":0.22800000000000001,"15min":0.218},"max":{"1min":102.242,"5min":102.242,"15min":102.242},"last":0.32200000000000001},{"interface":"front","average":{"1min":6.0330000000000004,"5min":1.9930000000000001,"15min":2.456},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20000000000000001},"max":{"1min":102.502,"5min":102.502,"15min":102.502},"last":0.29299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.7729999999999999,"15min":1.7729999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":23.649000000000001,"5min":23.649000000000001,"15min":23.649000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":23.920000000000002,"5min":23.920000000000002,"15min":23.920000000000002},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":1.169,"15min":1.7250000000000001},"min":{"1min":0.34300000000000003,"5min":0.224,"15min":0.224},"max":{"1min":1.173,"5min":52.808,"15min":52.808},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.77300000000000002,"5min":1.413,"15min":1.9350000000000001},"min":{"1min":0.27100000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.008,"5min":53.140000000000001,"15min":53.140000000000001},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.3100000000000001,"15min":4.726},"min":{"1min":0.31900000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.72499999999999998,"5min":29.824999999999999,"15min":250.804},"last":0.437},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.127,"15min":4.6589999999999998},"min":{"1min":0.35199999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.04,"5min":29.777999999999999,"15min":250.446},"last":1.454}]},{"osd":4,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.66500000000000004,"15min":0.99099999999999999},"min":{"1min":0.29799999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.50600000000000001,"5min":6.7030000000000003,"15min":9.1590000000000007},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.755,"15min":1.276},"min":{"1min":0.33900000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.70199999999999996,"5min":8.6859999999999999,"15min":9.8650000000000002},"last":0.38200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.70599999999999996,"15min":1.7949999999999999},"min":{"1min":0.33100000000000002,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":0.54900000000000004,"5min":9.3469999999999995,"15min":105.121},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.88800000000000001,"15min":1.8879999999999999},"min":{"1min":0.36299999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72999999999999998,"5min":9.5389999999999997,"15min":106.30500000000001},"last":0.33800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.84499999999999997,"15min":1.6519999999999999},"min":{"1min":0.38200000000000001,"5min":0.28199999999999997,"15min":0.22600000000000001},"max":{"1min":1.321,"5min":11.388999999999999,"15min":101.04600000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.504,"5min":1.087,"15min":1.6890000000000001},"min":{"1min":0.35799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67200000000000004,"5min":8.8010000000000002,"15min":101.081},"last":0.49199999999999999}]}]},{"osd":1,"up_from":276,"seq":1185410973722,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":393660,"kb_used_data":250228,"kb_used_omap":5,"kb_used_meta":143418,"kb_avail":93351492,"statfs":{"total":95995035648,"available":95591927808,"internally_reserved":0,"allocated":256233472,"data_stored":254855153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":146860771},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.374,"15min":0.374},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.3620000000000001,"5min":5.3620000000000001,"15min":5.3620000000000001},"last":0.40000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.165,"5min":3.165,"15min":3.165},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.95,"5min":1.95,"15min":1.95},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.496},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":0.44},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.37}]},{"osd":6,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.52700000000000002}]}]},{"osd":0,"up_from":92,"seq":395136991460,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3192384,"kb_used_data":2611064,"kb_used_omap":5,"kb_used_meta":581306,"kb_avail":90552768,"statfs":{"total":95995035648,"available":92726034432,"internally_reserved":0,"allocated":2673729536,"data_stored":2672408445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":595257978},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.223,"5min":0.184,"15min":0.184},"max":{"1min":2.8690000000000002,"5min":2.8690000000000002,"15min":2.8690000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.59099999999999997,"15min":0.626},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.68400000000000005,"5min":2.899,"15min":2.899},"last":0.52100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.68600000000000005,"15min":1.907},"min":{"1min":0.26400000000000001,"5min":0.161,"15min":0.161},"max":{"1min":0.94599999999999995,"5min":11.465,"15min":114.002},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.75600000000000001,"15min":2.552},"min":{"1min":0.28599999999999998,"5min":0.214,"15min":0.214},"max":{"1min":1.579,"5min":9.452,"15min":113.69499999999999},"last":0.93100000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.77200000000000002,"15min":3.7810000000000001},"min":{"1min":0.27700000000000002,"5min":0.183,"15min":0.183},"max":{"1min":3.2869999999999999,"5min":12.529,"15min":94.947999999999993},"last":2.0030000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.82199999999999995,"15min":3.895},"min":{"1min":0.27800000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.3180000000000001,"5min":15.176,"15min":94.382999999999996},"last":1.9590000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":1.0569999999999999,"15min":1.5960000000000001},"min":{"1min":0.29899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":3.0539999999999998,"5min":10.276,"15min":12.973000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.78400000000000003,"15min":1.278},"min":{"1min":0.33700000000000002,"5min":0.27100000000000002,"15min":0.26200000000000001},"max":{"1min":0.89300000000000002,"5min":11.493,"15min":12.461},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.0589999999999999,"15min":2.952},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.0819999999999999,"5min":10.451000000000001,"15min":154.30600000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.70999999999999996,"15min":2.294},"min":{"1min":0.35999999999999999,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":0.97499999999999998,"5min":8.2390000000000008,"15min":154.42400000000001},"last":0.38100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.82499999999999996,"15min":1.488},"min":{"1min":0.315,"5min":0.31,"15min":0.307},"max":{"1min":0.999,"5min":9.6660000000000004,"15min":14.037000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.625,"5min":0.84499999999999997,"15min":1.5920000000000001},"min":{"1min":0.33200000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.645,"5min":9.4030000000000005,"15min":20.399999999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.92700000000000005,"15min":2.7690000000000001},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0189999999999999,"5min":11.42,"15min":164.26400000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.872,"15min":2.9129999999999998},"min":{"1min":0.38,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.671,"5min":9.4849999999999994,"15min":164.23099999999999},"last":0.57699999999999996}]}]},{"osd":2,"up_from":129,"seq":554050781351,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1852600,"kb_used_data":1511664,"kb_used_omap":7,"kb_used_meta":340920,"kb_avail":91892552,"statfs":{"total":95995035648,"available":94097973248,"internally_reserved":0,"allocated":1547943936,"data_stored":1546628536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":349102161},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:21 2023","interfaces":[{"interface":"back","average":{"1min":0.32300000000000001,"5min":1.119,"15min":2.0920000000000001},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17599999999999999},"max":{"1min":0.47299999999999998,"5min":26.356000000000002,"15min":89.644999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":1.21,"15min":2.0489999999999999},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.186},"max":{"1min":3.1520000000000001,"5min":26.341000000000001,"15min":88.599999999999994},"last":0.36499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:58:48 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":4.5620000000000003,"5min":4.5620000000000003,"15min":4.5620000000000003},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.135,"5min":1.135,"15min":1.135},"last":0.26100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:59:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":0.64600000000000002,"15min":2.9119999999999999},"min":{"1min":0.28000000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.0410000000000004,"5min":6.0410000000000004,"15min":151.43799999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.68500000000000005,"15min":2.8380000000000001},"min":{"1min":0.23899999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.93400000000000005,"5min":5.5599999999999996,"15min":152.20699999999999},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:59:21 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.69099999999999995,"15min":1.3879999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63300000000000001,"5min":5.4850000000000003,"15min":9.9209999999999994},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.67500000000000004,"15min":1.522},"min":{"1min":0.371,"5min":0.30499999999999999,"15min":0.28399999999999997},"max":{"1min":3.21,"5min":3.8679999999999999,"15min":8.5350000000000001},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 23 20:59:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.80300000000000005,"15min":2.9500000000000002},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.27100000000000002},"max":{"1min":3.2749999999999999,"5min":9.0969999999999995,"15min":173.34},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.71799999999999997,"15min":2.8610000000000002},"min":{"1min":0.29899999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.9550000000000001,"5min":5.9550000000000001,"15min":173.15000000000001},"last":0.53300000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:59:21 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.67500000000000004,"15min":1.6830000000000001},"min":{"1min":0.32700000000000001,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":6.0090000000000003,"5min":6.0090000000000003,"15min":13.657},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.79400000000000004,"15min":1.835},"min":{"1min":0.38200000000000001,"5min":0.33000000000000002,"15min":0.32300000000000001},"max":{"1min":0.73099999999999998,"5min":5.5090000000000003,"15min":12.595000000000001},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:59:21 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.93500000000000005,"15min":2.9820000000000002},"min":{"1min":0.36499999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.75700000000000001,"5min":10.599,"15min":177.30799999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.78200000000000003,"15min":2.9750000000000001},"min":{"1min":0.40899999999999997,"5min":0.37,"15min":0.33600000000000002},"max":{"1min":3.2429999999999999,"5min":4.4480000000000004,"15min":177.40899999999999},"last":0.59999999999999998}]}]},{"osd":3,"up_from":148,"seq":635655159945,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1664444,"kb_used_data":1262452,"kb_used_omap":2,"kb_used_meta":401981,"kb_avail":92080708,"statfs":{"total":95995035648,"available":94290644992,"internally_reserved":0,"allocated":1292750848,"data_stored":1291367843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":411628597},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.90700000000000003,"15min":5.9930000000000003},"min":{"1min":0.23699999999999999,"5min":0.184,"15min":0.16700000000000001},"max":{"1min":2.2639999999999998,"5min":37.405000000000001,"15min":253.68100000000001},"last":0.222},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.1379999999999999,"15min":5.782},"min":{"1min":0.246,"5min":0.19500000000000001,"15min":0.17199999999999999},"max":{"1min":2.238,"5min":37.634999999999998,"15min":253.46199999999999},"last":0.29899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.73399999999999999,"15min":0.754},"min":{"1min":0.25600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":2.637,"5min":7.2869999999999999,"15min":7.2869999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.93899999999999995,"15min":0.995},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.0049999999999999,"5min":10.996,"15min":10.996},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.65800000000000003,"15min":1.0740000000000001},"min":{"1min":0.245,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":3.879,"5min":5.6429999999999998,"15min":12.859},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.81200000000000006,"15min":1.1779999999999999},"min":{"1min":0.27700000000000002,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":2.278,"5min":10.122,"15min":19.050999999999998},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.88500000000000001,"15min":1.4179999999999999},"min":{"1min":0.34899999999999998,"5min":0.27900000000000003,"15min":0.222},"max":{"1min":3.2469999999999999,"5min":11.087999999999999,"15min":11.087999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.72599999999999998,"15min":1.476},"min":{"1min":0.39100000000000001,"5min":0.28299999999999997,"15min":0.252},"max":{"1min":2.3210000000000002,"5min":5.2080000000000002,"15min":16.408999999999999},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.90100000000000002,"15min":1.593},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28499999999999998},"max":{"1min":2.3660000000000001,"5min":7.9020000000000001,"15min":64.528000000000006},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.88900000000000001,"15min":1.702},"min":{"1min":0.39000000000000001,"5min":0.29299999999999998,"15min":0.249},"max":{"1min":2.9630000000000001,"5min":11.116,"15min":14.054},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":1.0229999999999999,"15min":1.7250000000000001},"min":{"1min":0.37,"5min":0.315,"15min":0.315},"max":{"1min":3.306,"5min":11.148,"15min":19.568000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.94999999999999996,"15min":1.8300000000000001},"min":{"1min":0.41799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.3759999999999999,"5min":7.9870000000000001,"15min":16.451000000000001},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 20:59:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.94099999999999995,"15min":2.3980000000000001},"min":{"1min":0.39800000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3359999999999999,"5min":7.931,"15min":215.70099999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.0429999999999999,"15min":2.6339999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.984,"5min":11.052,"15min":222.11699999999999},"last":0.53100000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986563,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2516212,"kb_used_data":2058732,"kb_used_omap":0,"kb_used_meta":457472,"kb_avail":91228940,"statfs":{"total":95995035648,"available":93418434560,"internally_reserved":0,"allocated":2108141568,"data_stored":2107299292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":468451328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":3.0779999999999998,"15min":3.4780000000000002},"min":{"1min":0.223,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":1.002,"5min":102.358,"15min":146.25299999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.36599999999999999,"5min":3.1600000000000001,"15min":3.3479999999999999},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.0329999999999999,"5min":102.58199999999999,"15min":144.77000000000001},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Sep 23 20:58:46 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.8809999999999998,"5min":3.8809999999999998,"15min":3.8809999999999998},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.7829999999999999,"5min":3.7829999999999999,"15min":3.7829999999999999},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":2.3969999999999998,"15min":1.7769999999999999},"min":{"1min":0.27800000000000002,"5min":0.246,"15min":0.23100000000000001},"max":{"1min":3.395,"5min":99.274000000000001,"15min":99.274000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":2.3290000000000002,"15min":2.1539999999999999},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.3570000000000002,"5min":99.363,"15min":99.363},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:59:46 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.423,"15min":4.3559999999999999},"min":{"1min":0.34599999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.7559999999999998,"5min":60.720999999999997,"15min":142.28399999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.3959999999999999,"15min":4.3029999999999999},"min":{"1min":0.33300000000000002,"5min":0.29999999999999999,"15min":0.26700000000000002},"max":{"1min":2.4020000000000001,"5min":60.631999999999998,"15min":142.34700000000001},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.77000000000000002,"15min":1.2150000000000001},"min":{"1min":0.30299999999999999,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":0.51600000000000001,"5min":7.4720000000000004,"15min":72.304000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.60599999999999998,"15min":1.2709999999999999},"min":{"1min":0.254,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.60299999999999998,"5min":4.2279999999999998,"15min":72.290000000000006},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 20:59:22 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.68899999999999995,"15min":1.1279999999999999},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.497,"5min":5.4210000000000003,"15min":16.271000000000001},"last":2.681},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.64800000000000002,"15min":0.97299999999999998},"min":{"1min":0.34799999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57299999999999995,"5min":7.3239999999999998,"15min":11.718},"last":0.41799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.65800000000000003,"15min":1.3620000000000001},"min":{"1min":0.36299999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65000000000000002,"5min":7.6559999999999997,"15min":73.268000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.75700000000000001,"15min":1.355},"min":{"1min":0.38700000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61799999999999999,"5min":5.1740000000000004,"15min":69.760000000000005},"last":0.50900000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724927,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1830624,"kb_used_data":1311704,"kb_used_omap":0,"kb_used_meta":518912,"kb_avail":91914528,"statfs":{"total":95995035648,"available":94120476672,"internally_reserved":0,"allocated":1343184896,"data_stored":1342358820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":531365888},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:59:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":1.9970000000000001,"15min":2.0880000000000001},"min":{"1min":0.26800000000000002,"5min":0.23699999999999999,"15min":0.19400000000000001},"max":{"1min":0.58599999999999997,"5min":80.105999999999995,"15min":124.873},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.9750000000000001,"15min":2.2130000000000001},"min":{"1min":0.248,"5min":0.247,"15min":0.223},"max":{"1min":0.60999999999999999,"5min":80.152000000000001,"15min":123.554},"last":0.33500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:59:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.61099999999999999,"15min":1.536},"min":{"1min":0.31,"5min":0.254,"15min":0.22500000000000001},"max":{"1min":2.4550000000000001,"5min":5.0519999999999996,"15min":13.394},"last":0.312},{"interface":"front","average":{"1min":0.433,"5min":0.60999999999999999,"15min":1.3620000000000001},"min":{"1min":0.29599999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.98899999999999999,"5min":3.5390000000000001,"15min":14.263},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:59:27 2023","interfaces":[{"interface":"back","average":{"1min":5.1779999999999999,"5min":2.4660000000000002,"15min":3.8220000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":71.790000000000006,"5min":98.369,"15min":198.33099999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":4.6929999999999996,"5min":2.5379999999999998,"15min":3.7109999999999999},"min":{"1min":0.29699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":71.968000000000004,"5min":98.683000000000007,"15min":198.47300000000001},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.60999999999999999,"15min":0.93000000000000005},"min":{"1min":0.26000000000000001,"5min":0.247,"15min":0.24099999999999999},"max":{"1min":0.60099999999999998,"5min":5.0810000000000004,"15min":10.906000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.58899999999999997,"15min":1.159},"min":{"1min":0.30599999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.52500000000000002,"5min":3.9910000000000001,"15min":12.539},"last":0.443}]},{"osd":6,"last update":"Sat Sep 23 20:59:27 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.61699999999999999,"15min":1.099},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.57799999999999996,"5min":4.3029999999999999,"15min":12.727},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.624,"15min":1.1379999999999999},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.28599999999999998},"max":{"1min":0.67600000000000005,"5min":7.9589999999999996,"15min":12.593999999999999},"last":0.48399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":1.111,"15min":1.4770000000000001},"min":{"1min":0.34699999999999998,"5min":0.312,"15min":0.28199999999999997},"max":{"1min":1.5669999999999999,"5min":31.091999999999999,"15min":31.091999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.5,"5min":1.071,"15min":1.21},"min":{"1min":0.35099999999999998,"5min":0.29799999999999999,"15min":0.28699999999999998},"max":{"1min":0.63900000000000001,"5min":31.780000000000001,"15min":31.780000000000001},"last":0.52800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2667319296,"data_stored":2667315224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":251854848,"data_stored":251854848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1544159232,"data_stored":1544159232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1288175616,"data_stored":1288175616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2102988800,"data_stored":2102984728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1339162624,"data_stored":1339162624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1749291008,"data_stored":1749286936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1006239744,"data_stored":1006239744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:59:53.551 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:59:53.552 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-09-23T20:59:53.553 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:53.646 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:53.730 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:53.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:53.810 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:59:53.834 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:53.879 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:53.924 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:53.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:54.029 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:54.055 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:53 smithi123 ceph-mon[193903]: from='client.35916 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:54.056 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:53 smithi123 ceph-mon[193903]: pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30277/185505 objects degraded (16.321%); 847/185505 objects misplaced (0.457%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:54.058 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:54.103 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:54.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:54.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:54.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:53 smithi121 ceph-mon[219709]: from='client.35916 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:54.177 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:53 smithi121 ceph-mon[219709]: pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30277/185505 objects degraded (16.321%); 847/185505 objects misplaced (0.457%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:54.177 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:53 smithi121 ceph-mon[211526]: from='client.35916 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:59:54.177 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:53 smithi121 ceph-mon[211526]: pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30277/185505 objects degraded (16.321%); 847/185505 objects misplaced (0.457%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:59:54.181 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:54.209 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:54.245 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:54.401 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:54.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:54.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:54.505 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:54.540 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:54.786 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:54.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:54.890 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:54.915 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:54.917 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:54.940 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T20:59:54.962 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:54.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:54.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:54.972 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 20:59:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:20:59:54] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T20:59:54.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1187327960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:54.972 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:54.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1187327960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:54.973 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:55.065 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:55.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1187327960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:55.092 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:59:55.095 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:55.286 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:55.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:55.394 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:59:55.422 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:55.477 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T20:59:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:55.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:59:55.581 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:59:55.615 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:55.713 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T20:59:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:55.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:59:55.780 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T20:59:55.780 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T20:59:55.781 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T20:59:55.781 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T20:59:55.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:55.846 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 30207/183975 objects degraded (16.419%); 847/183975 objects misplaced (0.460%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: failed to encode map e301 with expected crc 2023-09-23T20:59:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T20:59:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: failed to encode map e301 with expected crc 2023-09-23T20:59:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: failed to encode map e301 with expected crc 2023-09-23T20:59:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:55 smithi123 ceph-mon[193903]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.001 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:56.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:56.105 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:56.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 30207/183975 objects degraded (16.419%); 847/183975 objects misplaced (0.460%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:56.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:56.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:56.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T20:59:56.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[219709]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 30207/183975 objects degraded (16.419%); 847/183975 objects misplaced (0.460%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:55 smithi121 ceph-mon[211526]: failed to encode map e301 with expected crc 2023-09-23T20:59:56.139 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:56.188 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:56.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:56.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:56.220 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T20:59:56.240 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T20:59:56.267 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T20:59:56.292 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:56.294 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T20:59:56.314 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T20:59:56.319 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:56.333 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T20:59:56.353 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T20:59:56.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T20:59:56.375 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.375 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.375 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.375 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.376 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.376 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.376 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.376 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:59:56.439 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:56.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:56.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:56.493 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:59:56.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:56.576 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:56.839 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:57.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:56 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/561968064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:57.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30207/183975 objects degraded (16.419%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:56 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/561968064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30207/183975 objects degraded (16.419%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:57.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:56 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/561968064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:59:57.226 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30207/183975 objects degraded (16.419%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T20:59:57.238 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:57.343 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:57.374 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:57.606 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:57.711 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:57.742 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:57.800 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:57.904 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:57.935 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:57.989 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:57.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:58.020 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:59:58.056 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:57 smithi121 ceph-mon[219709]: pgmap v953: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 30207/183690 objects degraded (16.445%); 847/183690 objects misplaced (0.461%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:59:58.056 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:58.056 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:57 smithi121 ceph-mon[211526]: pgmap v953: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 30207/183690 objects degraded (16.445%); 847/183690 objects misplaced (0.461%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:59:58.056 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:58.093 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:59:58.121 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:58.164 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T20:59:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:58.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:59:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:57 smithi123 ceph-mon[193903]: pgmap v953: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 30207/183690 objects degraded (16.445%); 847/183690 objects misplaced (0.461%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T20:59:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:59:58.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:58.298 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:58.405 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T20:59:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:58.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:59:58.420 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:59:58.473 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T20:59:58.474 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:59:58.510 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:59:58.538 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:58.634 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T20:59:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:58.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:59:58.738 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:59:58.768 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:58.814 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T20:59:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:58.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:59:58.918 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:59:58.937 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 10 pg_num to 26 2023-09-23T20:59:58.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3991072742' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:59:58.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:59:58.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3991072742' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:59:58.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:59:58.956 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:59.171 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:59:59.198 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3991072742' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:59:59.198 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:59:59.303 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:59:59.548 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:59:59.662 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T20:59:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:59.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:59:59.766 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:59:59.799 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:59:59.856 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T20:59:59.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:59:59.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:59:59.962 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:59:59.992 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.075 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:59 smithi123 ceph-mon[193903]: pgmap v954: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30143/182682 objects degraded (16.500%); 847/182682 objects misplaced (0.464%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:00:00.076 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:59 smithi123 ceph-mon[193903]: failed to encode map e302 with expected crc 2023-09-23T21:00:00.076 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:59 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:00:00.076 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:59 smithi123 ceph-mon[193903]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T21:00:00.076 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 20:59:59 smithi123 ceph-mon[193903]: failed to encode map e302 with expected crc 2023-09-23T21:00:00.217 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[211526]: pgmap v954: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30143/182682 objects degraded (16.500%); 847/182682 objects misplaced (0.464%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:00:00.217 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[211526]: failed to encode map e302 with expected crc 2023-09-23T21:00:00.217 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:00:00.217 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[211526]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T21:00:00.217 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[211526]: failed to encode map e302 with expected crc 2023-09-23T21:00:00.218 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[219709]: pgmap v954: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30143/182682 objects degraded (16.500%); 847/182682 objects misplaced (0.464%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:00:00.218 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[219709]: failed to encode map e302 with expected crc 2023-09-23T21:00:00.218 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:00:00.218 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[219709]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T21:00:00.218 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 20:59:59 smithi121 ceph-mon[219709]: failed to encode map e302 with expected crc 2023-09-23T21:00:00.218 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:00.327 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:00.365 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.393 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:00.434 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:00.538 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:00.569 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.665 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:00.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:00.773 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:00.808 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:00.876 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:00.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:00.980 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: failed to encode map e302 with expected crc 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 30143/182682 objects degraded (16.500%), 13 pgs degraded, 8 pgs undersized 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: [WRN] PG_DEGRADED: Degraded data redundancy: 30143/182682 objects degraded (16.500%), 13 pgs degraded, 8 pgs undersized 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 1.0 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,6] 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 3.6 is active+recovery_wait+degraded, acting [1,4,7] 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 3.b is active+recovery_wait+degraded, acting [3,4,1] 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 3.e is active+recovery_wait+degraded, acting [7,1,5] 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 3.f is active+recovery_wait+degraded, acting [7,4,1] 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 3.10 is active+recovery_wait+degraded, acting [6,5,3] 2023-09-23T21:00:01.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 3.1f is active+recovery_wait+degraded, acting [2,5,6] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 10.0 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 10.4 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 10.9 is stuck undersized for 2m, current state active+recovering+undersized+remapped, last acting [3,7] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 10.a is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 10.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: pg 10.d is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,4] 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: failed to encode map e302 with expected crc 2023-09-23T21:00:01.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 30143/182682 objects degraded (16.500%), 13 pgs degraded, 8 pgs undersized 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: [WRN] PG_DEGRADED: Degraded data redundancy: 30143/182682 objects degraded (16.500%), 13 pgs degraded, 8 pgs undersized 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 1.0 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,6] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 3.6 is active+recovery_wait+degraded, acting [1,4,7] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 3.b is active+recovery_wait+degraded, acting [3,4,1] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 3.e is active+recovery_wait+degraded, acting [7,1,5] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 3.f is active+recovery_wait+degraded, acting [7,4,1] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 3.10 is active+recovery_wait+degraded, acting [6,5,3] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 3.1f is active+recovery_wait+degraded, acting [2,5,6] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 10.0 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 10.4 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2023-09-23T21:00:01.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 10.9 is stuck undersized for 2m, current state active+recovering+undersized+remapped, last acting [3,7] 2023-09-23T21:00:01.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 10.a is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T21:00:01.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 10.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-09-23T21:00:01.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: pg 10.d is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,4] 2023-09-23T21:00:01.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:00:01.168 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:00 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:00:01.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: failed to encode map e302 with expected crc 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 30143/182682 objects degraded (16.500%), 13 pgs degraded, 8 pgs undersized 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: [WRN] PG_DEGRADED: Degraded data redundancy: 30143/182682 objects degraded (16.500%), 13 pgs degraded, 8 pgs undersized 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 1.0 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,6] 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 3.6 is active+recovery_wait+degraded, acting [1,4,7] 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 3.b is active+recovery_wait+degraded, acting [3,4,1] 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 3.e is active+recovery_wait+degraded, acting [7,1,5] 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 3.f is active+recovery_wait+degraded, acting [7,4,1] 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 3.10 is active+recovery_wait+degraded, acting [6,5,3] 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 3.1f is active+recovery_wait+degraded, acting [2,5,6] 2023-09-23T21:00:01.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 10.0 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 10.4 is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 10.9 is stuck undersized for 2m, current state active+recovering+undersized+remapped, last acting [3,7] 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 10.a is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 10.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: pg 10.d is stuck undersized for 81s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,4] 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:00:01.236 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:00 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:00:01.285 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:01.340 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:01.340 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:01.359 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:00:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: pgmap v956: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30143/182244 objects degraded (16.540%); 847/182244 objects misplaced (0.465%); 928 KiB/s, 14 objects/s recovering 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: failed to encode map e303 with expected crc 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: failed to encode map e303 with expected crc 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1361203784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30143/182244 objects degraded (16.540%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:01 smithi123 ceph-mon[193903]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.344 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:00:02.377 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: pgmap v956: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30143/182244 objects degraded (16.540%); 847/182244 objects misplaced (0.465%); 928 KiB/s, 14 objects/s recovering 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: failed to encode map e303 with expected crc 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: failed to encode map e303 with expected crc 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1361203784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30143/182244 objects degraded (16.540%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:02.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[219709]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: pgmap v956: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30143/182244 objects degraded (16.540%); 847/182244 objects misplaced (0.465%); 928 KiB/s, 14 objects/s recovering 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: failed to encode map e303 with expected crc 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: failed to encode map e303 with expected crc 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1361203784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:02.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:02.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30143/182244 objects degraded (16.540%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:02.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:02.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:02.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T21:00:02.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:01 smithi121 ceph-mon[211526]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.579 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:02.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:02.686 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:02.715 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:02.915 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:02.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:02.938 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[211526]: pgmap v960: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 8 unknown, 5 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 27438/181899 objects degraded (15.084%); 24 MiB/s, 430 objects/s recovering 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: failed to encode map e304 with expected crc 2023-09-23T21:00:02.939 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:02.940 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:02.940 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T21:00:02.940 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:02.940 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:02 smithi121 ceph-mon[219709]: pgmap v960: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 8 unknown, 5 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 27438/181899 objects degraded (15.084%); 24 MiB/s, 430 objects/s recovering 2023-09-23T21:00:02.957 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:03.019 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:03.029 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:00:03.048 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:00:03.059 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:03.068 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:00:03.088 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:00:03.104 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:03.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:00:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:03.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:03.127 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:00:03.147 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:00:03.167 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.169 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:00:03.214 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: failed to encode map e304 with expected crc 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: failed to encode map e304 with expected crc 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: failed to encode map e304 with expected crc 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:02 smithi123 ceph-mon[193903]: pgmap v960: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 8 unknown, 5 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 27438/181899 objects degraded (15.084%); 24 MiB/s, 430 objects/s recovering 2023-09-23T21:00:03.242 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:03.287 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:03.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:03.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:03.394 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:03.423 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:03.466 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:03.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:03.574 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:03.604 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:04.183 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:04.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2579823982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: failed to encode map e306 with expected crc 2023-09-23T21:00:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:03 smithi123 ceph-mon[193903]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T21:00:04.287 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:04.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2579823982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: failed to encode map e306 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[211526]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: failed to encode map e305 with expected crc 2023-09-23T21:00:04.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2579823982' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:04.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: failed to encode map e306 with expected crc 2023-09-23T21:00:04.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:03 smithi121 ceph-mon[219709]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T21:00:04.322 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:04.377 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:04.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:04.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:04.480 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:04.508 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:04.570 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:00:04.761 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:04.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:04.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:04.865 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:04.885 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:00:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:00:04] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T21:00:04.898 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:05.063 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:05.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:05.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:05.128 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:00:05.129 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:00:05.129 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:00:05.129 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:00:05.157 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:04 smithi121 ceph-mon[211526]: failed to encode map e306 with expected crc 2023-09-23T21:00:05.157 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:04 smithi121 ceph-mon[211526]: failed to encode map e306 with expected crc 2023-09-23T21:00:05.157 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:04 smithi121 ceph-mon[211526]: pgmap v962: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 1 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 26780/181155 objects degraded (14.783%); 28 MiB/s, 487 objects/s recovering 2023-09-23T21:00:05.158 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:04 smithi121 ceph-mon[219709]: failed to encode map e306 with expected crc 2023-09-23T21:00:05.158 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:04 smithi121 ceph-mon[219709]: failed to encode map e306 with expected crc 2023-09-23T21:00:05.158 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:04 smithi121 ceph-mon[219709]: pgmap v962: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 1 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 26780/181155 objects degraded (14.783%); 28 MiB/s, 487 objects/s recovering 2023-09-23T21:00:05.165 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:05.167 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:05.201 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:04 smithi123 ceph-mon[193903]: failed to encode map e306 with expected crc 2023-09-23T21:00:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:04 smithi123 ceph-mon[193903]: failed to encode map e306 with expected crc 2023-09-23T21:00:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:04 smithi123 ceph-mon[193903]: pgmap v962: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 1 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 26780/181155 objects degraded (14.783%); 28 MiB/s, 487 objects/s recovering 2023-09-23T21:00:05.245 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:00:05.245 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:00:05.245 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.28587436886481826 2023-09-23T21:00:05.245 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 0.28587436886481826 2023-09-23T21:00:05.356 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:05.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:05.460 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:05.502 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:05.547 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:05.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:05.652 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:05.683 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:05.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:00:06.085 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:06.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:06.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:06.189 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:06.217 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:06.262 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:06.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:06.277 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:06.370 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:06.399 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 26780/181155 objects degraded (14.783%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:06.546 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:06.548 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 26780/181155 objects degraded (14.783%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:06.548 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 26780/181155 objects degraded (14.783%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:06.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:06.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:06.649 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:06.676 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:06.972 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:06.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:06.985 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:00:07.017 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:07.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:07.114 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:07.249 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:07.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:07.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:07.354 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:07.402 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:07.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:07 smithi121 ceph-mon[211526]: pgmap v963: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 1 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 26780/181155 objects degraded (14.783%); 19 MiB/s, 329 objects/s recovering 2023-09-23T21:00:07.613 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:07 smithi121 ceph-mon[219709]: pgmap v963: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 1 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 26780/181155 objects degraded (14.783%); 19 MiB/s, 329 objects/s recovering 2023-09-23T21:00:07.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:07 smithi123 ceph-mon[193903]: pgmap v963: 99 pgs: 1 peering, 1 remapped+peering, 1 remapped, 1 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 26780/181155 objects degraded (14.783%); 19 MiB/s, 329 objects/s recovering 2023-09-23T21:00:08.332 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:08.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:08.438 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:08.482 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:08.752 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:08.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:08.794 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:00:08.801 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:08.868 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:08.898 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/183762712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/183762712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:08.957 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:08.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:09.063 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:09.095 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/183762712' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:09.098 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:09.143 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:09.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:09.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:09.248 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:09.289 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[211526]: failed to encode map e307 with expected crc 2023-09-23T21:00:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/183762712' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[211526]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T21:00:10.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[211526]: pgmap v965: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 240 KiB/s wr, 76 op/s; 30531/180555 objects degraded (16.910%) 2023-09-23T21:00:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[219709]: failed to encode map e307 with expected crc 2023-09-23T21:00:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/183762712' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[219709]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T21:00:10.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:09 smithi121 ceph-mon[219709]: pgmap v965: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 240 KiB/s wr, 76 op/s; 30531/180555 objects degraded (16.910%) 2023-09-23T21:00:10.179 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:10.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:10.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:09 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:09 smithi123 ceph-mon[193903]: failed to encode map e307 with expected crc 2023-09-23T21:00:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/183762712' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:09 smithi123 ceph-mon[193903]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T21:00:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:09 smithi123 ceph-mon[193903]: pgmap v965: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 240 KiB/s wr, 76 op/s; 30531/180555 objects degraded (16.910%) 2023-09-23T21:00:10.284 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:10.313 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:10.373 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:10.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:10.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:10.480 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:10.511 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:10.571 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:10.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:10.678 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:10.704 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:10.751 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:10.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:10.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:10.804 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.5 to 0.285874 (492f) 2023-09-23T21:00:10.856 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:10.889 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3365829982' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.28587436886481826}]: dispatch 2023-09-23T21:00:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3365829982' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.28587436886481826}]: dispatch 2023-09-23T21:00:11.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3365829982' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.28587436886481826}]: dispatch 2023-09-23T21:00:11.380 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:00:11.381 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:00:11.381 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:00:11.381 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:00:11.381 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:00:11.843 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:11.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:11.946 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:11.977 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: failed to encode map e308 with expected crc 2023-09-23T21:00:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3365829982' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.28587436886481826}]': finished 2023-09-23T21:00:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T21:00:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: failed to encode map e308 with expected crc 2023-09-23T21:00:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: pgmap v967: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 208 KiB/s wr, 77 op/s; 30531/180297 objects degraded (16.934%) 2023-09-23T21:00:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: 10.1 scrub starts 2023-09-23T21:00:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30531/180555 objects degraded (16.910%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[211526]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: failed to encode map e308 with expected crc 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3365829982' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.28587436886481826}]': finished 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: failed to encode map e308 with expected crc 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: pgmap v967: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 208 KiB/s wr, 77 op/s; 30531/180297 objects degraded (16.934%) 2023-09-23T21:00:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: 10.1 scrub starts 2023-09-23T21:00:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30531/180555 objects degraded (16.910%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T21:00:12.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:11 smithi121 ceph-mon[219709]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: failed to encode map e308 with expected crc 2023-09-23T21:00:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3365829982' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.28587436886481826}]': finished 2023-09-23T21:00:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T21:00:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: failed to encode map e308 with expected crc 2023-09-23T21:00:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: pgmap v967: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 208 KiB/s wr, 77 op/s; 30531/180297 objects degraded (16.934%) 2023-09-23T21:00:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: 10.1 scrub starts 2023-09-23T21:00:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30531/180555 objects degraded (16.910%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T21:00:12.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:11 smithi123 ceph-mon[193903]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.356 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:00:12.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:12.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:12.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:12.699 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:12.733 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: 10.3 scrub starts 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: failed to encode map e309 with expected crc 2023-09-23T21:00:12.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: 10.3 scrub starts 2023-09-23T21:00:12.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T21:00:12.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:13.179 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:13.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: failed to encode map e309 with expected crc 2023-09-23T21:00:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: failed to encode map e309 with expected crc 2023-09-23T21:00:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: failed to encode map e309 with expected crc 2023-09-23T21:00:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: failed to encode map e309 with expected crc 2023-09-23T21:00:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: failed to encode map e309 with expected crc 2023-09-23T21:00:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: failed to encode map e309 with expected crc 2023-09-23T21:00:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: 10.3 scrub starts 2023-09-23T21:00:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T21:00:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:13.283 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:13.318 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:13.378 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:00:13.402 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:13.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:13.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:13.508 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:13.569 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:13.650 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:13.688 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:13.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:13.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:13.789 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:00:13.794 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:13.815 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:00:13.836 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:13.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:00:13.881 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:13.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:13.937 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:00:13.953 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[211526]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[211526]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[211526]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[211526]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[211526]: 10.8 scrub starts 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[211526]: pgmap v970: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 79 op/s; 30472/180063 objects degraded (16.923%); 6.8 MiB/s, 123 objects/s recovering 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3657098644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[219709]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.954 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[219709]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.955 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[219709]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.955 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[219709]: failed to encode map e310 with expected crc 2023-09-23T21:00:13.955 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[219709]: 10.8 scrub starts 2023-09-23T21:00:13.955 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[219709]: pgmap v970: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 79 op/s; 30472/180063 objects degraded (16.923%); 6.8 MiB/s, 123 objects/s recovering 2023-09-23T21:00:13.955 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3657098644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:13.966 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:00:13.985 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:13.990 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:00:14.018 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:14.025 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:00:14.053 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:00:14.056 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:00:14.125 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:14.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:13 smithi123 ceph-mon[193903]: failed to encode map e310 with expected crc 2023-09-23T21:00:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:13 smithi123 ceph-mon[193903]: failed to encode map e310 with expected crc 2023-09-23T21:00:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:13 smithi123 ceph-mon[193903]: failed to encode map e310 with expected crc 2023-09-23T21:00:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:13 smithi123 ceph-mon[193903]: failed to encode map e310 with expected crc 2023-09-23T21:00:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:13 smithi123 ceph-mon[193903]: 10.8 scrub starts 2023-09-23T21:00:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:13 smithi123 ceph-mon[193903]: pgmap v970: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 79 op/s; 30472/180063 objects degraded (16.923%); 6.8 MiB/s, 123 objects/s recovering 2023-09-23T21:00:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3657098644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:14.230 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:14.258 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:14.300 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:14.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:14.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:14.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:14.437 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:15.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:00:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:00:14] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T21:00:15.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:14 smithi121 ceph-mon[219709]: 10.5 scrub starts 2023-09-23T21:00:15.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:14 smithi121 ceph-mon[211526]: 10.5 scrub starts 2023-09-23T21:00:15.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:14 smithi123 ceph-mon[193903]: 10.5 scrub starts 2023-09-23T21:00:16.000 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:16.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:16.107 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:16.145 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:16.201 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:16.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:15 smithi123 ceph-mon[193903]: pgmap v971: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 41975/177411 objects degraded (23.660%); 2328/177411 objects misplaced (1.312%); 7.1 MiB/s, 124 objects/s recovering 2023-09-23T21:00:16.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:15 smithi123 ceph-mon[193903]: 10.13 scrub starts 2023-09-23T21:00:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:16.306 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:16.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:15 smithi121 ceph-mon[211526]: pgmap v971: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 41975/177411 objects degraded (23.660%); 2328/177411 objects misplaced (1.312%); 7.1 MiB/s, 124 objects/s recovering 2023-09-23T21:00:16.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:15 smithi121 ceph-mon[211526]: 10.13 scrub starts 2023-09-23T21:00:16.334 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:15 smithi121 ceph-mon[219709]: pgmap v971: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 41975/177411 objects degraded (23.660%); 2328/177411 objects misplaced (1.312%); 7.1 MiB/s, 124 objects/s recovering 2023-09-23T21:00:16.334 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:15 smithi121 ceph-mon[219709]: 10.13 scrub starts 2023-09-23T21:00:16.336 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:16.484 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:16.494 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:16.603 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:16.630 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:16.674 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:16.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:16.781 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:16.807 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:16.849 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:16.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:16.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:16.956 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:16.990 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:17.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41975/177411 objects degraded (23.660%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:17.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41975/177411 objects degraded (23.660%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41975/177411 objects degraded (23.660%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:17.659 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:17.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:17.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:17.801 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:17.887 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:17.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:17.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:17.993 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:18.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.079 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:18.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:18.180 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:17 smithi121 ceph-mon[211526]: pgmap v972: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 41975/177411 objects degraded (23.660%); 2328/177411 objects misplaced (1.312%); 7.0 MiB/s, 122 objects/s recovering 2023-09-23T21:00:18.180 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:17 smithi121 ceph-mon[219709]: pgmap v972: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 41975/177411 objects degraded (23.660%); 2328/177411 objects misplaced (1.312%); 7.0 MiB/s, 122 objects/s recovering 2023-09-23T21:00:18.186 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:18.195 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:00:18.195 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:00:18.195 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T21:00:18.195 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:00:18.195 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:00:18.196 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T21:00:18.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:17 smithi123 ceph-mon[193903]: pgmap v972: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 41975/177411 objects degraded (23.660%); 2328/177411 objects misplaced (1.312%); 7.0 MiB/s, 122 objects/s recovering 2023-09-23T21:00:18.211 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.264 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:18.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:18.371 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:18.407 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.439 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:18.439 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:18.464 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:18.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:18.518 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:00:18.571 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:18.600 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.662 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:18.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:18.711 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:18.769 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:18.795 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:18.848 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:18.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:18.947 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:00:18.951 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:18.991 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:19.047 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:19.048 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/288886264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:19.151 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:19.212 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/288886264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:19.212 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/288886264' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:19.220 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:19.450 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:19.468 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:00:19.557 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:19.592 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:19.635 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:19.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:19.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:19.776 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:19.840 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:19.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:19.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:19.944 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:19.971 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:20.021 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:20.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:19 smithi123 ceph-mon[193903]: pgmap v973: 99 pgs: 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 41857/175479 objects degraded (23.853%); 2328/175479 objects misplaced (1.327%); 5.8 MiB/s, 100 objects/s recovering 2023-09-23T21:00:20.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:19 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:20.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:19 smithi123 ceph-mon[193903]: failed to encode map e311 with expected crc 2023-09-23T21:00:20.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:19 smithi123 ceph-mon[193903]: failed to encode map e311 with expected crc 2023-09-23T21:00:20.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/288886264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:20.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:19 smithi123 ceph-mon[193903]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T21:00:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:20.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:20.124 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:20.174 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:20.228 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:20.249 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:20.314 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:20.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[211526]: pgmap v973: 99 pgs: 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 41857/175479 objects degraded (23.853%); 2328/175479 objects misplaced (1.327%); 5.8 MiB/s, 100 objects/s recovering 2023-09-23T21:00:20.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:20.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[211526]: failed to encode map e311 with expected crc 2023-09-23T21:00:20.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[211526]: failed to encode map e311 with expected crc 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/288886264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[211526]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[219709]: pgmap v973: 99 pgs: 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 7 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 41857/175479 objects degraded (23.853%); 2328/175479 objects misplaced (1.327%); 5.8 MiB/s, 100 objects/s recovering 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[219709]: failed to encode map e311 with expected crc 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[219709]: failed to encode map e311 with expected crc 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/288886264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:20.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:19 smithi121 ceph-mon[219709]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T21:00:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:20.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:20.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:20.472 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:20.581 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:20.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:20.684 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:20.728 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:20.886 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:20.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:20.992 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:21.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:21.063 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:21.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:21.166 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:21.192 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:21.233 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:21.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:21.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:21.336 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:21.366 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:21.423 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:21.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:21.527 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:21.563 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:21.828 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:21.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:21.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:21.932 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:21.957 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:22.008 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:22.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:22.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:22.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:22.136 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:00:22.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:21 smithi121 ceph-mon[219709]: pgmap v975: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 41240/174534 objects degraded (23.629%); 2327/174534 objects misplaced (1.333%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:00:22.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41240/174534 objects degraded (23.629%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:22.149 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:22.149 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:21 smithi121 ceph-mon[211526]: pgmap v975: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 41240/174534 objects degraded (23.629%); 2327/174534 objects misplaced (1.333%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:00:22.149 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41240/174534 objects degraded (23.629%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:21 smithi123 ceph-mon[193903]: pgmap v975: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 41240/174534 objects degraded (23.629%); 2327/174534 objects misplaced (1.333%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:00:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41240/174534 objects degraded (23.629%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:22.291 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:22.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:22.397 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:22.425 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:22.474 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:22.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:22.580 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:22.606 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:22.657 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:22.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:22.721 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:22.721 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:22.760 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:22.793 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:22.840 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:22.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:22.944 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:22.978 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:23.583 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:00:23.583 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:00:23.648 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:23.680 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:23.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:23.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:23.688 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:00:23.784 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:23.823 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:23.993 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:00:23.994 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "" 2023-09-23T21:00:23.994 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:00:24.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:23 smithi123 ceph-mon[193903]: pgmap v976: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 41240/174168 objects degraded (23.678%); 2327/174168 objects misplaced (1.336%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:00:24.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4236195000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:24.266 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:24.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:24.269 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:23 smithi121 ceph-mon[219709]: pgmap v976: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 41240/174168 objects degraded (23.678%); 2327/174168 objects misplaced (1.336%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:00:24.270 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4236195000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:24.270 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:23 smithi121 ceph-mon[211526]: pgmap v976: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 41240/174168 objects degraded (23.678%); 2327/174168 objects misplaced (1.336%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:00:24.270 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4236195000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:24.371 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:24.429 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:24.541 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:24.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:24.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:24.649 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:24.680 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:24.723 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:24.766 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:24.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:24.871 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:24.901 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:24.950 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:24.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:25.058 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:25.084 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:25.126 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:25.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:25.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:25.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:00:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:00:24] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T21:00:25.234 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:25.262 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:25.309 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:25.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:25.415 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:25.445 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:25.503 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:25.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:25.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:25.614 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:25.666 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:25.735 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:25.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:25.839 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:25.876 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:26.054 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:26.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:26.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:25 smithi121 ceph-mon[211526]: pgmap v977: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 40991/169563 objects degraded (24.174%); 2265/169563 objects misplaced (1.336%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:00:26.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:25 smithi121 ceph-mon[219709]: pgmap v977: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 40991/169563 objects degraded (24.174%); 2265/169563 objects misplaced (1.336%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:00:26.169 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:26.196 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:25 smithi123 ceph-mon[193903]: pgmap v977: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 40991/169563 objects degraded (24.174%); 2265/169563 objects misplaced (1.336%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:00:26.244 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:26.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:26.351 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:26.379 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:26.425 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:26.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:26.432 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:00:26.530 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:26.559 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:26.610 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:26.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:26.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:26.717 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:26.780 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:26.899 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:26.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:27.009 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:27.094 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:27.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 40991/169563 objects degraded (24.174%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:27.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2679010448' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:27.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:26 smithi121 ceph-mon[219709]: pgmap v978: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 40991/169563 objects degraded (24.174%); 2265/169563 objects misplaced (1.336%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:00:27.094 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 40991/169563 objects degraded (24.174%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:27.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2679010448' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:27.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:26 smithi121 ceph-mon[211526]: pgmap v978: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 40991/169563 objects degraded (24.174%); 2265/169563 objects misplaced (1.336%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:00:27.101 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 40991/169563 objects degraded (24.174%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2679010448' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:00:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:26 smithi123 ceph-mon[193903]: pgmap v978: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 261 op/s; 40991/169563 objects degraded (24.174%); 2265/169563 objects misplaced (1.336%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:00:27.242 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:27.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:27.311 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:00:27.348 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:27.403 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:27.530 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:00:27.531 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:00:27.531 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:00:27.531 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:00:27.531 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:00:28.132 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:28.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:28.200 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:28.200 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2679010448' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:28.200 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[211526]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T21:00:28.200 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[211526]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[211526]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[211526]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2679010448' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[219709]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[219709]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[219709]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[219709]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.201 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:27 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2679010448' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:00:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:27 smithi123 ceph-mon[193903]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T21:00:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:27 smithi123 ceph-mon[193903]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:27 smithi123 ceph-mon[193903]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:27 smithi123 ceph-mon[193903]: failed to encode map e312 with expected crc 2023-09-23T21:00:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:28.237 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:28.272 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:28 smithi123 ceph-mon[193903]: failed to encode map e312 with expected crc 2023-09-23T21:00:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:28 smithi123 ceph-mon[193903]: pgmap v980: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 40777/166182 objects degraded (24.538%); 2265/166182 objects misplaced (1.363%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:00:29.280 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:28 smithi121 ceph-mon[219709]: failed to encode map e312 with expected crc 2023-09-23T21:00:29.281 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:28 smithi121 ceph-mon[219709]: pgmap v980: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 40777/166182 objects degraded (24.538%); 2265/166182 objects misplaced (1.363%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:00:29.281 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:28 smithi121 ceph-mon[211526]: failed to encode map e312 with expected crc 2023-09-23T21:00:29.281 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:28 smithi121 ceph-mon[211526]: pgmap v980: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 40777/166182 objects degraded (24.538%); 2265/166182 objects misplaced (1.363%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:00:29.308 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:00:29.637 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:29.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:29.743 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:29.793 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:29.829 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:00:29.829 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:00:29.829 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T21:00:29.829 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:00:29.829 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:00:29.831 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T21:00:29.860 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:29.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:29.964 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:30.049 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:30.115 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:30.186 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:30.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:30.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:30.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:00:30.272 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:00:30.291 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:30.295 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:00:30.317 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:30.321 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:00:30.348 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:00:30.363 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:30.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:30.373 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:00:30.400 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:00:30.426 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:00:30.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.429 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.429 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:00:30.441 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4006131232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:30.441 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4006131232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:30.468 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:30.498 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:30.548 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:30.553 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4006131232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:30.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:30.586 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:30.655 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:30.705 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:31.597 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:00:31.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:31 smithi121 ceph-mon[211526]: pgmap v981: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 330 op/s; 40670/164625 objects degraded (24.705%); 2220/164625 objects misplaced (1.349%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:00:31.623 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:31 smithi121 ceph-mon[219709]: pgmap v981: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 330 op/s; 40670/164625 objects degraded (24.705%); 2220/164625 objects misplaced (1.349%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:00:31.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:31 smithi123 ceph-mon[193903]: pgmap v981: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 330 op/s; 40670/164625 objects degraded (24.705%); 2220/164625 objects misplaced (1.349%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:00:32.008 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:32.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:32.113 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:32.166 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:32.446 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:32.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:32.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:32.536 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 40670/164625 objects degraded (24.705%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:32.536 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 40670/164625 objects degraded (24.705%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:32.554 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:32.579 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 40670/164625 objects degraded (24.705%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:32.582 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:32.631 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:32.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:32.738 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:32.812 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:32.972 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:32.973 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:33.044 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:33.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:33.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:33.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:33.176 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:33.221 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:33.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:33.326 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:33.361 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:33.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:33 smithi123 ceph-mon[193903]: pgmap v982: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 40670/164325 objects degraded (24.750%); 2220/164325 objects misplaced (1.351%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:00:33.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:33 smithi121 ceph-mon[211526]: pgmap v982: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 40670/164325 objects degraded (24.750%); 2220/164325 objects misplaced (1.351%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:00:33.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:33 smithi121 ceph-mon[219709]: pgmap v982: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 40670/164325 objects degraded (24.750%); 2220/164325 objects misplaced (1.351%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T21:00:33.779 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:00:33.847 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:33.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:33.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:33.943 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:33.957 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:33.981 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:00:33.984 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:34.029 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:34.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:34.134 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:34.163 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:34.213 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:34.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:34.319 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:34.364 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:34.778 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:00:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:00:34] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.43.0" 2023-09-23T21:00:35.315 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:00:35.315 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:00:35.315 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:00:35.848 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:35.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:35.920 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:35.922 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:35.931 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:35 smithi121 ceph-mon[211526]: pgmap v983: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 40329/159609 objects degraded (25.267%); 2166/159609 objects misplaced (1.357%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:35.932 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:35 smithi121 ceph-mon[219709]: pgmap v983: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 40329/159609 objects degraded (25.267%); 2166/159609 objects misplaced (1.357%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:35.953 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:35.980 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:36.025 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:36.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:36.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:36.130 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:36.162 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:36.222 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:36.225 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:35 smithi123 ceph-mon[193903]: pgmap v983: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 40329/159609 objects degraded (25.267%); 2166/159609 objects misplaced (1.357%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:36.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:36.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:36.327 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:36.355 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:36.404 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:36.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:36.511 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:36.575 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.022 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:37.141 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:37.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 40329/159609 objects degraded (25.267%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/308861158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 40329/159609 objects degraded (25.267%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/308861158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:37.167 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.209 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:37.214 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 40329/159609 objects degraded (25.267%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:37.214 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/308861158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:37.315 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:37.322 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:00:37.341 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.381 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:37.489 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:37.525 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.580 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:37.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:37.687 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:37.713 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.757 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:37.864 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:37.890 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:37.943 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:37.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:37.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:38.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:38.076 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:38.129 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:38.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:38.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[211526]: pgmap v984: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 40329/159609 objects degraded (25.267%); 2166/159609 objects misplaced (1.357%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:38.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/308861158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[211526]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[211526]: failed to encode map e313 with expected crc 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[211526]: failed to encode map e313 with expected crc 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[219709]: pgmap v984: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 40329/159609 objects degraded (25.267%); 2166/159609 objects misplaced (1.357%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/308861158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[219709]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T21:00:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[219709]: failed to encode map e313 with expected crc 2023-09-23T21:00:38.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:37 smithi121 ceph-mon[219709]: failed to encode map e313 with expected crc 2023-09-23T21:00:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:37 smithi123 ceph-mon[193903]: pgmap v984: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 40329/159609 objects degraded (25.267%); 2166/159609 objects misplaced (1.357%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:37 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/308861158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:37 smithi123 ceph-mon[193903]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T21:00:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:37 smithi123 ceph-mon[193903]: failed to encode map e313 with expected crc 2023-09-23T21:00:38.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:37 smithi123 ceph-mon[193903]: failed to encode map e313 with expected crc 2023-09-23T21:00:38.235 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:38.279 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:38.874 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:38.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:38.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:38.985 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:39.023 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.159 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:39.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:38 smithi123 ceph-mon[193903]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:38 smithi123 ceph-mon[193903]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:38 smithi123 ceph-mon[193903]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:38 smithi123 ceph-mon[193903]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:39.291 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.339 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:39.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[211526]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[211526]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[211526]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[211526]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.394 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[219709]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.395 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[219709]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.395 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[219709]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.395 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:38 smithi121 ceph-mon[219709]: failed to encode map e313 with expected crc 2023-09-23T21:00:39.446 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:39.473 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:39.474 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:39.476 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.526 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:39.634 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:39.667 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.675 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:00:39.712 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:00:39.716 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:39.750 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:00:39.824 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:39.853 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:39.898 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:39.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:39.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:39.974 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:00:39.997 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:00:40.002 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:40.016 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:00:40.027 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:40.035 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:00:40.055 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.057 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:00:40.069 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:40.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:40.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:39 smithi121 ceph-mon[211526]: pgmap v986: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 40075/155946 objects degraded (25.698%); 2166/155946 objects misplaced (1.389%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T21:00:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4254457011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:39 smithi121 ceph-mon[219709]: pgmap v986: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 40075/155946 objects degraded (25.698%); 2166/155946 objects misplaced (1.389%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T21:00:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4254457011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:40.173 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:40.203 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:39 smithi123 ceph-mon[193903]: pgmap v986: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 40075/155946 objects degraded (25.698%); 2166/155946 objects misplaced (1.389%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T21:00:40.203 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4254457011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:40.206 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:40.257 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:40.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:40.362 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:40.400 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:40.740 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:00:40.919 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:40.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:40.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:40.956 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:00:40.957 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:00:40.957 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T21:00:40.957 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:00:40.957 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:00:40.957 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T21:00:41.025 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:41.034 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:00:41.055 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:41.103 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:41.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:41.209 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:41.226 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:41.241 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:41.350 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:41.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:41.394 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:00:41.458 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:41.493 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:41.595 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:41.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:41.699 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:41.745 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:41.823 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:41.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:41.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:41.930 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:41.966 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.077 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:42.079 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:41 smithi121 ceph-mon[219709]: pgmap v987: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 39960/154218 objects degraded (25.911%); 2104/154218 objects misplaced (1.364%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:00:42.079 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39960/154218 objects degraded (25.911%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:42.079 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:41 smithi121 ceph-mon[211526]: pgmap v987: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 39960/154218 objects degraded (25.911%); 2104/154218 objects misplaced (1.364%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:00:42.079 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39960/154218 objects degraded (25.911%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:42.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:42.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:42.181 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:42.208 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:41 smithi123 ceph-mon[193903]: pgmap v987: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 39960/154218 objects degraded (25.911%); 2104/154218 objects misplaced (1.364%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:00:42.208 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39960/154218 objects degraded (25.911%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:42.212 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.255 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:42.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:42.359 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:42.400 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.678 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:42.686 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:42.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:42.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:42.796 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:42.829 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:42.979 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:42.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:42.993 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:00:43.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:43.113 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:43.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:43.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:00:43.263 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:43.263 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:00:43.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:43.263 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:00:43.737 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:43.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:43.753 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:00:43.753 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:00:43.753 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:00:43.753 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:00:43.753 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:00:43.843 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:43.886 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:43.956 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:43 smithi121 ceph-mon[211526]: pgmap v988: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 39960/153927 objects degraded (25.960%); 2104/153927 objects misplaced (1.367%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:00:43.956 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:43 smithi121 ceph-mon[219709]: pgmap v988: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 39960/153927 objects degraded (25.960%); 2104/153927 objects misplaced (1.367%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:00:44.094 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:43 smithi123 ceph-mon[193903]: pgmap v988: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 346 op/s; 39960/153927 objects degraded (25.960%); 2104/153927 objects misplaced (1.367%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:00:44.100 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:44.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:44.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:44.206 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:00:44.248 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:44.353 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:00:44.354 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "" 2023-09-23T21:00:44.354 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:00:44.408 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:00:44.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:44.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:00:44.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:44.546 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:44.653 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:00:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:00:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:00:44] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.43.0" 2023-09-23T21:00:45.105 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:45.144 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:45.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:45.248 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:45.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:44 smithi121 ceph-mon[219709]: pgmap v989: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 39680/149430 objects degraded (26.554%); 2055/149430 objects misplaced (1.375%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:00:45.296 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:44 smithi121 ceph-mon[211526]: pgmap v989: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 39680/149430 objects degraded (26.554%); 2055/149430 objects misplaced (1.375%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:00:45.296 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:45.388 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:44 smithi123 ceph-mon[193903]: pgmap v989: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 39680/149430 objects degraded (26.554%); 2055/149430 objects misplaced (1.375%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:00:45.635 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:45.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:45.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:45.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:45.772 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:45.925 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:45.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:46.005 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:00:46.033 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:46.068 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:46.221 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3211274023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:46.221 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3211274023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:46.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3211274023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:00:47.127 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:47.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:47.230 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:47.236 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:47.236 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.236 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.236 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3211274023' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:47.236 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T21:00:47.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39680/149430 objects degraded (26.554%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:47.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[219709]: pgmap v991: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 39680/149430 objects degraded (26.554%); 2055/149430 objects misplaced (1.375%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:00:47.238 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:47.238 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.238 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3211274023' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:47.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T21:00:47.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39680/149430 objects degraded (26.554%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:47.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:46 smithi121 ceph-mon[211526]: pgmap v991: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 39680/149430 objects degraded (26.554%); 2055/149430 objects misplaced (1.375%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:00:47.259 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:00:47.259 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.259 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.260 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3211274023' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:00:47.260 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T21:00:47.260 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: failed to encode map e314 with expected crc 2023-09-23T21:00:47.260 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39680/149430 objects degraded (26.554%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:47.260 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:46 smithi123 ceph-mon[193903]: pgmap v991: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 39680/149430 objects degraded (26.554%); 2055/149430 objects misplaced (1.375%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:00:47.263 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:47.282 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:47.318 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:47.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:47.424 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:47.452 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:47.503 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:47.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:47.608 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:47.668 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:48.059 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:48.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:48.139 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:00:48.164 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:48.180 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:00:48.220 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:00:48.242 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:48.263 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:00:48.298 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:00:48.329 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:00:48.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:00:48.385 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:00:48.394 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:00:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:00:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:00:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:00:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:00:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:00:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:00:48.395 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:00:49.027 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:49.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:49.131 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:49.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:49.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:49.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3268320872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:49.158 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:49.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:49.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:49.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3268320872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3268320872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:49.201 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:49.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:49.239 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:00:49.306 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:49.346 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:49.696 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:49.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:49.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:49.800 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:49.836 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:50.040 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:50.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:50.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:49 smithi121 ceph-mon[211526]: pgmap v992: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 39464/145665 objects degraded (27.092%); 2055/145665 objects misplaced (1.411%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T21:00:50.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:49 smithi121 ceph-mon[219709]: pgmap v992: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 39464/145665 objects degraded (27.092%); 2055/145665 objects misplaced (1.411%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T21:00:50.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:00:50.172 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:49 smithi123 ceph-mon[193903]: pgmap v992: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 39464/145665 objects degraded (27.092%); 2055/145665 objects misplaced (1.411%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T21:00:50.174 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:50.205 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:50.221 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:00:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:50.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:00:50.326 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:50.343 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:50.343 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:50.360 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:50.404 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:00:50.404 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:00:50.404 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:00:50.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T21:00:51.578 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:51.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:51.678 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:00:51.686 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:51.783 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:51.903 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:00:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:51 smithi121 ceph-mon[219709]: pgmap v993: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 39363/143901 objects degraded (27.354%); 1988/143901 objects misplaced (1.382%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:00:52.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39363/143901 objects degraded (27.354%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:51 smithi121 ceph-mon[211526]: pgmap v993: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 39363/143901 objects degraded (27.354%); 1988/143901 objects misplaced (1.382%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:00:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39363/143901 objects degraded (27.354%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:51 smithi123 ceph-mon[193903]: pgmap v993: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 39363/143901 objects degraded (27.354%); 1988/143901 objects misplaced (1.382%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:00:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39363/143901 objects degraded (27.354%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:52.428 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:52.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:52.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:52.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:00:52.594 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:52.811 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:52.949 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:00:52.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:52.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:00:53.060 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:53.118 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.247 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:53.248 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:53.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:53.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:53.256 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:00:53.357 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:53.408 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.543 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:53.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:53.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:53.653 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:53.705 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:53.790 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:53.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:53.903 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:53.943 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:54.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[211526]: pgmap v994: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 39363/143553 objects degraded (27.421%); 1988/143553 objects misplaced (1.385%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:00:54.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:54.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:54.074 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2539948350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:54.074 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[219709]: pgmap v994: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 39363/143553 objects degraded (27.421%); 1988/143553 objects misplaced (1.385%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:00:54.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:54.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:54.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2539948350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:54.172 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:00:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:53 smithi123 ceph-mon[193903]: pgmap v994: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 39363/143553 objects degraded (27.421%); 1988/143553 objects misplaced (1.385%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:00:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:54.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2539948350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:00:55.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:00:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:00:54] "GET /metrics HTTP/1.1" 200 36626 "" "Prometheus/2.43.0" 2023-09-23T21:00:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2539948350' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T21:00:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: failed to encode map e315 with expected crc 2023-09-23T21:00:55.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: failed to encode map e315 with expected crc 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2539948350' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: failed to encode map e315 with expected crc 2023-09-23T21:00:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: failed to encode map e315 with expected crc 2023-09-23T21:00:55.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:00:55.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:00:55.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:54 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:55.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:00:55.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2539948350' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:00:55.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T21:00:55.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: failed to encode map e315 with expected crc 2023-09-23T21:00:55.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: failed to encode map e315 with expected crc 2023-09-23T21:00:55.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:00:55.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:00:55.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:54 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:55 smithi121 ceph-mon[211526]: failed to encode map e315 with expected crc 2023-09-23T21:00:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:55 smithi121 ceph-mon[211526]: pgmap v996: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 39060/138564 objects degraded (28.189%); 1936/138564 objects misplaced (1.397%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:00:56.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:55 smithi121 ceph-mon[219709]: failed to encode map e315 with expected crc 2023-09-23T21:00:56.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:55 smithi121 ceph-mon[219709]: pgmap v996: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 39060/138564 objects degraded (28.189%); 1936/138564 objects misplaced (1.397%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:00:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:55 smithi123 ceph-mon[193903]: failed to encode map e315 with expected crc 2023-09-23T21:00:56.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:55 smithi123 ceph-mon[193903]: pgmap v996: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 39060/138564 objects degraded (28.189%); 1936/138564 objects misplaced (1.397%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:00:56.411 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:56.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:56.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:56.519 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:00:56.551 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:56.595 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:00:56.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:56.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:00:56.702 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:00:56.781 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:56.879 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:00:56.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:56.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:00:56.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39060/138564 objects degraded (28.189%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:56.942 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39060/138564 objects degraded (28.189%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:56.943 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39060/138564 objects degraded (28.189%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:00:56.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:57.024 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:57.507 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:00:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:57.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:00:57.521 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:00:57 smithi121 conmon[198376]: 2023-09-23T21:00:57.508+0000 7f479dcb1700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T21:00:57.527 INFO:teuthology.orchestra.run.smithi121.stdout:moved 4 / 297 (1.3468%) 2023-09-23T21:00:57.527 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:00:57.527 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 12.5144 -> 13.3458 (expected baseline 5.69951) 2023-09-23T21:00:57.527 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.5 with 10 -> 10 pgs (0.26936 -> 0.26936 * mean) 2023-09-23T21:00:57.527 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 52 -> 56 pgs (1.40067 -> 1.50842 * mean) 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0169 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0186 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.5554 -> 0.5054 2023-09-23T21:00:57.528 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.6890 -> 0.6390 2023-09-23T21:00:57.556 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:00:57.581 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:00:57.620 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:00:57.629 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:00:57.649 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:57.656 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:00:57.656 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:00:57.670 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:00:57.699 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:00:57.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:57.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:00:57.726 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:00:57.778 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:00:57.808 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:00:57.815 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:00:57.850 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:00:57.884 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:00:57.888 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:00:57.891 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:00:57.891 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:00:57.891 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:00:57.891 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:00:57.891 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:00:57.891 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:00:57.892 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:00:57.892 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:00:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[211526]: pgmap v997: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 39060/138564 objects degraded (28.189%); 1936/138564 objects misplaced (1.397%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:58.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:58.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2081482220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[219709]: pgmap v997: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 39060/138564 objects degraded (28.189%); 1936/138564 objects misplaced (1.397%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:58.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2081482220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:57 smithi123 ceph-mon[193903]: pgmap v997: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 39060/138564 objects degraded (28.189%); 1936/138564 objects misplaced (1.397%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T21:00:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:00:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:00:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2081482220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:00:58.595 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:00:59.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:58 smithi123 ceph-mon[193903]: from='client.56038 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:00:59.258 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:58 smithi121 ceph-mon[219709]: from='client.56038 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:00:59.258 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:58 smithi121 ceph-mon[211526]: from='client.56038 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:00:59.628 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:00:59.628 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-23T21:00:59.631 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:00:59.769 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:00:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:00:59.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:00:59.874 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:00:59.982 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.025 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:00.078 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:00.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:00.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:00:59 smithi121 ceph-mon[211526]: pgmap v998: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 38768/134223 objects degraded (28.883%); 1936/134223 objects misplaced (1.442%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:01:00.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:00:59 smithi121 ceph-mon[219709]: pgmap v998: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 38768/134223 objects degraded (28.883%); 1936/134223 objects misplaced (1.442%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:01:00.186 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:00.215 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:00:59 smithi123 ceph-mon[193903]: pgmap v998: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 38768/134223 objects degraded (28.883%); 1936/134223 objects misplaced (1.442%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:01:00.218 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.265 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:00.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:00.372 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:00.399 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.461 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:00.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:00.568 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:00.598 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.611 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:00.613 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:00.640 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:00.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:00.748 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:00.775 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:00.822 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:00.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:00.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:00.929 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:00.954 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:01.002 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:01.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:01.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:01.133 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:00 smithi123 ceph-mon[193903]: pgmap v999: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 389 op/s; 38642/132219 objects degraded (29.226%); 1866/132219 objects misplaced (1.411%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:01.141 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:01.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:00 smithi121 ceph-mon[211526]: pgmap v999: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 389 op/s; 38642/132219 objects degraded (29.226%); 1866/132219 objects misplaced (1.411%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:01.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:00 smithi121 ceph-mon[219709]: pgmap v999: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 389 op/s; 38642/132219 objects degraded (29.226%); 1866/132219 objects misplaced (1.411%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:01.676 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:01.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:01.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:01.783 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:01.817 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:01.867 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:01.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:01.918 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:01:01.973 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:02.005 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:02.050 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:02.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:02.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:02.156 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:02.180 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 38642/132219 objects degraded (29.226%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:02.183 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:02.229 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:02.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:02.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:02.295 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 38642/132219 objects degraded (29.226%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:02.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 38642/132219 objects degraded (29.226%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:02.336 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:02.365 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:02.414 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:02.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:02.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:02.520 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:02.545 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:02.589 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:02.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:02.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:02.695 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:02.755 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:02.943 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:02.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:03.047 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:03.075 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:03.124 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:03.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:03.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:03.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:02 smithi123 ceph-mon[193903]: pgmap v1000: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 38642/131838 objects degraded (29.310%); 1866/131838 objects misplaced (1.415%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:03.232 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:03.260 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:03.303 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:02 smithi121 ceph-mon[211526]: pgmap v1000: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 38642/131838 objects degraded (29.310%); 1866/131838 objects misplaced (1.415%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:03.303 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:02 smithi121 ceph-mon[219709]: pgmap v1000: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 390 op/s; 38642/131838 objects degraded (29.310%); 1866/131838 objects misplaced (1.415%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:03.314 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:03.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:03.421 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:03.455 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:03.535 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:03.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:03.641 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:03.686 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:03.815 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:03.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:03.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:03.919 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:04.018 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:04.105 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:04.147 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:04.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:04.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:04.289 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:04.329 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:04.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:04.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:04.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/717654902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:04.381 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:04.385 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout:moved 25 / 297 (8.41751%) 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 12.5144 -> 17.1496 (expected baseline 5.69951) 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.5 with 10 -> 11 pgs (0.26936 -> 0.296296 * mean) 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 52 -> 58 pgs (1.40067 -> 1.56229 * mean) 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:01:04.437 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:01:04.438 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:04.438 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0161 2023-09-23T21:01:04.438 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0177 2023-09-23T21:01:04.438 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.5554 -> 0.3154 2023-09-23T21:01:04.438 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.6890 -> 0.5238 2023-09-23T21:01:04.438 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:04.466 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:01:04.500 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:04.645 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/717654902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:04.645 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:04.645 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:04 smithi121 conmon[198376]: 2023-09-23T21:01:04.435+0000 7f479dcb1700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T21:01:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/717654902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:04.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:01:04] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.43.0" 2023-09-23T21:01:05.337 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:01:05.337 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:01:05.338 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:01:05.338 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:01:05.338 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:01:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T21:01:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: from='client.45992 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[211526]: pgmap v1002: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 38322/127086 objects degraded (30.154%); 1798/127086 objects misplaced (1.415%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: from='client.45992 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:05 smithi121 ceph-mon[219709]: pgmap v1002: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 38322/127086 objects degraded (30.154%); 1798/127086 objects misplaced (1.415%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T21:01:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T21:01:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: failed to encode map e316 with expected crc 2023-09-23T21:01:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: from='client.45992 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:05 smithi123 ceph-mon[193903]: pgmap v1002: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 38322/127086 objects degraded (30.154%); 1798/127086 objects misplaced (1.415%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T21:01:06.059 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:01:06.059 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T21:01:06.388 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 38322/127086 objects degraded (30.154%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:06.520 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:06.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:06.561 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:01:06.634 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:06.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 38322/127086 objects degraded (30.154%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:06.675 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:06.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 38322/127086 objects degraded (30.154%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:07.074 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:07.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:07.179 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:07.213 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:07.543 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:07.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:07.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:07.648 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:07.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:07 smithi121 ceph-mon[211526]: pgmap v1003: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 38322/127086 objects degraded (30.154%); 1798/127086 objects misplaced (1.415%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T21:01:07.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:07 smithi121 ceph-mon[219709]: pgmap v1003: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 38322/127086 objects degraded (30.154%); 1798/127086 objects misplaced (1.415%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T21:01:07.707 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:07.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:07 smithi123 ceph-mon[193903]: pgmap v1003: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 38322/127086 objects degraded (30.154%); 1798/127086 objects misplaced (1.415%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T21:01:07.775 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:07.801 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:07.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:07.839 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:07.906 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:07.938 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:07.986 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:07.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:07.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:08.092 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:08.132 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:08.246 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:08.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:08.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:08.260 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:01:08.298 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:01:08.332 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:01:08.355 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:08.366 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:01:08.382 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:08.395 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:01:08.428 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:01:08.437 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:08.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:08.454 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:01:08.481 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:01:08.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.487 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:01:08.510 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3228956598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:08.513 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3228956598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:08.545 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:08.601 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:08.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3228956598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:09.431 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:09.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:09.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:09.545 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:09.607 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:09.666 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:01:09.687 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:09 smithi121 ceph-mon[219709]: pgmap v1004: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 38066/123216 objects degraded (30.894%); 1798/123216 objects misplaced (1.459%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:09.690 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:09 smithi121 ceph-mon[211526]: pgmap v1004: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 38066/123216 objects degraded (30.894%); 1798/123216 objects misplaced (1.459%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:09 smithi123 ceph-mon[193903]: pgmap v1004: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 38066/123216 objects degraded (30.894%); 1798/123216 objects misplaced (1.459%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:10.015 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:10.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:10.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:10.119 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:10.151 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:10.703 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:10.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:10.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:10.814 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:10.846 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:10.901 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:10.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:10.932 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:11.005 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:11.036 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:11.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:11.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:11.191 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:11.222 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:11.281 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:11.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:11.387 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:11.404 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:11.404 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:11.415 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:11.470 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:11.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:11.576 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:11.623 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:11.693 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:11.695 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:01:11.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:11.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:11.797 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:11.815 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 297 (0%) 2023-09-23T21:01:11.815 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:11.815 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 12.5144 -> 12.5144 (expected baseline 5.69951) 2023-09-23T21:01:11.815 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.5 with 10 -> 10 pgs (0.26936 -> 0.26936 * mean) 2023-09-23T21:01:11.815 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 52 -> 52 pgs (1.40067 -> 1.40067 * mean) 2023-09-23T21:01:11.815 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:11.816 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:01:11.816 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T21:01:11.816 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:11.816 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0157 2023-09-23T21:01:11.816 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.1574 2023-09-23T21:01:11.817 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:01:11.822 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:11.864 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:11.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:11.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:11.970 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:12.000 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:12.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:11 smithi121 ceph-mon[211526]: pgmap v1005: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 37927/121371 objects degraded (31.249%); 1738/121371 objects misplaced (1.432%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:12.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 37927/121371 objects degraded (31.249%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:12.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:11 smithi121 ceph-mon[219709]: pgmap v1005: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 37927/121371 objects degraded (31.249%); 1738/121371 objects misplaced (1.432%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:12.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 37927/121371 objects degraded (31.249%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:12.049 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:12.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:12.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:12.155 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:12.177 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:11 smithi123 ceph-mon[193903]: pgmap v1005: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 37927/121371 objects degraded (31.249%); 1738/121371 objects misplaced (1.432%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:12.177 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 37927/121371 objects degraded (31.249%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:12.181 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:12.242 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:12.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:12.347 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:12.389 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:13.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:12 smithi121 ceph-mon[219709]: from='client.56059 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:13.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:13.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:12 smithi121 ceph-mon[211526]: from='client.56059 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:12 smithi123 ceph-mon[193903]: from='client.56059 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:13.265 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:13.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:13.371 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:13.406 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:13.570 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:01:13.573 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T21:01:13.574 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T21:01:13.601 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:13.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:13.720 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:13.784 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:13.793 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:13.976 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:14.032 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:14.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:14.140 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:14.178 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:14.211 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:13 smithi121 ceph-mon[211526]: pgmap v1006: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 37927/120963 objects degraded (31.354%); 1738/120963 objects misplaced (1.437%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:14.211 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:13 smithi121 ceph-mon[219709]: pgmap v1006: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 37927/120963 objects degraded (31.354%); 1738/120963 objects misplaced (1.437%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:14.213 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:13 smithi123 ceph-mon[193903]: pgmap v1006: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 37927/120963 objects degraded (31.354%); 1738/120963 objects misplaced (1.437%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:01:14.244 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:14.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:14.349 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:14.389 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:14.456 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:14.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:14.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:14.560 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:14.632 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:14.714 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:14.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:14.820 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:14.849 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:14.893 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:14.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:14.961 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:01:14.985 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4004148255' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:14.985 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4004148255' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:14.986 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:01:14] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.43.0" 2023-09-23T21:01:14.998 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:15.033 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:15.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4004148255' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:15.437 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:15.503 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:15.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:15.614 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:15.649 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:15.709 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:15.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:15.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:15.812 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:15.845 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:15.901 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:15.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:15.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:15.920 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:15.921 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:16.005 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:16.036 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[219709]: pgmap v1007: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 37555/115212 objects degraded (32.596%); 1672/115212 objects misplaced (1.451%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:01:16.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:16.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4004148255' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:16.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[219709]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T21:01:16.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[219709]: failed to encode map e317 with expected crc 2023-09-23T21:01:16.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[219709]: failed to encode map e317 with expected crc 2023-09-23T21:01:16.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[211526]: pgmap v1007: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 37555/115212 objects degraded (32.596%); 1672/115212 objects misplaced (1.451%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:01:16.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:16.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4004148255' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:16.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[211526]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T21:01:16.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[211526]: failed to encode map e317 with expected crc 2023-09-23T21:01:16.124 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:15 smithi121 ceph-mon[211526]: failed to encode map e317 with expected crc 2023-09-23T21:01:16.147 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:15 smithi123 ceph-mon[193903]: pgmap v1007: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 37555/115212 objects degraded (32.596%); 1672/115212 objects misplaced (1.451%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:01:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:15 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4004148255' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:16.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:15 smithi123 ceph-mon[193903]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T21:01:16.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:15 smithi123 ceph-mon[193903]: failed to encode map e317 with expected crc 2023-09-23T21:01:16.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:15 smithi123 ceph-mon[193903]: failed to encode map e317 with expected crc 2023-09-23T21:01:16.253 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:16.309 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.433 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:16.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:16.537 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:16.572 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.678 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:01:16.713 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:01:16.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:16.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:16.761 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:01:16.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:01:16.817 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:01:16.820 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:01:16.835 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:16.857 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:01:16.891 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:01:16.901 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:16.914 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:01:16.915 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.915 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.915 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.916 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.916 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.916 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.916 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.916 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:01:16.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:16.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:16.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:17.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:17.073 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:16 smithi123 ceph-mon[193903]: failed to encode map e317 with expected crc 2023-09-23T21:01:17.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:16 smithi123 ceph-mon[193903]: failed to encode map e317 with expected crc 2023-09-23T21:01:17.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 37555/115212 objects degraded (32.596%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:17.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2869390994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:17.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:16 smithi123 ceph-mon[193903]: pgmap v1009: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 37555/115212 objects degraded (32.596%); 1672/115212 objects misplaced (1.451%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:01:17.076 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:17.149 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:17.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[219709]: failed to encode map e317 with expected crc 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[219709]: failed to encode map e317 with expected crc 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 37555/115212 objects degraded (32.596%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2869390994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[219709]: pgmap v1009: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 37555/115212 objects degraded (32.596%); 1672/115212 objects misplaced (1.451%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[211526]: failed to encode map e317 with expected crc 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[211526]: failed to encode map e317 with expected crc 2023-09-23T21:01:17.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 37555/115212 objects degraded (32.596%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:17.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2869390994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:17.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:16 smithi121 ceph-mon[211526]: pgmap v1009: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 37555/115212 objects degraded (32.596%); 1672/115212 objects misplaced (1.451%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:01:17.244 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:17.255 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:17.290 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:17.396 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:17.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:17.505 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:17.555 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:18.082 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:18.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:18.154 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:01:18.188 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:18.231 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:18.334 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:18.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:18.445 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:18.483 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:18.558 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:18.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:18.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:18.667 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:18.733 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:18.835 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:18.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:18.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:18.864 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:18.939 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:19.002 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:19.060 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:01:19.060 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:01:19.060 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:01:19.060 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:01:19.060 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:01:19.097 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:19.204 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:19.242 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:19.356 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:19.460 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:19.520 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:19.584 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:19.689 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:19.716 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:19.958 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 297 (0%) 2023-09-23T21:01:19.958 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:19.958 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 12.5144 -> 12.5144 (expected baseline 5.69951) 2023-09-23T21:01:19.958 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.5 with 10 -> 10 pgs (0.26936 -> 0.26936 * mean) 2023-09-23T21:01:19.958 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 52 -> 52 pgs (1.40067 -> 1.40067 * mean) 2023-09-23T21:01:19.958 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:19.958 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:01:19.959 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:01:19.959 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:19.959 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0150 2023-09-23T21:01:19.959 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.1502 2023-09-23T21:01:19.959 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8427 -> 0.8500 2023-09-23T21:01:19.963 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:01:19.976 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:19 smithi123 ceph-mon[193903]: pgmap v1010: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 37277/110511 objects degraded (33.731%); 1672/110511 objects misplaced (1.513%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:01:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:19 smithi123 ceph-mon[193903]: from='client.36036 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:19 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]: dispatch 2023-09-23T21:01:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:19 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]: dispatch 2023-09-23T21:01:19.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:19.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:20.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:20.092 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:01:20.124 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:20.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[211526]: pgmap v1010: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 37277/110511 objects degraded (33.731%); 1672/110511 objects misplaced (1.513%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:01:20.365 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[211526]: from='client.36036 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:20.366 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]: dispatch 2023-09-23T21:01:20.366 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]: dispatch 2023-09-23T21:01:20.366 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[219709]: pgmap v1010: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 37277/110511 objects degraded (33.731%); 1672/110511 objects misplaced (1.513%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:01:20.366 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[219709]: from='client.36036 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:20.366 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]: dispatch 2023-09-23T21:01:20.366 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:19 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]: dispatch 2023-09-23T21:01:20.395 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:20.399 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:20.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:20.505 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:20.538 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:20.949 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:20.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:20.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:20.966 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:01:20.966 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T21:01:21.062 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:21.091 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.151 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:21.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:21.226 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[211526]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[211526]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]': finished 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[211526]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[211526]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[211526]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[211526]: pgmap v1012: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 37130/108372 objects degraded (34.262%); 1606/108372 objects misplaced (1.482%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[219709]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[219709]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]': finished 2023-09-23T21:01:21.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[219709]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T21:01:21.228 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[219709]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.228 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[219709]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.228 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:20 smithi121 ceph-mon[219709]: pgmap v1012: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 37130/108372 objects degraded (34.262%); 1606/108372 objects misplaced (1.482%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T21:01:21.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:20 smithi123 ceph-mon[193903]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:20 smithi123 ceph-mon[193903]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:20 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'55703'}"}]': finished 2023-09-23T21:01:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:20 smithi123 ceph-mon[193903]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T21:01:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:20 smithi123 ceph-mon[193903]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:20 smithi123 ceph-mon[193903]: failed to encode map e318 with expected crc 2023-09-23T21:01:21.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:20 smithi123 ceph-mon[193903]: pgmap v1012: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 37130/108372 objects degraded (34.262%); 1606/108372 objects misplaced (1.482%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T21:01:21.257 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:21.290 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.338 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:21.341 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:21.441 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:21.488 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.768 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:21.879 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:21.911 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:21.923 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:21.991 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:21.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:21.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:22.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 37130/108372 objects degraded (34.262%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:22.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/494564302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:22.055 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:22.055 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 37130/108372 objects degraded (34.262%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:22.055 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/494564302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:22.055 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:22.097 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:22.147 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:22.224 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:01:22.224 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:01:22.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:01:22.224 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:01:22.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.224 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136336 9.4 9:27d01416:::benchmark_data_smithi121_293770_object33448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T21:01:22.224 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.267616+0000", 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 531.95404278199999, 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.014674013, 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:22.225 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136336 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267616+0000", 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267616+0000", 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267613+0000", 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267623+0000", 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2199999999999998e-06 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267637+0000", 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3888e-05 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267649+0000", 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.236e-05 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267721+0000", 2023-09-23T21:01:22.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2181000000000006e-05 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.267732+0000", 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1311999999999999e-05 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.263921+0000", 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99618875900000003 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.268248+0000", 2023-09-23T21:01:22.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043266629999999997 2023-09-23T21:01:22.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3", 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.268344+0000", 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5913000000000001e-05 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271716+0000", 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003372618 2023-09-23T21:01:22.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:22.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.282134+0000", 2023-09-23T21:01:22.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010417277000000001 2023-09-23T21:01:22.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.282188+0000", 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4156000000000001e-05 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.282290+0000", 2023-09-23T21:01:22.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101767 2023-09-23T21:01:22.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.234 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.234 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.234 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54857 10.2 10:41d67633:::benchmark_data_smithi121_358375_object54856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:22.234 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.603795+0000", 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 368.617862925, 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25188782500000001, 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54857 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603795+0000", 2023-09-23T21:01:22.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603795+0000", 2023-09-23T21:01:22.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603794+0000", 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603826+0000", 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1313999999999999e-05 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603834+0000", 2023-09-23T21:01:22.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4160000000000004e-06 2023-09-23T21:01:22.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603839+0000", 2023-09-23T21:01:22.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8130000000000003e-06 2023-09-23T21:01:22.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.605361+0000", 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015223610000000001 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.605480+0000", 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011848700000000001 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.605621+0000", 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000141676 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609534+0000", 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039124939999999999 2023-09-23T21:01:22.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:22.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855505+0000", 2023-09-23T21:01:22.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24597127799999999 2023-09-23T21:01:22.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855568+0000", 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2665999999999999e-05 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855683+0000", 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115369 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.242 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54862 10.2 10:4ea029ac:::benchmark_data_smithi121_358375_object54861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.607325+0000", 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 368.61433319499997, 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24949592100000001, 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.243 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.244 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:22.244 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:22.244 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:22.244 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54862 2023-09-23T21:01:22.244 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.244 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607325+0000", 2023-09-23T21:01:22.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607325+0000", 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607324+0000", 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607361+0000", 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6653999999999998e-05 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607373+0000", 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2636000000000001e-05 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607380+0000", 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.426e-06 2023-09-23T21:01:22.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607479+0000", 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9697000000000004e-05 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607611+0000", 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013148599999999999 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:01:22.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607752+0000", 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014129999999999999 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610522+0000", 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027701330000000001 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856687+0000", 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.246164834 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856741+0000", 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3551000000000003e-05 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856821+0000", 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0419000000000004e-05 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57382 10.2 10:44e2fe36:::benchmark_data_smithi121_358375_object57381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.685040+0000", 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 366.53661817199998, 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24216987100000001, 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:22.251 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57382 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685040+0000", 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685040+0000", 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685039+0000", 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685073+0000", 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3844000000000003e-05 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685085+0000", 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2111e-05 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685091+0000", 2023-09-23T21:01:22.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4509999999999997e-06 2023-09-23T21:01:22.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685137+0000", 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6076999999999999e-05 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685274+0000", 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013711600000000001 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:01:22.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685418+0000", 2023-09-23T21:01:22.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000144029 2023-09-23T21:01:22.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:22.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.689479+0000", 2023-09-23T21:01:22.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004060536 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927051+0000", 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.237571753 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927111+0000", 2023-09-23T21:01:22.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0189000000000003e-05 2023-09-23T21:01:22.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927210+0000", 2023-09-23T21:01:22.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9098999999999994e-05 2023-09-23T21:01:22.259 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57383 10.2 10:40044bd6:::benchmark_data_smithi121_358375_object57382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.685106+0000", 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 366.53655254, 2023-09-23T21:01:22.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.248525258, 2023-09-23T21:01:22.261 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.261 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.261 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:22.261 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:22.261 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:22.261 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57383 2023-09-23T21:01:22.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.262 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685106+0000", 2023-09-23T21:01:22.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685106+0000", 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685105+0000", 2023-09-23T21:01:22.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:22.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685134+0000", 2023-09-23T21:01:22.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9094999999999998e-05 2023-09-23T21:01:22.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685141+0000", 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6680000000000004e-06 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685146+0000", 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7759999999999997e-06 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.687493+0000", 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023472459999999999 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.687611+0000", 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011831 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:01:22.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.687740+0000", 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012903599999999999 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690353+0000", 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026128660000000001 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933529+0000", 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24317556300000001 2023-09-23T21:01:22.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933566+0000", 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6981999999999999e-05 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933631+0000", 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5173999999999996e-05 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.268 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57398 10.2 10:4136b738:::benchmark_data_smithi121_358375_object57397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.694526+0000", 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 366.52713190100002, 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.239204737, 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:22.269 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57398 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694526+0000", 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694526+0000", 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694525+0000", 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694587+0000", 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2389000000000002e-05 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694611+0000", 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3608000000000002e-05 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694618+0000", 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.939e-06 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694663+0000", 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5444000000000001e-05 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694793+0000", 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012995199999999999 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694923+0000", 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000129803 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.698840+0000", 2023-09-23T21:01:22.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039173840000000003 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933653+0000", 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.234812778 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933685+0000", 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2156000000000002e-05 2023-09-23T21:01:22.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933731+0000", 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5748000000000003e-05 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57399 10.2 10:459beae3:::benchmark_data_smithi121_358375_object57398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:22.275 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 366.52702495699998, 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.241389191, 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:22.276 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57399 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:01:22.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694665+0000", 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1838999999999998e-05 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694673+0000", 2023-09-23T21:01:22.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.892e-06 2023-09-23T21:01:22.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694677+0000", 2023-09-23T21:01:22.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9289999999999997e-06 2023-09-23T21:01:22.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.697304+0000", 2023-09-23T21:01:22.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026263620000000001 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.697455+0000", 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015165799999999999 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.697595+0000", 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000139352 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.699022+0000", 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001426891 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.935859+0000", 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23683732599999999 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.935908+0000", 2023-09-23T21:01:22.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8865000000000001e-05 2023-09-23T21:01:22.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.936023+0000", 2023-09-23T21:01:22.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114645 2023-09-23T21:01:22.284 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.284 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.284 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.285 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:97215 10.d e286/285 10:b031210c:::benchmark_data_smithi121_358375_object12470:head v 286'5926, mlcod=0'0)", 2023-09-23T21:01:22.285 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:37.491122+0000", 2023-09-23T21:01:22.285 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 164.730535925, 2023-09-23T21:01:22.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0091874369999999997, 2023-09-23T21:01:22.285 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.285 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.285 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491122+0000", 2023-09-23T21:01:22.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491122+0000", 2023-09-23T21:01:22.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491122+0000", 2023-09-23T21:01:22.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:22.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491128+0000", 2023-09-23T21:01:22.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9499999999999998e-06 2023-09-23T21:01:22.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491136+0000", 2023-09-23T21:01:22.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4700000000000002e-06 2023-09-23T21:01:22.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491141+0000", 2023-09-23T21:01:22.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8609999999999997e-06 2023-09-23T21:01:22.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491615+0000", 2023-09-23T21:01:22.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00047385199999999998 2023-09-23T21:01:22.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491641+0000", 2023-09-23T21:01:22.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5602999999999999e-05 2023-09-23T21:01:22.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.500272+0000", 2023-09-23T21:01:22.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0086317779999999997 2023-09-23T21:01:22.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.500310+0000", 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7543000000000003e-05 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:120366 10.e e313/257 10:7b034b5c:::benchmark_data_smithi121_358375_object35621:head v 313'7466, mlcod=313'7464)", 2023-09-23T21:01:22.293 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:00:45.780738+0000", 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 36.440920546000001, 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0096437669999999993, 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780738+0000", 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780738+0000", 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780737+0000", 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780745+0000", 2023-09-23T21:01:22.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1790000000000008e-06 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780761+0000", 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5658000000000002e-05 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780766+0000", 2023-09-23T21:01:22.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4689999999999996e-06 2023-09-23T21:01:22.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780819+0000", 2023-09-23T21:01:22.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2450000000000001e-05 2023-09-23T21:01:22.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.780845+0000", 2023-09-23T21:01:22.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5978000000000001e-05 2023-09-23T21:01:22.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.790346+0000", 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095014560000000001 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:45.790382+0000", 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5559000000000003e-05 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.299 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:120612 10.d e314/285 10:bd0ba818:::benchmark_data_smithi121_358375_object35867:head v 314'7339, mlcod=231'737)", 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:00:46.710442+0000", 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 35.511216761999997, 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010073455, 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.300 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710442+0000", 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710442+0000", 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710441+0000", 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710450+0000", 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1789999999999997e-06 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710469+0000", 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9145000000000001e-05 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710475+0000", 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1449999999999996e-06 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710529+0000", 2023-09-23T21:01:22.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4023000000000001e-05 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.710559+0000", 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0309999999999999e-05 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.720467+0000", 2023-09-23T21:01:22.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0099078159999999998 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:46.720515+0000", 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7800000000000003e-05 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:132637 10.c e317/269 10:30538f8b:::benchmark_data_smithi121_358375_object47892:head v 317'8172, mlcod=317'8170)", 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:17.151058+0000", 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 5.0706005970000003, 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0087526050000000001, 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151058+0000", 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151058+0000", 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151057+0000", 2023-09-23T21:01:22.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151066+0000", 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3640000000000005e-06 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151082+0000", 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.562e-05 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151089+0000", 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5740000000000001e-06 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151149+0000", 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0168999999999999e-05 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.151171+0000", 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2387000000000001e-05 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.159767+0000", 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085956769999999995 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.159810+0000", 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3569000000000002e-05 2023-09-23T21:01:22.311 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:132968 10.e e317/257 10:7bce2e83:::benchmark_data_smithi121_358375_object48223:head v 317'8228, mlcod=317'8226)", 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:17.907054+0000", 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 4.3146042429999998, 2023-09-23T21:01:22.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0086529099999999998, 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907054+0000", 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907054+0000", 2023-09-23T21:01:22.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907053+0000", 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907062+0000", 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6670000000000002e-06 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907078+0000", 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6079e-05 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907085+0000", 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4659999999999996e-06 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907145+0000", 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0372999999999999e-05 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.907163+0000", 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8003999999999999e-05 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.915661+0000", 2023-09-23T21:01:22.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0084977750000000008 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:17.915707+0000", 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6315e-05 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.318 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:133239 10.c e317/269 10:3ec80fa0:::benchmark_data_smithi121_358375_object48494:head v 317'8206, mlcod=317'8204)", 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:18.533183+0000", 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 3.6884755280000001, 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010508328000000001, 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533183+0000", 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533183+0000", 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533182+0000", 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533201+0000", 2023-09-23T21:01:22.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9867000000000001e-05 2023-09-23T21:01:22.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533222+0000", 2023-09-23T21:01:22.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0659e-05 2023-09-23T21:01:22.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533241+0000", 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9153000000000001e-05 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533321+0000", 2023-09-23T21:01:22.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9945999999999997e-05 2023-09-23T21:01:22.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.533343+0000", 2023-09-23T21:01:22.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2240000000000001e-05 2023-09-23T21:01:22.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.543661+0000", 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01031761 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.543691+0000", 2023-09-23T21:01:22.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0141000000000002e-05 2023-09-23T21:01:22.325 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.325 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.325 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.325 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:133257 10.c e317/269 10:36804efa:::benchmark_data_smithi121_358375_object48512:head v 317'8208, mlcod=317'8206)", 2023-09-23T21:01:22.326 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:18.565368+0000", 2023-09-23T21:01:22.326 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 3.6562907720000002, 2023-09-23T21:01:22.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011001291999999999, 2023-09-23T21:01:22.326 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.326 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.326 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565368+0000", 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565368+0000", 2023-09-23T21:01:22.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565366+0000", 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565377+0000", 2023-09-23T21:01:22.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0973e-05 2023-09-23T21:01:22.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565394+0000", 2023-09-23T21:01:22.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6827e-05 2023-09-23T21:01:22.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565401+0000", 2023-09-23T21:01:22.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0330000000000003e-06 2023-09-23T21:01:22.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565442+0000", 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0958000000000003e-05 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565464+0000", 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1591999999999999e-05 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.576327+0000", 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010863705 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.576369+0000", 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1357999999999999e-05 2023-09-23T21:01:22.332 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:133258 10.c e317/269 10:3a119372:::benchmark_data_smithi121_358375_object48513:head v 317'8209, mlcod=317'8206)", 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:18.565594+0000", 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 3.656064497, 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010796418, 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565594+0000", 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565594+0000", 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565593+0000", 2023-09-23T21:01:22.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565599+0000", 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4879999999999998e-06 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565608+0000", 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1209999999999996e-06 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.565613+0000", 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0390000000000004e-06 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569250+0000", 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036373320000000001 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569282+0000", 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1461999999999998e-05 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.576372+0000", 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070901319999999999 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.576390+0000", 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8354000000000001e-05 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.338 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.339 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.339 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:133263 10.c e317/269 10:30fad199:::benchmark_data_smithi121_358375_object48518:head v 317'8210, mlcod=317'8206)", 2023-09-23T21:01:22.339 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:18.569167+0000", 2023-09-23T21:01:22.339 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 3.6524917280000002, 2023-09-23T21:01:22.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012215405, 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569167+0000", 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569167+0000", 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569166+0000", 2023-09-23T21:01:22.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569175+0000", 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1079999999999999e-06 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569187+0000", 2023-09-23T21:01:22.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1987999999999999e-05 2023-09-23T21:01:22.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.569193+0000", 2023-09-23T21:01:22.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2999999999999998e-06 2023-09-23T21:01:22.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.572894+0000", 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037008689999999999 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.572916+0000", 2023-09-23T21:01:22.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2252999999999999e-05 2023-09-23T21:01:22.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.581339+0000", 2023-09-23T21:01:22.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0084229130000000006 2023-09-23T21:01:22.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:18.581382+0000", 2023-09-23T21:01:22.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2809999999999998e-05 2023-09-23T21:01:22.346 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.346 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.347 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.347 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:133593 10.c e317/269 10:330c684a:::benchmark_data_smithi121_358375_object48848:head v 317'8239, mlcod=317'8236)", 2023-09-23T21:01:22.347 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:19.237405+0000", 2023-09-23T21:01:22.347 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.9842530040000002, 2023-09-23T21:01:22.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011554283, 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.237405+0000", 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.237405+0000", 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.237405+0000", 2023-09-23T21:01:22.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:22.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.237413+0000", 2023-09-23T21:01:22.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4840000000000004e-06 2023-09-23T21:01:22.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.242292+0000", 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048793550000000002 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.242301+0000", 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5280000000000003e-06 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.242349+0000", 2023-09-23T21:01:22.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7722000000000002e-05 2023-09-23T21:01:22.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.242380+0000", 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1269999999999997e-05 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.248907+0000", 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065266109999999999 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.248960+0000", 2023-09-23T21:01:22.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2986999999999999e-05 2023-09-23T21:01:22.354 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.354 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.354 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:133848 10.14 e318/303 10:2b97c9e4:::benchmark_data_smithi121_358375_object49103:head v 318'7598, mlcod=234'3073)", 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:19.980722+0000", 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.2409365139999999, 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092685289999999993, 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.980722+0000", 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.980722+0000", 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.980722+0000", 2023-09-23T21:01:22.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:01:22.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.980727+0000", 2023-09-23T21:01:22.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.417e-06 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.980737+0000", 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0064999999999999e-05 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.980743+0000", 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2430000000000002e-06 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.984226+0000", 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034831609999999998 2023-09-23T21:01:22.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.360 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.984263+0000", 2023-09-23T21:01:22.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6443000000000003e-05 2023-09-23T21:01:22.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.989937+0000", 2023-09-23T21:01:22.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056743339999999996 2023-09-23T21:01:22.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:19.989990+0000", 2023-09-23T21:01:22.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3075999999999998e-05 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:134707 10.e e318/257 10:71969be5:::benchmark_data_smithi121_358375_object49962:head v 318'8330, mlcod=318'8328)", 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:22.055335+0000", 2023-09-23T21:01:22.362 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.16632297400000001, 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010529116999999999, 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055335+0000", 2023-09-23T21:01:22.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055335+0000", 2023-09-23T21:01:22.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055335+0000", 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055343+0000", 2023-09-23T21:01:22.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7749999999999998e-06 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055358+0000", 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4421e-05 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055364+0000", 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5289999999999997e-06 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055411+0000", 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6417e-05 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055433+0000", 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2291000000000001e-05 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.065836+0000", 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010403145000000001 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.065864+0000", 2023-09-23T21:01:22.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8388999999999999e-05 2023-09-23T21:01:22.369 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.369 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.369 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.369 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:134708 10.c e318/269 10:3bbd4642:::benchmark_data_smithi121_358375_object49963:head v 318'8311, mlcod=318'8309)", 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:22.055436+0000", 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.166222389, 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0087439479999999997, 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055436+0000", 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055436+0000", 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:22.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055435+0000", 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055441+0000", 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9009999999999999e-06 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055451+0000", 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2289999999999992e-06 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055455+0000", 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7450000000000003e-06 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055495+0000", 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9205000000000002e-05 2023-09-23T21:01:22.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.055512+0000", 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7458000000000001e-05 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.064136+0000", 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0086244589999999993 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:22.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:22.064180+0000", 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3429e-05 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:22.375 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:01:22.376 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 37130/108372 objects degraded (34.262%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:22.376 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/494564302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:22.376 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:22.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:22.377 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:22.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:22.378 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:01:22.478 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:22.512 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:22.562 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:22.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:22.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:22.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:22.696 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:22.741 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:22.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:22.845 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:22.886 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:23.142 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:23.142 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[211526]: failed to encode map e319 with expected crc 2023-09-23T21:01:23.142 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:23.142 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[211526]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T21:01:23.142 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[211526]: pgmap v1014: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 37130/108042 objects degraded (34.366%); 1606/108042 objects misplaced (1.486%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:01:23.143 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:23.143 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[219709]: failed to encode map e319 with expected crc 2023-09-23T21:01:23.143 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:23.144 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[219709]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T21:01:23.144 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:22 smithi121 ceph-mon[219709]: pgmap v1014: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 37130/108042 objects degraded (34.366%); 1606/108042 objects misplaced (1.486%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:01:23.151 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:01:23.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:23.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:23.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:23.288 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:23.337 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:22 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:22 smithi123 ceph-mon[193903]: failed to encode map e319 with expected crc 2023-09-23T21:01:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:22 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:22 smithi123 ceph-mon[193903]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T21:01:23.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:22 smithi123 ceph-mon[193903]: pgmap v1014: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 37130/108042 objects degraded (34.366%); 1606/108042 objects misplaced (1.486%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:01:23.554 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:24.139 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:23 smithi123 ceph-mon[193903]: failed to encode map e319 with expected crc 2023-09-23T21:01:24.254 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:23 smithi121 ceph-mon[211526]: failed to encode map e319 with expected crc 2023-09-23T21:01:24.256 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:23 smithi121 ceph-mon[219709]: failed to encode map e319 with expected crc 2023-09-23T21:01:24.694 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:24.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:24.805 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:24.850 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:24.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:01:24] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.43.0" 2023-09-23T21:01:24.942 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:24.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:24.981 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:01:25.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:01:25.046 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:25.047 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:01:25.080 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:01:25.103 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.113 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:01:25.153 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:01:25.188 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:01:25.193 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:25.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:25.223 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:01:25.254 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:24 smithi121 ceph-mon[211526]: pgmap v1015: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 36663/102360 objects degraded (35.818%); 1537/102360 objects misplaced (1.502%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:01:25.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2365662018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:25.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:24 smithi121 ceph-mon[219709]: pgmap v1015: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 36663/102360 objects degraded (35.818%); 1537/102360 objects misplaced (1.502%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:01:25.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2365662018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:25.363 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:25.364 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:25.364 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:25.364 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:24 smithi123 ceph-mon[193903]: pgmap v1015: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 36663/102360 objects degraded (35.818%); 1537/102360 objects misplaced (1.502%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:01:25.365 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2365662018' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:25.401 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:moved 4 / 297 (1.3468%) 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 12.5144 -> 13.3458 (expected baseline 5.69951) 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.5 with 10 -> 10 pgs (0.26936 -> 0.26936 * mean) 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 52 -> 56 pgs (1.40067 -> 1.50842 * mean) 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:01:25.535 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:25.536 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0144 2023-09-23T21:01:25.536 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0158 2023-09-23T21:01:25.536 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.5554 -> 0.5054 2023-09-23T21:01:25.536 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.6890 -> 0.6390 2023-09-23T21:01:25.536 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.2859 -> 0.2524 2023-09-23T21:01:25.536 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8500 -> 0.8620 2023-09-23T21:01:25.537 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:01:25.558 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:25.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:25.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:25.663 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:25.690 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.736 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:25.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:25.843 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:25.874 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:25.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:25 smithi121 conmon[198376]: 2023-09-23T21:01:25.534+0000 7f479dcb1700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T21:01:26.116 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:01:26.117 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:26.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:26.124 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:25 smithi123 ceph-mon[193903]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:26.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:26.255 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:26.301 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:26.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:26.407 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:26.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:25 smithi121 ceph-mon[211526]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:26.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:25 smithi121 ceph-mon[219709]: from='client.46028 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:26.441 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:26.493 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:26.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:26.521 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:26.600 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:26.650 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:26.716 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:01:26.821 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:26.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:26.938 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:27.008 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:27 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 36663/102360 objects degraded (35.818%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:27.008 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:27 smithi123 ceph-mon[193903]: pgmap v1016: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 36663/102360 objects degraded (35.818%); 1537/102360 objects misplaced (1.502%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T21:01:27.020 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:27.105 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:27.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:27.109 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:27 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 36663/102360 objects degraded (35.818%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:27.109 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:27 smithi121 ceph-mon[211526]: pgmap v1016: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 36663/102360 objects degraded (35.818%); 1537/102360 objects misplaced (1.502%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T21:01:27.109 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:27 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 36663/102360 objects degraded (35.818%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:27.109 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:27 smithi121 ceph-mon[219709]: pgmap v1016: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 36663/102360 objects degraded (35.818%); 1537/102360 objects misplaced (1.502%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T21:01:27.210 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:27.243 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:27.289 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:27.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:27.308 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:27.393 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:27.421 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:27.740 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:27.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:27.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:27.847 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:27.888 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.058 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:28.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:28.163 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:28.192 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.235 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:28.257 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:28.257 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:28.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:28.338 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:28.372 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.481 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:28.504 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:01:28.541 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:28.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:28.585 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:28.632 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:28.686 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:28.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:28.801 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:28.841 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.079 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:01:29.085 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:29.191 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:29.247 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:29.268 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.373 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:29.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:29.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:29 smithi123 ceph-mon[193903]: pgmap v1017: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 36431/98118 objects degraded (37.130%); 1537/98118 objects misplaced (1.566%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:01:29.486 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:29.521 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.534 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:29.571 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:29 smithi121 ceph-mon[211526]: pgmap v1017: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 36431/98118 objects degraded (37.130%); 1537/98118 objects misplaced (1.566%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:01:29.571 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:29 smithi121 ceph-mon[219709]: pgmap v1017: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 36431/98118 objects degraded (37.130%); 1537/98118 objects misplaced (1.566%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:01:29.587 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:29.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:29.692 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:29.748 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:29.885 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:29.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:29.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:29.998 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:30.035 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.198 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:30.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:30.304 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:30.334 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.353 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:01:30.391 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:30.495 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:30.552 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1179138464' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:30 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1179138464' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:30 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1179138464' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:30 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:30.881 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:30.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:30.991 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:31.031 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:31.121 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:31.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:31.230 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:31.273 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:31.340 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:01:31.340 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.340 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:31.341 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T21:01:31.341 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.341 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.341 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.341 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: pgmap v1019: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 36211/96072 objects degraded (37.692%); 1476/96072 objects misplaced (1.536%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:01:31.341 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 36431/98118 objects degraded (37.130%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:31.343 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:31.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:31.375 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:31.375 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:31.447 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:31.493 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:31.609 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:31.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:31.617 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: pgmap v1019: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 36211/96072 objects degraded (37.692%); 1476/96072 objects misplaced (1.536%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 36431/98118 objects degraded (37.130%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:01:31.618 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.619 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:31.619 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T21:01:31.619 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.619 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.619 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: failed to encode map e320 with expected crc 2023-09-23T21:01:31.619 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: pgmap v1019: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 36211/96072 objects degraded (37.692%); 1476/96072 objects misplaced (1.536%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:01:31.619 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 36431/98118 objects degraded (37.130%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:31.714 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:31.743 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:31.842 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:31.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:31.852 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:01:31.852 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:01:31.852 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:01:31.852 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T21:01:31.895 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:01:31.916 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:01:31.946 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:31.958 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:01:31.984 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:32.016 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:01:32.050 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:01:32.078 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:01:32.116 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:01:32.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:01:32.224 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:32.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:32.275 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:01:32.327 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:32.395 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3047755426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:32.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3047755426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3047755426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:32.749 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:32.969 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:32.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:33.076 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:33.113 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:33.182 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:33.321 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:33.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:33.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:33.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:33 smithi121 ceph-mon[211526]: pgmap v1020: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 36211/95586 objects degraded (37.883%); 1476/95586 objects misplaced (1.544%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T21:01:33.425 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:33.463 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:33.518 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:01:33.519 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:33.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:33.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:33.622 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:33.630 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:33 smithi121 ceph-mon[219709]: pgmap v1020: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 36211/95586 objects degraded (37.883%); 1476/95586 objects misplaced (1.544%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T21:01:33.654 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:33 smithi123 ceph-mon[193903]: pgmap v1020: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 36211/95586 objects degraded (37.883%); 1476/95586 objects misplaced (1.544%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T21:01:33.655 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:33.729 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:33.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:33.836 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:33.866 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.013 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:34.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:34.118 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:34.147 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.466 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140295 10.0 10:00a6c52e:::benchmark_data_smithi121_358375_object55550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:34.546111+0000", 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.006122504, 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.006160085, 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:34.553 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "delayed", 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140295 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546111+0000", 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546111+0000", 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546110+0000", 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:34.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:34.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546116+0000", 2023-09-23T21:01:34.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0399999999999998e-06 2023-09-23T21:01:34.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546126+0000", 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0049e-05 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546131+0000", 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6779999999999997e-06 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:34.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546167+0000", 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5738000000000002e-05 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.550741+0000", 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045742869999999998 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.558 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140296 10.d 10:be9626f6:::benchmark_data_smithi121_358375_object55551:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:34.546489+0000", 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0057446160000000001, 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058650359999999997, 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140296 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:34.559 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546489+0000", 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546489+0000", 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:34.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546488+0000", 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546492+0000", 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4880000000000001e-06 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546502+0000", 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6469999999999993e-06 2023-09-23T21:01:34.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546507+0000", 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0329999999999999e-06 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.549057+0000", 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025496220000000001 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.549214+0000", 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015695699999999999 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.549389+0000", 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017571000000000001 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140297 10.d 10.4006469d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:34.546516+0000", 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0057170449999999996, 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059109410000000003, 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:34.564 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140297 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546516+0000", 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:34.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546516+0000", 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546516+0000", 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546518+0000", 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2299999999999998e-06 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546523+0000", 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8709999999999996e-06 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546526+0000", 2023-09-23T21:01:34.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2940000000000001e-06 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140300 10.d 10.b9ed7cd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:34.546709+0000", 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0055238199999999996, 2023-09-23T21:01:34.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0057728780000000004, 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140300 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:34.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546709+0000", 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546709+0000", 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.570 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546709+0000", 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546713+0000", 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2250000000000002e-06 2023-09-23T21:01:34.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546720+0000", 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9639999999999997e-06 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.546724+0000", 2023-09-23T21:01:34.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1779999999999996e-06 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140304 10.17 10:ec7eee30:::benchmark_data_smithi121_358375_object55559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:34.552243+0000", 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 4294967295.9999905, 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00030016899999999998, 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:01:34.573 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140304 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552243+0000", 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:34.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552243+0000", 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552241+0000", 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552247+0000", 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9490000000000001e-06 2023-09-23T21:01:34.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552269+0000", 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1522000000000002e-05 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552276+0000", 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.72e-06 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:34.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552315+0000", 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9279000000000001e-05 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:34.552459+0000", 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014463799999999999 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 5 2023-09-23T21:01:34.577 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:01:34.578 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:34.614 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.703 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:34.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:01:34] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-09-23T21:01:34.785 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:34.889 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:34.916 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:34.960 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:34.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:34.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:35.063 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:35.074 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:01:35.143 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.284 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:35.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:35.397 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:35.427 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:01:35.429 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.481 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:35.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:35.591 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:35.621 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.693 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:35.799 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:35.827 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:35.872 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:35.880 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:35.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:35.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:35.974 INFO:teuthology.orchestra.run.smithi121.stdout:moved 26 / 297 (8.75421%) 2023-09-23T21:01:35.974 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 12.5144 -> 17.496 (expected baseline 5.69951) 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.5 with 10 -> 10 pgs (0.26936 -> 0.26936 * mean) 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 52 -> 59 pgs (1.40067 -> 1.58923 * mean) 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0135 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0148 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.5554 -> 0.3050 2023-09-23T21:01:35.975 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.6890 -> 0.5329 2023-09-23T21:01:35.976 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.2859 -> 0.2417 2023-09-23T21:01:35.976 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8500 -> 0.8790 2023-09-23T21:01:35.985 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:36.011 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:36.105 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:36.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:36.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:36.209 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:36.237 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:35 smithi121 ceph-mon[211526]: pgmap v1021: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 35632/88524 objects degraded (40.251%); 1414/88524 objects misplaced (1.597%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:01:36.237 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:35 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]: dispatch 2023-09-23T21:01:36.238 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:35 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]: dispatch 2023-09-23T21:01:36.238 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:35 smithi121 ceph-mon[219709]: pgmap v1021: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 35632/88524 objects degraded (40.251%); 1414/88524 objects misplaced (1.597%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:01:36.238 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:35 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]: dispatch 2023-09-23T21:01:36.238 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:35 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]: dispatch 2023-09-23T21:01:36.241 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:36.263 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:35 smithi123 ceph-mon[193903]: pgmap v1021: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 35632/88524 objects degraded (40.251%); 1414/88524 objects misplaced (1.597%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:01:36.263 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:35 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]: dispatch 2023-09-23T21:01:36.264 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:35 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]: dispatch 2023-09-23T21:01:37.125 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:01:37.126 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T21:01:37.268 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:37.306 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:37.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:37.310 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: from='client.56101 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]': finished 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 35632/88524 objects degraded (40.251%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: failed to encode map e322 with expected crc 2023-09-23T21:01:37.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[211526]: pgmap v1024: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 35632/88524 objects degraded (40.251%); 1414/88524 objects misplaced (1.597%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: from='client.56101 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]': finished 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 35632/88524 objects degraded (40.251%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:37.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: failed to encode map e322 with expected crc 2023-09-23T21:01:37.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T21:01:37.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:36 smithi121 ceph-mon[219709]: pgmap v1024: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 35632/88524 objects degraded (40.251%); 1414/88524 objects misplaced (1.597%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T21:01:37.360 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: from='client.56101 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:37.360 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.360 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'19990','4':'34925','5':'15841','6':'57607'}"}]': finished 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: failed to encode map e321 with expected crc 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 35632/88524 objects degraded (40.251%), 14 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: failed to encode map e322 with expected crc 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T21:01:37.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:36 smithi123 ceph-mon[193903]: pgmap v1024: 99 pgs: 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 35632/88524 objects degraded (40.251%); 1414/88524 objects misplaced (1.597%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T21:01:37.412 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:37.529 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:37.633 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:01:37.634 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:01:37.634 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:01:37.634 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:01:37.634 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:01:37.790 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:37.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:37.897 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:37.972 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:38.143 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[211526]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[211526]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[211526]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[211526]: failed to encode map e323 with expected crc 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[211526]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[219709]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[219709]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[219709]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[219709]: failed to encode map e323 with expected crc 2023-09-23T21:01:38.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:37 smithi121 ceph-mon[219709]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T21:01:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:38.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:38.248 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:38.274 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:37 smithi123 ceph-mon[193903]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.274 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:37 smithi123 ceph-mon[193903]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.274 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:37 smithi123 ceph-mon[193903]: failed to encode map e322 with expected crc 2023-09-23T21:01:38.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:37 smithi123 ceph-mon[193903]: failed to encode map e323 with expected crc 2023-09-23T21:01:38.275 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:37 smithi123 ceph-mon[193903]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T21:01:38.278 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:38.323 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:38.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:38.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:38.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:38.459 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:38.608 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:01:38.660 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:38.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:38.763 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:38.788 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:38.838 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:38.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:38.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:38.944 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:38.957 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:38.970 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:38.996 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:01:39.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:38 smithi123 ceph-mon[193903]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:38 smithi123 ceph-mon[193903]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:38 smithi123 ceph-mon[193903]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:38 smithi123 ceph-mon[193903]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 320'8705 2023-09-23T21:01:39.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:38 smithi123 ceph-mon[193903]: 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 320'8855 2023-09-23T21:01:39.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/206983457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:39.379 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:38 smithi123 ceph-mon[193903]: pgmap v1026: 99 pgs: 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 32653/86127 objects degraded (37.913%); 5876/86127 objects misplaced (6.822%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:01:39.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[211526]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[211526]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[211526]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[211526]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 320'8705 2023-09-23T21:01:39.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[211526]: 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 320'8855 2023-09-23T21:01:39.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/206983457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[211526]: pgmap v1026: 99 pgs: 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 32653/86127 objects degraded (37.913%); 5876/86127 objects misplaced (6.822%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[219709]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[219709]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[219709]: failed to encode map e323 with expected crc 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[219709]: 10.c starting backfill to osd.3 from (0'0,0'0] MAX to 320'8705 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[219709]: 10.b starting backfill to osd.7 from (0'0,0'0] MAX to 320'8855 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/206983457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:01:39.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:38 smithi121 ceph-mon[219709]: pgmap v1026: 99 pgs: 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 32653/86127 objects degraded (37.913%); 5876/86127 objects misplaced (6.822%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:01:39.405 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:39.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:39.515 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:39.601 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:39.671 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:39.957 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:39.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:40.066 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:40.111 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:40.180 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:40.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:40.226 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:40.226 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[219709]: failed to encode map e324 with expected crc 2023-09-23T21:01:40.226 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/206983457' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:40.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[219709]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T21:01:40.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[219709]: failed to encode map e324 with expected crc 2023-09-23T21:01:40.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:40.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[211526]: failed to encode map e324 with expected crc 2023-09-23T21:01:40.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/206983457' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:40.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[211526]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T21:01:40.227 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:39 smithi121 ceph-mon[211526]: failed to encode map e324 with expected crc 2023-09-23T21:01:40.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:40.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:39 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:40.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:39 smithi123 ceph-mon[193903]: failed to encode map e324 with expected crc 2023-09-23T21:01:40.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/206983457' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:01:40.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:39 smithi123 ceph-mon[193903]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T21:01:40.320 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:39 smithi123 ceph-mon[193903]: failed to encode map e324 with expected crc 2023-09-23T21:01:40.327 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:40.438 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:40.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:40.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:40.617 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:01:40.650 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:40.660 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136312 9.0 9:087a316d:::benchmark_data_smithi121_293770_object33424:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.228796+0000", 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 550.43935486299995, 2023-09-23T21:01:40.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.026362005, 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136312 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T21:01:40.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228796+0000", 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228794+0000", 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228801+0000", 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6000000000000003e-06 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228812+0000", 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1517e-05 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.228821+0000", 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.089e-06 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230287+0000", 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014660960000000001 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230299+0000", 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2660000000000001e-05 2023-09-23T21:01:40.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255083+0000", 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247833159999999 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255158+0000", 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5184000000000004e-05 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136315 9.0 9:07b4e2b9:::benchmark_data_smithi121_293770_object33427:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.230807+0000", 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 550.437343887, 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0248562560000001, 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136315 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.677 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230807+0000", 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230806+0000", 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230811+0000", 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.879e-06 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230823+0000", 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.151e-05 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230830+0000", 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8460000000000002e-06 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230876+0000", 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6560999999999997e-05 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.230884+0000", 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8960000000000003e-06 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255603+0000", 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0247184060000001 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255663+0000", 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0364e-05 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136335 9.0 9:00bf7afd:::benchmark_data_smithi121_293770_object33447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.256436+0000", 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 550.41171441799997, 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0152258009999999, 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.682 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136335 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256436+0000", 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256434+0000", 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256453+0000", 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8349999999999999e-05 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256484+0000", 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0908999999999998e-05 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256494+0000", 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0479000000000001e-05 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256538+0000", 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3940999999999998e-05 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.256547+0000", 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6689999999999995e-06 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255731+0000", 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99918387799999997 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255863+0000", 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000131917 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2", 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.255979+0000", 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011618899999999999 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.267419+0000", 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011440723999999999 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271524+0000", 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041050169999999999 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271584+0000", 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9577999999999998e-05 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.271662+0000", 2023-09-23T21:01:40.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7960999999999994e-05 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90329 10.f 10:f35e32c5:::benchmark_data_smithi121_358375_object5584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.105215+0000", 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 267.56293609800002, 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80166978300000002, 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.690 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90329 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105215+0000", 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.691 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105215+0000", 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105214+0000", 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105217+0000", 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3589999999999999e-06 2023-09-23T21:01:40.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105228+0000", 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0261999999999999e-05 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105234+0000", 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2149999999999998e-06 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105290+0000", 2023-09-23T21:01:40.694 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5810000000000003e-05 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105298+0000", 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0020000000000006e-06 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.894981+0000", 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.789683778 2023-09-23T21:01:40.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.895080+0000", 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9035000000000003e-05 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.895162+0000", 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1553999999999999e-05 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900361+0000", 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005198846 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900369+0000", 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.388e-06 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906827+0000", 2023-09-23T21:01:40.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064581009999999999 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906859+0000", 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2100000000000001e-05 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906884+0000", 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4882e-05 2023-09-23T21:01:40.698 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90332 10.f 10:f123f416:::benchmark_data_smithi121_358375_object5587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.300759+0000", 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 267.367391215, 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.60945262700000002, 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.699 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90332 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300759+0000", 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300759+0000", 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300758+0000", 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300766+0000", 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9340000000000006e-06 2023-09-23T21:01:40.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300782+0000", 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.632e-05 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300791+0000", 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8529999999999998e-06 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300838+0000", 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6925000000000002e-05 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.300849+0000", 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1008e-05 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.897583+0000", 2023-09-23T21:01:40.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.59673357400000004 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.897653+0000", 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0284999999999995e-05 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.897718+0000", 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5127000000000002e-05 2023-09-23T21:01:40.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906113+0000", 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.008394594 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906163+0000", 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0469000000000001e-05 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910127+0000", 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039636309999999996 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910174+0000", 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6918e-05 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910212+0000", 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8473999999999998e-05 2023-09-23T21:01:40.706 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90333 10.f 10:f4af6242:::benchmark_data_smithi121_358375_object5588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.396755+0000", 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 267.27139585200001, 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.51361582500000003, 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.707 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90333 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396755+0000", 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.708 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396755+0000", 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396746+0000", 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999909 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396767+0000", 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1305999999999999e-05 2023-09-23T21:01:40.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396803+0000", 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6508999999999999e-05 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396817+0000", 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.33e-05 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396927+0000", 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110268 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.396953+0000", 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6139000000000002e-05 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.898950+0000", 2023-09-23T21:01:40.711 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.50199691899999999 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.899041+0000", 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.077e-05 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.899104+0000", 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3583000000000004e-05 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.712 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906117+0000", 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070128350000000002 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910092+0000", 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039743599999999997 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910283+0000", 2023-09-23T21:01:40.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000191053 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910330+0000", 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7914999999999998e-05 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910371+0000", 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0157000000000002e-05 2023-09-23T21:01:40.714 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90334 10.f 10:f59733bd:::benchmark_data_smithi121_358375_object5589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.505405+0000", 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 267.16274565499998, 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.407500266, 2023-09-23T21:01:40.715 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90334 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505405+0000", 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.716 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505405+0000", 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505404+0000", 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505414+0000", 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0674e-05 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505432+0000", 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7810000000000001e-05 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505439+0000", 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.708e-06 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505498+0000", 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9385999999999997e-05 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.505509+0000", 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1235e-05 2023-09-23T21:01:40.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900477+0000", 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.39496745700000002 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900562+0000", 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5111999999999994e-05 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900624+0000", 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2447999999999996e-05 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906121+0000", 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0054971009999999999 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.910246+0000", 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004124104 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.912851+0000", 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026054440000000002 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.912878+0000", 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6570000000000001e-05 2023-09-23T21:01:40.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.912905+0000", 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7699e-05 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97176 10.d 10:bf52c426:::benchmark_data_smithi121_358375_object12431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 185.168177783, 2023-09-23T21:01:40.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0090121330000001, 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97176 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.724 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499972+0000", 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:40.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499975+0000", 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2090000000000001e-06 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499984+0000", 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.422e-06 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499988+0000", 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8739999999999997e-06 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500030+0000", 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2366000000000001e-05 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500039+0000", 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5569999999999996e-06 2023-09-23T21:01:40.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242363+0000", 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.74232412699999994 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242370+0000", 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.041e-06 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482610+0000", 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2402398800000001 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482781+0000", 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000170884 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482955+0000", 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017400099999999999 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490821+0000", 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078662190000000007 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.493967+0000", 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031467309999999998 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506247+0000", 2023-09-23T21:01:40.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012279684000000001 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506289+0000", 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2329000000000001e-05 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.508985+0000", 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026955099999999999 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.731 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97184 10.a 10:54b30599:::benchmark_data_smithi121_358375_object12439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 185.08083356200001, 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.974572057, 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.732 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97184 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587316+0000", 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587321+0000", 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3270000000000002e-06 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587333+0000", 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2383e-05 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587338+0000", 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9189999999999998e-06 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587383+0000", 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5014e-05 2023-09-23T21:01:40.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587389+0000", 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9660000000000001e-06 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242357+0000", 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.65496830699999997 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242367+0000", 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0266e-05 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538285+0000", 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2959178060000001 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538406+0000", 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120308 2023-09-23T21:01:40.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538531+0000", 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000125761 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.549226+0000", 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010694789999999999 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551472+0000", 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002246315 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559794+0000", 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083212719999999993 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559841+0000", 2023-09-23T21:01:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7197999999999998e-05 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561889+0000", 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020482159999999998 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97187 10.0 10:057fb271:::benchmark_data_smithi121_358375_object12442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.740 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 185.05730390599999, 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8667573179999999, 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97187 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.741 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610846+0000", 2023-09-23T21:01:40.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610850+0000", 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2509999999999996e-06 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610863+0000", 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2995e-05 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.743 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610869+0000", 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1480000000000003e-06 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610945+0000", 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5284000000000006e-05 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.744 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610953+0000", 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3750000000000003e-06 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242784+0000", 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63183069599999997 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242792+0000", 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6510000000000007e-06 2023-09-23T21:01:40.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.434568+0000", 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1917759189999999 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.435371+0000", 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000802339 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.746 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451218+0000", 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015847757000000001 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451292+0000", 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3386000000000005e-05 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451393+0000", 2023-09-23T21:01:40.747 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101216 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462865+0000", 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011472379 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.466387+0000", 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035211050000000001 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.474961+0000", 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085745549999999993 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.475028+0000", 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7105000000000002e-05 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.749 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477604+0000", 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025758389999999999 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97197 10.d 10:b1ae4ef2:::benchmark_data_smithi121_358375_object12452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.769347+0000", 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 184.898803472, 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7457691610000001, 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.750 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97197 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769347+0000", 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.751 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769347+0000", 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769346+0000", 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769352+0000", 2023-09-23T21:01:40.752 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4269999999999999e-06 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769364+0000", 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2561999999999999e-05 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769371+0000", 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4520000000000002e-06 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.753 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769417+0000", 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6017999999999999e-05 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769425+0000", 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6619999999999994e-06 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242398+0000", 2023-09-23T21:01:40.754 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.47297264900000002 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242404+0000", 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5640000000000002e-06 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488192+0000", 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.245787612 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488310+0000", 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118377 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488416+0000", 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000105685 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.493980+0000", 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055637339999999999 2023-09-23T21:01:40.756 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.497354+0000", 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033737319999999999 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.512976+0000", 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015622575 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.757 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.513019+0000", 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2478999999999998e-05 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515116+0000", 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020976649999999999 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.758 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97205 10.0 10:0f792b11:::benchmark_data_smithi121_358375_object12460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.899978+0000", 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 184.76817292999999, 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5793486960000001, 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97205 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.759 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899978+0000", 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899978+0000", 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899977+0000", 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:40.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899981+0000", 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5700000000000003e-06 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899993+0000", 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1613e-05 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899999+0000", 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5969999999999997e-06 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.900045+0000", 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5772000000000002e-05 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.900053+0000", 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5580000000000003e-06 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242824+0000", 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34277136600000002 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242832+0000", 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4569999999999999e-06 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.436698+0000", 2023-09-23T21:01:40.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1938665129999999 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.437505+0000", 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00080636499999999997 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.456977+0000", 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019472245999999999 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.764 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.457059+0000", 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2544000000000001e-05 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.457160+0000", 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100435 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.466405+0000", 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092448630000000007 2023-09-23T21:01:40.765 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477677+0000", 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011272453999999999 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477732+0000", 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4910000000000001e-05 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.766 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477796+0000", 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4053000000000005e-05 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479326+0000", 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015303929999999999 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97215 10.d 10:b031210c:::benchmark_data_smithi121_358375_object12470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.767 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.106818+0000", 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 184.56133233400001, 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4100548179999999, 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97215 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.768 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106818+0000", 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106818+0000", 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106816+0000", 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106830+0000", 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4032000000000001e-05 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106869+0000", 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8349999999999997e-05 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106883+0000", 2023-09-23T21:01:40.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4243000000000001e-05 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106982+0000", 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8944000000000007e-05 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.107003+0000", 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1302999999999999e-05 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242425+0000", 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.13542152199999999 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242431+0000", 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1639999999999998e-06 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490848+0000", 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2484173919999999 2023-09-23T21:01:40.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490954+0000", 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010638500000000001 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491045+0000", 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0453999999999998e-05 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.773 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.497302+0000", 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062572529999999999 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.500461+0000", 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031586980000000001 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515185+0000", 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014723630999999999 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.774 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515224+0000", 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9691999999999999e-05 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.516873+0000", 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001648928 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.775 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97221 10.0 10:0777f148:::benchmark_data_smithi121_358375_object12476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.130321+0000", 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 184.537829636, 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3504028450000001, 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97221 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.776 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130321+0000", 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130321+0000", 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130320+0000", 2023-09-23T21:01:40.777 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130327+0000", 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.328e-06 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130344+0000", 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6699999999999999e-05 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.778 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130352+0000", 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1990000000000006e-06 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130408+0000", 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6060999999999997e-05 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130418+0000", 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0169999999999999e-05 2023-09-23T21:01:40.779 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242919+0000", 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.112500957 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242927+0000", 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7819999999999994e-06 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.437568+0000", 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.194641184 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.438303+0000", 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00073454399999999995 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462904+0000", 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024601067000000001 2023-09-23T21:01:40.781 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462984+0000", 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9919000000000005e-05 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.463135+0000", 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000151361 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.782 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479342+0000", 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016207200000000001 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479403+0000", 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0417000000000001e-05 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479456+0000", 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3798999999999998e-05 2023-09-23T21:01:40.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479512+0000", 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5714000000000002e-05 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.480724+0000", 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001211684 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.784 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97225 10.a 10:52fbb22c:::benchmark_data_smithi121_358375_object12480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.225436+0000", 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 184.442714822, 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3382990889999999, 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97225 2023-09-23T21:01:40.785 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225436+0000", 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225436+0000", 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225433+0000", 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225446+0000", 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2703e-05 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225471+0000", 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5225e-05 2023-09-23T21:01:40.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225481+0000", 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6050000000000004e-06 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225544+0000", 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2649000000000003e-05 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.788 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225553+0000", 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1549999999999996e-06 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242391+0000", 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016838199000000002 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242398+0000", 2023-09-23T21:01:40.789 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8959999999999997e-06 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541687+0000", 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.299289683 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541850+0000", 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016283399999999999 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.790 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541997+0000", 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014649800000000001 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.549235+0000", 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0072378959999999997 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551516+0000", 2023-09-23T21:01:40.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022813149999999999 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561929+0000", 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010412988999999999 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561972+0000", 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2719000000000002e-05 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.563735+0000", 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001763202 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97254 10.a 10:56bae3f9:::benchmark_data_smithi121_358375_object12509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.794906+0000", 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 183.87324484600001, 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77559789899999998, 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.793 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97254 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794906+0000", 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.794 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794906+0000", 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794905+0000", 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794911+0000", 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5320000000000003e-06 2023-09-23T21:01:40.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794926+0000", 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.447e-05 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794931+0000", 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.451e-06 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.795017+0000", 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5575000000000007e-05 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.795026+0000", 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5729999999999997e-06 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.544759+0000", 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.74973301299999995 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.797 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.544898+0000", 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000138985 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.545008+0000", 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000109745 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.798 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551524+0000", 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065155339999999999 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551553+0000", 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9068999999999999e-05 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.568290+0000", 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016737577 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.568339+0000", 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9119000000000002e-05 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570504+0000", 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002164329 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97257 10.a 10:548053b5:::benchmark_data_smithi121_358375_object12512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 183.74787433, 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.65229715300000002, 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97257 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.801 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.802 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920279+0000", 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8709999999999999e-06 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920289+0000", 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4919999999999999e-06 2023-09-23T21:01:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920293+0000", 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1359999999999999e-06 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920345+0000", 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2012999999999998e-05 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.804 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920352+0000", 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7309999999999996e-06 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547185+0000", 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.626832944 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547290+0000", 2023-09-23T21:01:40.805 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010505600000000001 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547383+0000", 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2948000000000005e-05 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551592+0000", 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042095969999999998 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.806 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551600+0000", 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3139999999999998e-06 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570562+0000", 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018962326000000002 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570625+0000", 2023-09-23T21:01:40.807 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3427000000000001e-05 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.572573+0000", 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019480890000000001 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140911 10.b 10:dec2be02:::benchmark_data_smithi121_358375_object56166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:01:40.808 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.147545+0000", 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 4.5206060990000001, 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4336918569999999, 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140911 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.809 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147545+0000", 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147545+0000", 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147543+0000", 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:40.810 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147551+0000", 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6820000000000003e-06 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147569+0000", 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8785999999999999e-05 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.811 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147576+0000", 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.567e-06 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147629+0000", 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3174000000000003e-05 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147640+0000", 2023-09-23T21:01:40.812 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0345e-05 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326914+0000", 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17927400499999999 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326925+0000", 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1165999999999999e-05 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.813 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.523743+0000", 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.196818079 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.524436+0000", 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00069355399999999998 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.560304+0000", 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.035867555000000002 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.560413+0000", 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010935200000000001 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.561076+0000", 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00066255899999999996 2023-09-23T21:01:40.815 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576376+0000", 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015300591000000001 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579829+0000", 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034525229999999999 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579864+0000", 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5165e-05 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579937+0000", 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2452000000000004e-05 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579997+0000", 2023-09-23T21:01:40.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0735e-05 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581236+0000", 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001239139 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140926 10.b 10:dac4289f:::benchmark_data_smithi121_358375_object56181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:01:40.818 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.230787+0000", 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 4.4373631949999996, 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3612417960000001, 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140926 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230787+0000", 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230787+0000", 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230785+0000", 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230799+0000", 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4133e-05 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230832+0000", 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2963999999999998e-05 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230840+0000", 2023-09-23T21:01:40.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7349999999999996e-06 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230916+0000", 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6191000000000003e-05 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230924+0000", 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9200000000000004e-06 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326944+0000", 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.096019652999999996 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326950+0000", 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9599999999999997e-06 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.524496+0000", 2023-09-23T21:01:40.823 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1975459370000001 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525163+0000", 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00066679700000000001 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.570349+0000", 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.045186562 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.824 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.570465+0000", 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115815 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.570641+0000", 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000175835 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579899+0000", 2023-09-23T21:01:40.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092581480000000008 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581655+0000", 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001755803 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.586132+0000", 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004477127 2023-09-23T21:01:40.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.590701+0000", 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045693959999999999 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.590750+0000", 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8501000000000002e-05 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.592029+0000", 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012795020000000001 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140937 10.b 10:dce73c29:::benchmark_data_smithi121_358375_object56192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.299885+0000", 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 4.3682653609999997, 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2932650050000001, 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:40.828 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140937 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299885+0000", 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:40.829 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299885+0000", 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299884+0000", 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299889+0000", 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1679999999999997e-06 2023-09-23T21:01:40.830 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299898+0000", 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1130000000000007e-06 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299903+0000", 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.831e-06 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.831 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299953+0000", 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0448999999999998e-05 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299960+0000", 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4189999999999996e-06 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326967+0000", 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027006183 2023-09-23T21:01:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326972+0000", 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8289999999999996e-06 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525195+0000", 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.198222087 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:40.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525848+0000", 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00065309500000000002 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576400+0000", 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.050552391000000002 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576470+0000", 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0252000000000004e-05 2023-09-23T21:01:40.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576547+0000", 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6408000000000004e-05 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581649+0000", 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051024900000000003 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:40.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581661+0000", 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1686e-05 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.586175+0000", 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045139739999999996 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.592090+0000", 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059147319999999998 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.592159+0000", 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9060000000000006e-05 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.593150+0000", 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0009916949999999999 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:40.838 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:40.838 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:01:40.838 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:01:40.838 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:01:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:40.839 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:40.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:40.840 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:01:40.840 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:01:40.845 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:01:40.863 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:01:40.864 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.864 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:01:40.941 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:40.968 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:41.011 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:41.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:41.114 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:41.144 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[219709]: failed to encode map e324 with expected crc 2023-09-23T21:01:41.144 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[219709]: failed to encode map e324 with expected crc 2023-09-23T21:01:41.144 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/97858833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:41.144 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[219709]: pgmap v1028: 99 pgs: 2 remapped+peering, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 43307/85143 objects degraded (50.864%); 7538/85143 objects misplaced (8.853%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T21:01:41.145 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:41.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[211526]: failed to encode map e324 with expected crc 2023-09-23T21:01:41.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[211526]: failed to encode map e324 with expected crc 2023-09-23T21:01:41.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/97858833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:41.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:40 smithi121 ceph-mon[211526]: pgmap v1028: 99 pgs: 2 remapped+peering, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 43307/85143 objects degraded (50.864%); 7538/85143 objects misplaced (8.853%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T21:01:41.467 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:40 smithi123 ceph-mon[193903]: failed to encode map e324 with expected crc 2023-09-23T21:01:41.467 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:40 smithi123 ceph-mon[193903]: failed to encode map e324 with expected crc 2023-09-23T21:01:41.467 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/97858833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:41.467 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:40 smithi123 ceph-mon[193903]: pgmap v1028: 99 pgs: 2 remapped+peering, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 43307/85143 objects degraded (50.864%); 7538/85143 objects misplaced (8.853%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T21:01:41.484 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:01:41.649 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:41.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:41.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:41.755 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:41.797 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:41.884 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:41.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:41.971 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:01:41.993 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:41.994 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:42.092 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:42.212 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 43307/85143 objects degraded (50.864%), 18 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:42.255 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 43307/85143 objects degraded (50.864%), 18 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:42.255 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 43307/85143 objects degraded (50.864%), 18 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:42.290 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:42.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:42.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:42.401 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:42.517 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:42.638 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:42.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:42.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:42.745 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:42.778 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:42.793 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:42.830 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:42.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:42.934 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:43.034 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:43.066 INFO:teuthology.orchestra.run.smithi121.stdout:moved 14 / 297 (4.7138%) 2023-09-23T21:01:43.067 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:43.067 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 17.496 -> 20.8533 (expected baseline 5.69951) 2023-09-23T21:01:43.067 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 9 -> 4 pgs (0.242424 -> 0.107744 * mean) 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 59 -> 60 pgs (1.58923 -> 1.61616 * mean) 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0133 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0146 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.3050 -> 0.1658 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.5329 -> 0.4118 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.2417 -> 0.2027 2023-09-23T21:01:43.068 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8790 -> 0.9230 2023-09-23T21:01:43.080 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:01:43.165 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:43.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:43.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:43.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:43 smithi123 ceph-mon[193903]: pgmap v1029: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 48635/84852 objects degraded (57.317%); 7538/84852 objects misplaced (8.884%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T21:01:43.280 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:43.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:43.317 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:43 smithi121 ceph-mon[211526]: pgmap v1029: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 48635/84852 objects degraded (57.317%); 7538/84852 objects misplaced (8.884%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T21:01:43.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:43.318 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:43 smithi121 ceph-mon[219709]: pgmap v1029: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 48635/84852 objects degraded (57.317%); 7538/84852 objects misplaced (8.884%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T21:01:43.319 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:43 smithi121 conmon[198376]: 2023-09-23T21:01:43.062+0000 7f479dcb1700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T21:01:43.368 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:43.897 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:43.898 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:44.190 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:44.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:44.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:44.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:44 smithi123 ceph-mon[193903]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:44.293 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:01:44.296 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:44.350 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:44.371 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:44 smithi121 ceph-mon[211526]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:44.378 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:44 smithi121 ceph-mon[219709]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:44.414 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:44.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:44.518 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:44.609 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:44.805 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:44.810 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:01:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:44.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:44.836 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:01:44.836 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T21:01:44.919 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:44.948 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:45.008 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:45.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:45.036 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:45 smithi121 ceph-mon[211526]: pgmap v1030: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 47752/78681 objects degraded (60.691%); 7241/78681 objects misplaced (9.203%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:01:45.040 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:01:44] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.43.0" 2023-09-23T21:01:45.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:45 smithi121 ceph-mon[219709]: pgmap v1030: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 47752/78681 objects degraded (60.691%); 7241/78681 objects misplaced (9.203%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:01:45.118 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:45.264 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:45.267 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:45.392 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:45.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:45.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:45 smithi123 ceph-mon[193903]: pgmap v1030: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 47752/78681 objects degraded (60.691%); 7241/78681 objects misplaced (9.203%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:01:45.499 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:45.511 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:45.611 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:45.790 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:45.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:45.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:45.901 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:45.940 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.008 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:46.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:46.059 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:01:46.112 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:46.117 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:46.136 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4107381490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:46.136 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:46 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:46.138 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4107381490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:46.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:46 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4107381490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:46.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:46 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:01:46.179 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:46.283 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:46.321 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.514 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:46.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:46.618 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:46.646 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:46.763 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:46.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:46.866 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:46.918 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:47.087 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:47.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:47.143 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:01:47.144 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 47752/78681 objects degraded (60.691%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:47.170 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:47 smithi123 ceph-mon[193903]: pgmap v1032: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 47752/78681 objects degraded (60.691%); 7241/78681 objects misplaced (9.203%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:01:47.192 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:47.222 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:47.387 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:47.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:47.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 47752/78681 objects degraded (60.691%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[219709]: pgmap v1032: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 47752/78681 objects degraded (60.691%); 7241/78681 objects misplaced (9.203%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: failed to encode map e325 with expected crc 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 47752/78681 objects degraded (60.691%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:47.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:47 smithi121 ceph-mon[211526]: pgmap v1032: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 310 op/s; 47752/78681 objects degraded (60.691%); 7241/78681 objects misplaced (9.203%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:01:47.491 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:47.517 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:47.561 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:47.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:47.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:47.664 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:47.748 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:47.873 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:47.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:47.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:47.940 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:01:48.009 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:48.051 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.134 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:48.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:48 smithi123 ceph-mon[193903]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:48 smithi123 ceph-mon[193903]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:48 smithi123 ceph-mon[193903]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:48 smithi123 ceph-mon[193903]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.243 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:48.285 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.354 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[219709]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[219709]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[219709]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[219709]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[211526]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[211526]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[211526]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:48 smithi121 ceph-mon[211526]: failed to encode map e325 with expected crc 2023-09-23T21:01:48.435 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:01:48.461 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:48.490 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:48.932 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:48.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:49.049 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:49.137 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:49.243 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:49.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:49.260 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:49 smithi121 ceph-mon[211526]: pgmap v1033: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 47311/73659 objects degraded (64.230%); 7086/73659 objects misplaced (9.620%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T21:01:49.263 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:49 smithi121 ceph-mon[219709]: pgmap v1033: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 47311/73659 objects degraded (64.230%); 7086/73659 objects misplaced (9.620%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T21:01:49.352 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:01:49.361 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:49.375 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:01:49.421 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:49.421 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:01:49.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:01:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:49 smithi123 ceph-mon[193903]: pgmap v1033: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 47311/73659 objects degraded (64.230%); 7086/73659 objects misplaced (9.620%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T21:01:49.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:01:49.557 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:49.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:49.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:49.568 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:01:49.605 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:01:49.643 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:01:49.669 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:49.759 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:49.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:01:49.950 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:49.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:50.068 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/393847974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:50.167 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/393847974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:50.214 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:50.252 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136320 9.d 9:b4eaf8c8:::benchmark_data_smithi121_293770_object33432:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214752+0000", 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 560.03514196100002, 2023-09-23T21:01:50.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.101782247, 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136320 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214752+0000", 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214752+0000", 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214751+0000", 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214758+0000", 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4969999999999998e-06 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214772+0000", 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.379e-05 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214778+0000", 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3770000000000001e-06 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214866+0000", 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7578999999999996e-05 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214992+0000", 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012608199999999999 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:01:50.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215115+0000", 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012328299999999999 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231351+0000", 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016235991000000002 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231358+0000", 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2620000000000001e-06 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296318+0000", 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0649602810000001 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296421+0000", 2023-09-23T21:01:50.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010260900000000001 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316433+0000", 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020011547000000001 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316535+0000", 2023-09-23T21:01:50.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010212700000000001 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136321 9.3 9:cbaf7339:::benchmark_data_smithi121_293770_object33433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214796+0000", 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 560.03509820399995, 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.113576339, 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.262 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136321 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214796+0000", 2023-09-23T21:01:50.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214796+0000", 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214796+0000", 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:50.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214799+0000", 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0259999999999999e-06 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214809+0000", 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0531e-05 2023-09-23T21:01:50.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214814+0000", 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4909999999999999e-06 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214940+0000", 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012664600000000001 2023-09-23T21:01:50.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215036+0000", 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5371999999999994e-05 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215132+0000", 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6465000000000006e-05 2023-09-23T21:01:50.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227009+0000", 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011876695 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237869+0000", 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010860071000000001 2023-09-23T21:01:50.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237876+0000", 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.063e-06 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.315925+0000", 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0780493440000001 2023-09-23T21:01:50.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316065+0000", 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013960900000000001 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.328296+0000", 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012231103 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.328373+0000", 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6440999999999995e-05 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136322 9.3 9:c9fc89fe:::benchmark_data_smithi121_293770_object33434:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T21:01:50.271 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214824+0000", 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 560.03507044699995, 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.125673253, 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136322 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.272 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214824+0000", 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214824+0000", 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214824+0000", 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214826+0000", 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.728e-06 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214835+0000", 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0850000000000003e-06 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214841+0000", 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9220000000000002e-06 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.222782+0000", 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079405770000000007 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.222913+0000", 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013101399999999999 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.223041+0000", 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128493 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237906+0000", 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014864308999999999 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237911+0000", 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3820000000000003e-06 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316294+0000", 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0783833570000001 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:01:50.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316394+0000", 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9423e-05 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340398+0000", 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024004376000000001 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340497+0000", 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8981000000000007e-05 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136324 9.3 9:c6c56bec:::benchmark_data_smithi121_293770_object33436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214846+0000", 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 560.03504880499997, 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1260450399999999, 2023-09-23T21:01:50.280 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136324 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214846+0000", 2023-09-23T21:01:50.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214846+0000", 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214845+0000", 2023-09-23T21:01:50.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214848+0000", 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3439999999999999e-06 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214853+0000", 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3499999999999996e-06 2023-09-23T21:01:50.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214856+0000", 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.568e-06 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227032+0000", 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012175333 2023-09-23T21:01:50.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227151+0000", 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119363 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.227272+0000", 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120783 2023-09-23T21:01:50.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237950+0000", 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010677657 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237954+0000", 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9249999999999997e-06 2023-09-23T21:01:50.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316421+0000", 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0784679429999999 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316514+0000", 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.221e-05 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340813+0000", 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024299313999999999 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340891+0000", 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7650000000000004e-05 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.288 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136326 9.3 9:c6acc5bf:::benchmark_data_smithi121_293770_object33438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214859+0000", 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 560.03503507300002, 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.125181303, 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.289 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136326 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214859+0000", 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214859+0000", 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214859+0000", 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214861+0000", 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.846e-06 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214864+0000", 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9560000000000002e-06 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214866+0000", 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.17e-06 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237967+0000", 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023100961999999999 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.237970+0000", 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.909e-06 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316540+0000", 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.078569551 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316636+0000", 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6572999999999998e-05 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3", 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.316711+0000", 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5104000000000005e-05 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.339845+0000", 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023133529 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.339951+0000", 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000106549 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.339984+0000", 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2450999999999998e-05 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.340041+0000", 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6918e-05 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136327 9.e 9:7476b4e6:::benchmark_data_smithi121_293770_object33439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214868+0000", 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 560.035026045, 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.094948848, 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:50.298 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136327 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214868+0000", 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214868+0000", 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214868+0000", 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214870+0000", 2023-09-23T21:01:50.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.699e-06 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214873+0000", 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7690000000000001e-06 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214875+0000", 2023-09-23T21:01:50.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2680000000000001e-06 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214909+0000", 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3696999999999999e-05 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215019+0000", 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010990800000000001 2023-09-23T21:01:50.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.215131+0000", 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000112131 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.224915+0000", 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0097839450000000005 2023-09-23T21:01:50.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231113+0000", 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.006198856 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231122+0000", 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2479999999999996e-06 2023-09-23T21:01:50.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.294619+0000", 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.063497175 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.294749+0000", 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000130407 2023-09-23T21:01:50.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.309762+0000", 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015012942 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.309817+0000", 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5009000000000001e-05 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.306 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45101.0:136329 9.d 9:bf8aaa61:::benchmark_data_smithi121_293770_object33441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:52:30.214884+0000", 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 560.03501066499996, 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1040498940000001, 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.307 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45101", 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 136329 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214884+0000", 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214884+0000", 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214883+0000", 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214886+0000", 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.17e-06 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214891+0000", 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5779999999999998e-06 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.214895+0000", 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9199999999999997e-06 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.219285+0000", 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004389736 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.219408+0000", 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122967 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,5", 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.219520+0000", 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000111798 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231384+0000", 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011864263 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:30.231389+0000", 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.412e-06 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296450+0000", 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.065061152 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.296538+0000", 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7361000000000006e-05 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.318855+0000", 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022317212999999999 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:52:31.318934+0000", 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8712000000000001e-05 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54855 10.1 10:8aabcc1d:::benchmark_data_smithi121_358375_object54854:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.603179+0000", 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 396.64671548000001, 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25036673199999998, 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.316 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54855 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603179+0000", 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603179+0000", 2023-09-23T21:01:50.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603178+0000", 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603235+0000", 2023-09-23T21:01:50.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6938999999999998e-05 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603257+0000", 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2316000000000001e-05 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603266+0000", 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6880000000000005e-06 2023-09-23T21:01:50.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603310+0000", 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4462000000000002e-05 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603429+0000", 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118477 2023-09-23T21:01:50.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603589+0000", 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016056499999999999 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610291+0000", 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0067014869999999999 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612288+0000", 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019972969999999999 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853436+0000", 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24114797099999999 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853501+0000", 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4787000000000002e-05 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.853546+0000", 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4966999999999998e-05 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.323 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54859 10.c 10:3d9b601b:::benchmark_data_smithi121_358375_object54858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.606114+0000", 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 396.64378033000003, 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25144838899999999, 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.324 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54859 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606114+0000", 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606114+0000", 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606113+0000", 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606167+0000", 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4460999999999999e-05 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606180+0000", 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2628e-05 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.606186+0000", 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2820000000000002e-06 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607593+0000", 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001406869 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607722+0000", 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012907700000000001 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607861+0000", 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013883799999999999 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.611809+0000", 2023-09-23T21:01:50.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039483060000000004 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.615227+0000", 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034176660000000002 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857476+0000", 2023-09-23T21:01:50.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24224914 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857525+0000", 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8582999999999999e-05 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.857562+0000", 2023-09-23T21:01:50.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7622e-05 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54864 10.c 10:3c18fb56:::benchmark_data_smithi121_358375_object54863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.607487+0000", 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 396.64240773699998, 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25261208400000001, 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.332 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54864 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607487+0000", 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607487+0000", 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607486+0000", 2023-09-23T21:01:50.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607521+0000", 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4876999999999998e-05 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607530+0000", 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9400000000000008e-06 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607536+0000", 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5260000000000001e-06 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609744+0000", 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022083789999999999 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609868+0000", 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000124067 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609971+0000", 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103428 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.612926+0000", 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029545740000000002 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.615266+0000", 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023401440000000002 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860009+0000", 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24474301600000001 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860061+0000", 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1554999999999997e-05 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.860099+0000", 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8111999999999997e-05 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.340 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140894 10.c 10:33443f8a:::benchmark_data_smithi121_358375_object56149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 14.289066798, 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5509230220000001, 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.341 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140894 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960827+0000", 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960833+0000", 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1330000000000004e-06 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960845+0000", 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2065999999999999e-05 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960850+0000", 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1599999999999997e-06 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960895+0000", 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5522999999999998e-05 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960988+0000", 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2343999999999995e-05 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.961068+0000", 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9864999999999995e-05 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.964285+0000", 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0032171370000000001 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.964723+0000", 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043790000000000002 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973518+0000", 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0087954379999999992 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973531+0000", 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2510999999999999e-05 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322237+0000", 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34870620200000002 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322246+0000", 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3449999999999995e-06 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511643+0000", 2023-09-23T21:01:50.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1893964910000001 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511751+0000", 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107996 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140895 10.c 10:32d93067:::benchmark_data_smithi121_358375_object56150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 14.284442887000001, 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.546411454, 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.351 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140895 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965450+0000", 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965457+0000", 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.297e-06 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965471+0000", 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4352000000000001e-05 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965476+0000", 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.198e-06 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965514+0000", 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7704999999999999e-05 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965608+0000", 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4121000000000002e-05 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965685+0000", 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7127999999999997e-05 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973585+0000", 2023-09-23T21:01:50.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078994029999999993 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973593+0000", 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2810000000000008e-06 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322280+0000", 2023-09-23T21:01:50.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34868769799999999 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322287+0000", 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1879999999999999e-06 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511780+0000", 2023-09-23T21:01:50.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.189493559 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511863+0000", 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2774000000000004e-05 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140896 10.1 10:86418234:::benchmark_data_smithi121_358375_object56151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 14.274858864, 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6998174590000001, 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.360 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140896 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975034+0000", 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975041+0000", 2023-09-23T21:01:50.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4139999999999999e-06 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975054+0000", 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3604999999999999e-05 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975061+0000", 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3419999999999996e-06 2023-09-23T21:01:50.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.976385+0000", 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0013248260000000001 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.976396+0000", 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.076e-05 2023-09-23T21:01:50.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320693+0000", 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34429716599999999 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320701+0000", 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6899999999999992e-06 2023-09-23T21:01:50.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.654437+0000", 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3337360949999999 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.655243+0000", 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00080610999999999996 2023-09-23T21:01:50.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668434+0000", 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013190676 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668502+0000", 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8547999999999995e-05 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668880+0000", 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00037727300000000002 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674052+0000", 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051727869999999999 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674083+0000", 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0043999999999999e-05 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674132+0000", 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9484000000000003e-05 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674191+0000", 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8567999999999999e-05 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674853+0000", 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000662445 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.370 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140900 10.c 10:3c2d050d:::benchmark_data_smithi121_358375_object56155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 14.252236855, 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5351996919999999, 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.371 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140900 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997656+0000", 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997665+0000", 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8799999999999997e-06 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997687+0000", 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1967000000000002e-05 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997699+0000", 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2074e-05 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997758+0000", 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9375e-05 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997768+0000", 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6749999999999997e-06 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322303+0000", 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.324535299 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322309+0000", 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5690000000000004e-06 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511887+0000", 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1895777999999999 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.512028+0000", 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014090500000000001 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.512592+0000", 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00056408399999999996 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525899+0000", 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013307461 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532566+0000", 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0066666620000000003 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532600+0000", 2023-09-23T21:01:50.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4233000000000002e-05 2023-09-23T21:01:50.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532699+0000", 2023-09-23T21:01:50.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8743e-05 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532754+0000", 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4941000000000001e-05 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532857+0000", 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103405 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.382 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140903 10.c 10:38af2a49:::benchmark_data_smithi121_358375_object56158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 14.230420625000001, 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.513614249, 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.383 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140903 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019472+0000", 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019481+0000", 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1039999999999996e-06 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019501+0000", 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9706000000000001e-05 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019513+0000", 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2092999999999999e-05 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019575+0000", 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2136000000000003e-05 2023-09-23T21:01:50.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019587+0000", 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1616e-05 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322325+0000", 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30273779699999998 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322334+0000", 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3780000000000007e-06 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520201+0000", 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1978669260000001 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520304+0000", 2023-09-23T21:01:50.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102685 2023-09-23T21:01:50.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:01:50.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520390+0000", 2023-09-23T21:01:50.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6036000000000001e-05 2023-09-23T21:01:50.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.528925+0000", 2023-09-23T21:01:50.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085353589999999993 2023-09-23T21:01:50.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532633+0000", 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037075699999999999 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532894+0000", 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00026169499999999999 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532952+0000", 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7717999999999999e-05 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532991+0000", 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9443999999999997e-05 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533088+0000", 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6669000000000006e-05 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140920 10.11 10:8e38667f:::benchmark_data_smithi121_358375_object56175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:01:50.393 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 14.049825528, 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4951127319999999, 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140920 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200068+0000", 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200075+0000", 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4109999999999999e-06 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200092+0000", 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6860000000000001e-05 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200099+0000", 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2559999999999996e-06 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200156+0000", 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6854000000000001e-05 2023-09-23T21:01:50.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200167+0000", 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0437e-05 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320655+0000", 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.120488398 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320668+0000", 2023-09-23T21:01:50.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3132999999999999e-05 2023-09-23T21:01:50.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.652475+0000", 2023-09-23T21:01:50.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3318069219999999 2023-09-23T21:01:50.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:01:50.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.653395+0000", 2023-09-23T21:01:50.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00092004800000000002 2023-09-23T21:01:50.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677570+0000", 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024174998999999999 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677680+0000", 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011025499999999999 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677936+0000", 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00025574300000000001 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.679730+0000", 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0017943849999999999 2023-09-23T21:01:50.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.692535+0000", 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012804519 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.694272+0000", 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001737326 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.694330+0000", 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7287999999999998e-05 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.695182+0000", 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00085209999999999995 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140940 10.c 10:3f6dfdef:::benchmark_data_smithi121_358375_object56195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:01:50.404 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 13.791609384000001, 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0750470560000001, 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140940 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458283+0000", 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:01:50.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458293+0000", 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9380000000000004e-06 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458312+0000", 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9083999999999999e-05 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458320+0000", 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2300000000000008e-06 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458374+0000", 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4116000000000002e-05 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458383+0000", 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4409999999999993e-06 2023-09-23T21:01:50.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.522747+0000", 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.064364592 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.522997+0000", 2023-09-23T21:01:50.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024926899999999999 2023-09-23T21:01:50.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:01:50.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.523081+0000", 2023-09-23T21:01:50.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4122999999999994e-05 2023-09-23T21:01:50.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532666+0000", 2023-09-23T21:01:50.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.009585484 2023-09-23T21:01:50.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532923+0000", 2023-09-23T21:01:50.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00025692899999999998 2023-09-23T21:01:50.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533121+0000", 2023-09-23T21:01:50.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000198209 2023-09-23T21:01:50.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533191+0000", 2023-09-23T21:01:50.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9379000000000001e-05 2023-09-23T21:01:50.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533229+0000", 2023-09-23T21:01:50.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8133999999999998e-05 2023-09-23T21:01:50.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533332+0000", 2023-09-23T21:01:50.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103198 2023-09-23T21:01:50.415 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.415 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.415 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.415 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140941 10.c 10:3b70ed7f:::benchmark_data_smithi121_358375_object56196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:01:50.416 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:01:50.416 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 13.790872307000001, 2023-09-23T21:01:50.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.075658805, 2023-09-23T21:01:50.416 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.416 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.416 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.416 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140941 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:01:50.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:01:50.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:01:50.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:50.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459026+0000", 2023-09-23T21:01:50.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9400000000000001e-06 2023-09-23T21:01:50.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459035+0000", 2023-09-23T21:01:50.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5290000000000008e-06 2023-09-23T21:01:50.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459039+0000", 2023-09-23T21:01:50.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0709999999999996e-06 2023-09-23T21:01:50.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459081+0000", 2023-09-23T21:01:50.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2103e-05 2023-09-23T21:01:50.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459087+0000", 2023-09-23T21:01:50.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3230000000000003e-06 2023-09-23T21:01:50.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525940+0000", 2023-09-23T21:01:50.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.066852771 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.526063+0000", 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012287299999999999 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:01:50.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.526177+0000", 2023-09-23T21:01:50.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113816 2023-09-23T21:01:50.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533150+0000", 2023-09-23T21:01:50.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0069729099999999997 2023-09-23T21:01:50.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533368+0000", 2023-09-23T21:01:50.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000218383 2023-09-23T21:01:50.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533381+0000", 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2839e-05 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534586+0000", 2023-09-23T21:01:50.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001204552 2023-09-23T21:01:50.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534618+0000", 2023-09-23T21:01:50.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2261999999999997e-05 2023-09-23T21:01:50.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534681+0000", 2023-09-23T21:01:50.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3054999999999999e-05 2023-09-23T21:01:50.429 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.429 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.429 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140942 10.c 10:3a89f6d8:::benchmark_data_smithi121_358375_object56197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 13.768256938, 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0531614490000001, 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:01:50.430 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:01:50.431 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:01:50.431 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:50.431 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140942 2023-09-23T21:01:50.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.431 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:01:50.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:01:50.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481636+0000", 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481643+0000", 2023-09-23T21:01:50.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6429999999999995e-06 2023-09-23T21:01:50.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481659+0000", 2023-09-23T21:01:50.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5274e-05 2023-09-23T21:01:50.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481665+0000", 2023-09-23T21:01:50.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3500000000000002e-06 2023-09-23T21:01:50.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481716+0000", 2023-09-23T21:01:50.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1357999999999999e-05 2023-09-23T21:01:50.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:01:50.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481726+0000", 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8099999999999992e-06 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.528966+0000", 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0472400340000001 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:01:50.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.529117+0000", 2023-09-23T21:01:50.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015073199999999999 2023-09-23T21:01:50.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:01:50.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.529274+0000", 2023-09-23T21:01:50.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000157431 2023-09-23T21:01:50.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533184+0000", 2023-09-23T21:01:50.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00390931 2023-09-23T21:01:50.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:01:50.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533387+0000", 2023-09-23T21:01:50.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000203301 2023-09-23T21:01:50.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533489+0000", 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102224 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534716+0000", 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012264190000000001 2023-09-23T21:01:50.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:01:50.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534744+0000", 2023-09-23T21:01:50.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8334999999999999e-05 2023-09-23T21:01:50.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534799+0000", 2023-09-23T21:01:50.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4917999999999998e-05 2023-09-23T21:01:50.443 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.443 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.443 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPull(10.9 e323/308 cost 1049576)", 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:37.336435+0000", 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 12.913459785000001, 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25303262599999998, 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:01:50.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:01:50.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.336435+0000", 2023-09-23T21:01:50.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:01:50.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.336435+0000", 2023-09-23T21:01:50.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:01:50.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:01:50.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.336433+0000", 2023-09-23T21:01:50.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:50.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.336455+0000", 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2158999999999999e-05 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:01:50.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.336468+0000", 2023-09-23T21:01:50.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2802e-05 2023-09-23T21:01:50.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:01:50.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.336475+0000", 2023-09-23T21:01:50.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.204e-06 2023-09-23T21:01:50.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:01:50.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.588953+0000", 2023-09-23T21:01:50.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25247786999999999 2023-09-23T21:01:50.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:01:50.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:01:50.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:01:50.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.589467+0000", 2023-09-23T21:01:50.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00051394400000000001 2023-09-23T21:01:50.450 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.450 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.450 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.450 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:01:50.450 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:01:50.451 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:01:50.451 INFO:teuthology.orchestra.run.smithi121.stdout:moved 14 / 297 (4.7138%) 2023-09-23T21:01:50.452 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:50.452 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 17.496 -> 20.4661 (expected baseline 5.69951) 2023-09-23T21:01:50.452 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 9 -> 4 pgs (0.242424 -> 0.107744 * mean) 2023-09-23T21:01:50.452 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 59 -> 60 pgs (1.58923 -> 1.61616 * mean) 2023-09-23T21:01:50.452 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:50.452 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:01:50.452 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T21:01:50.453 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:50.453 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0126 2023-09-23T21:01:50.453 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0139 2023-09-23T21:01:50.453 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.3050 -> 0.1606 2023-09-23T21:01:50.453 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.5329 -> 0.4214 2023-09-23T21:01:50.453 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.2417 -> 0.1948 2023-09-23T21:01:50.453 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8790 -> 0.9883 2023-09-23T21:01:50.454 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/393847974' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:01:50.454 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:01:50.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:50.455 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:50.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:50.556 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:50.631 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:50 smithi121 conmon[198376]: 2023-09-23T21:01:50.278+0000 7f479dcb1700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T21:01:50.635 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:51.070 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:51.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:51.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:51.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:01:51.176 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:51.305 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:51 smithi121 ceph-mon[211526]: from='client.56134 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:51 smithi121 ceph-mon[211526]: pgmap v1034: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 46786/71355 objects degraded (65.568%); 6887/71355 objects misplaced (9.652%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:01:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:51 smithi121 ceph-mon[219709]: from='client.56134 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:51.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:51 smithi121 ceph-mon[219709]: pgmap v1034: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 46786/71355 objects degraded (65.568%); 6887/71355 objects misplaced (9.652%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:01:51.470 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:51.477 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:01:51.478 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T21:01:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:51 smithi123 ceph-mon[193903]: from='client.56134 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:51 smithi123 ceph-mon[193903]: pgmap v1034: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 46786/71355 objects degraded (65.568%); 6887/71355 objects misplaced (9.652%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:01:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:51.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:51.596 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:51.597 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:51.630 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:51.689 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:51.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:51.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:51.794 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:51.874 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:51.997 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:52.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:52.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:52.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 46786/71355 objects degraded (65.568%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:52.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 46786/71355 objects degraded (65.568%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:52.195 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:52.326 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:52.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:52.447 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:52.477 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 46786/71355 objects degraded (65.568%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:52.479 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:52.550 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:52.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:52.661 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:52.688 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:52.730 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:52.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:52.833 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:52.860 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:52.922 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:52.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:53.027 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:53.107 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:53.145 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:01:53.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:53 smithi123 ceph-mon[193903]: pgmap v1035: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 46786/70899 objects degraded (65.990%); 6887/70899 objects misplaced (9.714%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:01:53.300 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:53.306 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:53.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:53.366 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:53.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:53 smithi121 ceph-mon[219709]: pgmap v1035: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 46786/70899 objects degraded (65.990%); 6887/70899 objects misplaced (9.714%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:01:53.415 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:01:53.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:53 smithi121 ceph-mon[211526]: pgmap v1035: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 46786/70899 objects degraded (65.990%); 6887/70899 objects misplaced (9.714%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:01:53.498 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:53.680 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:01:53.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:53.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:01:53.793 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:53.827 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:53.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:53.946 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:53.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:54.047 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:54.081 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:54.133 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:54.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:54.239 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:01:54.330 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:54.616 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:54 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:01:54.616 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:54 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:01:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:54 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:01:54.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:01:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:01:54] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.43.0" 2023-09-23T21:01:55.119 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:01:55.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:55.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:01:55.230 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:55.327 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:55.495 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:01:55.509 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:55.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:55.620 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:55.669 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:55 smithi123 ceph-mon[193903]: pgmap v1036: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 45853/64167 objects degraded (71.459%); 6531/64167 objects misplaced (10.178%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T21:01:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2379122950' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:55.790 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:55 smithi121 ceph-mon[219709]: pgmap v1036: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 45853/64167 objects degraded (71.459%); 6531/64167 objects misplaced (10.178%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T21:01:55.790 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2379122950' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:55.790 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:55.791 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:55 smithi121 ceph-mon[211526]: pgmap v1036: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 45853/64167 objects degraded (71.459%); 6531/64167 objects misplaced (10.178%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T21:01:55.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2379122950' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:55.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:01:55.863 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:55.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:55.967 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:56.005 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:56.683 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:01:56.760 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:56.775 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:56.775 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.775 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.775 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.775 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.775 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:56.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T21:01:56.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 45853/64167 objects degraded (71.459%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:56.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T21:01:56.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 45853/64167 objects degraded (71.459%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:56.878 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:56.965 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: failed to encode map e326 with expected crc 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T21:01:56.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 45853/64167 objects degraded (71.459%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:01:57.521 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:01:57.521 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:150089 10.4 e326/305 10:2467a447:::benchmark_data_smithi121_358375_object65344:head v 326'8205, mlcod=236'4115)", 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:57.509554+0000", 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0098365840000000006, 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098698989999999997, 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509554+0000", 2023-09-23T21:01:57.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509554+0000", 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509552+0000", 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509565+0000", 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.279e-05 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509589+0000", 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4015e-05 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509599+0000", 2023-09-23T21:01:57.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0210999999999999e-05 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509781+0000", 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000181461 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509805+0000", 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3845999999999998e-05 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:01:57.525 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.45470.0:150094 10.1 e326/322 10:806c62c1:::benchmark_data_smithi121_358375_object65349:head v 326'8267, mlcod=236'4166)", 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:57.509625+0000", 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0097650889999999994, 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098690370000000006, 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:01:57.526 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509625+0000", 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509625+0000", 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509625+0000", 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509630+0000", 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4489999999999998e-06 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509640+0000", 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8390000000000002e-06 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509645+0000", 2023-09-23T21:01:57.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4839999999999998e-06 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509689+0000", 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4505000000000002e-05 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.509733+0000", 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3835000000000003e-05 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:01:57.529 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:150099 10.2 10:4603b1a2:::benchmark_data_smithi121_358375_object65354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:57.512648+0000", 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0067425699999999998, 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069142980000000001, 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:01:57.530 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 150099 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.512648+0000", 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.512648+0000", 2023-09-23T21:01:57.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.512645+0000", 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.512654+0000", 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3809999999999999e-06 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.512669+0000", 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4688000000000001e-05 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.512678+0000", 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3570000000000004e-06 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.513126+0000", 2023-09-23T21:01:57.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00044776800000000003 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:57.514856+0000", 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001730597 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 3 2023-09-23T21:01:57.534 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:01:57.693 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:01:57.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:57.705 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:57 smithi121 ceph-mon[211526]: pgmap v1038: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 45853/64167 objects degraded (71.459%); 6531/64167 objects misplaced (10.178%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T21:01:57.705 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:57.705 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:57 smithi121 ceph-mon[219709]: pgmap v1038: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 45853/64167 objects degraded (71.459%); 6531/64167 objects misplaced (10.178%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T21:01:57.705 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:57.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:01:57.806 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:57.894 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:57 smithi123 ceph-mon[193903]: pgmap v1038: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 45853/64167 objects degraded (71.459%); 6531/64167 objects misplaced (10.178%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T21:01:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:01:58.262 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:58.370 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:01:58.414 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:58.480 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:01:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:01:58.599 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:01:58.634 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:58.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:01:58.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:01:58.733 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:01:58.808 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:01:58.844 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:58.903 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:01:58.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:58.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:01:59.016 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:01:59.050 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:59.093 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:01:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:59.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:01:59.200 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:01:59.226 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:01:59.357 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:59.357 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:59.358 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:59 smithi123 ceph-mon[193903]: from='client.36099 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:moved 5 / 297 (1.6835%) 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 17.496 -> 18.5637 (expected baseline 5.69951) 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 9 -> 8 pgs (0.242424 -> 0.215488 * mean) 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 59 -> 59 pgs (1.58923 -> 1.58923 * mean) 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:01:59.417 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:01:59.418 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0119 2023-09-23T21:01:59.418 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0131 2023-09-23T21:01:59.418 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.3050 -> 0.2550 2023-09-23T21:01:59.418 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.5329 -> 0.4829 2023-09-23T21:01:59.418 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.2417 -> 0.1917 2023-09-23T21:01:59.418 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8790 -> 0.9290 2023-09-23T21:01:59.435 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:01:59.497 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[211526]: from='client.36099 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]: dispatch 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]: dispatch 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[211526]: pgmap v1039: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 45358/58626 objects degraded (77.368%); 6378/58626 objects misplaced (10.879%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:01:59.498 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[219709]: from='client.36099 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:01:59.499 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]: dispatch 2023-09-23T21:01:59.499 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]: dispatch 2023-09-23T21:01:59.499 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:01:59 smithi121 ceph-mon[219709]: pgmap v1039: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 45358/58626 objects degraded (77.368%); 6378/58626 objects misplaced (10.879%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:01:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:59 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]: dispatch 2023-09-23T21:01:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]: dispatch 2023-09-23T21:01:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:01:59 smithi123 ceph-mon[193903]: pgmap v1039: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 45358/58626 objects degraded (77.368%); 6378/58626 objects misplaced (10.879%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:01:59.788 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:01:59.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:01:59.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:01:59.894 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:01:59.920 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:00.492 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:02:00.492 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T21:02:00.497 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:00.499 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[211526]: failed to encode map e327 with expected crc 2023-09-23T21:02:00.499 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]': finished 2023-09-23T21:02:00.499 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[211526]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T21:02:00.500 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[211526]: failed to encode map e327 with expected crc 2023-09-23T21:02:00.500 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[219709]: failed to encode map e327 with expected crc 2023-09-23T21:02:00.500 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]': finished 2023-09-23T21:02:00.500 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[219709]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T21:02:00.500 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:00 smithi121 ceph-mon[219709]: failed to encode map e327 with expected crc 2023-09-23T21:02:00.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:00.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:00.601 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:00.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:00.719 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:00 smithi123 ceph-mon[193903]: failed to encode map e327 with expected crc 2023-09-23T21:02:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:00 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'16714','4':'31649','5':'12565','6':'60883'}"}]': finished 2023-09-23T21:02:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:00 smithi123 ceph-mon[193903]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T21:02:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:00 smithi123 ceph-mon[193903]: failed to encode map e327 with expected crc 2023-09-23T21:02:00.842 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:00.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:00.932 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:00.948 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:00.988 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:01.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:02:01.094 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:02:01.117 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:02:01.144 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:01.145 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:02:01.150 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:01.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:01.178 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:02:01.200 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:02:01.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:02:01.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:02:01.250 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.250 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.250 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.250 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.251 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:02:01.254 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:01.290 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:01.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: pgmap v1042: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 44741/56148 objects degraded (79.684%); 6119/56148 objects misplaced (10.898%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3714695905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:01.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 45358/58626 objects degraded (77.368%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: pgmap v1042: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 44741/56148 objects degraded (79.684%); 6119/56148 objects misplaced (10.898%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3714695905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:01.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 45358/58626 objects degraded (77.368%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T21:02:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: failed to encode map e328 with expected crc 2023-09-23T21:02:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: pgmap v1042: 99 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 44741/56148 objects degraded (79.684%); 6119/56148 objects misplaced (10.898%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:02:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3714695905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 45358/58626 objects degraded (77.368%), 20 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:02.063 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:02.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:02.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:02.168 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:02.193 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:02.236 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:02.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:02.344 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:02.380 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:02 smithi123 ceph-mon[193903]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T21:02:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:02 smithi123 ceph-mon[193903]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:02 smithi123 ceph-mon[193903]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:02 smithi123 ceph-mon[193903]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[219709]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[219709]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[219709]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[219709]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[211526]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[211526]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[211526]: failed to encode map e329 with expected crc 2023-09-23T21:02:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:02 smithi121 ceph-mon[211526]: failed to encode map e329 with expected crc 2023-09-23T21:02:03.473 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:03 smithi121 ceph-mon[219709]: failed to encode map e329 with expected crc 2023-09-23T21:02:03.473 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:03 smithi121 ceph-mon[219709]: pgmap v1044: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 50110/55929 objects degraded (89.596%); 6119/55929 objects misplaced (10.941%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:02:03.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:03 smithi121 ceph-mon[211526]: failed to encode map e329 with expected crc 2023-09-23T21:02:03.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:03 smithi121 ceph-mon[211526]: pgmap v1044: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 50110/55929 objects degraded (89.596%); 6119/55929 objects misplaced (10.941%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:02:03.502 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:03.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:03.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:03.615 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:03.652 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:03.714 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:03.719 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:03 smithi123 ceph-mon[193903]: failed to encode map e329 with expected crc 2023-09-23T21:02:03.719 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:03 smithi123 ceph-mon[193903]: pgmap v1044: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 50110/55929 objects degraded (89.596%); 6119/55929 objects misplaced (10.941%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:02:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:03.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:03.821 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:03.859 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:03.919 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:03.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:03.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:04.029 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:04.061 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:04.116 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:04.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:04.225 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:04.318 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:04.554 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:04.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:04.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:04.671 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:04.759 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:04.901 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:04.909 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:02:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:02:04] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.43.0" 2023-09-23T21:02:04.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:05.005 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:05.005 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:05.010 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:05.139 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:05.223 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:02:05.554 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:05.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:05.661 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:05.764 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout:moved 14 / 297 (4.7138%) 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout:avg 37.125 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 18.5637 -> 21.8371 (expected baseline 5.69951) 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.5 with 7 -> 5 pgs (0.188552 -> 0.13468 * mean) 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 59 -> 61 pgs (1.58923 -> 1.6431 * mean) 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:02:06.018 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:02:06.019 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:02:06.019 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0114 2023-09-23T21:02:06.019 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0126 2023-09-23T21:02:06.019 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.2550 -> 0.1312 2023-09-23T21:02:06.019 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.1917 -> 0.1457 2023-09-23T21:02:06.019 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.4829 -> 0.3734 2023-09-23T21:02:06.019 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.9290 -> 1.0000 2023-09-23T21:02:06.082 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:06.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:06.200 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:06.229 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:05 smithi121 ceph-mon[211526]: pgmap v1045: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 55274/54096 objects degraded (102.178%); 5794/54096 objects misplaced (10.711%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T21:02:06.312 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]: dispatch 2023-09-23T21:02:06.312 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:05 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]: dispatch 2023-09-23T21:02:06.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:05 smithi121 ceph-mon[219709]: pgmap v1045: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 55274/54096 objects degraded (102.178%); 5794/54096 objects misplaced (10.711%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T21:02:06.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:05 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]: dispatch 2023-09-23T21:02:06.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:05 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]: dispatch 2023-09-23T21:02:06.329 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:02:06.443 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:05 smithi123 ceph-mon[193903]: pgmap v1045: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 55274/54096 objects degraded (102.178%); 5794/54096 objects misplaced (10.711%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T21:02:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:05 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]: dispatch 2023-09-23T21:02:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:05 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]: dispatch 2023-09-23T21:02:06.548 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:06.588 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.812 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:06.916 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:06.942 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:06.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:06.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:06.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:07.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:07.132 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: from='client.46076 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:02:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: failed to encode map e330 with expected crc 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: failed to encode map e330 with expected crc 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]': finished 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 55274/54096 objects degraded (102.178%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/934770048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:06 smithi123 ceph-mon[193903]: pgmap v1048: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 55274/54096 objects degraded (102.178%); 5794/54096 objects misplaced (10.711%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:02:07.348 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:02:07.383 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: from='client.46076 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:02:07.383 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: failed to encode map e330 with expected crc 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: failed to encode map e330 with expected crc 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]': finished 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 55274/54096 objects degraded (102.178%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/934770048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:07.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[219709]: pgmap v1048: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 55274/54096 objects degraded (102.178%); 5794/54096 objects misplaced (10.711%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:02:07.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: from='client.46076 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: failed to encode map e330 with expected crc 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: failed to encode map e330 with expected crc 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'8600','4':'24471','5':'9546','6':'65536'}"}]': finished 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 55274/54096 objects degraded (102.178%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:07.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/934770048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:07.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:07.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:06 smithi121 ceph-mon[211526]: pgmap v1048: 99 pgs: 2 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 55274/54096 objects degraded (102.178%); 5794/54096 objects misplaced (10.711%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:02:07.686 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:07.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:07.806 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:07.832 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:07.877 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:07.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:07.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:07.985 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:08.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:08.076 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:08.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:08.182 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:08.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.205 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.206 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.206 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.206 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:08.206 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:08.206 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T21:02:08.206 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e332 with expected crc 2023-09-23T21:02:08.206 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:08 smithi123 ceph-mon[193903]: failed to encode map e332 with expected crc 2023-09-23T21:02:08.207 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:08.248 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:08.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:08.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:08.354 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:08.390 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e332 with expected crc 2023-09-23T21:02:08.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[219709]: failed to encode map e332 with expected crc 2023-09-23T21:02:08.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e331 with expected crc 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e332 with expected crc 2023-09-23T21:02:08.423 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:08 smithi121 ceph-mon[211526]: failed to encode map e332 with expected crc 2023-09-23T21:02:08.446 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:08.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:08.551 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:08.662 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:08.970 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:08.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:08.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:09.032 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:09.059 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[219709]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.059 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[219709]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.059 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[219709]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.059 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[219709]: 10.a starting backfill to osd.3 from (0'0,0'0] MAX to 329'9466 2023-09-23T21:02:09.059 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[219709]: pgmap v1050: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 1.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 55942/50388 objects degraded (111.022%); 10925/50388 objects misplaced (21.682%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T21:02:09.060 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[211526]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.060 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[211526]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.060 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[211526]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.060 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[211526]: 10.a starting backfill to osd.3 from (0'0,0'0] MAX to 329'9466 2023-09-23T21:02:09.060 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:09 smithi121 ceph-mon[211526]: pgmap v1050: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 1.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 55942/50388 objects degraded (111.022%); 10925/50388 objects misplaced (21.682%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T21:02:09.080 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:09.082 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:09.188 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:09.462 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:09.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:09 smithi123 ceph-mon[193903]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:09 smithi123 ceph-mon[193903]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:09 smithi123 ceph-mon[193903]: failed to encode map e332 with expected crc 2023-09-23T21:02:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:09 smithi123 ceph-mon[193903]: 10.a starting backfill to osd.3 from (0'0,0'0] MAX to 329'9466 2023-09-23T21:02:09.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:09 smithi123 ceph-mon[193903]: pgmap v1050: 99 pgs: 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 1.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 55942/50388 objects degraded (111.022%); 10925/50388 objects misplaced (21.682%); 8.2 MiB/s, 130 objects/s recovering 2023-09-23T21:02:09.574 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:09.606 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:09.657 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:09.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:09.762 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:09.806 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:09.870 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:09.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:09.978 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:10.009 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:10.077 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:10.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:10.183 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:10.279 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:10.833 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:02:11.273 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:11.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:11.377 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:11.415 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:11.479 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:11.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:11.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:11.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:11.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:11.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[219709]: pgmap v1051: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 1022 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 54265/49098 objects degraded (110.524%); 10564/49098 objects misplaced (21.516%); 11 MiB/s, 1 keys/s, 174 objects/s recovering 2023-09-23T21:02:11.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 55942/50388 objects degraded (111.022%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:11.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:11.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:11.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[211526]: pgmap v1051: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 1022 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 54265/49098 objects degraded (110.524%); 10564/49098 objects misplaced (21.516%); 11 MiB/s, 1 keys/s, 174 objects/s recovering 2023-09-23T21:02:11.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 55942/50388 objects degraded (111.022%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:11.588 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:11.618 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:11.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:11 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:11.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:11 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:11.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:11 smithi123 ceph-mon[193903]: pgmap v1051: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 1022 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 277 op/s; 54265/49098 objects degraded (110.524%); 10564/49098 objects misplaced (21.516%); 11 MiB/s, 1 keys/s, 174 objects/s recovering 2023-09-23T21:02:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 55942/50388 objects degraded (111.022%), 24 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:12.315 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:12.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:12.439 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:12.607 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:12.777 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:12.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:12.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:12.895 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:12.927 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:12.989 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:12.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:12.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:13.095 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:13.158 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:13.175 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:13.176 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:13.197 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:13.493 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:02:13.493 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:02:13.493 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T21:02:13.493 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:02:13.493 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:02:13.493 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T21:02:13.569 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:02:13.616 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:02:13.659 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:02:13.691 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:02:13.726 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:13.726 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:13.726 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:02:13.726 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:02:13.726 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:13.726 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[211526]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 1004 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 283 op/s; 53981/48222 objects degraded (111.943%); 10564/48222 objects misplaced (21.907%); 11 MiB/s, 1 keys/s, 172 objects/s recovering 2023-09-23T21:02:13.726 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:13.727 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:13.727 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:02:13.727 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:02:13.727 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:13.727 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:13 smithi121 ceph-mon[219709]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 1004 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 283 op/s; 53981/48222 objects degraded (111.943%); 10564/48222 objects misplaced (21.907%); 11 MiB/s, 1 keys/s, 172 objects/s recovering 2023-09-23T21:02:13.727 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:02:13.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:13.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:13.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:02:13.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:02:13.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:13 smithi123 ceph-mon[193903]: pgmap v1052: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 1004 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 283 op/s; 53981/48222 objects degraded (111.943%); 10564/48222 objects misplaced (21.907%); 11 MiB/s, 1 keys/s, 172 objects/s recovering 2023-09-23T21:02:13.993 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:02:13.994 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:02:13.995 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:02:13.995 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:02:14.623 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:02:14.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3717865616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:14.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3717865616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:14.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3717865616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:14.869 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:14.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:14.975 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:15.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:15.081 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:15.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:15.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:15.168 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:02:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:02:14] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2023-09-23T21:02:15.189 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:15.273 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:15.473 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:15 smithi123 ceph-mon[193903]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 873 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 52962/41937 objects degraded (126.289%); 9658/41937 objects misplaced (23.030%); 16 MiB/s, 1 keys/s, 274 objects/s recovering 2023-09-23T21:02:15.509 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:15.520 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:15 smithi121 ceph-mon[211526]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 873 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 52962/41937 objects degraded (126.289%); 9658/41937 objects misplaced (23.030%); 16 MiB/s, 1 keys/s, 274 objects/s recovering 2023-09-23T21:02:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:15.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:15.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:15 smithi121 ceph-mon[219709]: pgmap v1053: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 873 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 52962/41937 objects degraded (126.289%); 9658/41937 objects misplaced (23.030%); 16 MiB/s, 1 keys/s, 274 objects/s recovering 2023-09-23T21:02:15.622 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:15.655 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:15.710 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:15.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:15.818 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:15.923 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:16.045 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:16.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:16.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:16.149 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:16.193 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:16.260 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:16.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:16.371 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:16.410 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:16.468 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:16.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:16.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:16.574 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:16.619 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:16.668 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:16.685 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 52962/41937 objects degraded (126.289%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:16.762 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 52962/41937 objects degraded (126.289%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:16.762 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 52962/41937 objects degraded (126.289%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:16.780 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:16.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:16.892 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:16.921 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:16.986 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:16.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:16.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:17.093 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:17.173 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:17.391 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:17.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:17.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:17.497 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:17.531 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:02:17.571 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:17 smithi121 ceph-mon[219709]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 873 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 52962/41937 objects degraded (126.289%); 9658/41937 objects misplaced (23.030%); 14 MiB/s, 0 keys/s, 235 objects/s recovering 2023-09-23T21:02:17.573 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:17 smithi121 ceph-mon[211526]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 873 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 52962/41937 objects degraded (126.289%); 9658/41937 objects misplaced (23.030%); 14 MiB/s, 0 keys/s, 235 objects/s recovering 2023-09-23T21:02:17.593 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:17.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:17 smithi123 ceph-mon[193903]: pgmap v1054: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 873 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 52962/41937 objects degraded (126.289%); 9658/41937 objects misplaced (23.030%); 14 MiB/s, 0 keys/s, 235 objects/s recovering 2023-09-23T21:02:18.094 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:18.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:18.200 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:18.231 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:18.295 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:18.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:18.403 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:18.432 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:18.473 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:18.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:18.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2681188102' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:18.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:18.524 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:02:18.579 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:18.660 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2681188102' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:18.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2681188102' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:18.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:02:19.385 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:19.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:19.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:19.489 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:19.564 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[211526]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[211526]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T21:02:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[211526]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[211526]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[211526]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 762 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 52116/36615 objects degraded (142.335%); 9364/36615 objects misplaced (25.574%); 15 MiB/s, 0 keys/s, 258 objects/s recovering 2023-09-23T21:02:19.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:19.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[219709]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:19.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[219709]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T21:02:19.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[219709]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[219709]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:19 smithi121 ceph-mon[219709]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 762 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 52116/36615 objects degraded (142.335%); 9364/36615 objects misplaced (25.574%); 15 MiB/s, 0 keys/s, 258 objects/s recovering 2023-09-23T21:02:19.972 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:19 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:19 smithi123 ceph-mon[193903]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:19 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:02:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:19 smithi123 ceph-mon[193903]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T21:02:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:19 smithi123 ceph-mon[193903]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:19 smithi123 ceph-mon[193903]: failed to encode map e333 with expected crc 2023-09-23T21:02:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:19 smithi123 ceph-mon[193903]: pgmap v1056: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering, 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 762 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 52116/36615 objects degraded (142.335%); 9364/36615 objects misplaced (25.574%); 15 MiB/s, 0 keys/s, 258 objects/s recovering 2023-09-23T21:02:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:19.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:20.082 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:20.134 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:20.385 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:20.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:20.491 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:20.528 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:20.645 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:20.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:20 smithi121 ceph-mon[219709]: failed to encode map e334 with expected crc 2023-09-23T21:02:20.655 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:20 smithi121 ceph-mon[219709]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T21:02:20.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:20.656 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:20 smithi121 ceph-mon[211526]: failed to encode map e334 with expected crc 2023-09-23T21:02:20.656 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:20 smithi121 ceph-mon[211526]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T21:02:20.757 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:20.846 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:20.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:20 smithi123 ceph-mon[193903]: failed to encode map e334 with expected crc 2023-09-23T21:02:20.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:20 smithi123 ceph-mon[193903]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T21:02:21.005 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:21.082 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:21.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:21.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:21.125 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:02:21.125 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:02:21.126 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:02:21.190 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:21.209 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:02:21.209 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:02:21.209 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:02:21.209 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:02:21.220 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:21.286 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:21.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:21.396 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:21.428 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:21.490 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:21.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:21.595 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:21.694 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[219709]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[219709]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T21:02:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[219709]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[219709]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[219709]: pgmap v1059: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 729 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 50654/35061 objects degraded (144.474%); 9892/35061 objects misplaced (28.214%); 9.2 MiB/s, 203 objects/s recovering 2023-09-23T21:02:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 52116/36615 objects degraded (142.335%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[211526]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[211526]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T21:02:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[211526]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[211526]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[211526]: pgmap v1059: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 729 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 50654/35061 objects degraded (144.474%); 9892/35061 objects misplaced (28.214%); 9.2 MiB/s, 203 objects/s recovering 2023-09-23T21:02:21.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 52116/36615 objects degraded (142.335%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:21 smithi123 ceph-mon[193903]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:21 smithi123 ceph-mon[193903]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T21:02:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:21 smithi123 ceph-mon[193903]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:21 smithi123 ceph-mon[193903]: failed to encode map e335 with expected crc 2023-09-23T21:02:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:21 smithi123 ceph-mon[193903]: pgmap v1059: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 729 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 50654/35061 objects degraded (144.474%); 9892/35061 objects misplaced (28.214%); 9.2 MiB/s, 203 objects/s recovering 2023-09-23T21:02:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 52116/36615 objects degraded (142.335%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:22.402 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:02:22.766 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:22.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:22.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:22.881 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:23.015 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:23.169 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:02:23.431 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:23.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:23.536 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:23.571 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:23.629 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:23.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:23.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:23.736 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:23.770 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:23.832 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:23.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:23.937 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:23.971 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:24.029 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:24.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:24.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:24.135 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:24.137 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:24.165 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:24 smithi123 ceph-mon[193903]: pgmap v1060: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 710 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 50311/34131 objects degraded (147.406%); 9892/34131 objects misplaced (28.982%); 11 MiB/s, 232 objects/s recovering 2023-09-23T21:02:24.168 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:24.220 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:24.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:24.325 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:24.357 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:24.364 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:24 smithi121 ceph-mon[211526]: pgmap v1060: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 710 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 50311/34131 objects degraded (147.406%); 9892/34131 objects misplaced (28.982%); 11 MiB/s, 232 objects/s recovering 2023-09-23T21:02:24.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:24 smithi121 ceph-mon[219709]: pgmap v1060: 99 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 710 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 50311/34131 objects degraded (147.406%); 9892/34131 objects misplaced (28.982%); 11 MiB/s, 232 objects/s recovering 2023-09-23T21:02:24.408 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:24.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:24.517 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:24.550 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:24.556 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:24.601 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:24.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:24.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:24.709 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:24.739 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:24.740 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:02:24.775 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:02:24.800 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:24.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:24.852 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:02:24.892 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:02:24.910 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:24.927 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:02:24.959 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:02:24.969 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:02:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:02:24] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2023-09-23T21:02:24.985 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:24.996 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:02:25.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1860631534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:25.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:25 smithi123 ceph-mon[193903]: pgmap v1061: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 576 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 49413/27708 objects degraded (178.335%); 8976/27708 objects misplaced (32.395%); 17 MiB/s, 409 objects/s recovering 2023-09-23T21:02:25.062 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:02:25.063 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.063 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.063 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.063 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.064 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.064 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.064 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.064 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:02:25.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1860631534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:25.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:25 smithi121 ceph-mon[211526]: pgmap v1061: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 576 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 49413/27708 objects degraded (178.335%); 8976/27708 objects misplaced (32.395%); 17 MiB/s, 409 objects/s recovering 2023-09-23T21:02:25.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1860631534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:25.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:25 smithi121 ceph-mon[219709]: pgmap v1061: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 576 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 49413/27708 objects degraded (178.335%); 8976/27708 objects misplaced (32.395%); 17 MiB/s, 409 objects/s recovering 2023-09-23T21:02:25.490 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:25.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:25.597 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:25.707 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 49413/27708 objects degraded (178.335%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 49413/27708 objects degraded (178.335%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:26.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 49413/27708 objects degraded (178.335%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:27.547 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:27.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:27.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:27 smithi121 ceph-mon[211526]: pgmap v1062: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 576 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 49413/27708 objects degraded (178.335%); 8976/27708 objects misplaced (32.395%); 11 MiB/s, 276 objects/s recovering 2023-09-23T21:02:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:27 smithi121 ceph-mon[219709]: pgmap v1062: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 576 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 49413/27708 objects degraded (178.335%); 8976/27708 objects misplaced (32.395%); 11 MiB/s, 276 objects/s recovering 2023-09-23T21:02:27.666 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:27 smithi123 ceph-mon[193903]: pgmap v1062: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 576 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 49413/27708 objects degraded (178.335%); 8976/27708 objects misplaced (32.395%); 11 MiB/s, 276 objects/s recovering 2023-09-23T21:02:27.745 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:27.941 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:02:27.942 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:02:27.942 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:02:27.942 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:02:27.942 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:02:27.942 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:02:28.113 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:28.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:28.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:28.256 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:28.310 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:28.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:28.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:28.416 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:28.524 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:28.858 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:28.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:28.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:28.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:28.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:28.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:28.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:28.966 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:29.119 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:29.271 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:29.333 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:29.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:29.445 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:29.482 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:29.548 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:29.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:29.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:29 smithi121 ceph-mon[211526]: pgmap v1063: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 448 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 46323/21540 objects degraded (215.056%); 8723/21540 objects misplaced (40.497%); 14 MiB/s, 312 objects/s recovering 2023-09-23T21:02:29.614 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:29 smithi121 ceph-mon[219709]: pgmap v1063: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 448 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 46323/21540 objects degraded (215.056%); 8723/21540 objects misplaced (40.497%); 14 MiB/s, 312 objects/s recovering 2023-09-23T21:02:29.656 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:29 smithi123 ceph-mon[193903]: pgmap v1063: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 448 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 46323/21540 objects degraded (215.056%); 8723/21540 objects misplaced (40.497%); 14 MiB/s, 312 objects/s recovering 2023-09-23T21:02:29.742 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:29.858 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:29.858 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:29.952 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:02:30.434 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:30.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:30.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:30.530 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:02:30.540 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:30.572 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:30.627 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:30.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:30.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1751899395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:30 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:30.733 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:30.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1751899395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:30.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:30 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:30.863 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1751899395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:30.863 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:30 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:02:30.872 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:31 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:31 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:31 smithi123 ceph-mon[193903]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T21:02:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:31 smithi123 ceph-mon[193903]: failed to encode map e336 with expected crc 2023-09-23T21:02:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:31 smithi123 ceph-mon[193903]: pgmap v1065: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 410 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 45886/19719 objects degraded (232.699%); 8255/19719 objects misplaced (41.863%); 14 MiB/s, 297 objects/s recovering 2023-09-23T21:02:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 46323/21540 objects degraded (215.056%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:31.804 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:31.813 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:31.813 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:31.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[211526]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T21:02:31.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[211526]: failed to encode map e336 with expected crc 2023-09-23T21:02:31.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[211526]: pgmap v1065: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 410 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 45886/19719 objects degraded (232.699%); 8255/19719 objects misplaced (41.863%); 14 MiB/s, 297 objects/s recovering 2023-09-23T21:02:31.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 46323/21540 objects degraded (215.056%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:31.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:31.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:02:31.814 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[219709]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T21:02:31.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[219709]: failed to encode map e336 with expected crc 2023-09-23T21:02:31.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[219709]: pgmap v1065: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 410 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 45886/19719 objects degraded (232.699%); 8255/19719 objects misplaced (41.863%); 14 MiB/s, 297 objects/s recovering 2023-09-23T21:02:31.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 46323/21540 objects degraded (215.056%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:31.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:31.916 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:31.944 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:32.450 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:32.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:32.554 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:32.582 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:32.625 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:32.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:32.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:32.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[211526]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[211526]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T21:02:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[211526]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[211526]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[219709]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[219709]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T21:02:32.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[219709]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:32 smithi121 ceph-mon[219709]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.730 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:32.809 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:32.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:32 smithi123 ceph-mon[193903]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:32 smithi123 ceph-mon[193903]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T21:02:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:32 smithi123 ceph-mon[193903]: failed to encode map e337 with expected crc 2023-09-23T21:02:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:32 smithi123 ceph-mon[193903]: failed to encode map e337 with expected crc 2023-09-23T21:02:33.016 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:02:33.079 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:33.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:33.196 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:33.225 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:33.283 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:33.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:33.391 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:33.439 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:33.471 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:33.586 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:33.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:33.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:33.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:33 smithi121 ceph-mon[211526]: pgmap v1067: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 392 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 45593/18858 objects degraded (241.770%); 8192/18858 objects misplaced (43.440%); 12 MiB/s, 241 objects/s recovering 2023-09-23T21:02:33.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:33 smithi121 ceph-mon[219709]: pgmap v1067: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 392 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 45593/18858 objects degraded (241.770%); 8192/18858 objects misplaced (43.440%); 12 MiB/s, 241 objects/s recovering 2023-09-23T21:02:33.785 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:33.947 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:02:33.947 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T21:02:33.947 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:02:33.977 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:33 smithi123 ceph-mon[193903]: pgmap v1067: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 392 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 45593/18858 objects degraded (241.770%); 8192/18858 objects misplaced (43.440%); 12 MiB/s, 241 objects/s recovering 2023-09-23T21:02:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:33.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:34.093 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:34.141 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:34.214 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:34.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:34.320 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:34.352 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:34.413 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:34.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:34.520 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:34.589 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:34.779 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:02:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:02:34] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.43.0" 2023-09-23T21:02:35.205 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:02:36.115 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:36.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:36.224 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:36.253 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:36.267 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:36 smithi121 ceph-mon[211526]: pgmap v1068: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 268 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 44690/12897 objects degraded (346.515%); 7769/12897 objects misplaced (60.239%); 16 MiB/s, 322 objects/s recovering 2023-09-23T21:02:36.271 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:36 smithi121 ceph-mon[219709]: pgmap v1068: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 268 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 44690/12897 objects degraded (346.515%); 7769/12897 objects misplaced (60.239%); 16 MiB/s, 322 objects/s recovering 2023-09-23T21:02:36.312 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:36.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:36.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:36.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:36 smithi123 ceph-mon[193903]: pgmap v1068: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 268 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 44690/12897 objects degraded (346.515%); 7769/12897 objects misplaced (60.239%); 16 MiB/s, 322 objects/s recovering 2023-09-23T21:02:36.317 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:36.417 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:36.450 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:36.514 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:36.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:36.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:36.620 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:36.652 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:36.709 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:36.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:36.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:36.910 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:37.029 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:37.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 44690/12897 objects degraded (346.515%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:37.179 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 44690/12897 objects degraded (346.515%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:37.263 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:37.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:37.373 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:37.447 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 44690/12897 objects degraded (346.515%), 18 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:37.521 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:02:37.521 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:02:37.521 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:02:37.521 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:02:37.521 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:02:37.564 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:02:37.609 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:02:37.656 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:02:37.898 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:02:37.898 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:02:37.898 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:02:37.899 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:02:37.899 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:02:37.899 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:02:37.899 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:02:37.899 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:38 smithi121 ceph-mon[211526]: pgmap v1069: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 268 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 44690/12897 objects degraded (346.515%); 7769/12897 objects misplaced (60.239%); 9.6 MiB/s, 212 objects/s recovering 2023-09-23T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3937222620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:38.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:38 smithi121 ceph-mon[219709]: pgmap v1069: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 268 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 44690/12897 objects degraded (346.515%); 7769/12897 objects misplaced (60.239%); 9.6 MiB/s, 212 objects/s recovering 2023-09-23T21:02:38.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3937222620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:38 smithi123 ceph-mon[193903]: pgmap v1069: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 268 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 44690/12897 objects degraded (346.515%); 7769/12897 objects misplaced (60.239%); 9.6 MiB/s, 212 objects/s recovering 2023-09-23T21:02:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3937222620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:38.585 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:38.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:38.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:38.695 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:38.792 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:39 smithi121 ceph-mon[219709]: pgmap v1070: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 130 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 43675/6273 objects degraded (696.238%); 7416/6273 objects misplaced (118.221%); 11 MiB/s, 246 objects/s recovering 2023-09-23T21:02:39.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:39 smithi121 ceph-mon[211526]: pgmap v1070: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 130 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 43675/6273 objects degraded (696.238%); 7416/6273 objects misplaced (118.221%); 11 MiB/s, 246 objects/s recovering 2023-09-23T21:02:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:39 smithi123 ceph-mon[193903]: pgmap v1070: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 130 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 43675/6273 objects degraded (696.238%); 7416/6273 objects misplaced (118.221%); 11 MiB/s, 246 objects/s recovering 2023-09-23T21:02:39.528 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:39.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:39.635 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:39.745 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:39.975 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:39.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:40.087 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:40.203 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:40.347 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:40.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:40.462 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:40.497 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:02:40.497 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:02:40.497 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175048 9.9 9:98e06086:::benchmark_data_smithi121_293770_object72160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.209862+0000", 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 556.28425345100004, 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43337474100000001, 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T21:02:40.498 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175048 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.499 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209862+0000", 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209865+0000", 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9799999999999998e-06 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.500 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209882+0000", 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7142e-05 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.209888+0000", 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.691e-06 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226284+0000", 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01639606 2023-09-23T21:02:40.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226451+0000", 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000167028 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.226596+0000", 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014432099999999999 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232724+0000", 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061281840000000001 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.235331+0000", 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026065340000000002 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643145+0000", 2023-09-23T21:02:40.503 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.40781475099999998 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643197+0000", 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1427000000000001e-05 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.643237+0000", 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0119e-05 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.504 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45101.0:175062 9.5 9:af0bbe63:::benchmark_data_smithi121_293770_object72174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e209)", 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:53:24.223421+0000", 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 556.27069450600004, 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41800432799999998, 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45101", 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/2719075507", 2023-09-23T21:02:40.505 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 175062 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223421+0000", 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223421+0000", 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.506 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223420+0000", 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223426+0000", 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4750000000000001e-06 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.507 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223440+0000", 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3971000000000001e-05 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223447+0000", 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5379999999999997e-06 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223496+0000", 2023-09-23T21:02:40.508 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8513000000000001e-05 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223640+0000", 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014410900000000001 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.509 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.223777+0000", 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000137279 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232561+0000", 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0087839670000000002 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.232568+0000", 2023-09-23T21:02:40.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9580000000000001e-06 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641331+0000", 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.4087635 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.511 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641388+0000", 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6258000000000002e-05 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:53:24.641425+0000", 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7688999999999998e-05 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74269 10.c 10:34f5f527:::benchmark_data_smithi121_358375_object74268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e238)", 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:30.911332+0000", 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 429.58278332200001, 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.170509689, 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.513 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74269 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911332+0000", 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.514 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911332+0000", 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911330+0000", 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911357+0000", 2023-09-23T21:02:40.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6041999999999999e-05 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911376+0000", 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9082000000000001e-05 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911384+0000", 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6349999999999995e-06 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.516 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911437+0000", 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2290999999999998e-05 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.911445+0000", 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8030000000000003e-06 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052751+0000", 2023-09-23T21:02:40.517 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1413060900000001 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.054736+0000", 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019844519999999998 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.054936+0000", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000200127 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.062180+0000", 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072435889999999999 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.519 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.062196+0000", 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.626e-05 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078622+0000", 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016426151999999999 2023-09-23T21:02:40.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078687+0000", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5153000000000005e-05 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.081842+0000", 2023-09-23T21:02:40.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031547390000000002 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82131 10.6 10:6abfd253:::benchmark_data_smithi121_358375_object82130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.473119+0000", 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 406.02099629499997, 2023-09-23T21:02:40.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0092071929999999, 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82131 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473119+0000", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473119+0000", 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473118+0000", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473164+0000", 2023-09-23T21:02:40.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6202999999999998e-05 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473179+0000", 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5135e-05 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.526 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473190+0000", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0943e-05 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473323+0000", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013271499999999999 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473334+0000", 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1313e-05 2023-09-23T21:02:40.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.472285+0000", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.998951276 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.474038+0000", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001752408 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:02:40.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.474147+0000", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000109298 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482179+0000", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080320080000000002 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482192+0000", 2023-09-23T21:02:40.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2642e-05 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482220+0000", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8218e-05 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482285+0000", 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4950999999999995e-05 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482326+0000", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1459999999999999e-05 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82133 10.6 10:63e8454a:::benchmark_data_smithi121_358375_object82132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 406.02090910300001, 2023-09-23T21:02:40.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.01243694, 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82133 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:02:40.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473255+0000", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9543999999999999e-05 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473263+0000", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.106e-06 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.534 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473278+0000", 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4651000000000001e-05 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473362+0000", 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4215999999999995e-05 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473371+0000", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9430000000000006e-06 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.476809+0000", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0034375790000001 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.476935+0000", 2023-09-23T21:02:40.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012644800000000001 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.477029+0000", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3652000000000004e-05 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482363+0000", 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053339640000000001 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482387+0000", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.421e-05 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485541+0000", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031537000000000002 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485602+0000", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0899000000000001e-05 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485643+0000", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1616000000000003e-05 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82167 10.6 10:628536db:::benchmark_data_smithi121_358375_object82166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:02:40.539 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.616057+0000", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 405.87805807500001, 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.87047633700000004, 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82167 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616057+0000", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616057+0000", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616056+0000", 2023-09-23T21:02:40.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616105+0000", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.867e-05 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616120+0000", 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5238e-05 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616128+0000", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5939999999999996e-06 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616192+0000", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3879000000000003e-05 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616202+0000", 2023-09-23T21:02:40.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0848e-05 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479337+0000", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86313474099999998 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479507+0000", 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016978800000000001 2023-09-23T21:02:40.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479677+0000", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000169586 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485520+0000", 2023-09-23T21:02:40.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0058433160000000003 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485651+0000", 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000131278 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486447+0000", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00079564700000000002 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486498+0000", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1010000000000001e-05 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486534+0000", 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5933999999999998e-05 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82175 10.6 10:6b1c2f60:::benchmark_data_smithi121_358375_object82174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.626165+0000", 2023-09-23T21:02:40.548 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 405.86795036299998, 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86051544000000002, 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82175 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.549 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626165+0000", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626165+0000", 2023-09-23T21:02:40.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626164+0000", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626188+0000", 2023-09-23T21:02:40.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3451999999999998e-05 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626197+0000", 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7440000000000002e-06 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626204+0000", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.781e-06 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626249+0000", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4608999999999999e-05 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626256+0000", 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5560000000000002e-06 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482405+0000", 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.85614919499999997 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482547+0000", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000141285 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482686+0000", 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013936499999999999 2023-09-23T21:02:40.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485656+0000", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029699399999999999 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485728+0000", 2023-09-23T21:02:40.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2163999999999996e-05 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486608+0000", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00087970500000000005 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486650+0000", 2023-09-23T21:02:40.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1770999999999997e-05 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486681+0000", 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0961999999999999e-05 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.558 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90547 10.c 10:38f673f1:::benchmark_data_smithi121_358375_object5802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.927985+0000", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 322.56613014700002, 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24451520700000001, 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90547 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927985+0000", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927985+0000", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927984+0000", 2023-09-23T21:02:40.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927989+0000", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3669999999999996e-06 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927998+0000", 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8799999999999997e-06 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928004+0000", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7620000000000001e-06 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928042+0000", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7648999999999998e-05 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928130+0000", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8039000000000002e-05 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928220+0000", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0078000000000001e-05 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.930554+0000", 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023342699999999998 2023-09-23T21:02:40.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.930699+0000", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014521900000000001 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932382+0000", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016824559999999999 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932413+0000", 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0815999999999998e-05 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172501+0000", 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24008792700000001 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.565 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.45470.0:90547 10.c e268/265 ondisk, result = 0)", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.931963+0000", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 322.56215274099998, 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24055167899999999, 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931963+0000", 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931963+0000", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931962+0000", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931965+0000", 2023-09-23T21:02:40.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9299999999999999e-06 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931970+0000", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8409999999999999e-06 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931974+0000", 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2890000000000002e-06 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932374+0000", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000400739 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932378+0000", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1690000000000002e-06 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172514+0000", 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24013602100000001 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90548 10.4 10:202f5be9:::benchmark_data_smithi121_358375_object5803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 322.56159425700002, 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243222459, 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.570 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90548 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932525+0000", 2023-09-23T21:02:40.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9750000000000001e-06 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172121+0000", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23959696899999999 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172147+0000", 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5612000000000001e-05 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172546+0000", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039850399999999999 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172644+0000", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8318000000000003e-05 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:02:40.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172716+0000", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1630000000000004e-05 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175320+0000", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026049089999999999 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175444+0000", 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012345800000000001 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175635+0000", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000191528 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175695+0000", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9590999999999999e-05 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175744+0000", 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8677000000000002e-05 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:110359 10.16 10:6ce1aa55:::benchmark_data_smithi121_358375_object25614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:02:40.577 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 149.661431674, 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7173976529999999, 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 110359 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832682+0000", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832691+0000", 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0270000000000001e-06 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832711+0000", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.999e-05 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832721+0000", 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0033e-05 2023-09-23T21:02:40.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832788+0000", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7268999999999996e-05 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832802+0000", 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3319000000000001e-05 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:11.299916+0000", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.467114749 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:11.299930+0000", 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3368e-05 2023-09-23T21:02:40.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.542785+0000", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2428551299999999 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.542973+0000", 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018853699999999999 2023-09-23T21:02:40.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.543153+0000", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000180065 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548645+0000", 2023-09-23T21:02:40.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054919060000000004 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548699+0000", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4026000000000001e-05 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548751+0000", 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2011000000000001e-05 2023-09-23T21:02:40.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548821+0000", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.97e-05 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.550081+0000", 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001260401 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151000 10.16 10:6fb6daa8:::benchmark_data_smithi121_358375_object66255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 41.084444632, 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.019180497, 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.588 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151000 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409669+0000", 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:02:40.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409680+0000", 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1647e-05 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409699+0000", 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8763e-05 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409711+0000", 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409930+0000", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021934400000000001 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410095+0000", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016493700000000001 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410262+0000", 2023-09-23T21:02:40.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016674600000000001 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.415289+0000", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050267660000000002 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428299+0000", 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013010763 2023-09-23T21:02:40.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428313+0000", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4064e-05 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428836+0000", 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0005229959999999 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428851+0000", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4960000000000001e-05 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151024 10.6 10.8ded8286 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.995183910999998, 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.930075294, 2023-09-23T21:02:40.596 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151024 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498930+0000", 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498938+0000", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3890000000000005e-06 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498955+0000", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6996000000000001e-05 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498965+0000", 2023-09-23T21:02:40.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0629999999999995e-06 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499015+0000", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0868000000000002e-05 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499026+0000", 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0883999999999999e-05 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428989+0000", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92996320600000004 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429007+0000", 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7419e-05 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151064 10.16 10.14ca33f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.858418147999998, 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79318550200000004, 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151064 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.602 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635696+0000", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635703+0000", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8220000000000001e-06 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635720+0000", 2023-09-23T21:02:40.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7116999999999999e-05 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635729+0000", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1929999999999999e-06 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635796+0000", 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7163999999999996e-05 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635810+0000", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4469e-05 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428874+0000", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79306313900000003 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428883+0000", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2390000000000008e-06 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151070 10.6 10.e7155d86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.807732446000003, 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74265465799999997, 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151070 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686381+0000", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686389+0000", 2023-09-23T21:02:40.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.204e-06 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686407+0000", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8130000000000001e-05 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686416+0000", 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2369999999999998e-06 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686477+0000", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1432000000000004e-05 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686488+0000", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1026000000000001e-05 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429028+0000", 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74253917700000005 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429038+0000", 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.007e-05 2023-09-23T21:02:40.612 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151071 10.16 10.6fc596f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.800496504999998, 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73529290700000005, 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.613 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151071 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693617+0000", 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693627+0000", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4120000000000007e-06 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693644+0000", 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7669999999999999e-05 2023-09-23T21:02:40.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693654+0000", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1980000000000007e-06 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693712+0000", 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8579999999999998e-05 2023-09-23T21:02:40.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693726+0000", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3847e-05 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428903+0000", 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73517710400000003 2023-09-23T21:02:40.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428912+0000", 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7469999999999994e-06 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.619 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151082 10.16 10.dd3bf356 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.780478412000001, 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71530539400000004, 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.620 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151082 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713635+0000", 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713645+0000", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0113e-05 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713670+0000", 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4507000000000001e-05 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713679+0000", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1710000000000008e-06 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713749+0000", 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0607999999999999e-05 2023-09-23T21:02:40.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713761+0000", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1841999999999999e-05 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428934+0000", 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71517288000000001 2023-09-23T21:02:40.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428942+0000", 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3040000000000005e-06 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151084 10.6 10.dfef7e86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.626 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.773730477000001, 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70867917800000002, 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151084 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720383+0000", 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:40.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720392+0000", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1699999999999997e-06 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720411+0000", 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9681e-05 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720422+0000", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0587999999999999e-05 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720488+0000", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.648e-05 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720504+0000", 2023-09-23T21:02:40.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6172999999999999e-05 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429056+0000", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70855175100000001 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429064+0000", 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0150000000000003e-06 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.631 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151096 10.16 10.c63710d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.703660298999999, 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63851427199999999, 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.632 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151096 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790453+0000", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790471+0000", 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7876999999999998e-05 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790488+0000", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6642000000000001e-05 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790498+0000", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6569999999999992e-06 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790553+0000", 2023-09-23T21:02:40.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5707e-05 2023-09-23T21:02:40.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790563+0000", 2023-09-23T21:02:40.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6520000000000001e-06 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428961+0000", 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63839818400000004 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428969+0000", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3799999999999994e-06 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151153 10.6 10.b41dbb46 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:02:40.638 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 40.452587956999999, 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.387565296, 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151153 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:02:40.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041526+0000", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041533+0000", 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1250000000000004e-06 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041546+0000", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2289e-05 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041552+0000", 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041613+0000", 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0640000000000002e-05 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041623+0000", 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0309000000000001e-05 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429084+0000", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.38746082799999998 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429093+0000", 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6910000000000003e-06 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.644 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:02:40.645 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:40.668 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:40.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:40.779 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:40.819 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:40.865 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:40.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:40.968 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:41.016 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:41.247 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 2, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:02:41.247 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:02:41.248 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 3, 4, 5, 6, 2, 7] 2023-09-23T21:02:41.248 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T21:02:41.550 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:41.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:41.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:41.555 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 84742 objects 2023-09-23T21:02:41.555 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :384.494 2023-09-23T21:02:41.656 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:41.681 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:41.723 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:41.724 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:02:41.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:41.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:41.830 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:41.857 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:41.903 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T21:02:41.903 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T21:02:42.304 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:42 smithi121 ceph-mon[211526]: pgmap v1071: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 96 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 43190/4647 objects degraded (929.417%); 7252/4647 objects misplaced (156.058%); 11 MiB/s, 238 objects/s recovering 2023-09-23T21:02:42.304 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 43190/4647 objects degraded (929.417%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:42.305 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:42 smithi121 ceph-mon[219709]: pgmap v1071: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 96 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 43190/4647 objects degraded (929.417%); 7252/4647 objects misplaced (156.058%); 11 MiB/s, 238 objects/s recovering 2023-09-23T21:02:42.305 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 43190/4647 objects degraded (929.417%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:42.316 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:42.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:42.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:42.329 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:02:42.422 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:42.459 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:42 smithi123 ceph-mon[193903]: pgmap v1071: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 96 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 43190/4647 objects degraded (929.417%); 7252/4647 objects misplaced (156.058%); 11 MiB/s, 238 objects/s recovering 2023-09-23T21:02:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 43190/4647 objects degraded (929.417%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:42.662 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:42.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:42.685 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:42.767 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:42.799 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:42.822 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:42.823 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:42.846 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:42.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:42.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:42.951 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:42.979 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:43.023 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:43.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:43.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:43.150 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:43.162 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:43.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:44.037 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:02:44.397 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:44 smithi121 ceph-mon[219709]: pgmap v1072: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 85 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 42950/4137 objects degraded (1038.192%); 7252/4137 objects misplaced (175.296%); 11 MiB/s, 262 objects/s recovering 2023-09-23T21:02:44.398 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3564446188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:44.398 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:44 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:44.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:44 smithi121 ceph-mon[211526]: pgmap v1072: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 85 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 42950/4137 objects degraded (1038.192%); 7252/4137 objects misplaced (175.296%); 11 MiB/s, 262 objects/s recovering 2023-09-23T21:02:44.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3564446188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:44.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:44 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:44.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:44 smithi123 ceph-mon[193903]: pgmap v1072: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 85 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 42950/4137 objects degraded (1038.192%); 7252/4137 objects misplaced (175.296%); 11 MiB/s, 262 objects/s recovering 2023-09-23T21:02:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3564446188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:44 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:02:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:02:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:02:44] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.43.0" 2023-09-23T21:02:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[211526]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:02:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[211526]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T21:02:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[211526]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[211526]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[211526]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[219709]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[219709]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[219709]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[219709]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:45 smithi121 ceph-mon[219709]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:45 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:02:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:45 smithi123 ceph-mon[193903]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:45 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:02:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:45 smithi123 ceph-mon[193903]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T21:02:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:45 smithi123 ceph-mon[193903]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:45 smithi123 ceph-mon[193903]: failed to encode map e338 with expected crc 2023-09-23T21:02:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:45 smithi123 ceph-mon[193903]: failed to encode map e338 with expected crc 2023-09-23T21:02:46.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:46 smithi121 ceph-mon[211526]: pgmap v1074: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 42329/756 objects degraded (5599.074%); 6969/756 objects misplaced (921.825%); 9.5 MiB/s, 227 objects/s recovering 2023-09-23T21:02:46.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:46 smithi121 ceph-mon[219709]: pgmap v1074: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 42329/756 objects degraded (5599.074%); 6969/756 objects misplaced (921.825%); 9.5 MiB/s, 227 objects/s recovering 2023-09-23T21:02:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:46 smithi123 ceph-mon[193903]: pgmap v1074: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 42329/756 objects degraded (5599.074%); 6969/756 objects misplaced (921.825%); 9.5 MiB/s, 227 objects/s recovering 2023-09-23T21:02:47.213 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:47.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:47.318 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:47.342 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:47 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 42329/756 objects degraded (5599.074%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:47.342 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:47 smithi123 ceph-mon[193903]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T21:02:47.342 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:47 smithi123 ceph-mon[193903]: failed to encode map e339 with expected crc 2023-09-23T21:02:47.342 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:47 smithi123 ceph-mon[193903]: pgmap v1076: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 42329/756 objects degraded (5599.074%); 6969/756 objects misplaced (921.825%); 6.6 MiB/s, 169 objects/s recovering 2023-09-23T21:02:47.344 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:47.390 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:47.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:47.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 42329/756 objects degraded (5599.074%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[211526]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T21:02:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[211526]: failed to encode map e339 with expected crc 2023-09-23T21:02:47.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[211526]: pgmap v1076: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 42329/756 objects degraded (5599.074%); 6969/756 objects misplaced (921.825%); 6.6 MiB/s, 169 objects/s recovering 2023-09-23T21:02:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 42329/756 objects degraded (5599.074%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[219709]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T21:02:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[219709]: failed to encode map e339 with expected crc 2023-09-23T21:02:47.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:47 smithi121 ceph-mon[219709]: pgmap v1076: 99 pgs: 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 42329/756 objects degraded (5599.074%); 6969/756 objects misplaced (921.825%); 6.6 MiB/s, 169 objects/s recovering 2023-09-23T21:02:47.495 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:47.529 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[219709]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[219709]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[219709]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[219709]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[219709]: failed to encode map e340 with expected crc 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[219709]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[219709]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[211526]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[211526]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[211526]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[211526]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T21:02:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[211526]: failed to encode map e340 with expected crc 2023-09-23T21:02:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[211526]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:48 smithi121 ceph-mon[211526]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:48 smithi123 ceph-mon[193903]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:48 smithi123 ceph-mon[193903]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:48 smithi123 ceph-mon[193903]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:48 smithi123 ceph-mon[193903]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T21:02:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:48 smithi123 ceph-mon[193903]: failed to encode map e340 with expected crc 2023-09-23T21:02:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:48 smithi123 ceph-mon[193903]: failed to encode map e339 with expected crc 2023-09-23T21:02:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:48 smithi123 ceph-mon[193903]: failed to encode map e339 with expected crc 2023-09-23T21:02:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:49 smithi121 ceph-mon[211526]: failed to encode map e340 with expected crc 2023-09-23T21:02:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:49 smithi121 ceph-mon[211526]: failed to encode map e340 with expected crc 2023-09-23T21:02:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:49 smithi121 ceph-mon[211526]: pgmap v1078: 99 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 39627/756 objects degraded (5241.667%); 6969/756 objects misplaced (921.825%); 4.4 MiB/s, 437 objects/s recovering 2023-09-23T21:02:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:49 smithi121 ceph-mon[219709]: failed to encode map e340 with expected crc 2023-09-23T21:02:49.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:49 smithi121 ceph-mon[219709]: failed to encode map e340 with expected crc 2023-09-23T21:02:49.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:49 smithi121 ceph-mon[219709]: pgmap v1078: 99 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 39627/756 objects degraded (5241.667%); 6969/756 objects misplaced (921.825%); 4.4 MiB/s, 437 objects/s recovering 2023-09-23T21:02:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:49 smithi123 ceph-mon[193903]: failed to encode map e340 with expected crc 2023-09-23T21:02:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:49 smithi123 ceph-mon[193903]: failed to encode map e340 with expected crc 2023-09-23T21:02:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:49 smithi123 ceph-mon[193903]: pgmap v1078: 99 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 39627/756 objects degraded (5241.667%); 6969/756 objects misplaced (921.825%); 4.4 MiB/s, 437 objects/s recovering 2023-09-23T21:02:50.567 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:50.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:50.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:50.672 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:50.705 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:51.472 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:51.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:51.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:51.608 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.106 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:52.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:52.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:52.210 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:52.236 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:52 smithi121 ceph-mon[219709]: pgmap v1079: 99 pgs: 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 35232/756 objects degraded (4660.317%); 6969/756 objects misplaced (921.825%); 1.1 MiB/s, 473 objects/s recovering 2023-09-23T21:02:52.265 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 35232/756 objects degraded (4660.317%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:52.265 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:52 smithi121 ceph-mon[211526]: pgmap v1079: 99 pgs: 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 35232/756 objects degraded (4660.317%); 6969/756 objects misplaced (921.825%); 1.1 MiB/s, 473 objects/s recovering 2023-09-23T21:02:52.265 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 35232/756 objects degraded (4660.317%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:52.282 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:52.286 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:52 smithi123 ceph-mon[193903]: pgmap v1079: 99 pgs: 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 35232/756 objects degraded (4660.317%); 6969/756 objects misplaced (921.825%); 1.1 MiB/s, 473 objects/s recovering 2023-09-23T21:02:52.286 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 35232/756 objects degraded (4660.317%), 14 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:02:52.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:52.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:52.388 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:52.415 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.462 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:52.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:52.569 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:52.611 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.664 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:52.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:52.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:52.768 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:52.800 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:52.897 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:02:52.906 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:52.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:52.920 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:02:52.944 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:02:52.968 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:02:52.995 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:02:53.011 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:53.020 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:02:53.038 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:53.056 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:02:53.079 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:02:53.081 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.081 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.081 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.082 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.082 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.082 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.082 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.082 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:02:53.084 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:53.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:53.134 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:02:53.134 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:02:53.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2752609220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2752609220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:53.193 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:53.220 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:53.264 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:53.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:53.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:53.371 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:02:53.404 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2752609220' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:02:53.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:53 smithi121 conmon[211495]: 2023-09-23T21:02:53.608+0000 7fdc50c66700 -1 mon.a@0(leader).osd e340 definitely_dead 0 2023-09-23T21:02:54.057 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:02:54.124 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.3. 2023-09-23T21:02:54.126 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_5' removed 2023-09-23T21:02:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[211526]: pgmap v1080: 99 pgs: 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 35232/756 objects degraded (4660.317%); 6969/756 objects misplaced (921.825%); 0 B/s, 394 objects/s recovering 2023-09-23T21:02:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3954943590' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:02:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/842615157' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T21:02:54.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[211526]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T21:02:54.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[219709]: pgmap v1080: 99 pgs: 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 35232/756 objects degraded (4660.317%); 6969/756 objects misplaced (921.825%); 0 B/s, 394 objects/s recovering 2023-09-23T21:02:54.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3954943590' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:02:54.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/842615157' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T21:02:54.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:54 smithi121 ceph-mon[219709]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T21:02:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:54 smithi123 ceph-mon[193903]: pgmap v1080: 99 pgs: 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 15 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 35232/756 objects degraded (4660.317%); 6969/756 objects misplaced (921.825%); 0 B/s, 394 objects/s recovering 2023-09-23T21:02:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3954943590' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:02:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/842615157' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T21:02:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:54 smithi123 ceph-mon[193903]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T21:02:55.061 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:02:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:02:54] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.43.0" 2023-09-23T21:02:55.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3954943590' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:02:55.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/842615157' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T21:02:55.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: osdmap e341: 8 total, 8 up, 7 in 2023-09-23T21:02:55.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: pgmap v1082: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:55.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[211526]: osdmap e342: 8 total, 8 up, 7 in 2023-09-23T21:02:55.422 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.422 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.422 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3954943590' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:02:55.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/842615157' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T21:02:55.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: osdmap e341: 8 total, 8 up, 7 in 2023-09-23T21:02:55.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: pgmap v1082: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:55.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:55 smithi121 ceph-mon[219709]: osdmap e342: 8 total, 8 up, 7 in 2023-09-23T21:02:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3954943590' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:02:55.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/842615157' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T21:02:55.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: osdmap e341: 8 total, 8 up, 7 in 2023-09-23T21:02:55.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: failed to encode map e341 with expected crc 2023-09-23T21:02:55.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: pgmap v1082: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:55.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:55 smithi123 ceph-mon[193903]: osdmap e342: 8 total, 8 up, 7 in 2023-09-23T21:02:56.120 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:02:56.121 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:02:56.121 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-09-23T21:02:56.121 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-09-23T21:02:56.252 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:02:56.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:56.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:02:56.356 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:56.362 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.362 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.362 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.362 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.362 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T21:02:56.363 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35232/756 objects degraded (4660.317%), 14 pgs degraded, 6 pgs undersized) 2023-09-23T21:02:56.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.363 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T21:02:56.364 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:56 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35232/756 objects degraded (4660.317%), 14 pgs degraded, 6 pgs undersized) 2023-09-23T21:02:56.365 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:02:56.386 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: failed to encode map e342 with expected crc 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T21:02:56.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:56 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35232/756 objects degraded (4660.317%), 14 pgs degraded, 6 pgs undersized) 2023-09-23T21:02:56.389 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:56.438 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:56.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:56.495 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:56.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:56.575 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:56.645 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:56.779 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:56.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:56.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:56.909 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:57.059 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:02:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:57.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:02:57.163 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:02:57.195 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:57.240 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:02:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:57.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:02:57.343 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:02:57.376 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:57.425 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:02:57.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:57.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:02:57.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:02:57.560 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:57 smithi121 ceph-mon[211526]: pgmap v1084: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:57 smithi121 ceph-mon[219709]: pgmap v1084: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:57.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:57 smithi123 ceph-mon[193903]: pgmap v1084: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:57.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:02:58.401 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:58.460 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:02:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:02:58.528 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:02:58.568 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:02:58.600 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:58.625 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:58.626 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:58.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:58.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:58.648 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:02:58.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:02:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:02:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:02:58.756 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:58.785 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:58.831 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:58.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:58.936 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:02:58.963 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:59.005 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:02:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:59.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:02:59.112 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:02:59.138 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:59.178 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:02:59.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:02:59.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:02:59.283 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:02:59.316 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:02:59.510 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_6' created 2023-09-23T21:02:59.538 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:02:59 smithi121 ceph-mon[211526]: pgmap v1085: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:59.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:02:59 smithi121 ceph-mon[219709]: pgmap v1085: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:02:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:02:59 smithi123 ceph-mon[193903]: pgmap v1085: 73 pgs: 14 peering, 59 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:00.026 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:00.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:00.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:00.129 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:00.157 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:00.201 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:00.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:00.306 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:00.334 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:00.375 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:00.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:00.481 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:00.506 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:00.513 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:03:00.552 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:00.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:00.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:00.658 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:00.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3124855908' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T21:03:00.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/361240948' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: osdmap e343: 8 total, 8 up, 7 in 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[211526]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3124855908' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/361240948' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: osdmap e343: 8 total, 8 up, 7 in 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:00 smithi121 ceph-mon[219709]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.686 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3124855908' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/361240948' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: osdmap e343: 8 total, 8 up, 7 in 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:00 smithi123 ceph-mon[193903]: failed to encode map e343 with expected crc 2023-09-23T21:03:00.933 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:00.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:01.037 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:01.063 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:01.444 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:03:01.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:03:01.444 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:03:01.444 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T21:03:01.601 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[211526]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[211526]: osdmap e344: 8 total, 8 up, 7 in 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[211526]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[211526]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[211526]: pgmap v1088: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 B/s, 1 objects/s recovering 2023-09-23T21:03:01.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:01.723 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[219709]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.723 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:01.723 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[219709]: osdmap e344: 8 total, 8 up, 7 in 2023-09-23T21:03:01.723 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[219709]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.723 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[219709]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.723 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:01 smithi121 ceph-mon[219709]: pgmap v1088: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 B/s, 1 objects/s recovering 2023-09-23T21:03:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:01 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:01 smithi123 ceph-mon[193903]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:01 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:01 smithi123 ceph-mon[193903]: osdmap e344: 8 total, 8 up, 7 in 2023-09-23T21:03:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:01 smithi123 ceph-mon[193903]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:01 smithi123 ceph-mon[193903]: failed to encode map e344 with expected crc 2023-09-23T21:03:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:01 smithi123 ceph-mon[193903]: pgmap v1088: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 B/s, 1 objects/s recovering 2023-09-23T21:03:02.011 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:02.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:02.094 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:02.094 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:02.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:02.149 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:02.359 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:02.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:02.464 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:02.491 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: failed to encode map e344 with expected crc 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: osdmap e345: 8 total, 8 up, 7 in 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[211526]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: failed to encode map e344 with expected crc 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: osdmap e345: 8 total, 8 up, 7 in 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:02 smithi121 ceph-mon[219709]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: failed to encode map e344 with expected crc 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: osdmap e345: 8 total, 8 up, 7 in 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: failed to encode map e345 with expected crc 2023-09-23T21:03:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:02 smithi123 ceph-mon[193903]: failed to encode map e345 with expected crc 2023-09-23T21:03:03.013 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:03:03.139 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:03.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:03.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:03.196 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:03.196 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:03:03.211 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:03.225 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:03:03.243 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:03.260 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:03:03.275 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:03.282 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:03:03.304 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:03:03.324 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:03:03.326 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:03.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:03.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:03.343 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:03:03.364 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:03:03.366 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.366 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.366 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.366 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.366 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.367 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.367 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.367 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:03:03.431 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:03.458 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:03.502 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:03.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:03.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:03.609 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:03.637 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:03.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:03 smithi121 ceph-mon[211526]: pgmap v1090: 89 pgs: 11 unknown, 78 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 B/s, 1 objects/s recovering 2023-09-23T21:03:03.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2981409922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:03 smithi121 ceph-mon[219709]: pgmap v1090: 89 pgs: 11 unknown, 78 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 B/s, 1 objects/s recovering 2023-09-23T21:03:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2981409922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:03.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:03 smithi123 ceph-mon[193903]: pgmap v1090: 89 pgs: 11 unknown, 78 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 118 B/s, 1 objects/s recovering 2023-09-23T21:03:03.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2981409922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:04.073 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:03:04.593 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:04.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:04.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:04.695 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:04.723 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:04.819 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:04.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:04.901 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:03:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:03:04] "GET /metrics HTTP/1.1" 200 34162 "" "Prometheus/2.43.0" 2023-09-23T21:03:04.922 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:04.944 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:04.953 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:05.267 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:05.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:05.370 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:05.398 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:05.751 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:05.894 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:05.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:05.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:05.998 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:06.030 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:06.079 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:06.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:06 smithi121 ceph-mon[211526]: pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 26 B/s, 0 objects/s recovering 2023-09-23T21:03:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:06 smithi121 ceph-mon[219709]: pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 26 B/s, 0 objects/s recovering 2023-09-23T21:03:06.186 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:06.216 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:06.264 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:06.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:06.325 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 267 (0%) 2023-09-23T21:03:06.326 INFO:teuthology.orchestra.run.smithi121.stdout:avg 38.1429 2023-09-23T21:03:06.326 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 24.908 -> 24.908 (expected baseline 5.71786) 2023-09-23T21:03:06.326 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 2 -> 2 pgs (0.0524345 -> 0.0524345 * mean) 2023-09-23T21:03:06.326 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 69 -> 69 pgs (1.80899 -> 1.80899 * mean) 2023-09-23T21:03:06.327 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:03:06.327 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:03:06.327 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:03:06.327 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:03:06.327 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0077 2023-09-23T21:03:06.327 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0771 2023-09-23T21:03:06.328 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:03:06.368 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:06.395 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:06 smithi123 ceph-mon[193903]: pgmap v1091: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 26 B/s, 0 objects/s recovering 2023-09-23T21:03:06.696 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:03:07.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:07 smithi121 ceph-mon[211526]: from='client.36174 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:07.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:07 smithi121 ceph-mon[211526]: pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:07.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:07 smithi121 ceph-mon[219709]: from='client.36174 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:07.528 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:07 smithi121 ceph-mon[219709]: pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:07.654 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:07.655 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:03:07.656 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T21:03:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:07.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:07.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:07 smithi123 ceph-mon[193903]: from='client.36174 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:07.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:07 smithi123 ceph-mon[193903]: pgmap v1092: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:07.758 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:07.788 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:07.835 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:07.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:07.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:07.941 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:07.967 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.378 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-09-23T21:03:08.416 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:08.523 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:08.555 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.605 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:08.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:08.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4256116303' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:03:08.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4256116303' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:03:08.709 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4256116303' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:03:08.737 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.781 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:08.886 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:08.913 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:08.955 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:08.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:09.062 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:09.080 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:09.094 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.295 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:09.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:09.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[211526]: failed to encode map e346 with expected crc 2023-09-23T21:03:09.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4256116303' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:03:09.380 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[211526]: osdmap e346: 8 total, 8 up, 7 in 2023-09-23T21:03:09.381 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[211526]: failed to encode map e346 with expected crc 2023-09-23T21:03:09.400 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:09.428 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.471 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:09.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:09.518 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:03:09.518 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74228 10.8 10:11677ef2:::benchmark_data_smithi121_358375_object74227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.710247+0000", 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.80589368400001, 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3347776470000001, 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.519 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74228 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710247+0000", 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710247+0000", 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710246+0000", 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710282+0000", 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5995999999999999e-05 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.710293+0000", 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0908e-05 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880012+0000", 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16971977099999999 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880067+0000", 2023-09-23T21:03:09.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4975000000000001e-05 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880189+0000", 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012128700000000001 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880330+0000", 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014141699999999999 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.885871+0000", 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005541308 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899625+0000", 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01375358 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899644+0000", 2023-09-23T21:03:09.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8882e-05 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881429+0000", 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98178518699999995 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881444+0000", 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5021000000000001e-05 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.044904+0000", 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1634600930000001 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.045025+0000", 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012044099999999999 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74235 10.8 10:185a920d:::benchmark_data_smithi121_358375_object74234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880053+0000", 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.63608736399999, 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1684344960000002, 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74235 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.527 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880053+0000", 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880053+0000", 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880052+0000", 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880083+0000", 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1236999999999999e-05 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880095+0000", 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1749999999999999e-05 2023-09-23T21:03:09.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880102+0000", 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4189999999999996e-06 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.882848+0000", 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027460869999999999 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.882982+0000", 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000133292 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.883144+0000", 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016197600000000001 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899953+0000", 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016809233 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.899965+0000", 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2680000000000001e-05 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881491+0000", 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98152574000000004 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881501+0000", 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6569999999999992e-06 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048339+0000", 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.166837954 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048488+0000", 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014888999999999999 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.533 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74237 10.8 10:14c3954e:::benchmark_data_smithi121_358375_object74236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880108+0000", 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.63603261399999, 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1685115160000001, 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.534 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74237 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880108+0000", 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880108+0000", 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880107+0000", 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880132+0000", 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4752999999999998e-05 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880138+0000", 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2709999999999998e-06 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880143+0000", 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.045e-06 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.885893+0000", 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057495690000000004 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.886029+0000", 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013593500000000001 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.886174+0000", 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014497800000000001 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900009+0000", 2023-09-23T21:03:09.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013835439999999999 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900020+0000", 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0975000000000001e-05 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881537+0000", 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151694 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881547+0000", 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5840000000000001e-06 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048531+0000", 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.166983981 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048619+0000", 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8630999999999999e-05 2023-09-23T21:03:09.540 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74240 10.8 10:168ffde2:::benchmark_data_smithi121_358375_object74239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880148+0000", 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.63599252799997, 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.168593628, 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.541 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74240 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880148+0000", 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880148+0000", 2023-09-23T21:03:09.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880148+0000", 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880170+0000", 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2884999999999999e-05 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880177+0000", 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.015e-06 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880182+0000", 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4190000000000001e-06 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.888741+0000", 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085586689999999997 2023-09-23T21:03:09.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.888897+0000", 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015591699999999999 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.889065+0000", 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168944 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900058+0000", 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010992388000000001 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900067+0000", 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4390000000000006e-06 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881583+0000", 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151589500000003 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881592+0000", 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2969999999999992e-06 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048657+0000", 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1670643629999999 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048742+0000", 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4847999999999998e-05 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74248 10.8 10:13ca07d3:::benchmark_data_smithi121_358375_object74247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880233+0000", 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.63590769500001, 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.168624393, 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.548 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74248 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880233+0000", 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880233+0000", 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880232+0000", 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880261+0000", 2023-09-23T21:03:09.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8649999999999998e-05 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880274+0000", 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2418e-05 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880280+0000", 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6159999999999998e-06 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.895079+0000", 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014798412 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.895189+0000", 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110474 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.895291+0000", 2023-09-23T21:03:09.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000101677 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900103+0000", 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048123050000000002 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900112+0000", 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3470000000000005e-06 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881624+0000", 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151193800000003 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881634+0000", 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2459999999999992e-06 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048773+0000", 2023-09-23T21:03:09.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1671391209999999 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048857+0000", 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4610000000000005e-05 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74251 10.8 10:1df39893:::benchmark_data_smithi121_358375_object74250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880285+0000", 2023-09-23T21:03:09.555 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.63585576600002, 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1923718449999998, 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74251 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880285+0000", 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880285+0000", 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880284+0000", 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880308+0000", 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4032999999999998e-05 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880314+0000", 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0329999999999997e-06 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880319+0000", 2023-09-23T21:03:09.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8620000000000002e-06 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900140+0000", 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019820676999999998 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900147+0000", 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.306e-06 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881661+0000", 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98151351099999995 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881669+0000", 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9899999999999997e-06 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.048884+0000", 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1672154159999999 2023-09-23T21:03:09.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.049012+0000", 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012822300000000001 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.049189+0000", 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017644399999999999 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.561 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.057969+0000", 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0087801760000000006 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.057984+0000", 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4987000000000001e-05 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.069380+0000", 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011396007 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.069449+0000", 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9227000000000007e-05 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.072657+0000", 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032074130000000001 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74255 10.8 10:1bd97acb:::benchmark_data_smithi121_358375_object74254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.880323+0000", 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.635817213, 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1955500450000001, 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74255 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.564 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880323+0000", 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880323+0000", 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880323+0000", 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880348+0000", 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5286000000000002e-05 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880354+0000", 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.711e-06 2023-09-23T21:03:09.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.880358+0000", 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4410000000000003e-06 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900178+0000", 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019819973000000001 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900186+0000", 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4220000000000003e-06 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881695+0000", 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98150895500000002 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881703+0000", 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1429999999999998e-06 2023-09-23T21:03:09.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052047+0000", 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1703437619999999 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052194+0000", 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000146883 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.052399+0000", 2023-09-23T21:03:09.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020511699999999999 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.058109+0000", 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057098679999999999 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.059092+0000", 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0009838780000000001 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.072722+0000", 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013629743999999999 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.072795+0000", 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2441999999999995e-05 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.571 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.075873+0000", 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030788339999999999 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:74261 10.8 10:11b87711:::benchmark_data_smithi121_358375_object74260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:29.887012+0000", 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 459.62912849200001, 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1945222019999999, 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.572 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 74261 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887012+0000", 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887012+0000", 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887010+0000", 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887046+0000", 2023-09-23T21:03:09.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5676000000000002e-05 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887061+0000", 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4713e-05 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.887069+0000", 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3310000000000004e-06 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900305+0000", 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013235716 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:29.900313+0000", 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9240000000000007e-06 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881728+0000", 2023-09-23T21:03:09.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98141572399999999 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:30.881736+0000", 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6279999999999996e-06 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.054985+0000", 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1732493509999999 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.055149+0000", 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000164117 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5", 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.055369+0000", 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021944 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.058192+0000", 2023-09-23T21:03:09.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028229190000000001 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.061206+0000", 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030145749999999998 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078474+0000", 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017268002000000001 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.078539+0000", 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4863999999999994e-05 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:32.081534+0000", 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002994994 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.580 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89730 10.9 10.996162f9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885131+0000", 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63100965799998, 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1030085839999999, 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89730 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885131+0000", 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885131+0000", 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885129+0000", 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:09.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885138+0000", 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5590000000000006e-06 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885155+0000", 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.77e-05 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885167+0000", 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1764e-05 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885280+0000", 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011285600000000001 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885293+0000", 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2933e-05 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988127+0000", 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1028341589999999 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988140+0000", 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2449000000000001e-05 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89747 10.9 10.85e9dae9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885181+0000", 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63095946999999, 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102989947, 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89747 2023-09-23T21:03:09.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885181+0000", 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885181+0000", 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885181+0000", 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885184+0000", 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.422e-06 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885191+0000", 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8449999999999997e-06 2023-09-23T21:03:09.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885196+0000", 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5160000000000002e-06 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885337+0000", 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014073699999999999 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885344+0000", 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0820000000000001e-06 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988161+0000", 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102816861 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988171+0000", 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9760000000000007e-06 2023-09-23T21:03:09.590 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89811 10.9 10.259a2b39 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885200+0000", 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63094024399999, 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1030167200000001, 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.591 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89811 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885200+0000", 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885200+0000", 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885200+0000", 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885202+0000", 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8670000000000001e-06 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885206+0000", 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7519999999999998e-06 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885208+0000", 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.717e-06 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885376+0000", 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016799100000000001 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885384+0000", 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1459999999999996e-06 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988206+0000", 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1028212310000001 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988217+0000", 2023-09-23T21:03:09.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1331999999999999e-05 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89822 10.9 10.a18a0aa9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885210+0000", 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.630930089, 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1030388840000001, 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.596 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89822 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885210+0000", 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885210+0000", 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885210+0000", 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885212+0000", 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4959999999999999e-06 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885217+0000", 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4999999999999999e-06 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885222+0000", 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6630000000000001e-06 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885417+0000", 2023-09-23T21:03:09.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019500600000000001 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885423+0000", 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4320000000000004e-06 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988240+0000", 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102816367 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988249+0000", 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6399999999999992e-06 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89838 10.9 10.c26773e9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885286+0000", 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63085485599998, 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.103001481, 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.601 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89838 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885286+0000", 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885286+0000", 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885285+0000", 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885290+0000", 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2100000000000001e-06 2023-09-23T21:03:09.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885300+0000", 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0438999999999999e-05 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885309+0000", 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0940000000000008e-06 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885445+0000", 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000136124 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885452+0000", 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2060000000000001e-06 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988277+0000", 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.102825033 2023-09-23T21:03:09.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988287+0000", 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0288e-05 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89860 10.9 10.69271d69 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885332+0000", 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63080857199998, 2023-09-23T21:03:09.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.10314979, 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89860 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885332+0000", 2023-09-23T21:03:09.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885332+0000", 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885332+0000", 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885334+0000", 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9610000000000001e-06 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885341+0000", 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5030000000000002e-06 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885347+0000", 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7969999999999997e-06 2023-09-23T21:03:09.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885481+0000", 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013390799999999999 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885487+0000", 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4320000000000004e-06 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988458+0000", 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029713109999999 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988482+0000", 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3383e-05 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.611 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89867 10.9 10.1fecf3c9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885351+0000", 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63078975299999, 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1031650850000001, 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89867 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885351+0000", 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885351+0000", 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885351+0000", 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885353+0000", 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2230000000000001e-06 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885358+0000", 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6409999999999999e-06 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885373+0000", 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4381e-05 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885509+0000", 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013643300000000001 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885516+0000", 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6209999999999998e-06 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988505+0000", 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029893470000001 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988516+0000", 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0743e-05 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89889 10.9 10.48cc6ac9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885378+0000", 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63076218700002, 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.103172399, 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89889 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885378+0000", 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885378+0000", 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885378+0000", 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885381+0000", 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7439999999999999e-06 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885387+0000", 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6900000000000003e-06 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885392+0000", 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.425e-06 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885540+0000", 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000147995 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885546+0000", 2023-09-23T21:03:09.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2620000000000004e-06 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988541+0000", 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029948940000001 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988551+0000", 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.842e-06 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.621 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89903 10.9 10.32ebf39 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885421+0000", 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63071973799998, 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1031602439999999, 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89903 2023-09-23T21:03:09.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885421+0000", 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885421+0000", 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885420+0000", 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885423+0000", 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.932e-06 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885431+0000", 2023-09-23T21:03:09.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7179999999999996e-06 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885437+0000", 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6420000000000004e-06 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885794+0000", 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00035752599999999999 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885803+0000", 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1500000000000005e-06 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988572+0000", 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1027685359999999 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988581+0000", 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2560000000000008e-06 2023-09-23T21:03:09.626 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89915 10.9 10.e3da3b99 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885441+0000", 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63069973199998, 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1031719099999999, 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89915 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885441+0000", 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885441+0000", 2023-09-23T21:03:09.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885440+0000", 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885443+0000", 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.413e-06 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885449+0000", 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9800000000000003e-06 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885454+0000", 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7709999999999997e-06 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885829+0000", 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00037520899999999999 2023-09-23T21:03:09.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885837+0000", 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8730000000000007e-06 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988603+0000", 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1027659190000001 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988613+0000", 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0141e-05 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90001 10.9 10.6f241309 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:51.885922+0000", 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 377.63021876099998, 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1029280859999999, 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.632 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90001 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885922+0000", 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885922+0000", 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885922+0000", 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885923+0000", 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.903e-06 2023-09-23T21:03:09.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885929+0000", 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2270000000000001e-06 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885933+0000", 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.279e-06 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.885993+0000", 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0291999999999998e-05 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:51.886003+0000", 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3840000000000003e-06 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988840+0000", 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1028373460000001 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.988850+0000", 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.927e-06 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90540 10.8 10:16053130:::benchmark_data_smithi121_358375_object5795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.937140+0000", 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 354.57900071500001, 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0512755149999999, 2023-09-23T21:03:09.637 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90540 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937140+0000", 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937140+0000", 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937139+0000", 2023-09-23T21:03:09.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:03:09.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937144+0000", 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3750000000000002e-06 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937154+0000", 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8230000000000006e-06 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937159+0000", 2023-09-23T21:03:09.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.242e-06 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937213+0000", 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3680000000000001e-05 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.937224+0000", 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1433999999999999e-05 2023-09-23T21:03:09.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.889907+0000", 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.952682696 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.889921+0000", 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3973e-05 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:09.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.983157+0000", 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0932360480000001 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.983290+0000", 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013333799999999999 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,5,7", 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.983423+0000", 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000132583 2023-09-23T21:03:09.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.986986+0000", 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003562548 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.987037+0000", 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1159000000000003e-05 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:09.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.987300+0000", 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00026364500000000001 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.988288+0000", 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00098737 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.988325+0000", 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7636999999999999e-05 2023-09-23T21:03:09.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:16.988415+0000", 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9988999999999995e-05 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:09.647 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:03:09.647 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:09.650 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:09.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[211526]: pgmap v1094: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[219709]: failed to encode map e346 with expected crc 2023-09-23T21:03:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4256116303' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:03:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[219709]: osdmap e346: 8 total, 8 up, 7 in 2023-09-23T21:03:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[219709]: failed to encode map e346 with expected crc 2023-09-23T21:03:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:09 smithi121 ceph-mon[219709]: pgmap v1094: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:09.682 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.726 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:09 smithi123 ceph-mon[193903]: failed to encode map e346 with expected crc 2023-09-23T21:03:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4256116303' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:03:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:09 smithi123 ceph-mon[193903]: osdmap e346: 8 total, 8 up, 7 in 2023-09-23T21:03:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:09 smithi123 ceph-mon[193903]: failed to encode map e346 with expected crc 2023-09-23T21:03:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:09 smithi123 ceph-mon[193903]: pgmap v1094: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:09.829 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:09.855 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:09.897 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:09.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:09.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:10.003 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:10.031 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:10.110 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-09-23T21:03:10.112 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:03:10.113 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:03:10.199 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T21:03:10.199 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:03:10.199 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_477989 2023-09-23T21:03:10.199 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:03:10.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:03:10.382 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3443234535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:10.382 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:10 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:10.402 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:03:10.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3443234535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:10.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:10 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:10.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3443234535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:10.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:10 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:11.199 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 4299 4283 267.674 267.688 0.0024322 0.00372413 2023-09-23T21:03:11.442 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:11.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:11.547 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:11.596 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T21:03:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[211526]: pgmap v1096: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:11 smithi121 ceph-mon[219709]: pgmap v1096: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:11.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: failed to encode map e347 with expected crc 2023-09-23T21:03:11.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:11 smithi123 ceph-mon[193903]: pgmap v1096: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:03:11.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:03:12.199 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 8103 8087 252.7 237.75 0.00457232 0.00394964 2023-09-23T21:03:12.439 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:12.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:12.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:12.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:12.589 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:13.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 10940 10924 227.563 177.312 0.00394997 0.00424674 2023-09-23T21:03:13.646 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:13.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:03:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:13 smithi123 ceph-mon[193903]: pgmap v1097: 89 pgs: 89 active+clean; 189 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 23 MiB/s wr, 375 op/s 2023-09-23T21:03:13.756 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:13.802 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:13.802 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:13.802 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:03:13.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:13.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[211526]: pgmap v1097: 89 pgs: 89 active+clean; 189 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 23 MiB/s wr, 375 op/s 2023-09-23T21:03:13.803 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:13.803 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:13.803 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:03:13.803 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:13.803 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:13 smithi121 ceph-mon[219709]: pgmap v1097: 89 pgs: 89 active+clean; 189 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 23 MiB/s wr, 375 op/s 2023-09-23T21:03:13.809 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:14.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 13732 13716 214.293 174.5 0.00248948 0.00465945 2023-09-23T21:03:15.167 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:03:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:03:14] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T21:03:15.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 16384 16368 204.581 165.75 0.00596889 0.00488295 2023-09-23T21:03:15.349 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:15.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:15.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:15.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:15.515 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:15.697 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:15.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:15.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:15.706 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:15.804 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:15.808 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:03:15.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:03:15.859 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:15.888 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:03:15.926 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:03:15.952 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:15.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:15.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:15.961 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:03:16.000 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:03:16.045 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:03:16.061 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:16.072 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:03:16.079 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.079 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.079 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.079 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.079 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.079 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.080 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.080 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:03:16.115 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:16.198 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:16.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 18544 18528 192.983 135 0.00748412 0.0051727 2023-09-23T21:03:16.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:16.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:16.218 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:16 smithi121 ceph-mon[211526]: pgmap v1098: 89 pgs: 89 active+clean; 734 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:03:16.218 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2913523158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:16.219 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:16 smithi121 ceph-mon[219709]: pgmap v1098: 89 pgs: 89 active+clean; 734 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:03:16.219 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2913523158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:16.305 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:16.353 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:16 smithi123 ceph-mon[193903]: pgmap v1098: 89 pgs: 89 active+clean; 734 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:03:16.354 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2913523158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:16.358 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:16.549 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:16.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:16.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:16.654 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:16.723 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:16.787 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:16.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:16.902 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:16.961 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:17.050 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:17.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:17.158 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:17.186 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:17.198 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:03:17.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 20696 20680 184.626 134.5 0.00522023 0.00534751 2023-09-23T21:03:17.294 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:17.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:17.369 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:17 smithi123 ceph-mon[193903]: from='client.46169 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:17.369 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]: dispatch 2023-09-23T21:03:17.369 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]: dispatch 2023-09-23T21:03:17.369 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:17 smithi123 ceph-mon[193903]: pgmap v1099: 89 pgs: 89 active+clean; 734 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-09-23T21:03:17.378 INFO:teuthology.orchestra.run.smithi121.stdout:moved 1 / 267 (0.374532%) 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:avg 38.1429 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 24.908 -> 24.7007 (expected baseline 5.71786) 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 2 -> 2 pgs (0.0524345 -> 0.0524345 * mean) 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 69 -> 69 pgs (1.80899 -> 1.80899 * mean) 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0101 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0111 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.1312 -> 0.0937 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.3734 -> 0.3996 2023-09-23T21:03:17.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.1457 -> 0.1504 2023-09-23T21:03:17.404 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:17.451 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:17.605 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:17.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:17.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[211526]: from='client.46169 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]: dispatch 2023-09-23T21:03:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]: dispatch 2023-09-23T21:03:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[211526]: pgmap v1099: 89 pgs: 89 active+clean; 734 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-09-23T21:03:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[219709]: from='client.46169 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]: dispatch 2023-09-23T21:03:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]: dispatch 2023-09-23T21:03:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:17 smithi121 ceph-mon[219709]: pgmap v1099: 89 pgs: 89 active+clean; 734 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s 2023-09-23T21:03:17.713 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:17.754 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:18.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 21746 21730 169.75 65.625 0.00310283 0.00588601 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]': finished 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]': finished 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:18.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:18.713 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:18.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:18.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'6140','4':'26185','5':'9856'}"}]': finished 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: failed to encode map e348 with expected crc 2023-09-23T21:03:18.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:18.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:18.820 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:18.869 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:19.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 23811 23795 165.228 129.062 0.00402961 0.00604698 2023-09-23T21:03:19.503 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:19.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:19.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:19.609 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:19.655 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T21:03:19.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:19 smithi123 ceph-mon[193903]: pgmap v1102: 89 pgs: 89 active+clean; 1.3 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 161 MiB/s wr, 2.58k op/s 2023-09-23T21:03:19.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T21:03:19.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[211526]: pgmap v1102: 89 pgs: 89 active+clean; 1.3 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 161 MiB/s wr, 2.58k op/s 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: failed to encode map e349 with expected crc 2023-09-23T21:03:19.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:19 smithi121 ceph-mon[219709]: pgmap v1102: 89 pgs: 89 active+clean; 1.3 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 161 MiB/s wr, 2.58k op/s 2023-09-23T21:03:19.659 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:19.686 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T21:03:19.687 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T21:03:19.746 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:19.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:19.852 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:19.958 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:20.110 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:20.155 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:20.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:20.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:20.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 25905 25889 161.774 130.875 0.0148923 0.00617674 2023-09-23T21:03:20.263 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:20.263 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:20.271 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:20.343 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:20.517 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:03:20.607 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:20.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:20.713 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:20.793 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:21.022 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:21.206 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 27888 27872 158.274 123.938 0.00537443 0.00631221 2023-09-23T21:03:22.206 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 30313 30297 157.714 151.562 0.00666673 0.00633551 2023-09-23T21:03:22.374 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:22 smithi121 ceph-mon[219709]: pgmap v1103: 89 pgs: 89 active+clean; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 153 MiB/s wr, 2.45k op/s 2023-09-23T21:03:22.374 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:22 smithi121 ceph-mon[211526]: pgmap v1103: 89 pgs: 89 active+clean; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 153 MiB/s wr, 2.45k op/s 2023-09-23T21:03:22.381 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:22.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:22.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:22 smithi123 ceph-mon[193903]: pgmap v1103: 89 pgs: 89 active+clean; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 153 MiB/s wr, 2.45k op/s 2023-09-23T21:03:22.488 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:22.538 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:22.728 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:22.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:22.836 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:22.893 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:22.986 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:22.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:22.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:23.093 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:23.141 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:23.206 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 32396 32380 155.59 130.188 0.00742116 0.00642237 2023-09-23T21:03:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:23 smithi123 ceph-mon[193903]: pgmap v1104: 89 pgs: 89 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T21:03:23.404 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:23.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:23 smithi121 ceph-mon[211526]: pgmap v1104: 89 pgs: 89 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T21:03:23.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:23 smithi121 ceph-mon[219709]: pgmap v1104: 89 pgs: 89 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T21:03:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:23.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:23.449 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:03:23.523 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:23.588 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:23.687 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:23.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:23.792 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:23.796 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:23.824 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:23.907 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:23.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:23.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:24.013 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:24.073 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:24.206 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 34212 34196 152.584 113.5 0.00699715 0.00654858 2023-09-23T21:03:25.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:03:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:03:24] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T21:03:25.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:25.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:25.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 36196 36180 150.678 124 0.00537041 0.00663153 2023-09-23T21:03:25.290 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:25.402 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:25.493 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:25.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:25.601 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:25.660 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:25.917 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:25.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:26.022 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:26.023 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:26 smithi121 ceph-mon[211526]: pgmap v1105: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-09-23T21:03:26.024 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1303566221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:26.048 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:03:26.076 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.175 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:26.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:26.206 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 37756 37740 147.355 97.5 0.00685239 0.00678161 2023-09-23T21:03:26.286 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:26.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:26 smithi123 ceph-mon[193903]: pgmap v1105: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-09-23T21:03:26.336 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1303566221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:26.339 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:26 smithi121 ceph-mon[219709]: pgmap v1105: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-09-23T21:03:26.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1303566221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:26.442 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:26.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:26.552 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:26.603 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.681 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:26.789 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:26.842 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:26.940 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:26.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:26.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:27.049 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:27.104 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:27.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 39699 39683 145.83 121.438 0.00484727 0.00685291 2023-09-23T21:03:27.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:27 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:27.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:27 smithi123 ceph-mon[193903]: failed to encode map e350 with expected crc 2023-09-23T21:03:27.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1303566221' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:27.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:27 smithi123 ceph-mon[193903]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T21:03:27.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:27 smithi123 ceph-mon[193903]: pgmap v1107: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T21:03:27.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[211526]: failed to encode map e350 with expected crc 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1303566221' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[211526]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[211526]: pgmap v1107: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[219709]: failed to encode map e350 with expected crc 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1303566221' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[219709]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T21:03:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:27 smithi121 ceph-mon[219709]: pgmap v1107: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T21:03:28.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 41569 41553 144.222 116.875 0.00706684 0.00692931 2023-09-23T21:03:28.332 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:28.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:28.439 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:28.474 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:28.492 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:28.539 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:28.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:28.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:28.646 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:28.703 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:29.077 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:29.189 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:29.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 15 43184 43169 141.947 101 0.00475726 0.00703866 2023-09-23T21:03:29.239 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:29.321 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:29.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout:moved 6 / 267 (2.24719%) 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout:avg 38.1429 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 24.7007 -> 23.4608 (expected baseline 5.71786) 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 2 -> 2 pgs (0.0524345 -> 0.0524345 * mean) 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 69 -> 68 pgs (1.80899 -> 1.78277 * mean) 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:03:29.389 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:03:29.390 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:03:29.390 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0161 2023-09-23T21:03:29.390 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.1611 2023-09-23T21:03:29.390 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.1504 -> 0.2004 2023-09-23T21:03:29.390 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.0937 -> 0.1189 2023-09-23T21:03:29.390 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 weight 0.3996 -> 0.4495 2023-09-23T21:03:29.428 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:29.478 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:29.529 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:29.533 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:29 smithi123 ceph-mon[193903]: from='client.36216 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:29.533 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:29 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]: dispatch 2023-09-23T21:03:29.533 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:29 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]: dispatch 2023-09-23T21:03:29.533 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:29 smithi123 ceph-mon[193903]: pgmap v1108: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-09-23T21:03:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:29.535 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:03:29.555 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:03:29.573 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:03:29.594 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:03:29.612 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:03:29.631 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:03:29.635 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:29.650 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:03:29.660 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:29.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[211526]: from='client.36216 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:29.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]: dispatch 2023-09-23T21:03:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]: dispatch 2023-09-23T21:03:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[211526]: pgmap v1108: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-09-23T21:03:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[219709]: from='client.36216 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]: dispatch 2023-09-23T21:03:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]: dispatch 2023-09-23T21:03:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:29 smithi121 ceph-mon[219709]: pgmap v1108: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-09-23T21:03:29.668 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:03:29.669 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.669 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.670 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.670 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.670 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.670 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.670 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.670 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:03:29.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:29.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:29.807 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:29.838 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:30.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:03:30.207947+0000 min lat: 0.00123114 max lat: 0.208715 avg lat: 0.00706255 2023-09-23T21:03:30.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:03:30.208 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 43631 43615 136.245 27.875 0.00208513 0.00706255 2023-09-23T21:03:30.436 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:03:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[211526]: failed to encode map e351 with expected crc 2023-09-23T21:03:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[211526]: failed to encode map e351 with expected crc 2023-09-23T21:03:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]': finished 2023-09-23T21:03:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[211526]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T21:03:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1560289037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[219709]: failed to encode map e351 with expected crc 2023-09-23T21:03:30.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[219709]: failed to encode map e351 with expected crc 2023-09-23T21:03:30.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]': finished 2023-09-23T21:03:30.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[219709]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T21:03:30.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1560289037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:30.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:30 smithi123 ceph-mon[193903]: failed to encode map e351 with expected crc 2023-09-23T21:03:30.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:30 smithi123 ceph-mon[193903]: failed to encode map e351 with expected crc 2023-09-23T21:03:30.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:30 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'7793','4':'29461','5':'13132'}"}]': finished 2023-09-23T21:03:30.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:30 smithi123 ceph-mon[193903]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T21:03:30.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1560289037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:31.093 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:31.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:31.199 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:31.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 43631 43615 129.759 0 - 0.00706255 2023-09-23T21:03:31.229 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.284 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:31.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:31.389 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:31.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[219709]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[219709]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T21:03:31.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[219709]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[219709]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[219709]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.419 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.467 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:31.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:31 smithi123 ceph-mon[193903]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:31 smithi123 ceph-mon[193903]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T21:03:31.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:31 smithi123 ceph-mon[193903]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:31 smithi123 ceph-mon[193903]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:31 smithi123 ceph-mon[193903]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.472 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:31 smithi123 ceph-mon[193903]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.473 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:31 smithi123 ceph-mon[193903]: pgmap v1111: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T21:03:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:31.513 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:03:31.514 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T21:03:31.574 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:31.599 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.648 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:31.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[219709]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[219709]: pgmap v1111: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[211526]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[211526]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[211526]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[211526]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[211526]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[211526]: failed to encode map e352 with expected crc 2023-09-23T21:03:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:31 smithi121 ceph-mon[211526]: pgmap v1111: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s 2023-09-23T21:03:31.757 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:31.785 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:31.983 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:31.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:31.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:32.025 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:32.087 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:32.164 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:32.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 43794 43778 124.325 5.09375 0.00392264 0.00803713 2023-09-23T21:03:32.265 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:32.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:32.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:32.307 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:32.373 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:32.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[211526]: failed to encode map e353 with expected crc 2023-09-23T21:03:32.393 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[211526]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T21:03:32.393 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[211526]: failed to encode map e353 with expected crc 2023-09-23T21:03:32.393 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:32.393 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:32.418 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:32 smithi123 ceph-mon[193903]: failed to encode map e353 with expected crc 2023-09-23T21:03:32.418 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:32 smithi123 ceph-mon[193903]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T21:03:32.418 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:32 smithi123 ceph-mon[193903]: failed to encode map e353 with expected crc 2023-09-23T21:03:32.419 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:32 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:32.419 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:32 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:32.431 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:32.486 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:03:32.486 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:03:32.486 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:03:32.486 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:03:32.487 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:03:32.560 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:32.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:32.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:32.657 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[219709]: failed to encode map e353 with expected crc 2023-09-23T21:03:32.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[219709]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T21:03:32.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[219709]: failed to encode map e353 with expected crc 2023-09-23T21:03:32.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:32.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:32 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:32.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:32.742 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:32.879 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:32.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:32.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:32.992 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:33.037 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:33.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 44978 44962 122.138 74 0.00373669 0.00818136 2023-09-23T21:03:33.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:03:33.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:03:33.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:33.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[211526]: pgmap v1113: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:33.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:33 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:03:33.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:33 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:03:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:33 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:33 smithi123 ceph-mon[193903]: pgmap v1113: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:33.622 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:33.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:33.641 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:03:33.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:03:33.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:03:33.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:33.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:33 smithi121 ceph-mon[219709]: pgmap v1113: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:03:33.727 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:33.783 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:33.864 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:33.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:33.975 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:34.032 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:34.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 46007 45991 119.729 64.3125 0.0115604 0.00834703 2023-09-23T21:03:34.512 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:34.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:34.619 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:34.684 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:34.790 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:34.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:34.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:34.906 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:34.923 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:03:34.949 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:35.024 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:35.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:35.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:35.076 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:03:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:03:34] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.43.0" 2023-09-23T21:03:35.076 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:35 smithi121 ceph-mon[211526]: pgmap v1114: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 2594/135663 objects misplaced (1.912%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:03:35.135 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:35.202 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:35.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 47231 47215 117.964 76.5 0.024268 0.0084716 2023-09-23T21:03:35.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:35 smithi123 ceph-mon[193903]: pgmap v1114: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 2594/135663 objects misplaced (1.912%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:03:35.294 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:35.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:35.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:35.402 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:35.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:35 smithi121 ceph-mon[219709]: pgmap v1114: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 2594/135663 objects misplaced (1.912%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:03:35.450 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:36.068 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:36.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:36.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:36.215 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 48686 48670 116.925 90.9375 0.00903937 0.008547 2023-09-23T21:03:36.228 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:36.339 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:36.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:36.445 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:36.496 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:36.509 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:36.542 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:36.583 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:36.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:36.690 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:36.746 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:36.839 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:36.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:36.953 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:37.022 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:37.104 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:37.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:37.215 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 49889 49873 115.379 75.1875 0.010683 0.00866151 2023-09-23T21:03:37.219 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:37.269 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:37.357 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:37.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:37.390 INFO:teuthology.orchestra.run.smithi121.stdout:moved 49 / 267 (18.3521%) 2023-09-23T21:03:37.390 INFO:teuthology.orchestra.run.smithi121.stdout:avg 38.1429 2023-09-23T21:03:37.390 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 23.4608 -> 17.5505 (expected baseline 5.71786) 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 2 -> 4 pgs (0.0524345 -> 0.104869 * mean) 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 68 -> 47 pgs (1.78277 -> 1.23221 * mean) 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0196 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0216 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.2004 -> 0.5202 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 weight 1.0000 -> 0.6751 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 weight 1.0000 -> 0.7159 2023-09-23T21:03:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.1189 -> 0.1965 2023-09-23T21:03:37.470 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:37.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[211526]: from='client.46187 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:37.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]: dispatch 2023-09-23T21:03:37.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]: dispatch 2023-09-23T21:03:37.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[211526]: pgmap v1115: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 26 MiB/s wr, 422 op/s; 2594/135663 objects misplaced (1.912%); 843 KiB/s, 13 objects/s recovering 2023-09-23T21:03:37.496 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[219709]: from='client.46187 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:37.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]: dispatch 2023-09-23T21:03:37.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]: dispatch 2023-09-23T21:03:37.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:37 smithi121 ceph-mon[219709]: pgmap v1115: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 26 MiB/s wr, 422 op/s; 2594/135663 objects misplaced (1.912%); 843 KiB/s, 13 objects/s recovering 2023-09-23T21:03:37.500 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:37 smithi123 ceph-mon[193903]: from='client.46187 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:37 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]: dispatch 2023-09-23T21:03:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:37 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]: dispatch 2023-09-23T21:03:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:37 smithi123 ceph-mon[193903]: pgmap v1115: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 26 MiB/s wr, 422 op/s; 2594/135663 objects misplaced (1.912%); 843 KiB/s, 13 objects/s recovering 2023-09-23T21:03:38.215 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 50132 50116 111.803 15.1875 0.00228865 0.00870745 2023-09-23T21:03:38.404 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:03:38.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[211526]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]': finished 2023-09-23T21:03:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[211526]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T21:03:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[211526]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[211526]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1203278469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:38.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[219709]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]': finished 2023-09-23T21:03:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[219709]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T21:03:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[219709]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[219709]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1203278469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:38 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:38.602 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:38.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:38.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:38.707 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:38.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:38 smithi123 ceph-mon[193903]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:38 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'12878','1':'44241','5':'34095','7':'46915'}"}]': finished 2023-09-23T21:03:38.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:38 smithi123 ceph-mon[193903]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T21:03:38.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:38 smithi123 ceph-mon[193903]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:38 smithi123 ceph-mon[193903]: failed to encode map e354 with expected crc 2023-09-23T21:03:38.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1203278469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:38.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:38 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:03:38.736 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:38.784 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:38.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:38.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:38.891 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:38.920 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:39.215 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 50132 50116 107.949 0 - 0.00870745 2023-09-23T21:03:39.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T21:03:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[219709]: pgmap v1118: 89 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 9 peering, 73 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s; 2194/148062 objects misplaced (1.482%); 4.0 MiB/s, 66 objects/s recovering 2023-09-23T21:03:39.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:39.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:39 smithi121 ceph-mon[211526]: pgmap v1118: 89 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 9 peering, 73 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s; 2194/148062 objects misplaced (1.482%); 4.0 MiB/s, 66 objects/s recovering 2023-09-23T21:03:39.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:39.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:03:39.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T21:03:39.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: failed to encode map e355 with expected crc 2023-09-23T21:03:39.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:39 smithi123 ceph-mon[193903]: pgmap v1118: 89 pgs: 1 active+recovering+undersized+remapped, 6 remapped+peering, 9 peering, 73 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 968 op/s; 2194/148062 objects misplaced (1.482%); 4.0 MiB/s, 66 objects/s recovering 2023-09-23T21:03:40.215 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 50776 50760 105.694 20.125 0.0214039 0.00945474 2023-09-23T21:03:40.658 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:40 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T21:03:40.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:40 smithi123 ceph-mon[193903]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:40 smithi123 ceph-mon[193903]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:40 smithi123 ceph-mon[193903]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T21:03:40.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:40 smithi123 ceph-mon[193903]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:40 smithi123 ceph-mon[193903]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T21:03:40.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[211526]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[211526]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[211526]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T21:03:40.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[211526]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[211526]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T21:03:40.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[219709]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[219709]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[219709]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T21:03:40.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[219709]: failed to encode map e356 with expected crc 2023-09-23T21:03:40.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:40 smithi121 ceph-mon[219709]: failed to encode map e356 with expected crc 2023-09-23T21:03:41.215 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 51579 51563 103.904 50.1875 0.0047697 0.00960651 2023-09-23T21:03:41.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[219709]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[219709]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T21:03:41.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[219709]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[219709]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.405 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[219709]: pgmap v1121: 89 pgs: 1 active+recovering+undersized+remapped, 7 remapped+peering, 9 peering, 72 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2194/150426 objects misplaced (1.459%); 6.0 MiB/s, 100 objects/s recovering 2023-09-23T21:03:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[211526]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[211526]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T21:03:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[211526]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[211526]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:41 smithi121 ceph-mon[211526]: pgmap v1121: 89 pgs: 1 active+recovering+undersized+remapped, 7 remapped+peering, 9 peering, 72 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2194/150426 objects misplaced (1.459%); 6.0 MiB/s, 100 objects/s recovering 2023-09-23T21:03:41.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:41 smithi123 ceph-mon[193903]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:41 smithi123 ceph-mon[193903]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T21:03:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:41 smithi123 ceph-mon[193903]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:41 smithi123 ceph-mon[193903]: failed to encode map e357 with expected crc 2023-09-23T21:03:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:41 smithi123 ceph-mon[193903]: pgmap v1121: 89 pgs: 1 active+recovering+undersized+remapped, 7 remapped+peering, 9 peering, 72 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2194/150426 objects misplaced (1.459%); 6.0 MiB/s, 100 objects/s recovering 2023-09-23T21:03:41.909 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:41.914 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:03:41.915 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-23T21:03:41.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:41.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:42.037 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:42.114 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 52263 52247 101.993 42.75 0.0158475 0.00979882 2023-09-23T21:03:42.224 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:42.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:42.335 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:42.395 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.421 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:42 smithi123 ceph-mon[193903]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.421 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:42 smithi123 ceph-mon[193903]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.421 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:42 smithi123 ceph-mon[193903]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T21:03:42.421 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:42 smithi123 ceph-mon[193903]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89934 10.9 10:99add1d4:::benchmark_data_smithi121_358375_object5189:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992357+0000", 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 409.59387635899998, 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.125213574, 2023-09-23T21:03:42.591 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89934 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992357+0000", 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992360+0000", 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.999e-06 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992365+0000", 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4210000000000003e-06 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992369+0000", 2023-09-23T21:03:42.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8870000000000002e-06 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992624+0000", 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00025508600000000002 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992633+0000", 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1339999999999993e-06 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.982585+0000", 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 17.989951529999999 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:10.991708+0000", 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091229280000000006 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.001314+0000", 2023-09-23T21:03:42.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096056409999999998 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.021104+0000", 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019790161000000001 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.076468+0000", 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.055364728000000002 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.081776+0000", 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053078259999999999 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.086509+0000", 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047326829999999997 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.093298+0000", 2023-09-23T21:03:42.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067891399999999999 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.117571+0000", 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024272848 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89935 10.9 10:99d64049:::benchmark_data_smithi121_358375_object5190:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992376+0000", 2023-09-23T21:03:42.599 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 409.59385787999997, 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.185285084, 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89935 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992376+0000", 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992377+0000", 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.635e-06 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992381+0000", 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2710000000000001e-06 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992383+0000", 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7920000000000001e-06 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992670+0000", 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00028668500000000001 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992679+0000", 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9819999999999997e-06 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.118245+0000", 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.125566417000002 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.129062+0000", 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010817108000000001 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T21:03:42.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.138856+0000", 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097938880000000006 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.155528+0000", 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016671136999999999 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.157350+0000", 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018224879999999999 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.157942+0000", 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00059154200000000002 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.159783+0000", 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018418969999999999 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.162715+0000", 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029311860000000001 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.177661+0000", 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014946291 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.607 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:89981 10.9 10:994f28fe:::benchmark_data_smithi121_358375_object5236:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992398+0000", 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 409.59383604700002, 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.146739247999999, 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89981 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.608 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992398+0000", 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992398+0000", 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992397+0000", 2023-09-23T21:03:42.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992399+0000", 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5099999999999999e-06 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992402+0000", 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1970000000000001e-06 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992405+0000", 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9790000000000002e-06 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992715+0000", 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00030991800000000001 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992724+0000", 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5080000000000005e-06 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.147118+0000", 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.154393974000001 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.153688+0000", 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065708340000000002 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.203940+0000", 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050252012999999998 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.210878+0000", 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069378920000000002 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.987311+0000", 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.77643294500000004 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.992743+0000", 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054319370000000004 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.002286+0000", 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095426230000000001 2023-09-23T21:03:42.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.089643+0000", 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087357120999999996 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.115322+0000", 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025679124000000001 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.118245+0000", 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029228069999999999 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.120479+0000", 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002233938 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.124182+0000", 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037032990000000002 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.139137+0000", 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014954857 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90001 10.9 10:90c824f6:::benchmark_data_smithi121_358375_object5256:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:56:52.992408+0000", 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 409.59382612899998, 2023-09-23T21:03:42.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.173996376000002, 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90001 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992408+0000", 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992408+0000", 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992407+0000", 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992409+0000", 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.449e-06 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992412+0000", 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.853e-06 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992414+0000", 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6070000000000002e-06 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992748+0000", 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000333247 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:56:52.992756+0000", 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8699999999999998e-06 2023-09-23T21:03:42.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.211516+0000", 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.218759336000002 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.220416+0000", 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089002730000000002 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.264440+0000", 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.044023962 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:11.271591+0000", 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071505479999999996 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.038619+0000", 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.76702871699999997 2023-09-23T21:03:42.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.043853+0000", 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052333670000000001 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.053317+0000", 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094646399999999999 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.115885+0000", 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.062567431000000007 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.141137+0000", 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025252508 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.143947+0000", 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028093950000000001 2023-09-23T21:03:42.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.146421+0000", 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024744300000000001 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.149991+0000", 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035700340000000001 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.166404+0000", 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016412930999999999 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90301 10.9 10:95826f21:::benchmark_data_smithi121_358375_object5556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:12.506054+0000", 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 390.08017937099999, 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 23.226904614999999, 2023-09-23T21:03:42.627 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90301 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506054+0000", 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506053+0000", 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506062+0000", 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7669999999999998e-06 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506077+0000", 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4562999999999999e-05 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.506088+0000", 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0779e-05 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.665171+0000", 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15908349199999999 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.672174+0000", 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070033109999999999 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.759608+0000", 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087433832000000003 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:12.770158+0000", 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010549433 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998499+0000", 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2283415890000002 2023-09-23T21:03:42.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998505+0000", 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6810000000000003e-06 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892231+0000", 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372554500000001 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892240+0000", 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4369999999999996e-06 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.600926+0000", 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.708686151999999 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.608105+0000", 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071792979999999998 2023-09-23T21:03:42.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.617691+0000", 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095853580000000004 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.654316+0000", 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036625188000000003 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.709183+0000", 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.054866572000000002 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.712311+0000", 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031283980000000001 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.714254+0000", 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001943005 2023-09-23T21:03:42.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.717912+0000", 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036581460000000001 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.732959+0000", 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015046838 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.637 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90304 10.9 10:95d08781:::benchmark_data_smithi121_358375_object5559:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.000789+0000", 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 389.58544452400002, 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.706967114000001, 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90304 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000789+0000", 2023-09-23T21:03:42.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000787+0000", 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000797+0000", 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6229999999999992e-06 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000815+0000", 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8391999999999999e-05 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.000827+0000", 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1934e-05 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.076647+0000", 2023-09-23T21:03:42.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.075820121000000004 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083118+0000", 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064707690000000003 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.125639+0000", 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.042520998999999997 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.133609+0000", 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079701110000000002 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998466+0000", 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8648562869999998 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998472+0000", 2023-09-23T21:03:42.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8070000000000002e-06 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892193+0000", 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372040100000005 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892202+0000", 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4790000000000002e-06 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.565782+0000", 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.673579972999999 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.572988+0000", 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072056059999999998 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.582601+0000", 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096135609999999996 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.596634+0000", 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014032559999999999 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.598447+0000", 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001812924 2023-09-23T21:03:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.600285+0000", 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018377759999999999 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.676784+0000", 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.076499811000000001 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.680024+0000", 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032395729999999999 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.707756+0000", 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027732273000000002 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90319 10.9 10:9c23793e:::benchmark_data_smithi121_358375_object5574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.072224+0000", 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 389.51400974799998, 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.227369583000002, 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90319 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072224+0000", 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072222+0000", 2023-09-23T21:03:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072231+0000", 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2539999999999992e-06 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072248+0000", 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7422e-05 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.072260+0000", 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2139e-05 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.083953+0000", 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011692759 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.090352+0000", 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063994910000000002 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.175493+0000", 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.085140748000000002 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.187526+0000", 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01203284 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998112+0000", 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8105862510000001 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998120+0000", 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.418e-06 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891970+0000", 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89384994900000003 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891979+0000", 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6320000000000003e-06 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.208032+0000", 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.316052247000002 2023-09-23T21:03:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.217965+0000", 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.009933753 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.229822+0000", 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011856366 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.248481+0000", 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018659464000000001 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.278359+0000", 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.029878236999999998 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.280279+0000", 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019199919999999999 2023-09-23T21:03:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.282082+0000", 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001802367 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.285240+0000", 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031586299999999999 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.299594+0000", 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014353223999999999 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90350 10.9 10:9252fbfb:::benchmark_data_smithi121_358375_object5605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.682468+0000", 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.90376579100001, 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.226020969, 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90350 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682468+0000", 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682466+0000", 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682473+0000", 2023-09-23T21:03:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7800000000000003e-06 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682487+0000", 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.402e-05 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.682496+0000", 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7860000000000002e-06 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.827509+0000", 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.145013317 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.834217+0000", 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067071600000000002 2023-09-23T21:03:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.884874+0000", 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050657455999999997 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.902020+0000", 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017146446999999999 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998607+0000", 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.096586142 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998613+0000", 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7830000000000001e-06 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892350+0000", 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373614700000004 2023-09-23T21:03:42.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892359+0000", 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3430000000000002e-06 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.797647+0000", 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.905288455000001 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.805039+0000", 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0073917929999999998 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.814794+0000", 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097547199999999997 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.846753+0000", 2023-09-23T21:03:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.031959301000000002 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.848628+0000", 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001875172 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.850451+0000", 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018229469999999999 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.887256+0000", 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.036804591999999997 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.890406+0000", 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003149982 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.908489+0000", 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018083054000000001 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90379 10.9 10:951c4b03:::benchmark_data_smithi121_358375_object5634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.928906+0000", 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.65732804200002, 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.839626358, 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90379 2023-09-23T21:03:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928906+0000", 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928904+0000", 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928913+0000", 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5439999999999999e-06 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928927+0000", 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4329e-05 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.928935+0000", 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9729999999999997e-06 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.954600+0000", 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.025665337 2023-09-23T21:03:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.964941+0000", 2023-09-23T21:03:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010340223000000001 2023-09-23T21:03:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.088603+0000", 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12366287400000001 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.095790+0000", 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0071866090000000001 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998534+0000", 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.90274379599999999 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998541+0000", 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1199999999999996e-06 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892274+0000", 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373288500000003 2023-09-23T21:03:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892284+0000", 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9399999999999997e-06 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.625055+0000", 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.732770945999999 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.633921+0000", 2023-09-23T21:03:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088660279999999998 2023-09-23T21:03:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.644084+0000", 2023-09-23T21:03:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010163669 2023-09-23T21:03:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.733850+0000", 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089765882000000005 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.736155+0000", 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023041950000000002 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.738094+0000", 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001939276 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.740271+0000", 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021772879999999999 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.744526+0000", 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042552850000000001 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.768532+0000", 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024005678999999999 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90424 10.9 10:980e8aaf:::benchmark_data_smithi121_358375_object5679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.981543+0000", 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.60469106300002, 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.531567125999999, 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90424 2023-09-23T21:03:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981543+0000", 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981541+0000", 2023-09-23T21:03:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981548+0000", 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1489999999999997e-06 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981561+0000", 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2448e-05 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.981569+0000", 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1499999999999999e-06 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.224643+0000", 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243074701 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.237465+0000", 2023-09-23T21:03:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012821737 2023-09-23T21:03:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.280151+0000", 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.04268632 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.286562+0000", 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064109060000000001 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998360+0000", 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71179806400000001 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998368+0000", 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5050000000000001e-06 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892082+0000", 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89371398099999999 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892091+0000", 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1409999999999994e-06 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.424130+0000", 2023-09-23T21:03:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.532038986 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.435357+0000", 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011226817 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.448022+0000", 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.01266514 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.486130+0000", 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.038108329000000003 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.487944+0000", 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018138589999999999 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.489799+0000", 2023-09-23T21:03:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001854332 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.492508+0000", 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027092589999999999 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.496591+0000", 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040830900000000002 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.513110+0000", 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016518877000000001 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90429 10.9 10:9b165b02:::benchmark_data_smithi121_358375_object5684:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.988639+0000", 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.59759450799999, 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.409704737999999, 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90429 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988639+0000", 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988638+0000", 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988645+0000", 2023-09-23T21:03:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2840000000000001e-06 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988661+0000", 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6053999999999999e-05 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.988669+0000", 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9419999999999995e-06 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.238554+0000", 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24988542499999999 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.246423+0000", 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0078688479999999995 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.311852+0000", 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.065429004999999998 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.318196+0000", 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063438970000000003 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998162+0000", 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.67996552200000004 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998172+0000", 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0208e-05 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892007+0000", 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89383517700000004 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892016+0000", 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4290000000000007e-06 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.301575+0000", 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.409558344000001 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.309095+0000", 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075199719999999998 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.319256+0000", 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010161106 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.376130+0000", 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.056873817 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.378120+0000", 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019904839999999998 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.378762+0000", 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00064167 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.381391+0000", 2023-09-23T21:03:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026288940000000001 2023-09-23T21:03:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.384518+0000", 2023-09-23T21:03:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031269750000000002 2023-09-23T21:03:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.398344+0000", 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013826188 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90434 10.9 10:90493f14:::benchmark_data_smithi121_358375_object5689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.994036+0000", 2023-09-23T21:03:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.59219812999999, 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.989979289000001, 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90434 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994036+0000", 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994034+0000", 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994041+0000", 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4219999999999997e-06 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994052+0000", 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.994059+0000", 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4060000000000002e-06 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.247296+0000", 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25323742700000001 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.254030+0000", 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067339619999999996 2023-09-23T21:03:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.341839+0000", 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087808943 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.348061+0000", 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006222216 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998641+0000", 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65057928700000001 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998645+0000", 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8470000000000003e-06 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892386+0000", 2023-09-23T21:03:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89374099500000004 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892396+0000", 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5119999999999997e-06 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.851132+0000", 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.958735591 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.858452+0000", 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0073205420000000002 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.868309+0000", 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098567020000000002 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.884649+0000", 2023-09-23T21:03:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016340086 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.886658+0000", 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020087680000000002 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.913375+0000", 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026717060000000001 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.962703+0000", 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.049328300999999998 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.965943+0000", 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003240368 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.984015+0000", 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018071474000000001 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90436 10.9 10:925ee57b:::benchmark_data_smithi121_358375_object5691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:13.995691+0000", 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.59054235100001, 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.84919506, 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90436 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995691+0000", 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995690+0000", 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995697+0000", 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5919999999999997e-06 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995714+0000", 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7144999999999999e-05 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:13.995724+0000", 2023-09-23T21:03:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0030000000000001e-05 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.350440+0000", 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.354715958 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.356739+0000", 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062994560000000002 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.388748+0000", 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032009263000000003 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.395007+0000", 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006258726 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998567+0000", 2023-09-23T21:03:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.60355961700000005 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998573+0000", 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8619999999999999e-06 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892312+0000", 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373954700000002 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892321+0000", 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9789999999999999e-06 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.769494+0000", 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.877172975000001 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.779991+0000", 2023-09-23T21:03:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010496959 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.790177+0000", 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010185801 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.822235+0000", 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032058013000000003 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.824316+0000", 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002081252 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.826167+0000", 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001850862 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.828002+0000", 2023-09-23T21:03:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018350759999999999 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.831036+0000", 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003033446 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.844886+0000", 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013850843999999999 2023-09-23T21:03:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90448 10.9 10:9dd18145:::benchmark_data_smithi121_358375_object5703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.027499+0000", 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.55873455599999, 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.248909286, 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90448 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027499+0000", 2023-09-23T21:03:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027497+0000", 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027505+0000", 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0879999999999997e-06 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027521+0000", 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6795000000000001e-05 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.027531+0000", 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3649999999999993e-06 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.359164+0000", 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.331633185 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.365457+0000", 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006293267 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.418540+0000", 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.053082826999999999 2023-09-23T21:03:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.429706+0000", 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011166376 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998075+0000", 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.56836851300000002 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998082+0000", 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5100000000000001e-06 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891929+0000", 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89384610799999997 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891940+0000", 2023-09-23T21:03:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1442e-05 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.172182+0000", 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.280241611000001 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.179816+0000", 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076342859999999997 2023-09-23T21:03:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.193670+0000", 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013853811000000001 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.246965+0000", 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.05329536 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.247710+0000", 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00074513800000000001 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.250763+0000", 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030532549999999999 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.252934+0000", 2023-09-23T21:03:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002170866 2023-09-23T21:03:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.256527+0000", 2023-09-23T21:03:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035923309999999998 2023-09-23T21:03:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.276408+0000", 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019881827000000001 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90471 10.9 10:9fd1a800:::benchmark_data_smithi121_358375_object5726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.106399+0000", 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.479834342, 2023-09-23T21:03:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.062235608000002, 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90471 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T21:03:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106399+0000", 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106398+0000", 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106407+0000", 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9089999999999995e-06 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106423+0000", 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5962999999999999e-05 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.106433+0000", 2023-09-23T21:03:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1589999999999999e-06 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.431330+0000", 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32489747699999999 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.444379+0000", 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013049432999999999 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.479473+0000", 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.035093725999999999 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.485924+0000", 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064507230000000002 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997976+0000", 2023-09-23T21:03:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.51205186199999997 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.997987+0000", 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1669e-05 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891608+0000", 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89362018099999996 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.891623+0000", 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5492e-05 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.072757+0000", 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.181133587000001 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for readable", 2023-09-23T21:03:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.087146+0000", 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014389206 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.107028+0000", 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019882529999999999 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.114642+0000", 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076134410000000003 2023-09-23T21:03:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.124398+0000", 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0097560830000000005 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.146747+0000", 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022348922 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.148740+0000", 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019930780000000001 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.149391+0000", 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00065059799999999995 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.151298+0000", 2023-09-23T21:03:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019071190000000001 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.154367+0000", 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030688310000000002 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.168635+0000", 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014268471 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90489 10.9 10:97f30eb8:::benchmark_data_smithi121_358375_object5744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.210753+0000", 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.375480482, 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.329841758000001, 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90489 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210753+0000", 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210752+0000", 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210761+0000", 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9970000000000004e-06 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210778+0000", 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7181999999999999e-05 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.210789+0000", 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0818e-05 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.446681+0000", 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23589284299999999 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.453972+0000", 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072908900000000004 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.511615+0000", 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057642634999999998 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.518011+0000", 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006395815 2023-09-23T21:03:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998399+0000", 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.480388125 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998406+0000", 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4789999999999999e-06 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892119+0000", 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.893712795 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892129+0000", 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3829999999999998e-06 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.457483+0000", 2023-09-23T21:03:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.565354894999999 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.466620+0000", 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0091370259999999995 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.476805+0000", 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010184886000000001 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.491775+0000", 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.014969623 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.513722+0000", 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.021947326999999999 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.516436+0000", 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027137620000000002 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.519247+0000", 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028114009999999998 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.523691+0000", 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0044435380000000003 2023-09-23T21:03:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.540595+0000", 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016903975000000002 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90505 10.9 10:99aea1ee:::benchmark_data_smithi121_358375_object5760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.268396+0000", 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 388.31783759199999, 2023-09-23T21:03:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.154937368999999, 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90505 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268396+0000", 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268395+0000", 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268402+0000", 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2389999999999996e-06 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268415+0000", 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2850999999999999e-05 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.268422+0000", 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.644e-06 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.520421+0000", 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.251998897 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.526858+0000", 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064371389999999997 2023-09-23T21:03:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.557595+0000", 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.030736329 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.563877+0000", 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006281989 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998305+0000", 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.43442851399999999 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998317+0000", 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1959e-05 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892044+0000", 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89372699 2023-09-23T21:03:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892054+0000", 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3389999999999999e-06 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.328034+0000", 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.435980598 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.336888+0000", 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088543010000000002 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.347140+0000", 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010251873999999999 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.379404+0000", 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032263781999999998 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.400240+0000", 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.020836174999999998 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.402072+0000", 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018316369999999999 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.403897+0000", 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001824579 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.406955+0000", 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030584589999999999 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.423333+0000", 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016378508 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90514 10.9 10:96e73e59:::benchmark_data_smithi121_358375_object5769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.661559+0000", 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 387.924674498, 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 21.014633163999999, 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90514 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661559+0000", 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661558+0000", 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661567+0000", 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.183e-06 2023-09-23T21:03:42.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661583+0000", 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6585000000000001e-05 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.661593+0000", 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6190000000000006e-06 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.825075+0000", 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16348148200000001 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.835611+0000", 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010536495 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.936853+0000", 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10124217200000001 2023-09-23T21:03:42.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.943598+0000", 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067444640000000004 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998433+0000", 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.054835452 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998439+0000", 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6470000000000003e-06 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892156+0000", 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89371742200000004 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892165+0000", 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1449999999999997e-06 2023-09-23T21:03:42.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.541432+0000", 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.649266821000001 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.548683+0000", 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072504270000000003 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.558326+0000", 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0096435710000000001 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.591157+0000", 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.032831143 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.593006+0000", 2023-09-23T21:03:42.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001848948 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.594819+0000", 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001812501 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.654981+0000", 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.060162001999999999 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.658403+0000", 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034224960000000001 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.676192+0000", 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017789158999999999 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90523 10.9 10:976e84cf:::benchmark_data_smithi121_358375_object5778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:14.743128+0000", 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 387.84310573699997, 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 22.046086119999998, 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90523 2023-09-23T21:03:42.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743128+0000", 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743128+0000", 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743126+0000", 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743133+0000", 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0520000000000004e-06 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743149+0000", 2023-09-23T21:03:42.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5480000000000001e-05 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.743156+0000", 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3000000000000004e-06 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998680+0000", 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25552361299999998 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:14.998687+0000", 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1949999999999997e-06 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892421+0000", 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89373391700000004 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:15.892428+0000", 2023-09-23T21:03:42.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5700000000000004e-06 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.875874+0000", 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 19.983445199999998 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.882670+0000", 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006796814 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.985057+0000", 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.102386677 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.993718+0000", 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086608669999999992 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.710912+0000", 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71719427999999996 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.716497+0000", 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0055847630000000004 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.726083+0000", 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095864479999999992 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.763944+0000", 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.037860317999999997 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.766094+0000", 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002150683 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.766698+0000", 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00060379499999999996 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.769891+0000", 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031930330000000001 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.773125+0000", 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032341840000000002 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.789214+0000", 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.016088683999999999 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90564 10.9 10:9972dda2:::benchmark_data_smithi121_358375_object5819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:18.227558+0000", 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 384.35867596200001, 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 18.678191501000001, 2023-09-23T21:03:42.800 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90564 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227558+0000", 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227558+0000", 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227556+0000", 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227562+0000", 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2580000000000001e-06 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227573+0000", 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0593e-05 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.227580+0000", 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4280000000000001e-06 2023-09-23T21:03:42.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:35.994895+0000", 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 17.767315675999999 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.010431+0000", 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015535731000000001 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.049810+0000", 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.039379499999999998 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for rw locks", 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.056645+0000", 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.006834311 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.815438+0000", 2023-09-23T21:03:42.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.758792841 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.821101+0000", 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005663413 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5,7", 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.830907+0000", 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0098062710000000001 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.882525+0000", 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.051617730000000001 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.883121+0000", 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00059645399999999995 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.885048+0000", 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001926456 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.886943+0000", 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018950320000000001 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.890040+0000", 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030973580000000001 2023-09-23T21:03:42.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:36.905749+0000", 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015708976999999999 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:03:42.809 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:03:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:42.810 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:42.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[211526]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[211526]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[211526]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T21:03:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[211526]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.812 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[219709]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.812 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[219709]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.812 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[219709]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T21:03:42.812 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:42 smithi121 ceph-mon[219709]: failed to encode map e358 with expected crc 2023-09-23T21:03:42.812 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:03:42.812 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:03:42.827 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:03:42.868 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:03:42.911 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:03:42.911 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:42.938 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:03:42.973 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:42.974 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:03:42.999 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:03:43.000 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.001 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.001 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.001 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.001 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.001 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.001 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.001 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:03:43.089 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:43.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:43.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:43.204 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:43.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 53226 53210 100.727 60.1875 0.0191622 0.00992228 2023-09-23T21:03:43.287 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:43.399 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:43.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:43.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:43.516 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:43.582 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:43.582 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:43.582 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/610116714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:43.582 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:43 smithi123 ceph-mon[193903]: pgmap v1123: 89 pgs: 1 active+recovering+undersized+remapped, 7 remapped+peering, 81 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 2194/152397 objects misplaced (1.440%); 5.2 MiB/s, 87 objects/s recovering 2023-09-23T21:03:43.596 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:43.714 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:43.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:43.831 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:43.907 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/610116714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[211526]: pgmap v1123: 89 pgs: 1 active+recovering+undersized+remapped, 7 remapped+peering, 81 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 2194/152397 objects misplaced (1.440%); 5.2 MiB/s, 87 objects/s recovering 2023-09-23T21:03:43.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/610116714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:43 smithi121 ceph-mon[219709]: pgmap v1123: 89 pgs: 1 active+recovering+undersized+remapped, 7 remapped+peering, 81 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 2194/152397 objects misplaced (1.440%); 5.2 MiB/s, 87 objects/s recovering 2023-09-23T21:03:43.942 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:03:44.014 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:44.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:44.129 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:44.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 54216 54200 99.5844 61.875 0.0161784 0.0100359 2023-09-23T21:03:44.217 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:44.270 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:44.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:44.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:44.376 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:44.422 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:44.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:03:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:03:44] "GET /metrics HTTP/1.1" 200 36636 "" "Prometheus/2.43.0" 2023-09-23T21:03:44.831 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:44.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:44.869 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:03:44.878 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:44.937 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:45.019 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:45.043 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:45.141 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:45.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:45.217 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 55148 55132 98.4036 58.25 0.0163243 0.0101567 2023-09-23T21:03:45.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:45 smithi123 ceph-mon[193903]: pgmap v1124: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 18877/160875 objects degraded (11.734%); 1820/160875 objects misplaced (1.131%); 4.0 MiB/s, 62 objects/s recovering 2023-09-23T21:03:45.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:45 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 18877/160875 objects degraded (11.734%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:45.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:45 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:03:45.264 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:45.307 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:45.397 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:45 smithi121 ceph-mon[211526]: pgmap v1124: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 18877/160875 objects degraded (11.734%); 1820/160875 objects misplaced (1.131%); 4.0 MiB/s, 62 objects/s recovering 2023-09-23T21:03:45.397 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:45 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 18877/160875 objects degraded (11.734%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:45.397 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:45 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:03:45.398 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:45 smithi121 ceph-mon[219709]: pgmap v1124: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 711 op/s; 18877/160875 objects degraded (11.734%); 1820/160875 objects misplaced (1.131%); 4.0 MiB/s, 62 objects/s recovering 2023-09-23T21:03:45.398 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:45 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 18877/160875 objects degraded (11.734%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:45.398 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:45 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:03:45.631 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:46.217 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 56080 56064 97.2885 58.25 0.00444877 0.0102586 2023-09-23T21:03:46.690 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:46.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:46.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:46.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:46.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:03:46.863 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:47.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 57013 56997 96.2354 58.3125 0.0147853 0.0103853 2023-09-23T21:03:47.445 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:47.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:47.552 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:47.623 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:47.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:47 smithi121 ceph-mon[211526]: pgmap v1125: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 29 MiB/s wr, 459 op/s; 18877/160875 objects degraded (11.734%); 1820/160875 objects misplaced (1.131%); 3.2 MiB/s, 49 objects/s recovering 2023-09-23T21:03:47.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:47 smithi121 ceph-mon[219709]: pgmap v1125: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 29 MiB/s wr, 459 op/s; 18877/160875 objects degraded (11.734%); 1820/160875 objects misplaced (1.131%); 3.2 MiB/s, 49 objects/s recovering 2023-09-23T21:03:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:47 smithi123 ceph-mon[193903]: pgmap v1125: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 29 MiB/s wr, 459 op/s; 18877/160875 objects degraded (11.734%); 1820/160875 objects misplaced (1.131%); 3.2 MiB/s, 49 objects/s recovering 2023-09-23T21:03:47.863 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:47.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:47.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:47.975 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:47.992 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:48.052 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:48.176 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:48.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:48.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 57998 57982 95.3231 61.5625 0.0102681 0.010485 2023-09-23T21:03:48.289 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:48.352 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:48.378 INFO:teuthology.orchestra.run.smithi121.stdout:moved 46 / 267 (17.2285%) 2023-09-23T21:03:48.378 INFO:teuthology.orchestra.run.smithi121.stdout:avg 38.1429 2023-09-23T21:03:48.378 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 17.5505 -> 17.6346 (expected baseline 5.71786) 2023-09-23T21:03:48.378 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.0 with 4 -> 13 pgs (0.104869 -> 0.340824 * mean) 2023-09-23T21:03:48.378 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.6 with 64 -> 63 pgs (1.6779 -> 1.65169 * mean) 2023-09-23T21:03:48.378 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:03:48.378 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0219 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0241 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 weight 0.5202 -> 1.0000 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 weight 0.6751 -> 0.4430 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 weight 0.1965 -> 0.3810 2023-09-23T21:03:48.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 weight 0.7159 -> 0.5063 2023-09-23T21:03:48.394 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:03:48.459 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:48.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:48.567 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:48.632 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:48.666 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:03:48 smithi121 conmon[198376]: 2023-09-23T21:03:48.377+0000 7f479dcb1700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T21:03:48.769 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:48.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:48.877 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:03:48.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:48.934 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:48.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2459670287' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:49.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2459670287' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:49.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2459670287' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:03:49.218 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 58897 58881 94.3148 56.1875 0.0261843 0.0105972 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[219709]: from='client.46202 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[219709]: failed to encode map e359 with expected crc 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[219709]: failed to encode map e359 with expected crc 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2459670287' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[219709]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[219709]: pgmap v1127: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 57 MiB/s wr, 913 op/s; 18877/172353 objects degraded (10.953%); 1496/172353 objects misplaced (0.868%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[211526]: from='client.46202 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:50.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[211526]: failed to encode map e359 with expected crc 2023-09-23T21:03:50.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[211526]: failed to encode map e359 with expected crc 2023-09-23T21:03:50.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2459670287' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:50.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[211526]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T21:03:50.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:49 smithi121 ceph-mon[211526]: pgmap v1127: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 57 MiB/s wr, 913 op/s; 18877/172353 objects degraded (10.953%); 1496/172353 objects misplaced (0.868%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T21:03:50.218 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:03:50.219382+0000 min lat: 0.00123114 max lat: 2.63541 avg lat: 0.0106662 2023-09-23T21:03:50.218 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:03:50.218 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 59792 59776 93.3556 55.9375 0.0122076 0.0106662 2023-09-23T21:03:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:49 smithi123 ceph-mon[193903]: from='client.46202 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:03:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:49 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:03:50.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:49 smithi123 ceph-mon[193903]: failed to encode map e359 with expected crc 2023-09-23T21:03:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:49 smithi123 ceph-mon[193903]: failed to encode map e359 with expected crc 2023-09-23T21:03:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2459670287' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:03:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:49 smithi123 ceph-mon[193903]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T21:03:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:49 smithi123 ceph-mon[193903]: pgmap v1127: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 57 MiB/s wr, 913 op/s; 18877/172353 objects degraded (10.953%); 1496/172353 objects misplaced (0.868%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T21:03:50.467 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:50.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:50.571 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:50.611 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:51.114 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:51.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:51.219 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 60671 60655 92.4188 54.9375 0.0109408 0.010815 2023-09-23T21:03:51.220 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:51.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:51 smithi123 ceph-mon[193903]: pgmap v1128: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 18877/174933 objects degraded (10.791%); 1496/174933 objects misplaced (0.855%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:03:51.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18877/174933 objects degraded (10.791%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:51.285 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:51.388 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:51 smithi121 ceph-mon[219709]: pgmap v1128: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 18877/174933 objects degraded (10.791%); 1496/174933 objects misplaced (0.855%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:03:51.388 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18877/174933 objects degraded (10.791%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:51.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:51 smithi121 ceph-mon[211526]: pgmap v1128: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 53 MiB/s wr, 853 op/s; 18877/174933 objects degraded (10.791%); 1496/174933 objects misplaced (0.855%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:03:51.392 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18877/174933 objects degraded (10.791%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:51.420 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:51.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:51.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:51.530 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:51.576 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:51.653 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:51.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:51.696 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:03:51.696 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:03:51.757 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:51.812 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:51.875 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:51.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:51.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:51.948 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:03:51.972 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:03:51.981 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:52.003 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:03:52.033 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:03:52.046 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:52.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:03:52.117 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:03:52.160 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:03:52.173 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:52.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:52.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:52.219 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 61561 61545 91.5429 55.625 0.00601789 0.0109121 2023-09-23T21:03:52.281 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:03:52.308 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:52.350 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:03:52.358 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:03:52.358 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.358 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.359 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.359 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.359 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.359 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.359 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.359 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:03:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:52.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:03:52.374 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1333538461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:52.474 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:52.546 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:52.623 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:52.628 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1333538461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:52.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:52.630 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1333538461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:03:52.730 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:52.808 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:03:52.834 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:52.887 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:52.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:52.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:52.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:53.020 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:53.071 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:53.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:53.178 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:53.219 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 62129 62113 90.24 35.5 0.0069505 0.0110753 2023-09-23T21:03:53.244 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:53.351 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:53.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:53.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:53.409 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:53.420 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:03:53.461 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:53.540 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:53.618 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:53.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:53.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:53.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:53 smithi121 ceph-mon[219709]: pgmap v1129: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 18877/178374 objects degraded (10.583%); 1496/178374 objects misplaced (0.839%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T21:03:53.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:53 smithi121 ceph-mon[211526]: pgmap v1129: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 18877/178374 objects degraded (10.583%); 1496/178374 objects misplaced (0.839%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T21:03:53.728 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:53.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:53 smithi123 ceph-mon[193903]: pgmap v1129: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 18877/178374 objects degraded (10.583%); 1496/178374 objects misplaced (0.839%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T21:03:53.768 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:53.843 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:53.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:53.951 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:53.995 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:54.021 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:54.080 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:54.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:54.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:54.188 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:54.219 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 63038 63022 89.4804 56.8125 0.0184188 0.0111688 2023-09-23T21:03:54.260 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:54.404 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:54.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:54.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:54.516 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:54.545 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:54.643 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:54.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:54.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:54.747 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:54.837 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:55.045 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:55.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:55.076 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:03:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:03:54] "GET /metrics HTTP/1.1" 200 36636 "" "Prometheus/2.43.0" 2023-09-23T21:03:55.076 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:55 smithi121 ceph-mon[211526]: pgmap v1130: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 18877/187680 objects degraded (10.058%); 1106/187680 objects misplaced (0.589%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:03:55.155 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:03:55.180 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:55 smithi123 ceph-mon[193903]: pgmap v1130: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 18877/187680 objects degraded (10.058%); 1106/187680 objects misplaced (0.589%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:03:55.184 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:55.219 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 63560 63544 88.2174 32.625 0.00712258 0.0112902 2023-09-23T21:03:55.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:03:55.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:55.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:03:55.331 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:03:55.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:55 smithi121 ceph-mon[219709]: pgmap v1130: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 18877/187680 objects degraded (10.058%); 1106/187680 objects misplaced (0.589%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:03:55.437 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:55.503 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:03:55.597 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:03:55.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:55.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:03:55.701 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:03:55.701 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:03:55.701 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:03:55.701 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:03:55.701 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:03:55.705 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:55.780 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:55.970 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:55.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:55.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:56.074 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:56.125 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:03:56.125 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:03:56.125 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:03:56.125 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.358438 2023-09-23T21:03:56.125 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0.35843763613146606 2023-09-23T21:03:56.136 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:56.219 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 64410 64394 87.4546 53.125 0.0190359 0.0114291 2023-09-23T21:03:56.334 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:56.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:56.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:56.437 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:56.443 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:03:56.529 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:56.592 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:03:56.616 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:03:56.621 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18877/187680 objects degraded (10.058%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:56.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18877/187680 objects degraded (10.058%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:56.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:56.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:03:56.654 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18877/187680 objects degraded (10.058%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:03:56.723 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:56.831 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:56.970 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:56.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:57.046 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:03:57.092 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:57.148 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:57.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 65196 65180 86.6382 49.125 0.0106088 0.0115368 2023-09-23T21:03:57.308 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:57.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:57.380 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:03:57.419 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:57.447 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:03:57.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/126268948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:57.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:57.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:57 smithi121 ceph-mon[211526]: pgmap v1131: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 18877/187680 objects degraded (10.058%); 1106/187680 objects misplaced (0.589%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:03:57.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/126268948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:57.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:57 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:57.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:57 smithi121 ceph-mon[219709]: pgmap v1131: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 18877/187680 objects degraded (10.058%); 1106/187680 objects misplaced (0.589%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:03:57.505 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:57.690 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:57.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:57.700 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/126268948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:57.700 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:57 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:03:57.700 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:57 smithi123 ceph-mon[193903]: pgmap v1131: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 18877/187680 objects degraded (10.058%); 1106/187680 objects misplaced (0.589%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:03:57.800 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:57.828 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:58.118 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:58.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:58.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 65945 65929 85.8086 46.8125 0.023397 0.0116482 2023-09-23T21:03:58.231 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:03:58.285 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:58.452 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:58 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:58.453 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:58.453 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:58.453 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:58 smithi123 ceph-mon[193903]: failed to encode map e360 with expected crc 2023-09-23T21:03:58.453 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:58 smithi123 ceph-mon[193903]: failed to encode map e360 with expected crc 2023-09-23T21:03:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[211526]: failed to encode map e360 with expected crc 2023-09-23T21:03:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[211526]: failed to encode map e360 with expected crc 2023-09-23T21:03:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:58.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[211526]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T21:03:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:03:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:03:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:03:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[219709]: failed to encode map e360 with expected crc 2023-09-23T21:03:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[219709]: failed to encode map e360 with expected crc 2023-09-23T21:03:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:58 smithi121 ceph-mon[219709]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T21:03:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:03:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:58 smithi123 ceph-mon[193903]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T21:03:58.824 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:03:58.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:58.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:03:58.932 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:03:59.044 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:59.163 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:03:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:59.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:03:59.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 66830 66814 85.1863 55.3125 0.018288 0.0117337 2023-09-23T21:03:59.268 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:59.359 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:59.466 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:03:59.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:59.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:03:59.574 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:03:59.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:03:59.604 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:03:59.632 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:03:59.648 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:03:59.682 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:03:59.706 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:03:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:03:59.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54857 10.2 10:41d67633:::benchmark_data_smithi121_358375_object54856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.603795+0000", 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 526.11561155000004, 2023-09-23T21:03:59.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25188782500000001, 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54857 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603795+0000", 2023-09-23T21:03:59.724 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603795+0000", 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603794+0000", 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603826+0000", 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1313999999999999e-05 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603834+0000", 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4160000000000004e-06 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.603839+0000", 2023-09-23T21:03:59.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8130000000000003e-06 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.605361+0000", 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015223610000000001 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.605480+0000", 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011848700000000001 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.605621+0000", 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000141676 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.609534+0000", 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039124939999999999 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:59.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855505+0000", 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24597127799999999 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855568+0000", 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2665999999999999e-05 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.855683+0000", 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115369 2023-09-23T21:03:59.729 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:54862 10.2 10:4ea029ac:::benchmark_data_smithi121_358375_object54861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:13.607325+0000", 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 526.11208181999996, 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24949592100000001, 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:03:59.730 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 54862 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607325+0000", 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607325+0000", 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607324+0000", 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607361+0000", 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6653999999999998e-05 2023-09-23T21:03:59.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607373+0000", 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2636000000000001e-05 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607380+0000", 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.426e-06 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607479+0000", 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9697000000000004e-05 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607611+0000", 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013148599999999999 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.607752+0000", 2023-09-23T21:03:59.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014129999999999999 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.610522+0000", 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027701330000000001 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856687+0000", 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.246164834 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856741+0000", 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3551000000000003e-05 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:13.856821+0000", 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0419000000000004e-05 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57382 10.2 10:44e2fe36:::benchmark_data_smithi121_358375_object57381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.685040+0000", 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 524.03436679699996, 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24216987100000001, 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57382 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.737 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685040+0000", 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685040+0000", 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685039+0000", 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685073+0000", 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3844000000000003e-05 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685085+0000", 2023-09-23T21:03:59.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2111e-05 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685091+0000", 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4509999999999997e-06 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685137+0000", 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6076999999999999e-05 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685274+0000", 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013711600000000001 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685418+0000", 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000144029 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:03:59.741 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.689479+0000", 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004060536 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927051+0000", 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.237571753 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927111+0000", 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0189000000000003e-05 2023-09-23T21:03:59.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.927210+0000", 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9098999999999994e-05 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57383 10.2 10:40044bd6:::benchmark_data_smithi121_358375_object57382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.685106+0000", 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 524.03430116499999, 2023-09-23T21:03:59.743 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.248525258, 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57383 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685106+0000", 2023-09-23T21:03:59.744 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685106+0000", 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685105+0000", 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.745 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685134+0000", 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9094999999999998e-05 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685141+0000", 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6680000000000004e-06 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.685146+0000", 2023-09-23T21:03:59.746 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7759999999999997e-06 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.687493+0000", 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023472459999999999 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.687611+0000", 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011831 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.747 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.687740+0000", 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012903599999999999 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.690353+0000", 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026128660000000001 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933529+0000", 2023-09-23T21:03:59.748 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24317556300000001 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933566+0000", 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6981999999999999e-05 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933631+0000", 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5173999999999996e-05 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.749 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57398 10.2 10:4136b738:::benchmark_data_smithi121_358375_object57397:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.694526+0000", 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 524.02488052599995, 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.239204737, 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:59.750 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57398 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694526+0000", 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694526+0000", 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694525+0000", 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694587+0000", 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2389000000000002e-05 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.752 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694611+0000", 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3608000000000002e-05 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694618+0000", 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.939e-06 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694663+0000", 2023-09-23T21:03:59.753 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5444000000000001e-05 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694793+0000", 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012995199999999999 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694923+0000", 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000129803 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.754 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.698840+0000", 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039173840000000003 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933653+0000", 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.234812778 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.755 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933685+0000", 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2156000000000002e-05 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.933731+0000", 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5748000000000003e-05 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:57399 10.2 10:459beae3:::benchmark_data_smithi121_358375_object57398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e235)", 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 524.02477358199997, 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.241389191, 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57399 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.757 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694633+0000", 2023-09-23T21:03:59.758 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694665+0000", 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1838999999999998e-05 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694673+0000", 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.892e-06 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.759 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.694677+0000", 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9289999999999997e-06 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.697304+0000", 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026263620000000001 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.697455+0000", 2023-09-23T21:03:59.760 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015165799999999999 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0", 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.697595+0000", 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000139352 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.699022+0000", 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001426891 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.761 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.935859+0000", 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.23683732599999999 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.935908+0000", 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8865000000000001e-05 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.762 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:15.936023+0000", 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114645 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:43605 11.d e353/352 11:b9421be7:::benchmark_data_smithi121_477989_object43604:head v 353'2707, mlcod=0'0)", 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:32.073861+0000", 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.645545473999999, 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021940576, 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.763 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.073861+0000", 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.073861+0000", 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.764 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.073861+0000", 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.073914+0000", 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3328000000000002e-05 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.765 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.073929+0000", 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5267000000000001e-05 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.073935+0000", 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9229999999999999e-06 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.082002+0000", 2023-09-23T21:03:59.766 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080664719999999999 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.082030+0000", 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8422e-05 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.095760+0000", 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013730213999999999 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.767 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.095802+0000", 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1758000000000002e-05 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:43631 11.d e353/352 11:b7d5beb3:::benchmark_data_smithi121_477989_object43630:head v 353'2710, mlcod=0'0)", 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:32.078104+0000", 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 27.641302743000001, 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024838589000000001, 2023-09-23T21:03:59.768 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.078104+0000", 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.078104+0000", 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.769 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.078096+0000", 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999919 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.078183+0000", 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6971000000000005e-05 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.770 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.078199+0000", 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6293000000000001e-05 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.078205+0000", 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0039999999999996e-06 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.771 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.092232+0000", 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01402684 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.092266+0000", 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4022000000000001e-05 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.102904+0000", 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010637594 2023-09-23T21:03:59.772 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:32.102943+0000", 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8909000000000003e-05 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:45060 11.d e353/352 11:b689308d:::benchmark_data_smithi121_477989_object45059:head v 353'2808, mlcod=347'59)", 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:33.532932+0000", 2023-09-23T21:03:59.773 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 26.186475095999999, 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023110107000000001, 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.532932+0000", 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.774 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.532932+0000", 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.532931+0000", 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.533820+0000", 2023-09-23T21:03:59.775 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00088938899999999998 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.533833+0000", 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.296e-05 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.533839+0000", 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1419999999999998e-06 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.776 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.540939+0000", 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070995340000000002 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.540961+0000", 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2521999999999999e-05 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.777 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.555998+0000", 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015036796 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:33.556042+0000", 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3903999999999998e-05 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.778 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:46531 11.d e353/352 11:bd8b7c3e:::benchmark_data_smithi121_477989_object46530:head v 353'2901, mlcod=347'154)", 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:34.655109+0000", 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 25.064298255000001, 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021077307, 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.655109+0000", 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.779 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.655109+0000", 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.655108+0000", 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.780 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.655144+0000", 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5210999999999999e-05 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.655153+0000", 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4499999999999993e-06 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.655157+0000", 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4089999999999996e-06 2023-09-23T21:03:59.781 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.663684+0000", 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085263600000000002 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.663709+0000", 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5398999999999999e-05 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.782 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.676136+0000", 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012427016000000001 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:34.676186+0000", 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9836999999999997e-05 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.783 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:49932 11.d e353/352 11:b57721e3:::benchmark_data_smithi121_477989_object49931:head v 353'3114, mlcod=347'294)", 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.261063+0000", 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 22.458344457999999, 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020758279000000001, 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.261063+0000", 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.784 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.261063+0000", 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.261062+0000", 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.785 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.261120+0000", 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7846999999999997e-05 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.261127+0000", 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5669999999999997e-06 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.261132+0000", 2023-09-23T21:03:59.786 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5530000000000003e-06 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.262536+0000", 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001403636 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.262556+0000", 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0622e-05 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.281791+0000", 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019234879999999999 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.281821+0000", 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9730000000000002e-05 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.788 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:50763 11.d e356/352 11:b9699dbf:::benchmark_data_smithi121_477989_object50762:head v 356'3174, mlcod=347'648)", 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:40.211485+0000", 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 19.507921964000001, 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023479450999999998, 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211485+0000", 2023-09-23T21:03:59.789 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211485+0000", 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211485+0000", 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.790 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211521+0000", 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5890999999999999e-05 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211527+0000", 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8859999999999998e-06 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211531+0000", 2023-09-23T21:03:59.791 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7110000000000001e-06 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.214866+0000", 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033345480000000001 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.214890+0000", 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3819999999999999e-05 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.792 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.234914+0000", 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020024301000000001 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.234964+0000", 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0594999999999999e-05 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.793 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:50765 11.d e356/352 11:bb474e09:::benchmark_data_smithi121_477989_object50764:head v 356'3175, mlcod=347'648)", 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:40.211605+0000", 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 19.507802286, 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023385040999999999, 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211605+0000", 2023-09-23T21:03:59.794 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211605+0000", 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211604+0000", 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.795 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211691+0000", 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7100000000000003e-05 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211701+0000", 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4660000000000005e-06 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.796 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.211706+0000", 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0119999999999996e-06 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.218017+0000", 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0063109739999999996 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.218039+0000", 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2138e-05 2023-09-23T21:03:59.797 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.234968+0000", 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016929340000000001 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.234990+0000", 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1681000000000001e-05 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.798 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:50769 11.d e356/352 11:b7396e68:::benchmark_data_smithi121_477989_object50768:head v 356'3176, mlcod=347'648)", 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:40.213758+0000", 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 19.505649075000001, 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021256244000000001, 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.799 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.213758+0000", 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.213758+0000", 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.213757+0000", 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.800 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.213832+0000", 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4777000000000006e-05 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.213847+0000", 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5033e-05 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.801 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.213854+0000", 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1130000000000003e-06 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.221208+0000", 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0073544270000000002 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.221231+0000", 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2979000000000001e-05 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.234995+0000", 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013763295 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:40.235014+0000", 2023-09-23T21:03:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.95e-05 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(11.d 359/352 [PushOp(11:bbfdb74e:::benchmark_data_smithi121_477989_object23171:head, version: 349'1437, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:bbfdb74e:::benchmark_data_smithi121_477989_object23171:head@349'1437, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 7.5000835669999999, 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.12563809300000001, 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:03:59.804 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.805 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219386+0000", 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3328000000000001e-05 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219395+0000", 2023-09-23T21:03:59.806 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4940000000000003e-06 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219400+0000", 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4770000000000002e-06 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219450+0000", 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9388999999999998e-05 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.807 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219454+0000", 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4379999999999997e-06 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.344961+0000", 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.12550729599999999 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.808 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:64133 11.d e359/352 11:b918c121:::benchmark_data_smithi121_477989_object64132:head v 359'4014, mlcod=349'1818)", 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:55.919176+0000", 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 3.8002312809999998, 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025187890000000001, 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919176+0000", 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.809 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919176+0000", 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919172+0000", 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999957 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.810 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919275+0000", 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010307899999999999 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919296+0000", 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1319999999999999e-05 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919303+0000", 2023-09-23T21:03:59.811 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4900000000000003e-06 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919369+0000", 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5654000000000005e-05 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.919394+0000", 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4609000000000001e-05 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.812 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.944306+0000", 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024912370999999999 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.944364+0000", 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7439999999999999e-05 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.813 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65111 11.6 e359/343 11:67de6a4e:::benchmark_data_smithi121_477989_object65110:head v 359'3966, mlcod=359'3964)", 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.124216+0000", 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.5951910680000001, 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022943887, 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124216+0000", 2023-09-23T21:03:59.814 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124216+0000", 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124215+0000", 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124252+0000", 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6656000000000003e-05 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124278+0000", 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6525000000000001e-05 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124286+0000", 2023-09-23T21:03:59.816 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.96e-06 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124350+0000", 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3721999999999997e-05 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.124375+0000", 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4975999999999999e-05 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.147123+0000", 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022748517999999999 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.147160+0000", 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6570999999999999e-05 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65113 11.6 e359/343 11:6338a99b:::benchmark_data_smithi121_477989_object65112:head v 359'3967, mlcod=359'3964)", 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.126513+0000", 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.5928940840000001, 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020671002000000001, 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126513+0000", 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126513+0000", 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126512+0000", 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126550+0000", 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8111999999999997e-05 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126564+0000", 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3867e-05 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126571+0000", 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4510000000000003e-06 2023-09-23T21:03:59.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126730+0000", 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015880899999999999 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.126759+0000", 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8839e-05 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.147164+0000", 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020404678999999998 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.147184+0000", 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0117999999999999e-05 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.823 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65195 11.d e359/352 11:b5e1d923:::benchmark_data_smithi121_477989_object65194:head v 359'4080, mlcod=349'1915)", 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.224124+0000", 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.4952833769999998, 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023229402, 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.224124+0000", 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.824 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.224124+0000", 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.224123+0000", 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.224175+0000", 2023-09-23T21:03:59.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2417000000000003e-05 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.224186+0000", 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1039999999999999e-05 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.224193+0000", 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9709999999999998e-06 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.826 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.230042+0000", 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058491740000000004 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.230076+0000", 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3074000000000002e-05 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.247314+0000", 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.017238628999999998 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.247353+0000", 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8826999999999999e-05 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.828 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65273 11.7 e359/343 11:ef6acf60:::benchmark_data_smithi121_477989_object65272:head v 359'4173, mlcod=359'4171)", 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.309470+0000", 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 2.4099374500000001, 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026437103, 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309470+0000", 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.829 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309470+0000", 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309468+0000", 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309517+0000", 2023-09-23T21:03:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8739999999999998e-05 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309533+0000", 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5959000000000001e-05 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309540+0000", 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2880000000000004e-06 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.831 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309604+0000", 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3613000000000002e-05 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.309637+0000", 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2577999999999999e-05 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:03:59.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.335854+0000", 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026216898999999998 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.335907+0000", 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3093000000000001e-05 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:03:59.833 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:03:59.834 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:03:59.834 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:03:59.835 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:03:59 smithi123 ceph-mon[193903]: pgmap v1133: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 18877/196950 objects degraded (9.585%); 596/196950 objects misplaced (0.303%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:03:59.835 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.836 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.837 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.837 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:03:59.846 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:03:59 smithi121 ceph-mon[211526]: pgmap v1133: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 18877/196950 objects degraded (9.585%); 596/196950 objects misplaced (0.303%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:03:59.846 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:03:59 smithi121 ceph-mon[219709]: pgmap v1133: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 18877/196950 objects degraded (9.585%); 596/196950 objects misplaced (0.303%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:03:59.901 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:00.015 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:00.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:00.129 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:00.176 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:00.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 67623 67607 84.474 49.5625 0.00400774 0.0118174 2023-09-23T21:04:00.712 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/739932646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:00.731 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:04:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/739932646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/739932646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:00.924 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:00.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:00.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:00.962 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:04:01.028 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:01.124 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:01.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 68247 68231 83.5826 39 0.0132638 0.0119581 2023-09-23T21:04:01.261 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:01.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:01.376 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:01.475 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:01.481 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:01.576 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:01.630 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:01.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:01.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:01 smithi121 ceph-mon[211526]: pgmap v1134: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 18877/199056 objects degraded (9.483%); 596/199056 objects misplaced (0.299%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:04:01.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18877/196950 objects degraded (9.585%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:01.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:01 smithi121 ceph-mon[219709]: pgmap v1134: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 18877/199056 objects degraded (9.483%); 596/199056 objects misplaced (0.299%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:04:01.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18877/196950 objects degraded (9.585%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:01 smithi123 ceph-mon[193903]: pgmap v1134: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 18877/199056 objects degraded (9.483%); 596/199056 objects misplaced (0.299%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:04:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18877/196950 objects degraded (9.585%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:01.748 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:01.857 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:01.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:01.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:01.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:02.098 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:02.156 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:02.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 69073 69057 82.9681 51.625 0.00483074 0.0120474 2023-09-23T21:04:02.338 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:02.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:02.444 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:02.476 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3673910889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]: dispatch 2023-09-23T21:04:02.477 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:02 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]: dispatch 2023-09-23T21:04:02.496 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.4 primary-affinity to 0.358438 (8234902) 2023-09-23T21:04:02.511 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:02.558 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:02.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:02.661 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:02.686 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:02.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:02.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:02.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:02.832 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:02.869 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3673910889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]: dispatch 2023-09-23T21:04:02.870 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:02 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]: dispatch 2023-09-23T21:04:02.870 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3673910889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]: dispatch 2023-09-23T21:04:02.870 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:02 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]: dispatch 2023-09-23T21:04:02.873 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:03.064 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:04:03.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 69717 69701 82.1623 40.25 0.0104498 0.0121658 2023-09-23T21:04:03.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]': finished 2023-09-23T21:04:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[219709]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T21:04:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[219709]: failed to encode map e361 with expected crc 2023-09-23T21:04:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[219709]: pgmap v1136: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 591 op/s; 18877/201879 objects degraded (9.351%); 596/201879 objects misplaced (0.295%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:04:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]': finished 2023-09-23T21:04:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[211526]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T21:04:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[211526]: failed to encode map e361 with expected crc 2023-09-23T21:04:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:03 smithi121 ceph-mon[211526]: pgmap v1136: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 591 op/s; 18877/201879 objects degraded (9.351%); 596/201879 objects misplaced (0.295%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:04:03.964 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:03.964 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:03.970 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:03.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:03.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:03.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.35843763613146606}]': finished 2023-09-23T21:04:03.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:03 smithi123 ceph-mon[193903]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T21:04:03.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:03 smithi123 ceph-mon[193903]: failed to encode map e361 with expected crc 2023-09-23T21:04:03.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:03 smithi123 ceph-mon[193903]: pgmap v1136: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 591 op/s; 18877/201879 objects degraded (9.351%); 596/201879 objects misplaced (0.295%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:04:04.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:04.126 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:04.207 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:04.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 70395 70379 81.4256 42.375 0.0284129 0.0122754 2023-09-23T21:04:04.296 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:04.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:04.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:04.401 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:04.456 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:04.537 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:04.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:04.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[211526]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.568 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[211526]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.568 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[211526]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T21:04:04.568 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[211526]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[219709]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[219709]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[219709]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T21:04:04.569 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:04 smithi121 ceph-mon[219709]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.645 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:04.729 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:04.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:04 smithi123 ceph-mon[193903]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:04 smithi123 ceph-mon[193903]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:04 smithi123 ceph-mon[193903]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T21:04:04.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:04 smithi123 ceph-mon[193903]: failed to encode map e362 with expected crc 2023-09-23T21:04:04.878 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:04:04.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:04:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:04:04] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T21:04:05.052 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:05.160 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:05.187 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.220 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 71222 71206 80.885 51.6875 0.0321875 0.0123448 2023-09-23T21:04:05.229 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:05.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:05.332 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:05.409 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.420 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:05.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2376478727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:05.497 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:05.497 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:05 smithi121 ceph-mon[211526]: pgmap v1138: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s; 18877/209853 objects degraded (8.995%); 155/209853 objects misplaced (0.074%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:04:05.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2376478727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:05.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:05.497 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:05 smithi121 ceph-mon[219709]: pgmap v1138: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s; 18877/209853 objects degraded (8.995%); 155/209853 objects misplaced (0.074%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:04:05.516 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:05.530 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:04:05.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:05.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:05.688 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2376478727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:05.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:05.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:05 smithi123 ceph-mon[193903]: pgmap v1138: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 980 op/s; 18877/209853 objects degraded (8.995%); 155/209853 objects misplaced (0.074%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:04:05.874 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:05.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:05.979 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:06.051 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:06.221 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 71752 71736 80.0323 33.125 0.0166408 0.0124862 2023-09-23T21:04:06.326 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:06.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:06.433 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:06.487 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:06.506 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:06.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:06.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[211526]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T21:04:06.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[211526]: failed to encode map e363 with expected crc 2023-09-23T21:04:06.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[211526]: failed to encode map e363 with expected crc 2023-09-23T21:04:06.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18877/209853 objects degraded (8.995%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:06.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[211526]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T21:04:06.507 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:06.508 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:06.508 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[219709]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T21:04:06.508 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[219709]: failed to encode map e363 with expected crc 2023-09-23T21:04:06.508 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[219709]: failed to encode map e363 with expected crc 2023-09-23T21:04:06.508 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18877/209853 objects degraded (8.995%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:06.508 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:06 smithi121 ceph-mon[219709]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T21:04:06.598 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:06.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:06 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:06.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:06 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:06.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:06 smithi123 ceph-mon[193903]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T21:04:06.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:06 smithi123 ceph-mon[193903]: failed to encode map e363 with expected crc 2023-09-23T21:04:06.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:06 smithi123 ceph-mon[193903]: failed to encode map e363 with expected crc 2023-09-23T21:04:06.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18877/209853 objects degraded (8.995%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:06.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:06 smithi123 ceph-mon[193903]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T21:04:06.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:06.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:06.717 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:06.769 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:06.961 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:04:06.983 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:04:07.005 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:04:07.027 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:04:07.049 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:04:07.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:04:07.092 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:04:07.114 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:04:07.115 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.115 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.116 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:04:07.126 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:07.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:07.221 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 72248 72232 79.1723 31 0.0031081 0.0125083 2023-09-23T21:04:07.231 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:07.261 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:07.359 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:07.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:07.464 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:07.514 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:07.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:07.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:07.690 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:04:07.690 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:04:07.690 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:04:07.690 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:04:07.690 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:04:07.701 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:07.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2164703161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:07.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: pgmap v1141: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 18877/209853 objects degraded (8.995%); 155/209853 objects misplaced (0.074%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T21:04:07.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:07 smithi123 ceph-mon[193903]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T21:04:07.774 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:07.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2164703161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: pgmap v1141: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 18877/209853 objects degraded (8.995%); 155/209853 objects misplaced (0.074%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[219709]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: failed to encode map e364 with expected crc 2023-09-23T21:04:07.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2164703161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:07.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: pgmap v1141: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 18877/209853 objects degraded (8.995%); 155/209853 objects misplaced (0.074%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T21:04:07.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:07 smithi121 ceph-mon[211526]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T21:04:07.976 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:07.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:07.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:08.080 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:08.118 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:08.223 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 72793 72777 78.3911 34.0625 0.0221749 0.0127513 2023-09-23T21:04:08.513 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:08.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:08.562 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:04:08.622 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:08.676 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:08.772 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:08.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:08 smithi121 ceph-mon[219709]: failed to encode map e365 with expected crc 2023-09-23T21:04:08.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:08 smithi121 ceph-mon[219709]: failed to encode map e365 with expected crc 2023-09-23T21:04:08.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:08.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:08.782 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:08 smithi121 ceph-mon[211526]: failed to encode map e365 with expected crc 2023-09-23T21:04:08.782 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:08 smithi121 ceph-mon[211526]: failed to encode map e365 with expected crc 2023-09-23T21:04:08.834 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:08 smithi123 ceph-mon[193903]: failed to encode map e365 with expected crc 2023-09-23T21:04:08.834 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:08 smithi123 ceph-mon[193903]: failed to encode map e365 with expected crc 2023-09-23T21:04:08.835 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:04:08.883 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:08.940 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:08.984 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:04:08.984 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:04:08.984 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T21:04:08.984 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:04:08.984 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:04:08.985 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T21:04:09.089 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:09.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:09.159 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:09.207 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:09.225 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 73623 73607 77.9392 51.875 0.0210833 0.0128239 2023-09-23T21:04:09.272 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:09.363 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:09.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:09.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:09.467 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:09.601 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:09.619 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:09.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:09 smithi121 ceph-mon[211526]: pgmap v1143: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s; 15740/217917 objects degraded (7.223%); 2942/217917 objects misplaced (1.350%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T21:04:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:09 smithi121 ceph-mon[219709]: pgmap v1143: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s; 15740/217917 objects degraded (7.223%); 2942/217917 objects misplaced (1.350%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T21:04:09.751 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:09.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:09.854 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:09.946 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:09 smithi123 ceph-mon[193903]: pgmap v1143: 89 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s; 15740/217917 objects degraded (7.223%); 2942/217917 objects misplaced (1.350%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T21:04:09.949 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:10.084 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:10.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:10.192 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:10.226 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:04:10.226467+0000 min lat: 0.00123114 max lat: 2.63541 avg lat: 0.0129076 2023-09-23T21:04:10.226 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:04:10.226 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 74394 74378 77.4434 48.1875 0.0108391 0.0129076 2023-09-23T21:04:10.237 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:10.321 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:10.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:10.431 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:10.476 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:10.529 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:04:10.558 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:10.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:10.666 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:10.731 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:04:10 smithi123 conmon[172845]: logger=cleanup t=2023-09-23T21:04:10.439208626Z level=info msg="Completed cleanup jobs" duration=190.607661ms 2023-09-23T21:04:10.741 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:11.041 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:11.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:11.147 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:11.181 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:11.184 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:11.231 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 75201 75185 76.9935 50.4375 0.0226917 0.012982 2023-09-23T21:04:11.287 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:11.297 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:11 smithi121 ceph-mon[219709]: pgmap v1144: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 540 op/s; 12657/219594 objects degraded (5.764%); 5855/219594 objects misplaced (2.666%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:04:11.298 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:11 smithi121 ceph-mon[211526]: pgmap v1144: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 540 op/s; 12657/219594 objects degraded (5.764%); 5855/219594 objects misplaced (2.666%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:04:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:11.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:11.336 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:11.400 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:11.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:11 smithi123 ceph-mon[193903]: pgmap v1144: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 540 op/s; 12657/219594 objects degraded (5.764%); 5855/219594 objects misplaced (2.666%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:04:11.470 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:11.626 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:11.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:11.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:11.841 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:12.072 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:12.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:12.096 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12657/219594 objects degraded (5.764%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:12.186 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:12.231 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 75943 75927 76.4998 46.375 0.0117725 0.0130659 2023-09-23T21:04:12.257 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:12.346 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12657/219594 objects degraded (5.764%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:12.406 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:04:12.406 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:04:12.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:04:12.406 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:04:12.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.406 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74225 10.9 10:9b7c5c1b:::benchmark_data_smithi121_358375_object74224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.710286+0000", 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.69264896000004, 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3339274059999999, 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74225 2023-09-23T21:04:12.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710286+0000", 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710286+0000", 2023-09-23T21:04:12.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710282+0000", 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999957 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710348+0000", 2023-09-23T21:04:12.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6082999999999997e-05 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710359+0000", 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1491000000000001e-05 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710364+0000", 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.206e-06 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710407+0000", 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3637000000000003e-05 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710560+0000", 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015250499999999999 2023-09-23T21:04:12.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.710732+0000", 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017187699999999999 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.718766+0000", 2023-09-23T21:04:12.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080340979999999999 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.727854+0000", 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0090881409999999992 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885871+0000", 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.15801679499999999 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885883+0000", 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1973999999999999e-05 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899218+0000", 2023-09-23T21:04:12.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.013335289 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899229+0000", 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1124000000000001e-05 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.044111+0000", 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1448817710000001 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.044214+0000", 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102935 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74232 10.9 10:9163f21b:::benchmark_data_smithi121_358375_object74231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.716015+0000", 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.68692058700003, 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.329761704, 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:12.417 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74232 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716015+0000", 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716015+0000", 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716013+0000", 2023-09-23T21:04:12.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716234+0000", 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022076299999999999 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716248+0000", 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3532e-05 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716259+0000", 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1138000000000001e-05 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716302+0000", 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2920000000000002e-05 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716418+0000", 2023-09-23T21:04:12.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115862 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.716541+0000", 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012331199999999999 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.718820+0000", 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022795559999999999 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.729911+0000", 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011090645 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885918+0000", 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.15600684300000001 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.885925+0000", 2023-09-23T21:04:12.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3590000000000002e-06 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899269+0000", 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0133434050000001 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899275+0000", 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0410000000000003e-06 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045708+0000", 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.146433093 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045776+0000", 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8604999999999998e-05 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74256 10.9 10:9d9325d5:::benchmark_data_smithi121_358375_object74255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.874623+0000", 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.52831205300004, 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1713063940000001, 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74256 2023-09-23T21:04:12.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874623+0000", 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874623+0000", 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874622+0000", 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874668+0000", 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5856000000000003e-05 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874680+0000", 2023-09-23T21:04:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2401999999999999e-05 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874685+0000", 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6380000000000003e-06 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874732+0000", 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6709999999999998e-05 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.874873+0000", 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014162800000000001 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.875031+0000", 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015807899999999999 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:12.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.877600+0000", 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025683099999999999 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.878245+0000", 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00064534499999999997 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.892611+0000", 2023-09-23T21:04:12.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014365562 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.892621+0000", 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0226e-05 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899301+0000", 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0066803090000001 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899307+0000", 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6520000000000003e-06 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045830+0000", 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.146523298 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045930+0000", 2023-09-23T21:04:12.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9457999999999995e-05 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74271 10.9 10:993b4897:::benchmark_data_smithi121_358375_object74270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.909039+0000", 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.49389654399999, 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1620222980000001, 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.434 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74271 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909039+0000", 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909039+0000", 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909037+0000", 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909098+0000", 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0850999999999997e-05 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909118+0000", 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0117999999999999e-05 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909126+0000", 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2110000000000001e-06 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909186+0000", 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0157e-05 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.909198+0000", 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1755e-05 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899327+0000", 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.990129386 2023-09-23T21:04:12.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899333+0000", 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6910000000000002e-06 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.045959+0000", 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.14662636 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.046111+0000", 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015205500000000001 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.046294+0000", 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000182938 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.056140+0000", 2023-09-23T21:04:12.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0098462289999999997 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061791+0000", 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056506589999999997 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.067344+0000", 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055534979999999996 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.067403+0000", 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9017000000000001e-05 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071061+0000", 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036576600000000001 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74276 10.9 10:916b3091:::benchmark_data_smithi121_358375_object74275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.915396+0000", 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.48753891599995, 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.159243085, 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74276 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.443 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915396+0000", 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915396+0000", 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915395+0000", 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915445+0000", 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9669000000000002e-05 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915456+0000", 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1321000000000001e-05 2023-09-23T21:04:12.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915461+0000", 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0590000000000002e-06 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915508+0000", 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7253999999999999e-05 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915517+0000", 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1599999999999998e-06 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899353+0000", 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98383633000000004 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899359+0000", 2023-09-23T21:04:12.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3860000000000004e-06 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048395+0000", 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1490360150000001 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048548+0000", 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015308499999999999 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.048724+0000", 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017566700000000001 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061802+0000", 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013078227 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061864+0000", 2023-09-23T21:04:12.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2067000000000001e-05 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071113+0000", 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092486760000000008 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.071163+0000", 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9515999999999998e-05 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074639+0000", 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003476884 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74277 10.9 10:902456f8:::benchmark_data_smithi121_358375_object74276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.915870+0000", 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.48706480600003, 2023-09-23T21:04:12.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1624961859999998, 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74277 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915870+0000", 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915934+0000", 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4109999999999994e-05 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915944+0000", 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7089999999999997e-06 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915948+0000", 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.33e-06 2023-09-23T21:04:12.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915982+0000", 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4347000000000002e-05 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.915990+0000", 2023-09-23T21:04:12.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8930000000000005e-06 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899380+0000", 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98338946299999996 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899385+0000", 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4439999999999999e-06 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.050951+0000", 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1515660750000001 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.051065+0000", 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113613 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:04:12.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.051231+0000", 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016582399999999999 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061811+0000", 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010579965 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061902+0000", 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1799000000000007e-05 2023-09-23T21:04:12.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074825+0000", 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012922417 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.074866+0000", 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0949999999999999e-05 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078367+0000", 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035008359999999998 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:74279 10.9 10:9ea30411:::benchmark_data_smithi121_358375_object74278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:55:29.920027+0000", 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.48290819099998, 2023-09-23T21:04:12.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.161850388, 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 74279 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.461 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920027+0000", 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920027+0000", 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920026+0000", 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920073+0000", 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6397000000000003e-05 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920085+0000", 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.287e-05 2023-09-23T21:04:12.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920091+0000", 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1490000000000002e-06 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920130+0000", 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9684999999999997e-05 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:29.920138+0000", 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5689999999999999e-06 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899415+0000", 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97927704699999996 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:30.899421+0000", 2023-09-23T21:04:12.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9020000000000004e-06 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053407+0000", 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1539860749999999 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053558+0000", 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000151216 2023-09-23T21:04:12.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.053723+0000", 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165048 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061817+0000", 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080936069999999992 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.061971+0000", 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000154582 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078458+0000", 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016486923000000001 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.078518+0000", 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0068000000000002e-05 2023-09-23T21:04:12.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:55:32.081877+0000", 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033591329999999998 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.45470.0:110371 10.a e310/285 10:5ba2eb9b:::benchmark_data_smithi121_358375_object25626:head v 310'6855, mlcod=231'751)", 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:00:12.617320+0000", 2023-09-23T21:04:12.469 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 239.78561494300001, 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032769197999999999, 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617320+0000", 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617320+0000", 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617319+0000", 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617328+0000", 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4069999999999993e-06 2023-09-23T21:04:12.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617343+0000", 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5352000000000001e-05 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617354+0000", 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.132e-05 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617407+0000", 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2920000000000002e-05 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.617436+0000", 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8853999999999999e-05 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.650046+0000", 2023-09-23T21:04:12.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032609856999999999 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:00:12.650090+0000", 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3426000000000001e-05 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.474 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:62939 11.a e359/355 11:58d53990:::benchmark_data_smithi121_477989_object62938:head v 359'3883, mlcod=0'0)", 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:54.110710+0000", 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 18.292225565999999, 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026481767, 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.110710+0000", 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.110710+0000", 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.110709+0000", 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.110734+0000", 2023-09-23T21:04:12.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4885999999999999e-05 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.110748+0000", 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4124999999999999e-05 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.110753+0000", 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3319999999999999e-06 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.114920+0000", 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041660910000000002 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.114953+0000", 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3358000000000001e-05 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.137137+0000", 2023-09-23T21:04:12.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022183886 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.137191+0000", 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4644000000000001e-05 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:62944 11.a e359/355 11:5700d05a:::benchmark_data_smithi121_477989_object62943:head v 359'3884, mlcod=0'0)", 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:54.123634+0000", 2023-09-23T21:04:12.479 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 18.279300915, 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025255829, 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.123634+0000", 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.123634+0000", 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.123634+0000", 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.123654+0000", 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9808000000000001e-05 2023-09-23T21:04:12.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.123660+0000", 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0780000000000001e-06 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.123664+0000", 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8519999999999997e-06 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.127892+0000", 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004228552 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.127919+0000", 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6996e-05 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.148678+0000", 2023-09-23T21:04:12.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020758281999999999 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:54.148890+0000", 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021260300000000001 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:63696 11.3 e359/355 11:cb46dcaa:::benchmark_data_smithi121_477989_object63695:head v 359'4083, mlcod=0'0)", 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:55.412278+0000", 2023-09-23T21:04:12.484 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 16.990656842, 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031193292000000001, 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412278+0000", 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412278+0000", 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412277+0000", 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412308+0000", 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0060999999999999e-05 2023-09-23T21:04:12.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412323+0000", 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5546999999999999e-05 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412330+0000", 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7819999999999996e-06 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.417651+0000", 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053208609999999996 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.417717+0000", 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6445000000000005e-05 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.443411+0000", 2023-09-23T21:04:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025693822000000002 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.443472+0000", 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0713e-05 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:63698 11.3 e359/355 11:cc866dc1:::benchmark_data_smithi121_477989_object63697:head v 359'4084, mlcod=0'0)", 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:55.412340+0000", 2023-09-23T21:04:12.489 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 16.990595171999999, 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031156745999999999, 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412340+0000", 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412340+0000", 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412340+0000", 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412396+0000", 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5878999999999998e-05 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412406+0000", 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9790000000000005e-06 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.412410+0000", 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5589999999999999e-06 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.423021+0000", 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010610682 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.423051+0000", 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0226000000000002e-05 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.443476+0000", 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020424827 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:55.443497+0000", 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0891e-05 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:64688 11.a e359/355 11:57db8fd0:::benchmark_data_smithi121_477989_object64687:head v 359'3999, mlcod=0'0)", 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:56.598374+0000", 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.804560839000001, 2023-09-23T21:04:12.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.035253815000000001, 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.598374+0000", 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.598374+0000", 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.598374+0000", 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.598398+0000", 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4085000000000001e-05 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.598405+0000", 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.776e-06 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.598410+0000", 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4680000000000003e-06 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.607175+0000", 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0087655770000000001 2023-09-23T21:04:12.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.607198+0000", 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.34e-05 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.633581+0000", 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026382790999999999 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.633628+0000", 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6959000000000002e-05 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:64701 11.a e359/355 11:537b53f8:::benchmark_data_smithi121_477989_object64700:head v 359'4000, mlcod=0'0)", 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:56.605984+0000", 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.796951039, 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027668635, 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.499 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.605984+0000", 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.605984+0000", 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.605983+0000", 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.606009+0000", 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5626000000000002e-05 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.606026+0000", 2023-09-23T21:04:12.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6900000000000001e-05 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.606032+0000", 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7750000000000004e-06 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.611452+0000", 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005419208 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.611477+0000", 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5310999999999999e-05 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.633632+0000", 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.02215541 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:56.633653+0000", 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0491000000000001e-05 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65265 11.3 e359/355 11:c932d248:::benchmark_data_smithi121_477989_object65264:head v 359'4181, mlcod=0'0)", 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.304417+0000", 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.098518513, 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032135150000000001, 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.504 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304417+0000", 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304417+0000", 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304415+0000", 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304443+0000", 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7169999999999999e-05 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304469+0000", 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5922999999999999e-05 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304476+0000", 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5329999999999997e-06 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304539+0000", 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3311000000000004e-05 2023-09-23T21:04:12.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.304565+0000", 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5449999999999999e-05 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.336508+0000", 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031943088000000001 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.336552+0000", 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3964000000000001e-05 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65267 11.3 e359/355 11:c0024337:::benchmark_data_smithi121_477989_object65266:head v 359'4182, mlcod=0'0)", 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.311947+0000", 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.090988197, 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024626466, 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.509 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311947+0000", 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311947+0000", 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311947+0000", 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311973+0000", 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6230000000000001e-05 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311982+0000", 2023-09-23T21:04:12.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2590000000000006e-06 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311986+0000", 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8929999999999998e-06 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.318939+0000", 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0069524390000000004 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.318965+0000", 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6222999999999999e-05 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.336555+0000", 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.017590306 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.336574+0000", 2023-09-23T21:04:12.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8376000000000002e-05 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65275 11.3 e359/355 11:c6d453c2:::benchmark_data_smithi121_477989_object65274:head v 359'4183, mlcod=0'0)", 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.311992+0000", 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.090943547, 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024600647, 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.514 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311992+0000", 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311992+0000", 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.311991+0000", 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.312012+0000", 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0191999999999999e-05 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.312020+0000", 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0919999999999998e-06 2023-09-23T21:04:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.312023+0000", 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3160000000000001e-06 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.323283+0000", 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011259899 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.323306+0000", 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2727000000000001e-05 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.336576+0000", 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013270534 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.336592+0000", 2023-09-23T21:04:12.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6121e-05 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:65282 11.3 e359/355 11:caa96743:::benchmark_data_smithi121_477989_object65281:head v 359'4185, mlcod=0'0)", 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:57.316973+0000", 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 15.085962186, 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024357275000000001, 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.519 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.316973+0000", 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.316973+0000", 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.520 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.316972+0000", 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.316998+0000", 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5216e-05 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.317015+0000", 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7034999999999999e-05 2023-09-23T21:04:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.317021+0000", 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4280000000000001e-06 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.331799+0000", 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014778003 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.522 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.331827+0000", 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7922000000000001e-05 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.341287+0000", 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0094597150000000005 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:57.341330+0000", 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3639e-05 2023-09-23T21:04:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:66504 11.3 e360/355 11:cabe9912:::benchmark_data_smithi121_477989_object66503:head v 360'4261, mlcod=0'0)", 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:58.851992+0000", 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 13.550943483999999, 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025604614000000001, 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.851992+0000", 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.851992+0000", 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.525 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.851991+0000", 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852017+0000", 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6298000000000001e-05 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852030+0000", 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3339000000000001e-05 2023-09-23T21:04:12.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852037+0000", 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0940000000000002e-06 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.858840+0000", 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0068028769999999997 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.858870+0000", 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0093000000000001e-05 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.877548+0000", 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018677546999999999 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.877596+0000", 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8537e-05 2023-09-23T21:04:12.528 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:66506 11.3 e360/355 11:c7dad3d7:::benchmark_data_smithi121_477989_object66505:head v 360'4262, mlcod=0'0)", 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:58.852048+0000", 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 13.550887475, 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025566057999999999, 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:12.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852048+0000", 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852048+0000", 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:12.530 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852048+0000", 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852068+0000", 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0197e-05 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852074+0000", 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0809999999999999e-06 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.852078+0000", 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9820000000000002e-06 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.863130+0000", 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011052630000000001 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.863156+0000", 2023-09-23T21:04:12.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5816000000000002e-05 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.877599+0000", 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014443190999999999 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:58.877614+0000", 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4431e-05 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.533 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.534 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.534 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:12.534 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:12.534 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:04:12.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12657/219594 objects degraded (5.764%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:12.840 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:12.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:12.946 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:13.029 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:13.097 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:13.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:13.106 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:04:13.128 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2379791877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:13.128 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:13.128 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:13.128 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[211526]: pgmap v1145: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 12657/222633 objects degraded (5.685%); 5855/222633 objects misplaced (2.630%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T21:04:13.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2379791877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:13.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:13.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:13.129 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:13 smithi121 ceph-mon[219709]: pgmap v1145: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 12657/222633 objects degraded (5.685%); 5855/222633 objects misplaced (2.630%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T21:04:13.201 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:13.229 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:13.231 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 76712 76696 76.0485 48.0625 0.00359616 0.0131121 2023-09-23T21:04:13.278 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:13.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:13.384 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:13.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2379791877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:13.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:13.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:13.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:13 smithi123 ceph-mon[193903]: pgmap v1145: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 12657/222633 objects degraded (5.685%); 5855/222633 objects misplaced (2.630%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T21:04:13.477 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:13.713 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:13.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:13.761 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:04:13.828 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:13.872 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[211526]: failed to encode map e366 with expected crc 2023-09-23T21:04:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[211526]: failed to encode map e366 with expected crc 2023-09-23T21:04:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2379791877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[211526]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T21:04:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[219709]: failed to encode map e366 with expected crc 2023-09-23T21:04:14.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[219709]: failed to encode map e366 with expected crc 2023-09-23T21:04:14.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2379791877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:14.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:14 smithi121 ceph-mon[219709]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T21:04:14.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 77535 77519 75.6641 51.4375 0.0352905 0.0132114 2023-09-23T21:04:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:14 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:14 smithi123 ceph-mon[193903]: failed to encode map e366 with expected crc 2023-09-23T21:04:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:14 smithi123 ceph-mon[193903]: failed to encode map e366 with expected crc 2023-09-23T21:04:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2379791877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:14 smithi123 ceph-mon[193903]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T21:04:14.480 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:15.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:15 smithi121 ceph-mon[211526]: pgmap v1147: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 12657/231354 objects degraded (5.471%); 5340/231354 objects misplaced (2.308%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T21:04:15.115 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:04:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:04:14] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.43.0" 2023-09-23T21:04:15.162 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:15.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:15.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 78346 78330 75.2799 50.6875 0.0142745 0.0132793 2023-09-23T21:04:15.271 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:15.328 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:15.395 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:15 smithi121 ceph-mon[219709]: pgmap v1147: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 12657/231354 objects degraded (5.471%); 5340/231354 objects misplaced (2.308%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T21:04:15.417 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:15.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:15.425 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:15 smithi123 ceph-mon[193903]: pgmap v1147: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 12657/231354 objects degraded (5.471%); 5340/231354 objects misplaced (2.308%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T21:04:15.526 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:15.595 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:15.678 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:15.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:15.783 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:15.850 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:15.915 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:15.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:15.925 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:04:15.948 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:04:15.969 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:04:15.992 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:04:16.019 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:16.028 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:04:16.051 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:04:16.052 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:16.105 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:16.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:16.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:16.142 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:04:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T21:04:16.142 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:04:16.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1492403183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:16.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1492403183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:16.210 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:16.225 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:04:16.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 78936 78920 74.6982 36.875 0.0141603 0.0133824 2023-09-23T21:04:16.269 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:04:16.270 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:16.273 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.273 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.273 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.274 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.274 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.274 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.274 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.274 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:04:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1492403183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:17.190 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:17.190 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:17.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:17 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12657/231354 objects degraded (5.471%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:17.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:17 smithi123 ceph-mon[193903]: pgmap v1148: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 746 op/s; 12657/231354 objects degraded (5.471%); 5340/231354 objects misplaced (2.308%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:04:17.229 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:17.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 79703 79687 74.2989 47.9375 0.0224625 0.0134538 2023-09-23T21:04:17.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:17.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:17.334 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:17.403 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:04:17.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:17 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12657/231354 objects degraded (5.471%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:17 smithi121 ceph-mon[211526]: pgmap v1148: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 746 op/s; 12657/231354 objects degraded (5.471%); 5340/231354 objects misplaced (2.308%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:04:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:17 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12657/231354 objects degraded (5.471%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:17 smithi121 ceph-mon[219709]: pgmap v1148: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 746 op/s; 12657/231354 objects degraded (5.471%); 5340/231354 objects misplaced (2.308%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:04:17.428 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:17.529 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:17.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:17.640 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:17.687 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:17.787 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:17.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:17.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:17.890 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:17.918 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:18.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 80401 80385 73.8479 43.625 0.14243 0.0135364 2023-09-23T21:04:18.340 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:04:18.441 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:18.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:18.550 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:18.586 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:18.651 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:18.753 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:18.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:18.867 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:18.948 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:19.052 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:19.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:19.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:19.075 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:19.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:19 smithi121 ceph-mon[211526]: pgmap v1149: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/240270 objects degraded (5.268%); 5035/240270 objects misplaced (2.096%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:04:19.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:19 smithi121 ceph-mon[219709]: pgmap v1149: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/240270 objects degraded (5.268%); 5035/240270 objects misplaced (2.096%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:04:19.166 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:19.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:19 smithi123 ceph-mon[193903]: pgmap v1149: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/240270 objects degraded (5.268%); 5035/240270 objects misplaced (2.096%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:04:19.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 81199 81183 73.5005 49.875 0.0127802 0.0135998 2023-09-23T21:04:19.234 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:19.385 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:19.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:19.492 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:19.579 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:19.795 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:19.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:19.906 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:19.975 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:20.079 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:20.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:20.190 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:20.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 82078 82062 73.2354 54.9375 0.0106454 0.0136502 2023-09-23T21:04:20.252 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:20.355 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:20.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:20.451 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:04:20.462 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:20.487 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:20.529 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:20.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:20.632 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:20.721 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:20.821 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:20.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:20.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:20.928 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:20.979 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:21.064 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:21.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:21.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:21.073 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:21.170 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:21.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:21 smithi123 ceph-mon[193903]: pgmap v1150: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 761 op/s; 12657/242448 objects degraded (5.221%); 4790/242448 objects misplaced (1.976%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T21:04:21.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1986909371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:21.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:21.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 82738 82722 72.7851 41.25 0.0254636 0.0137342 2023-09-23T21:04:21.238 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:21.365 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:21.369 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:21 smithi121 ceph-mon[211526]: pgmap v1150: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 761 op/s; 12657/242448 objects degraded (5.221%); 4790/242448 objects misplaced (1.976%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T21:04:21.369 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1986909371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:21.369 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:21.369 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:21 smithi121 ceph-mon[219709]: pgmap v1150: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 761 op/s; 12657/242448 objects degraded (5.221%); 4790/242448 objects misplaced (1.976%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T21:04:21.369 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1986909371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:21.369 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:21.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:21.477 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:21.523 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:21.653 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:21.654 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:21.816 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:21.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:21.921 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:22.016 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.094 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:04:22.112 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:22.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:22.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:22.232 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 83548 83532 72.4773 50.625 0.0161005 0.0137789 2023-09-23T21:04:22.252 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.295 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:22.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:22.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:22.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12657/242448 objects degraded (5.221%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:22.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:22 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:22.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:22 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:22.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:22 smithi123 ceph-mon[193903]: osdmap e367: 8 total, 8 up, 7 in 2023-09-23T21:04:22.399 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12657/242448 objects degraded (5.221%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[219709]: osdmap e367: 8 total, 8 up, 7 in 2023-09-23T21:04:22.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12657/242448 objects degraded (5.221%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:04:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:22.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:22 smithi121 ceph-mon[211526]: osdmap e367: 8 total, 8 up, 7 in 2023-09-23T21:04:22.426 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:22.819 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:04:23.093 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:23 smithi123 ceph-mon[193903]: failed to encode map e367 with expected crc 2023-09-23T21:04:23.094 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:23 smithi123 ceph-mon[193903]: failed to encode map e367 with expected crc 2023-09-23T21:04:23.094 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:23 smithi123 ceph-mon[193903]: pgmap v1152: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s; 12657/245196 objects degraded (5.162%); 4790/245196 objects misplaced (1.954%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:04:23.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 84179 84163 72.0248 39.4375 0.035789 0.0138782 2023-09-23T21:04:23.398 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:04:23.399 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:04:23.399 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 4, 5, 6, 2, 7] 2023-09-23T21:04:23.399 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T21:04:23.401 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:23.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:23 smithi121 ceph-mon[211526]: failed to encode map e367 with expected crc 2023-09-23T21:04:23.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:23 smithi121 ceph-mon[211526]: failed to encode map e367 with expected crc 2023-09-23T21:04:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:23 smithi121 ceph-mon[211526]: pgmap v1152: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s; 12657/245196 objects degraded (5.162%); 4790/245196 objects misplaced (1.954%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:04:23.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:23 smithi121 ceph-mon[219709]: failed to encode map e367 with expected crc 2023-09-23T21:04:23.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:23 smithi121 ceph-mon[219709]: failed to encode map e367 with expected crc 2023-09-23T21:04:23.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:23 smithi121 ceph-mon[219709]: pgmap v1152: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s; 12657/245196 objects degraded (5.162%); 4790/245196 objects misplaced (1.954%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:04:23.848 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:23.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:23.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:23.957 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:24.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.178 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:24.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 84978 84962 71.7264 49.9375 0.0185741 0.0139366 2023-09-23T21:04:24.284 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:24.345 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.437 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:24.502 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:24.543 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:24.576 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.646 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:24.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:24.666 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:04:24.696 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:04:24.742 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:04:24.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:24.782 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:04:24.822 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:04:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:04:24] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.43.0" 2023-09-23T21:04:24.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2476967593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:24.823 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2476967593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:24.824 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:24.826 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:04:24.865 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:04:24.911 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:04:24.948 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:04:24.955 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.955 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.955 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.955 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.955 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.955 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.955 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.956 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:04:24.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2476967593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:25.139 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:25.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:25.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 85682 85666 71.3565 44 0.00799547 0.0140087 2023-09-23T21:04:25.244 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:25.333 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:25.560 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:25.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:25.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:25.670 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:25.717 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:25.831 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:90329 10.f 10:f35e32c5:::benchmark_data_smithi121_358375_object5584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:57:13.105215+0000", 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 432.72346971600001, 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80166978300000002, 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.832 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 90329 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105215+0000", 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105215+0000", 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105214+0000", 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105217+0000", 2023-09-23T21:04:25.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3589999999999999e-06 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105228+0000", 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0261999999999999e-05 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105234+0000", 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2149999999999998e-06 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105290+0000", 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5810000000000003e-05 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.105298+0000", 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0020000000000006e-06 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.894981+0000", 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.789683778 2023-09-23T21:04:25.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.895080+0000", 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9035000000000003e-05 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.895162+0000", 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1553999999999999e-05 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900361+0000", 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005198846 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.900369+0000", 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.388e-06 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906827+0000", 2023-09-23T21:04:25.838 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064581009999999999 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906859+0000", 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2100000000000001e-05 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:57:13.906884+0000", 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4882e-05 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97176 10.d 10:bf52c426:::benchmark_data_smithi121_358375_object12431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 350.32871140100002, 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0090121330000001, 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.840 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97176 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499972+0000", 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499975+0000", 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2090000000000001e-06 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499984+0000", 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.422e-06 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499988+0000", 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8739999999999997e-06 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500030+0000", 2023-09-23T21:04:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2366000000000001e-05 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500039+0000", 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5569999999999996e-06 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242363+0000", 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.74232412699999994 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.844 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242370+0000", 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.041e-06 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482610+0000", 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2402398800000001 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482781+0000", 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000170884 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482955+0000", 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017400099999999999 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490821+0000", 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078662190000000007 2023-09-23T21:04:25.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.493967+0000", 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031467309999999998 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506247+0000", 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012279684000000001 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.847 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506289+0000", 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2329000000000001e-05 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.508985+0000", 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026955099999999999 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.848 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97184 10.a 10:54b30599:::benchmark_data_smithi121_358375_object12439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 350.24136718, 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.974572057, 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97184 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.849 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.850 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587316+0000", 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587321+0000", 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3270000000000002e-06 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587333+0000", 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2383e-05 2023-09-23T21:04:25.851 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587338+0000", 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9189999999999998e-06 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587383+0000", 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5014e-05 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.852 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587389+0000", 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9660000000000001e-06 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242357+0000", 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.65496830699999997 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242367+0000", 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0266e-05 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.853 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538285+0000", 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2959178060000001 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538406+0000", 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120308 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:04:25.854 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538531+0000", 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000125761 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.549226+0000", 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010694789999999999 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551472+0000", 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002246315 2023-09-23T21:04:25.855 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559794+0000", 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083212719999999993 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559841+0000", 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7197999999999998e-05 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.856 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561889+0000", 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020482159999999998 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97187 10.0 10:057fb271:::benchmark_data_smithi121_358375_object12442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 350.217837524, 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8667573179999999, 2023-09-23T21:04:25.857 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97187 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.858 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610846+0000", 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.859 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610850+0000", 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2509999999999996e-06 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610863+0000", 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2995e-05 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610869+0000", 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1480000000000003e-06 2023-09-23T21:04:25.860 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610945+0000", 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5284000000000006e-05 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610953+0000", 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3750000000000003e-06 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.861 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242784+0000", 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63183069599999997 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242792+0000", 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6510000000000007e-06 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.434568+0000", 2023-09-23T21:04:25.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1917759189999999 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.435371+0000", 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000802339 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451218+0000", 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015847757000000001 2023-09-23T21:04:25.863 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451292+0000", 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3386000000000005e-05 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451393+0000", 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101216 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462865+0000", 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011472379 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.466387+0000", 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035211050000000001 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.474961+0000", 2023-09-23T21:04:25.865 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085745549999999993 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.475028+0000", 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7105000000000002e-05 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477604+0000", 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025758389999999999 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.866 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97197 10.d 10:b1ae4ef2:::benchmark_data_smithi121_358375_object12452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.769347+0000", 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 350.05933708999999, 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7457691610000001, 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.867 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97197 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769347+0000", 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769347+0000", 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769346+0000", 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769352+0000", 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4269999999999999e-06 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769364+0000", 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2561999999999999e-05 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769371+0000", 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4520000000000002e-06 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769417+0000", 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6017999999999999e-05 2023-09-23T21:04:25.870 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.769425+0000", 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6619999999999994e-06 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242398+0000", 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.47297264900000002 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.871 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242404+0000", 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5640000000000002e-06 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488192+0000", 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.245787612 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488310+0000", 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118377 2023-09-23T21:04:25.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.488416+0000", 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000105685 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.493980+0000", 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055637339999999999 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.497354+0000", 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033737319999999999 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.512976+0000", 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015622575 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.513019+0000", 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2478999999999998e-05 2023-09-23T21:04:25.874 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515116+0000", 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020976649999999999 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97205 10.0 10:0f792b11:::benchmark_data_smithi121_358375_object12460:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.899978+0000", 2023-09-23T21:04:25.875 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 349.92870654799998, 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5793486960000001, 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97205 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899978+0000", 2023-09-23T21:04:25.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899978+0000", 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899977+0000", 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899981+0000", 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5700000000000003e-06 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899993+0000", 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1613e-05 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.899999+0000", 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5969999999999997e-06 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.900045+0000", 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5772000000000002e-05 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.900053+0000", 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5580000000000003e-06 2023-09-23T21:04:25.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242824+0000", 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34277136600000002 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242832+0000", 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4569999999999999e-06 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.436698+0000", 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1938665129999999 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.437505+0000", 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00080636499999999997 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.456977+0000", 2023-09-23T21:04:25.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019472245999999999 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.457059+0000", 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2544000000000001e-05 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.457160+0000", 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100435 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.466405+0000", 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092448630000000007 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477677+0000", 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011272453999999999 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477732+0000", 2023-09-23T21:04:25.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4910000000000001e-05 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477796+0000", 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4053000000000005e-05 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479326+0000", 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015303929999999999 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.884 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97215 10.d 10:b031210c:::benchmark_data_smithi121_358375_object12470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.106818+0000", 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 349.72186595199997, 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4100548179999999, 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.885 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97215 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106818+0000", 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106818+0000", 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106816+0000", 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106830+0000", 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4032000000000001e-05 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106869+0000", 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8349999999999997e-05 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106883+0000", 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4243000000000001e-05 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.106982+0000", 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8944000000000007e-05 2023-09-23T21:04:25.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.107003+0000", 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1302999999999999e-05 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242425+0000", 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.13542152199999999 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242431+0000", 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1639999999999998e-06 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490848+0000", 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2484173919999999 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490954+0000", 2023-09-23T21:04:25.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010638500000000001 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.491045+0000", 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0453999999999998e-05 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.497302+0000", 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062572529999999999 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.500461+0000", 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031586980000000001 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515185+0000", 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014723630999999999 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.515224+0000", 2023-09-23T21:04:25.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9691999999999999e-05 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.516873+0000", 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001648928 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97221 10.0 10:0777f148:::benchmark_data_smithi121_358375_object12476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.130321+0000", 2023-09-23T21:04:25.893 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 349.69836325400001, 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3504028450000001, 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97221 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.894 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130321+0000", 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130321+0000", 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130320+0000", 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:25.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130327+0000", 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.328e-06 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130344+0000", 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6699999999999999e-05 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130352+0000", 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1990000000000006e-06 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130408+0000", 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6060999999999997e-05 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.130418+0000", 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0169999999999999e-05 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242919+0000", 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.112500957 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242927+0000", 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7819999999999994e-06 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.437568+0000", 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.194641184 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.438303+0000", 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00073454399999999995 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462904+0000", 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024601067000000001 2023-09-23T21:04:25.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462984+0000", 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9919000000000005e-05 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.463135+0000", 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000151361 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479342+0000", 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016207200000000001 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479403+0000", 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0417000000000001e-05 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479456+0000", 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3798999999999998e-05 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.479512+0000", 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5714000000000002e-05 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.480724+0000", 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001211684 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97225 10.a 10:52fbb22c:::benchmark_data_smithi121_358375_object12480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.225436+0000", 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 349.60324844000002, 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3382990889999999, 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97225 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225436+0000", 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225436+0000", 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225433+0000", 2023-09-23T21:04:25.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225446+0000", 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2703e-05 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225471+0000", 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5225e-05 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.905 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225481+0000", 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6050000000000004e-06 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225544+0000", 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2649000000000003e-05 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.225553+0000", 2023-09-23T21:04:25.906 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1549999999999996e-06 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242391+0000", 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016838199000000002 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242398+0000", 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8959999999999997e-06 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541687+0000", 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.299289683 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541850+0000", 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016283399999999999 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.541997+0000", 2023-09-23T21:04:25.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014649800000000001 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.549235+0000", 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0072378959999999997 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551516+0000", 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022813149999999999 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561929+0000", 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010412988999999999 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561972+0000", 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2719000000000002e-05 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.563735+0000", 2023-09-23T21:04:25.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001763202 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97254 10.a 10:56bae3f9:::benchmark_data_smithi121_358375_object12509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.794906+0000", 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 349.03377846400002, 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77559789899999998, 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.911 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97254 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794906+0000", 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794906+0000", 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794905+0000", 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794911+0000", 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5320000000000003e-06 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794926+0000", 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.447e-05 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.794931+0000", 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.451e-06 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.914 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.795017+0000", 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5575000000000007e-05 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.795026+0000", 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5729999999999997e-06 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.544759+0000", 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.74973301299999995 2023-09-23T21:04:25.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.544898+0000", 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000138985 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.545008+0000", 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000109745 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551524+0000", 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065155339999999999 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551553+0000", 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9068999999999999e-05 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.568290+0000", 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016737577 2023-09-23T21:04:25.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.568339+0000", 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9119000000000002e-05 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570504+0000", 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002164329 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.918 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97257 10.a 10:548053b5:::benchmark_data_smithi121_358375_object12512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 348.90840794799999, 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.65229715300000002, 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97257 2023-09-23T21:04:25.919 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920276+0000", 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920279+0000", 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8709999999999999e-06 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920289+0000", 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4919999999999999e-06 2023-09-23T21:04:25.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920293+0000", 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1359999999999999e-06 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920345+0000", 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2012999999999998e-05 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.922 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.920352+0000", 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7309999999999996e-06 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547185+0000", 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.626832944 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547290+0000", 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010505600000000001 2023-09-23T21:04:25.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.547383+0000", 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2948000000000005e-05 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551592+0000", 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042095969999999998 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.924 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551600+0000", 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3139999999999998e-06 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570562+0000", 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018962326000000002 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.570625+0000", 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3427000000000001e-05 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.572573+0000", 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019480890000000001 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140911 10.b 10:dec2be02:::benchmark_data_smithi121_358375_object56166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.147545+0000", 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.68113971700001, 2023-09-23T21:04:25.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4336918569999999, 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140911 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147545+0000", 2023-09-23T21:04:25.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147545+0000", 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147543+0000", 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147551+0000", 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6820000000000003e-06 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147569+0000", 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8785999999999999e-05 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147576+0000", 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.567e-06 2023-09-23T21:04:25.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147629+0000", 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3174000000000003e-05 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.147640+0000", 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0345e-05 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326914+0000", 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17927400499999999 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326925+0000", 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1165999999999999e-05 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.523743+0000", 2023-09-23T21:04:25.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.196818079 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.524436+0000", 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00069355399999999998 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.560304+0000", 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.035867555000000002 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.560413+0000", 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010935200000000001 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.561076+0000", 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00066255899999999996 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576376+0000", 2023-09-23T21:04:25.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015300591000000001 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579829+0000", 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034525229999999999 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579864+0000", 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5165e-05 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579937+0000", 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2452000000000004e-05 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579997+0000", 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0735e-05 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581236+0000", 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001239139 2023-09-23T21:04:25.935 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140926 10.b 10:dac4289f:::benchmark_data_smithi121_358375_object56181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.230787+0000", 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.59789681300001, 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3612417960000001, 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.936 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140926 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230787+0000", 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230787+0000", 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230785+0000", 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230799+0000", 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4133e-05 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230832+0000", 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2963999999999998e-05 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230840+0000", 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7349999999999996e-06 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230916+0000", 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6191000000000003e-05 2023-09-23T21:04:25.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.230924+0000", 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9200000000000004e-06 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326944+0000", 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.096019652999999996 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326950+0000", 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9599999999999997e-06 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.524496+0000", 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1975459370000001 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525163+0000", 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00066679700000000001 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.570349+0000", 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.045186562 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.570465+0000", 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115815 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T21:04:25.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.570641+0000", 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000175835 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.579899+0000", 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0092581480000000008 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581655+0000", 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001755803 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.586132+0000", 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004477127 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.590701+0000", 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045693959999999999 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.590750+0000", 2023-09-23T21:04:25.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8501000000000002e-05 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.592029+0000", 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012795020000000001 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140937 10.b 10:dce73c29:::benchmark_data_smithi121_358375_object56192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:25.945 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.299885+0000", 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.52879897899999, 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2932650050000001, 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140937 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.946 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299885+0000", 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299885+0000", 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299884+0000", 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299889+0000", 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1679999999999997e-06 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299898+0000", 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1130000000000007e-06 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299903+0000", 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.831e-06 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299953+0000", 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0448999999999998e-05 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.299960+0000", 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4189999999999996e-06 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326967+0000", 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027006183 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.326972+0000", 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8289999999999996e-06 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525195+0000", 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.198222087 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525848+0000", 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00065309500000000002 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576400+0000", 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.050552391000000002 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576470+0000", 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0252000000000004e-05 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.576547+0000", 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6408000000000004e-05 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581649+0000", 2023-09-23T21:04:25.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051024900000000003 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.581661+0000", 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1686e-05 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.586175+0000", 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045139739999999996 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.592090+0000", 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059147319999999998 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.592159+0000", 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9060000000000006e-05 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.593150+0000", 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0009916949999999999 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50073 11.b 11:da39f2df:::benchmark_data_smithi121_477989_object50072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.423645, 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2188515999999998, 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:25.955 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50073 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405103+0000", 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3901999999999999e-05 2023-09-23T21:04:25.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405111+0000", 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4460000000000001e-06 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405116+0000", 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0629999999999996e-06 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416824+0000", 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011707555 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416833+0000", 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1540000000000008e-06 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400621+0000", 2023-09-23T21:04:25.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98378816000000002 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400630+0000", 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6950000000000006e-06 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.622813+0000", 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.222183198 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.623891+0000", 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001077929 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50086 11.b 11:d8a7cc4c:::benchmark_data_smithi121_477989_object50085:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.423558061000001, 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2482345829999999, 2023-09-23T21:04:25.961 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50086 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405188+0000", 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2147e-05 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405196+0000", 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1799999999999996e-06 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405201+0000", 2023-09-23T21:04:25.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9339999999999997e-06 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416857+0000", 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011656025 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416865+0000", 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8329999999999994e-06 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400651+0000", 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98378581600000004 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400658+0000", 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.08e-06 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.623919+0000", 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2232614909999999 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.624169+0000", 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024978700000000001 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.624334+0000", 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165315 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631814+0000", 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.007479482 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634635+0000", 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028213819999999999 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.650585+0000", 2023-09-23T21:04:25.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015949393999999999 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.650633+0000", 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8591999999999998e-05 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653361+0000", 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002727602 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.969 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50100 11.3 11:c9d587e5:::benchmark_data_smithi121_477989_object50099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.400084743000001, 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2031901700000001, 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:25.970 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50100 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428599+0000", 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428631+0000", 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1956999999999999e-05 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428639+0000", 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8269999999999998e-06 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428644+0000", 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3550000000000004e-06 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428697+0000", 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2799000000000001e-05 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428706+0000", 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0659999999999993e-06 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.399543+0000", 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97083674099999995 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.399551+0000", 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5429999999999994e-06 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607385+0000", 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2078334150000001 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607502+0000", 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117125 2023-09-23T21:04:25.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607664+0000", 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016177800000000001 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.614069+0000", 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064056510000000001 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629191+0000", 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015121252 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629222+0000", 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1405000000000002e-05 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629265+0000", 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2824000000000002e-05 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631790+0000", 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025248559999999998 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50110 11.b 11:d197f0fb:::benchmark_data_smithi121_477989_object50109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.394147494999999, 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2210893860000001, 2023-09-23T21:04:25.978 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50110 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434536+0000", 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434604+0000", 2023-09-23T21:04:25.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8382999999999999e-05 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434616+0000", 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2139e-05 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434623+0000", 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6510000000000004e-06 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434802+0000", 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017905300000000001 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434811+0000", 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3820000000000004e-06 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400678+0000", 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96586726099999998 2023-09-23T21:04:25.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400684+0000", 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2689999999999996e-06 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.625756+0000", 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2250717149999999 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.625892+0000", 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000135933 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.626051+0000", 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015894499999999999 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631821+0000", 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0057699200000000004 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634669+0000", 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028480839999999999 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653436+0000", 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018767817999999999 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653479+0000", 2023-09-23T21:04:25.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2070000000000002e-05 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655626+0000", 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002147599 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50121 11.b 11:d6d11928:::benchmark_data_smithi121_477989_object50120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:25.986 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.203694941999998, 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.033318296, 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50121 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624988+0000", 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625021+0000", 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3223999999999999e-05 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625034+0000", 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3191e-05 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625041+0000", 2023-09-23T21:04:25.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1130000000000003e-06 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625217+0000", 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000175921 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625227+0000", 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8390000000000002e-06 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400706+0000", 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77547898800000004 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400713+0000", 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9789999999999996e-06 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627606+0000", 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.226892662 2023-09-23T21:04:25.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627745+0000", 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013900300000000001 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627884+0000", 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000138932 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:25.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634608+0000", 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0067245739999999997 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634714+0000", 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010603300000000001 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655900+0000", 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021185137999999999 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655945+0000", 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5346000000000003e-05 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.658308+0000", 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002362762 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:25.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50132 11.a 11:593fc135:::benchmark_data_smithi121_477989_object50131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 48.185947759000001, 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9071047490000002, 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50132 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.995 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642735+0000", 2023-09-23T21:04:25.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642763+0000", 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7504e-05 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642777+0000", 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3684e-05 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:25.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642781+0000", 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9110000000000001e-06 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642843+0000", 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1890999999999994e-05 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642855+0000", 2023-09-23T21:04:25.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1999e-05 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.396214+0000", 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75335825499999998 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.396226+0000", 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2177e-05 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:25.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540693+0000", 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.144467481 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540826+0000", 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013311 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540929+0000", 2023-09-23T21:04:26.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010215799999999999 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547150+0000", 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062215100000000004 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547194+0000", 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3538999999999998e-05 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:26.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547204+0000", 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0873000000000001e-05 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547271+0000", 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6261999999999997e-05 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:26.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.549841+0000", 2023-09-23T21:04:26.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025705569999999998 2023-09-23T21:04:26.003 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:26.003 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:26.003 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:26.003 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:26.003 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:26.003 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:04:26.003 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:25 smithi123 ceph-mon[193903]: pgmap v1153: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/253689 objects degraded (4.989%); 4246/253689 objects misplaced (1.674%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T21:04:26.045 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:04:26.094 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:25 smithi121 ceph-mon[211526]: pgmap v1153: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/253689 objects degraded (4.989%); 4246/253689 objects misplaced (1.674%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T21:04:26.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:25 smithi121 ceph-mon[219709]: pgmap v1153: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/253689 objects degraded (4.989%); 4246/253689 objects misplaced (1.674%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T21:04:26.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 86339 86323 70.958 41.0625 0.0149695 0.0140837 2023-09-23T21:04:26.833 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:26.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:26.945 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:26.967 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:04:26.968 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12657/253689 objects degraded (4.989%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:27.027 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12657/253689 objects degraded (4.989%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:27.029 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12657/253689 objects degraded (4.989%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:04:27.041 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:27.041 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:27.149 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:27.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:27.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 86887 86871 70.4814 34.25 0.0301892 0.0141812 2023-09-23T21:04:27.255 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:27.323 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:27.446 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:27.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:27.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:27.560 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:27.632 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:27.644 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:27.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:27 smithi121 ceph-mon[211526]: pgmap v1154: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/253689 objects degraded (4.989%); 4246/253689 objects misplaced (1.674%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T21:04:27.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:27.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:27.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:27 smithi121 ceph-mon[219709]: pgmap v1154: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/253689 objects degraded (4.989%); 4246/253689 objects misplaced (1.674%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T21:04:27.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:27.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:27.995 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:27 smithi123 ceph-mon[193903]: pgmap v1154: 89 pgs: 1 active+clean+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s; 12657/253689 objects degraded (4.989%); 4246/253689 objects misplaced (1.674%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T21:04:27.995 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:27.995 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:28.021 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:28.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:28.128 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:28.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:27 smithi121 conmon[211495]: 2023-09-23T21:04:27.958+0000 7fdc50c66700 -1 mon.a@0(leader).osd e367 definitely_dead 0 2023-09-23T21:04:28.192 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:28.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 87698 87682 70.2276 50.6875 0.0260658 0.0142343 2023-09-23T21:04:28.291 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:28.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:28.396 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:28.449 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:28.521 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:28.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:28.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:28.629 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:28.671 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:28.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2025121837' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:04:28.750 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:28 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:04:28.750 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:28 smithi121 ceph-mon[211526]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:04:28.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2025121837' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:04:28.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:28 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:04:28.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:28 smithi121 ceph-mon[219709]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:04:28.764 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.7. 2023-09-23T21:04:28.857 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:04:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2025121837' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:04:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:28 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T21:04:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:28 smithi123 ceph-mon[193903]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T21:04:29.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 88150 88134 69.6964 28.25 0.0117868 0.0142573 2023-09-23T21:04:29.402 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:29.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:29.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:29.506 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:29.537 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:29.637 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:29.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:29.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:29.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:29.783 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:29.871 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:29.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:29.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:29.878 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:29 smithi123 ceph-mon[193903]: failed to encode map e368 with expected crc 2023-09-23T21:04:29.878 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:29 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:04:29.879 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:29 smithi123 ceph-mon[193903]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T21:04:29.879 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:29 smithi123 ceph-mon[193903]: failed to encode map e368 with expected crc 2023-09-23T21:04:29.879 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:29 smithi123 ceph-mon[193903]: pgmap v1156: 89 pgs: 2 remapped+peering, 1 active+clean+remapped, 1 unknown, 19 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 847 op/s; 6406/262797 objects degraded (2.438%); 5287/262797 objects misplaced (2.012%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T21:04:29.961 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:04:29.961 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:04:29.961 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:04:29.961 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:04:29.979 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:30.006 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.021 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[211526]: failed to encode map e368 with expected crc 2023-09-23T21:04:30.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:04:30.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[211526]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T21:04:30.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[211526]: failed to encode map e368 with expected crc 2023-09-23T21:04:30.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[211526]: pgmap v1156: 89 pgs: 2 remapped+peering, 1 active+clean+remapped, 1 unknown, 19 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 847 op/s; 6406/262797 objects degraded (2.438%); 5287/262797 objects misplaced (2.012%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T21:04:30.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[219709]: failed to encode map e368 with expected crc 2023-09-23T21:04:30.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T21:04:30.022 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[219709]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T21:04:30.023 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[219709]: failed to encode map e368 with expected crc 2023-09-23T21:04:30.023 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:29 smithi121 ceph-mon[219709]: pgmap v1156: 89 pgs: 2 remapped+peering, 1 active+clean+remapped, 1 unknown, 19 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 847 op/s; 6406/262797 objects degraded (2.438%); 5287/262797 objects misplaced (2.012%); 6.6 MiB/s, 104 objects/s recovering 2023-09-23T21:04:30.161 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:30.211 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:30.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:04:30.234273+0000 min lat: 0.00123114 max lat: 2.63541 avg lat: 0.0142573 2023-09-23T21:04:30.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:04:30.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 88150 88134 68.8255 0 - 0.0142573 2023-09-23T21:04:30.315 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:30.344 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.444 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:30.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:30.549 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:30.575 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.626 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:30.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:30.729 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:30.757 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:30.769 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:04:30.839 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:30.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:30.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T21:04:30.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.842 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3681561529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:30.843 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:30 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:30.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3681561529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:30.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: failed to encode map e369 with expected crc 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3681561529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:30.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:30 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:04:30.944 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:31.034 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:31.100 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:31.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:31.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:31.207 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:31.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 88155 88139 67.9799 0.15625 0.0301153 0.0143076 2023-09-23T21:04:31.235 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:31.389 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:31.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:31.451 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:04:31.496 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:31.525 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:31.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:31.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: failed to encode map e370 with expected crc 2023-09-23T21:04:31.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: failed to encode map e370 with expected crc 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: failed to encode map e370 with expected crc 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: 11.9 starting backfill to osd.5 from (0'0,0'0] MAX to 367'5559 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: pgmap v1159: 89 pgs: 3 remapped+peering, 1 active+clean+remapped, 1 unknown, 23 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 34 MiB/s wr, 537 op/s; 6406/263367 objects degraded (2.432%); 3061/263367 objects misplaced (1.162%); 5.4 MiB/s, 84 objects/s recovering 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6406/262797 objects degraded (2.438%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:31.816 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: failed to encode map e370 with expected crc 2023-09-23T21:04:31.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:04:31.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T21:04:31.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: failed to encode map e370 with expected crc 2023-09-23T21:04:31.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: failed to encode map e370 with expected crc 2023-09-23T21:04:31.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: 11.9 starting backfill to osd.5 from (0'0,0'0] MAX to 367'5559 2023-09-23T21:04:31.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: pgmap v1159: 89 pgs: 3 remapped+peering, 1 active+clean+remapped, 1 unknown, 23 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 34 MiB/s wr, 537 op/s; 6406/263367 objects degraded (2.432%); 3061/263367 objects misplaced (1.162%); 5.4 MiB/s, 84 objects/s recovering 2023-09-23T21:04:31.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6406/262797 objects degraded (2.438%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:04:31.845 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:32.051 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:32.051 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: failed to encode map e370 with expected crc 2023-09-23T21:04:32.051 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:04:32.051 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T21:04:32.052 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: failed to encode map e370 with expected crc 2023-09-23T21:04:32.052 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: failed to encode map e370 with expected crc 2023-09-23T21:04:32.052 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: 11.9 starting backfill to osd.5 from (0'0,0'0] MAX to 367'5559 2023-09-23T21:04:32.052 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: pgmap v1159: 89 pgs: 3 remapped+peering, 1 active+clean+remapped, 1 unknown, 23 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 34 MiB/s wr, 537 op/s; 6406/263367 objects degraded (2.432%); 3061/263367 objects misplaced (1.162%); 5.4 MiB/s, 84 objects/s recovering 2023-09-23T21:04:32.052 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6406/262797 objects degraded (2.438%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:04:32.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 88186 88170 67.1748 1.9375 0.00611097 0.0145162 2023-09-23T21:04:32.729 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:32.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:32.837 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:32.867 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:32.929 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:32.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:32.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:33.040 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:33.071 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:33.130 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:33.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:33.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:32 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:04:33.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:32 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:04:33.199 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:04:33.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:32 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:04:33.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 88186 88170 66.3657 0 - 0.0145162 2023-09-23T21:04:33.234 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:33.267 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:33.325 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:33.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:33.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:33.430 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:33.462 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:33.520 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:33.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:33.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:33.624 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:33.656 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:33.705 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:33.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:33.809 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:33.837 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:33.891 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:33.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:33.996 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:34.033 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:34.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:33 smithi121 ceph-mon[219709]: pgmap v1160: 89 pgs: 3 remapped+peering, 1 active+clean+remapped, 1 unknown, 23 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 34 MiB/s wr, 537 op/s; 6406/263367 objects degraded (2.432%); 3061/263367 objects misplaced (1.162%); 5.4 MiB/s, 84 objects/s recovering 2023-09-23T21:04:34.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3134331595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:33 smithi121 ceph-mon[211526]: pgmap v1160: 89 pgs: 3 remapped+peering, 1 active+clean+remapped, 1 unknown, 23 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 34 MiB/s wr, 537 op/s; 6406/263367 objects degraded (2.432%); 3061/263367 objects misplaced (1.162%); 5.4 MiB/s, 84 objects/s recovering 2023-09-23T21:04:34.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3134331595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:34.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:33 smithi123 ceph-mon[193903]: pgmap v1160: 89 pgs: 3 remapped+peering, 1 active+clean+remapped, 1 unknown, 23 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 34 MiB/s wr, 537 op/s; 6406/263367 objects degraded (2.432%); 3061/263367 objects misplaced (1.162%); 5.4 MiB/s, 84 objects/s recovering 2023-09-23T21:04:34.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3134331595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:04:34.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 88186 88170 65.5759 0 - 0.0145162 2023-09-23T21:04:34.293 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:34.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:34.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:34.343 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:04:34.343 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:04:34.343 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:04:34.343 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:04:34.343 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:04:34.402 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:34.435 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:34.490 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:34.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:34.594 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:34.625 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:34.706 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:04:34.870 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:04:34.871 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:04:34.871 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T21:04:34.871 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T21:04:35.091 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:35 smithi121 ceph-mon[211526]: pgmap v1161: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 24 MiB/s wr, 377 op/s; 6406/263382 objects degraded (2.432%); 6934/263382 objects misplaced (2.633%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:04:35.091 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:35 smithi121 ceph-mon[211526]: Health check update: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:35.091 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:04:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:04:34] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.43.0" 2023-09-23T21:04:35.092 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:35 smithi121 ceph-mon[219709]: pgmap v1161: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 24 MiB/s wr, 377 op/s; 6406/263382 objects degraded (2.432%); 6934/263382 objects misplaced (2.633%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:04:35.092 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:35 smithi121 ceph-mon[219709]: Health check update: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:35 smithi123 ceph-mon[193903]: pgmap v1161: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 24 MiB/s wr, 377 op/s; 6406/263382 objects degraded (2.432%); 6934/263382 objects misplaced (2.633%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T21:04:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:35 smithi123 ceph-mon[193903]: Health check update: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:35.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 88186 88170 64.8046 0 - 0.0145162 2023-09-23T21:04:35.483 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:35.509 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:35.734 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:04:35.776 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:35.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:35.882 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:35.914 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:36.181 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:04:36.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 88186 88170 64.0513 0 - 0.0145162 2023-09-23T21:04:36.466 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:36.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:36.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:36.570 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:36.605 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:36.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6406/263382 objects degraded (2.432%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:04:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6406/263382 objects degraded (2.432%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:04:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6406/263382 objects degraded (2.432%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:04:37.009 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:37.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:37.115 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:37.145 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:37.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 88186 88170 63.3153 0 - 0.0145162 2023-09-23T21:04:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:37 smithi123 ceph-mon[193903]: pgmap v1162: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 1.5 MiB/s wr, 24 op/s; 6406/263382 objects degraded (2.432%); 6934/263382 objects misplaced (2.633%); 3.3 MiB/s, 1 keys/s, 52 objects/s recovering 2023-09-23T21:04:37.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:37 smithi121 ceph-mon[211526]: pgmap v1162: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 1.5 MiB/s wr, 24 op/s; 6406/263382 objects degraded (2.432%); 6934/263382 objects misplaced (2.633%); 3.3 MiB/s, 1 keys/s, 52 objects/s recovering 2023-09-23T21:04:37.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:37 smithi121 ceph-mon[219709]: pgmap v1162: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 1.5 MiB/s wr, 24 op/s; 6406/263382 objects degraded (2.432%); 6934/263382 objects misplaced (2.633%); 3.3 MiB/s, 1 keys/s, 52 objects/s recovering 2023-09-23T21:04:38.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 88186 88170 62.5961 0 - 0.0145162 2023-09-23T21:04:38.815 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:04:38.815 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:04:38.815 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:04:38.815 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:04:38.815 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.815 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140894 10.c 10:33443f8a:::benchmark_data_smithi121_358375_object56149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:04:38.815 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.85143402, 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5509230220000001, 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140894 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960827+0000", 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:38.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960833+0000", 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1330000000000004e-06 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960845+0000", 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2065999999999999e-05 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.818 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960850+0000", 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1599999999999997e-06 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960895+0000", 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5522999999999998e-05 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960988+0000", 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2343999999999995e-05 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.961068+0000", 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9864999999999995e-05 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.964285+0000", 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0032171370000000001 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.964723+0000", 2023-09-23T21:04:38.820 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043790000000000002 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973518+0000", 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0087954379999999992 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973531+0000", 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2510999999999999e-05 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322237+0000", 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34870620200000002 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322246+0000", 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3449999999999995e-06 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511643+0000", 2023-09-23T21:04:38.822 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1893964910000001 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511751+0000", 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107996 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140895 10.c 10:32d93067:::benchmark_data_smithi121_358375_object56150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:04:38.823 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.84681010899999, 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.546411454, 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140895 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:04:38.824 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965450+0000", 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.825 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965457+0000", 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.297e-06 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965471+0000", 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4352000000000001e-05 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965476+0000", 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.198e-06 2023-09-23T21:04:38.826 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965514+0000", 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7704999999999999e-05 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965608+0000", 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4121000000000002e-05 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:04:38.827 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965685+0000", 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7127999999999997e-05 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973585+0000", 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078994029999999993 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973593+0000", 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2810000000000008e-06 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.828 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322280+0000", 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34868769799999999 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322287+0000", 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1879999999999999e-06 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511780+0000", 2023-09-23T21:04:38.829 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.189493559 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511863+0000", 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2774000000000004e-05 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140896 10.1 10:86418234:::benchmark_data_smithi121_358375_object56151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:04:38.830 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.83722608599999, 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6998174590000001, 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140896 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.831 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975034+0000", 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975041+0000", 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4139999999999999e-06 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975054+0000", 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3604999999999999e-05 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975061+0000", 2023-09-23T21:04:38.833 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3419999999999996e-06 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.976385+0000", 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0013248260000000001 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.976396+0000", 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.076e-05 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.834 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320693+0000", 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34429716599999999 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320701+0000", 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6899999999999992e-06 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.654437+0000", 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3337360949999999 2023-09-23T21:04:38.835 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.655243+0000", 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00080610999999999996 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668434+0000", 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013190676 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.836 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668502+0000", 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8547999999999995e-05 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668880+0000", 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00037727300000000002 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674052+0000", 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051727869999999999 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674083+0000", 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0043999999999999e-05 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674132+0000", 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9484000000000003e-05 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.838 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674191+0000", 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8567999999999999e-05 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674853+0000", 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000662445 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140900 10.c 10:3c2d050d:::benchmark_data_smithi121_358375_object56155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.81460407700001, 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5351996919999999, 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140900 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.840 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997656+0000", 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997665+0000", 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8799999999999997e-06 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997687+0000", 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1967000000000002e-05 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997699+0000", 2023-09-23T21:04:38.842 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2074e-05 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997758+0000", 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9375e-05 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997768+0000", 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6749999999999997e-06 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322303+0000", 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.324535299 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322309+0000", 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5690000000000004e-06 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511887+0000", 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1895777999999999 2023-09-23T21:04:38.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.512028+0000", 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014090500000000001 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.512592+0000", 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00056408399999999996 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525899+0000", 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013307461 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532566+0000", 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0066666620000000003 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532600+0000", 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4233000000000002e-05 2023-09-23T21:04:38.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532699+0000", 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8743e-05 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532754+0000", 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4941000000000001e-05 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.847 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532857+0000", 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103405 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140903 10.c 10:38af2a49:::benchmark_data_smithi121_358375_object56158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.792787847, 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.513614249, 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.848 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140903 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.849 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019472+0000", 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019481+0000", 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1039999999999996e-06 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.850 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019501+0000", 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9706000000000001e-05 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019513+0000", 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2092999999999999e-05 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019575+0000", 2023-09-23T21:04:38.851 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2136000000000003e-05 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019587+0000", 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1616e-05 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322325+0000", 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30273779699999998 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.852 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322334+0000", 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3780000000000007e-06 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520201+0000", 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1978669260000001 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520304+0000", 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102685 2023-09-23T21:04:38.853 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520390+0000", 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6036000000000001e-05 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.528925+0000", 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085353589999999993 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.854 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532633+0000", 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037075699999999999 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532894+0000", 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00026169499999999999 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532952+0000", 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7717999999999999e-05 2023-09-23T21:04:38.855 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532991+0000", 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9443999999999997e-05 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533088+0000", 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6669000000000006e-05 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140920 10.11 10:8e38667f:::benchmark_data_smithi121_358375_object56175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.61219274999999, 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4951127319999999, 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140920 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.857 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.858 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200068+0000", 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200075+0000", 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4109999999999999e-06 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200092+0000", 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6860000000000001e-05 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.859 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200099+0000", 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2559999999999996e-06 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200156+0000", 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6854000000000001e-05 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200167+0000", 2023-09-23T21:04:38.860 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0437e-05 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320655+0000", 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.120488398 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320668+0000", 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3132999999999999e-05 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.861 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.652475+0000", 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3318069219999999 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.653395+0000", 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00092004800000000002 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677570+0000", 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024174998999999999 2023-09-23T21:04:38.862 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677680+0000", 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011025499999999999 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677936+0000", 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00025574300000000001 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.863 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.679730+0000", 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0017943849999999999 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.692535+0000", 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012804519 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.694272+0000", 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001737326 2023-09-23T21:04:38.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.694330+0000", 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7287999999999998e-05 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.695182+0000", 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00085209999999999995 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.865 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140940 10.c 10:3f6dfdef:::benchmark_data_smithi121_358375_object56195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.353976606, 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0750470560000001, 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140940 2023-09-23T21:04:38.866 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.867 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458283+0000", 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458293+0000", 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9380000000000004e-06 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458312+0000", 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9083999999999999e-05 2023-09-23T21:04:38.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458320+0000", 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2300000000000008e-06 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458374+0000", 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4116000000000002e-05 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458383+0000", 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4409999999999993e-06 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.522747+0000", 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.064364592 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.522997+0000", 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024926899999999999 2023-09-23T21:04:38.870 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.523081+0000", 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4122999999999994e-05 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532666+0000", 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.009585484 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.871 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532923+0000", 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00025692899999999998 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533121+0000", 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000198209 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533191+0000", 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9379000000000001e-05 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533229+0000", 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8133999999999998e-05 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533332+0000", 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103198 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.873 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140941 10.c 10:3b70ed7f:::benchmark_data_smithi121_358375_object56196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.35323952900001, 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.075658805, 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140941 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.874 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:04:38.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459026+0000", 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9400000000000001e-06 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459035+0000", 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5290000000000008e-06 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459039+0000", 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0709999999999996e-06 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459081+0000", 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2103e-05 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459087+0000", 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3230000000000003e-06 2023-09-23T21:04:38.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525940+0000", 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.066852771 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.526063+0000", 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012287299999999999 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.526177+0000", 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113816 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533150+0000", 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0069729099999999997 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533368+0000", 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000218383 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533381+0000", 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2839e-05 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534586+0000", 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001204552 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534618+0000", 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2261999999999997e-05 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534681+0000", 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3054999999999999e-05 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140942 10.c 10:3a89f6d8:::benchmark_data_smithi121_358375_object56197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:04:38.881 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 182.33062416000001, 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0531614490000001, 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140942 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481636+0000", 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481643+0000", 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6429999999999995e-06 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481659+0000", 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5274e-05 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481665+0000", 2023-09-23T21:04:38.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3500000000000002e-06 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481716+0000", 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1357999999999999e-05 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481726+0000", 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8099999999999992e-06 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.885 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.528966+0000", 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0472400340000001 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.529117+0000", 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015073199999999999 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.529274+0000", 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000157431 2023-09-23T21:04:38.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533184+0000", 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00390931 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533387+0000", 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000203301 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533489+0000", 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102224 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534716+0000", 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012264190000000001 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534744+0000", 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8334999999999999e-05 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534799+0000", 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4917999999999998e-05 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50076 11.5 11:a3fd1c51:::benchmark_data_smithi121_477989_object50075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.420177144999997, 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1126377949999999, 2023-09-23T21:04:38.889 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50076 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392083+0000", 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392127+0000", 2023-09-23T21:04:38.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3806e-05 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392141+0000", 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4394e-05 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392150+0000", 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1650000000000006e-06 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392234+0000", 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4027e-05 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392388+0000", 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015483000000000001 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,6", 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392554+0000", 2023-09-23T21:04:38.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016592399999999999 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.398697+0000", 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061424319999999998 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.403213+0000", 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045163499999999997 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.894 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.403226+0000", 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2938999999999999e-05 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394392+0000", 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99116629099999998 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394406+0000", 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3896999999999999e-05 2023-09-23T21:04:38.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504613+0000", 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.110206864 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504722+0000", 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010918500000000001 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50104 11.5 11:ac5fad04:::benchmark_data_smithi121_477989_object50103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.382611318999999, 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0882670750000001, 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50104 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.897 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429649+0000", 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429803+0000", 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015389900000000001 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429823+0000", 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9340000000000001e-05 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429833+0000", 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0005e-05 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429893+0000", 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0324e-05 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429904+0000", 2023-09-23T21:04:38.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0781000000000001e-05 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394433+0000", 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96452942900000005 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394443+0000", 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4879999999999996e-06 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504746+0000", 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.110303627 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504903+0000", 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015637600000000001 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.505086+0000", 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000183516 2023-09-23T21:04:38.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.512945+0000", 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078586579999999993 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515525+0000", 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025799350000000002 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515589+0000", 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3974999999999996e-05 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515654+0000", 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4867000000000001e-05 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.517917+0000", 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022637120000000002 2023-09-23T21:04:38.904 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50113 11.5 11:ab5e54a5:::benchmark_data_smithi121_477989_object50112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.372572683999998, 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0819687199999999, 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.905 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50113 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439688+0000", 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439777+0000", 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9221000000000005e-05 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439790+0000", 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3013999999999999e-05 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439795+0000", 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2310000000000004e-06 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439871+0000", 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6036000000000002e-05 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439883+0000", 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.186e-05 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394464+0000", 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95458152699999999 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394472+0000", 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8790000000000003e-06 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.506903+0000", 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1124306429999999 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.507065+0000", 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016177499999999999 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.507239+0000", 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000174191 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515535+0000", 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0082961040000000003 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.517971+0000", 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024364360000000002 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.519321+0000", 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001349814 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.519388+0000", 2023-09-23T21:04:38.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6958999999999994e-05 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.521658+0000", 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022694899999999999 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50122 11.5 11:a0d5e578:::benchmark_data_smithi121_477989_object50121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:04:38.913 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.186328240000002, 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.905001269, 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50122 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:04:38.914 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.915 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625982+0000", 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9774999999999997e-05 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625994+0000", 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1986e-05 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625999+0000", 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9980000000000003e-06 2023-09-23T21:04:38.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.626045+0000", 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5528999999999997e-05 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.626052+0000", 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5159999999999997e-06 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394493+0000", 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76844040899999999 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394501+0000", 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5059999999999995e-06 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509037+0000", 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1145361 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509179+0000", 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014180599999999999 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509350+0000", 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017093699999999999 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515543+0000", 2023-09-23T21:04:38.919 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061929389999999997 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.522051+0000", 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065082919999999997 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.528556+0000", 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065045950000000002 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.528631+0000", 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4646000000000003e-05 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.530935+0000", 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023040959999999998 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.921 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50126 11.5 11:a51a9ec8:::benchmark_data_smithi121_477989_object50125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.183438586999998, 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9041968090000001, 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50126 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628822+0000", 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628877+0000", 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5653999999999999e-05 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628891+0000", 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3808000000000001e-05 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.924 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628897+0000", 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1460000000000001e-06 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628958+0000", 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0720000000000001e-05 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628968+0000", 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6229999999999992e-06 2023-09-23T21:04:38.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394522+0000", 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76555460600000003 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394530+0000", 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9019999999999999e-06 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511058+0000", 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1165273579999999 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511192+0000", 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013458099999999999 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511361+0000", 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016914299999999999 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515550+0000", 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041891139999999999 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.522105+0000", 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065545459999999996 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.531002+0000", 2023-09-23T21:04:38.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088968529999999997 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.531064+0000", 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2463000000000001e-05 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.533020+0000", 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019555419999999998 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.929 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78881 11.4 11:2f950127:::benchmark_data_smithi121_477989_object78880:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 22.926945602, 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30758963700000003, 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78881 2023-09-23T21:04:38.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885361+0000", 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5769000000000002e-05 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885384+0000", 2023-09-23T21:04:38.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3087000000000001e-05 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885390+0000", 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7540000000000003e-06 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.887935+0000", 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025446190000000001 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.933 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.888057+0000", 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122161 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.888200+0000", 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014262500000000001 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893701+0000", 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005501695 2023-09-23T21:04:38.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.905399+0000", 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011697991 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192819+0000", 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.28741952999999998 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192867+0000", 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8605e-05 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192906+0000", 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8302e-05 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78885 11.4 11:2a17f6c4:::benchmark_data_smithi121_477989_object78884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:04:38.936 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 22.926856615999998, 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30759584699999998, 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78885 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:38.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885437+0000", 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2227999999999997e-05 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885444+0000", 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6939999999999995e-06 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885449+0000", 2023-09-23T21:04:38.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3950000000000003e-06 2023-09-23T21:04:38.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.941 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.941 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890915+0000", 2023-09-23T21:04:38.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0054663869999999996 2023-09-23T21:04:38.941 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.891027+0000", 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011214600000000001 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.891156+0000", 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012827499999999999 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893843+0000", 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026873719999999999 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.905447+0000", 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011604353 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192935+0000", 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.28748794300000002 2023-09-23T21:04:38.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192973+0000", 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7407999999999998e-05 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.193001+0000", 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8166000000000001e-05 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78889 11.5 11:a04ebbe0:::benchmark_data_smithi121_477989_object78888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 22.925822339, 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30786717499999999, 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78889 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.946 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886438+0000", 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886479+0000", 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0985000000000001e-05 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886493+0000", 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4032999999999999e-05 2023-09-23T21:04:38.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886499+0000", 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1859999999999997e-06 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890610+0000", 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041114760000000002 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890765+0000", 2023-09-23T21:04:38.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000154184 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890939+0000", 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017454800000000001 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893385+0000", 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002445984 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.899338+0000", 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059533499999999996 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192227+0000", 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.29288863399999998 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192279+0000", 2023-09-23T21:04:38.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2392999999999998e-05 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.194306+0000", 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020270679999999999 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88133 11.5 11:a2e8b6ce:::benchmark_data_smithi121_477989_object88132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:04:38.951 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 10.064345736, 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.015216702, 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88133 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:04:38.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747914+0000", 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747958+0000", 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.333e-05 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747970+0000", 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2914999999999999e-05 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747977+0000", 2023-09-23T21:04:38.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4620000000000001e-06 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752486+0000", 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045088680000000001 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752629+0000", 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014352899999999999 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.955 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752804+0000", 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017513500000000001 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.776368+0000", 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023563562999999999 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.776380+0000", 2023-09-23T21:04:38.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2476999999999999e-05 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763117+0000", 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98673686100000002 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763133+0000", 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5291e-05 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.957 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88146 11.5 11.f29ad9f5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 10.039427927, 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99033201000000004, 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88146 2023-09-23T21:04:38.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772833+0000", 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772873+0000", 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0082e-05 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772885+0000", 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2529e-05 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772893+0000", 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3520000000000001e-06 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.777718+0000", 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048257910000000003 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.777727+0000", 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4780000000000008e-06 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763156+0000", 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98542964799999999 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763166+0000", 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2620000000000004e-06 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88149 11.5 11.fd969135 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 10.023124779, 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97405727799999997, 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:04:38.963 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88149 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:04:38.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789135+0000", 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789175+0000", 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9626000000000003e-05 2023-09-23T21:04:38.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789193+0000", 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7737999999999999e-05 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789199+0000", 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4500000000000001e-06 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789265+0000", 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5572000000000002e-05 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789281+0000", 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6612e-05 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763186+0000", 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.973904412 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:04:38.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763194+0000", 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5080000000000005e-06 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:04:38.968 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:04:39.087 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:39 smithi123 ceph-mon[193903]: pgmap v1163: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 35 KiB/s wr, 0 op/s; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.9 MiB/s, 1 keys/s, 62 objects/s recovering 2023-09-23T21:04:39.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 88186 88170 61.8929 0 - 0.0145162 2023-09-23T21:04:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:39 smithi121 ceph-mon[211526]: pgmap v1163: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 35 KiB/s wr, 0 op/s; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.9 MiB/s, 1 keys/s, 62 objects/s recovering 2023-09-23T21:04:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:39 smithi121 ceph-mon[219709]: pgmap v1163: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 35 KiB/s wr, 0 op/s; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.9 MiB/s, 1 keys/s, 62 objects/s recovering 2023-09-23T21:04:39.752 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:39.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:39.862 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:39.900 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:39.950 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:39.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:39.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:40.056 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:40.089 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 90 16 88186 88170 61.2055 0 - 0.0145162 2023-09-23T21:04:40.247 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:40.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:40.300 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:40 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:40.300 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:40 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:40.300 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:40 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:40.300 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:40 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:40.351 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:40.385 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.427 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:04:40.428 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:40 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:40.428 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:40 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:40.441 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:40.545 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:40.579 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.636 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:40.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:40.766 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:40 smithi121 conmon[211495]: 2023-09-23T21:04:40.763+0000 7fdc50c66700 -1 mon.a@0(leader).osd e370 definitely_dead 0 2023-09-23T21:04:40.769 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:40.817 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:40.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:40.848 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:40.924 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:40.952 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:41.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1184270038' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:41.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:41.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1038827197' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:04:41.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:41.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:41.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[211526]: pgmap v1164: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 31 KiB/s wr, 0 op/s; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.5 MiB/s, 0 keys/s, 56 objects/s recovering 2023-09-23T21:04:41.175 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:04:41.176 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.3. 2023-09-23T21:04:41.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 91 16 88186 88170 60.5331 0 - 0.0145162 2023-09-23T21:04:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1184270038' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1038827197' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:04:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:41 smithi121 ceph-mon[219709]: pgmap v1164: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 31 KiB/s wr, 0 op/s; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.5 MiB/s, 0 keys/s, 56 objects/s recovering 2023-09-23T21:04:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1184270038' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:04:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1038827197' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:04:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:41 smithi123 ceph-mon[193903]: pgmap v1164: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 31 KiB/s wr, 0 op/s; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.5 MiB/s, 0 keys/s, 56 objects/s recovering 2023-09-23T21:04:42.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 92 16 88186 88170 59.8753 0 - 0.0145162 2023-09-23T21:04:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: failed to encode map e371 with expected crc 2023-09-23T21:04:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: failed to encode map e371 with expected crc 2023-09-23T21:04:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1038827197' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:04:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6406/263382 objects degraded (2.432%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: failed to encode map e371 with expected crc 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: failed to encode map e371 with expected crc 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1038827197' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:42.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6406/263382 objects degraded (2.432%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:04:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: failed to encode map e371 with expected crc 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: failed to encode map e371 with expected crc 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1038827197' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: osdmap e371: 8 total, 8 up, 7 in 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: osdmap e372: 8 total, 8 up, 7 in 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: failed to encode map e372 with expected crc 2023-09-23T21:04:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:04:42.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:04:42.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:42.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6406/263382 objects degraded (2.432%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:04:42.652 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:42.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:42.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:42.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:42.848 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:43.015 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T21:04:43.051 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:43.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:43.155 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:43.205 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:43.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 93 16 88186 88170 59.2317 0 - 0.0145162 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: failed to encode map e372 with expected crc 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: failed to encode map e372 with expected crc 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: 11.9 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5561 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: 11.a starting backfill to osd.3 from (0'0,0'0] MAX to 370'5437 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: 11.b starting backfill to osd.3 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: failed to encode map e373 with expected crc 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: failed to encode map e373 with expected crc 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: 11.8 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:04:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: 11.2 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5491 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: 11.e starting backfill to osd.3 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[219709]: pgmap v1168: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: failed to encode map e372 with expected crc 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: failed to encode map e372 with expected crc 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: 11.9 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5561 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: 11.a starting backfill to osd.3 from (0'0,0'0] MAX to 370'5437 2023-09-23T21:04:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: 11.b starting backfill to osd.3 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: failed to encode map e373 with expected crc 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: failed to encode map e373 with expected crc 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: 11.8 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: 11.2 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5491 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: 11.e starting backfill to osd.3 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:43.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:43 smithi121 ceph-mon[211526]: pgmap v1168: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:04:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: failed to encode map e372 with expected crc 2023-09-23T21:04:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: failed to encode map e372 with expected crc 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: 11.9 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5561 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: 11.a starting backfill to osd.3 from (0'0,0'0] MAX to 370'5437 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: 11.b starting backfill to osd.3 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: failed to encode map e373 with expected crc 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: failed to encode map e373 with expected crc 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: osdmap e373: 8 total, 8 up, 7 in 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: 11.8 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: 11.2 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5491 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: 11.e starting backfill to osd.3 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:43 smithi123 ceph-mon[193903]: pgmap v1168: 89 pgs: 2 remapped+peering, 4 active+clean+remapped, 9 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 6406/263382 objects degraded (2.432%); 6562/263382 objects misplaced (2.491%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:04:43.904 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:43.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:43.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:44.007 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:44.036 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:44.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 94 16 88186 88170 58.6018 0 - 0.0145162 2023-09-23T21:04:44.330 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:44.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:44.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:44 smithi121 ceph-mon[211526]: 11.1 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5452 2023-09-23T21:04:44.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:44 smithi121 ceph-mon[219709]: 11.1 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5452 2023-09-23T21:04:44.436 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:44.468 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:44.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:44 smithi123 ceph-mon[193903]: 11.1 starting backfill to osd.3 from (0'0,0'0] MAX to 370'5452 2023-09-23T21:04:44.550 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:44.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:44.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:44.653 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:44.686 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:44.839 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:44.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:44.906 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88161 11.f 11.2377ddf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:31.400197+0000", 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 13.506124918999999, 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 13.506146862, 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:04:44.907 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88161 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.400197+0000", 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.400197+0000", 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.400195+0000", 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.400225+0000", 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9677999999999999e-05 2023-09-23T21:04:44.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.400237+0000", 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1762e-05 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.400245+0000", 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3480000000000004e-06 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:04:44.910 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88140 11.7 11.af9103e7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.748403+0000", 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 16.157919254999999, 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 16.157983181999999, 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:04:44.911 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88140 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.748403+0000", 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.748403+0000", 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.748401+0000", 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.748433+0000", 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1914e-05 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.748445+0000", 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2666e-05 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.748456+0000", 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0525e-05 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.760332+0000", 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011875847 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.760344+0000", 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1979e-05 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760168+0000", 2023-09-23T21:04:44.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99982417099999998 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760179+0000", 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1005999999999999e-05 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.916 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88134 11.f 11.4c58527f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.759686+0000", 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 16.146635970999998, 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 16.146746643, 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88134 2023-09-23T21:04:44.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759686+0000", 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759686+0000", 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759684+0000", 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759711+0000", 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6857000000000001e-05 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759726+0000", 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4849e-05 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759735+0000", 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7140000000000005e-06 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.760504+0000", 2023-09-23T21:04:44.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00076919800000000002 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.760513+0000", 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.602e-06 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760202+0000", 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99968884000000002 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760209+0000", 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1169999999999998e-06 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88136 11.f 11.451785ef (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.759742+0000", 2023-09-23T21:04:44.922 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 16.146579306, 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 16.146734608999999, 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88136 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759742+0000", 2023-09-23T21:04:44.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759742+0000", 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759742+0000", 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:04:44.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759761+0000", 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9381e-05 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759768+0000", 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.985e-06 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.759773+0000", 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.967e-06 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.760565+0000", 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00079180400000000001 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.760575+0000", 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.009e-05 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760229+0000", 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99965401499999995 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760236+0000", 2023-09-23T21:04:44.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.956e-06 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88179 11.f 11.d565c18f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e370)", 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:31.459722+0000", 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 13.446600221000001, 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 13.446800791999999, 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T21:04:44.928 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88179 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.459722+0000", 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.459722+0000", 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.459720+0000", 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:04:44.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.459746+0000", 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6291999999999998e-05 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.459757+0000", 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0868e-05 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.459764+0000", 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1949999999999997e-06 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.931 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88150 11.f 11.6421ab5f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.790924+0000", 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 16.115397959999999, 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 16.115631012000001, 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:04:44.932 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88150 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.790924+0000", 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.790924+0000", 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:04:44.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.790922+0000", 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.790945+0000", 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2575e-05 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.790959+0000", 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4455e-05 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.790966+0000", 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0140000000000001e-06 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.791010+0000", 2023-09-23T21:04:44.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4075000000000001e-05 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.791020+0000", 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6809999999999993e-06 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760259+0000", 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.969238657 2023-09-23T21:04:44.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.760276+0000", 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7422e-05 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 6 2023-09-23T21:04:44.937 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:04:44.944 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:44.971 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.018 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:45.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:45.144 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:04:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:04:44] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-09-23T21:04:45.156 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:04:45.172 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:45 smithi123 ceph-mon[193903]: pgmap v1169: 89 pgs: 2 remapped+peering, 14 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 96 KiB/s wr, 0 op/s; 9749/263400 objects degraded (3.701%); 11697/263400 objects misplaced (4.441%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:04:45.229 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:45.234 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 95 16 88186 88170 57.9851 0 - 0.0145162 2023-09-23T21:04:45.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:45.361 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.392 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:04:45.392 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:04:45.392 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:04:45.392 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:04:45.408 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:45.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:45 smithi121 ceph-mon[219709]: pgmap v1169: 89 pgs: 2 remapped+peering, 14 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 96 KiB/s wr, 0 op/s; 9749/263400 objects degraded (3.701%); 11697/263400 objects misplaced (4.441%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:04:45.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:45 smithi121 ceph-mon[211526]: pgmap v1169: 89 pgs: 2 remapped+peering, 14 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 96 KiB/s wr, 0 op/s; 9749/263400 objects degraded (3.701%); 11697/263400 objects misplaced (4.441%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T21:04:45.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:45.514 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:45.542 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.592 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:45.697 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:45.724 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.774 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:45.794 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:04:45.880 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:45.909 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:45.966 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:45.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:46.071 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:46.102 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.154 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:04:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:04:46.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 96 10 88186 88176 57.3851 0.0267857 16.6532 0.0155853 2023-09-23T21:04:46.244 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:46.259 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:46.285 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.329 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:46.354 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:04:46.435 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:46.461 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.505 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:46.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:46.611 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:46.635 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:46 smithi123 ceph-mon[193903]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:46.637 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:46 smithi121 ceph-mon[211526]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:46.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:46 smithi121 ceph-mon[219709]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T21:04:46.681 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:46.789 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:46.815 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:46.863 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:46.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:46.971 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:47.002 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.049 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:47.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:47.153 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:47.179 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.222 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:47.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 97 10 88186 88176 56.7937 0 - 0.0155853 2023-09-23T21:04:47.327 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:47.356 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.403 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:47.506 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:47.537 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.588 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:47.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:47.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:47 smithi121 ceph-mon[219709]: pgmap v1170: 89 pgs: 2 activating+undersized+degraded+remapped, 2 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 4.1 MiB/s wr, 64 op/s; 37293/264552 objects degraded (14.097%); 11700/264552 objects misplaced (4.423%); 3.0 MiB/s, 46 objects/s recovering 2023-09-23T21:04:47.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:47 smithi121 ceph-mon[211526]: pgmap v1170: 89 pgs: 2 activating+undersized+degraded+remapped, 2 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 4.1 MiB/s wr, 64 op/s; 37293/264552 objects degraded (14.097%); 11700/264552 objects misplaced (4.423%); 3.0 MiB/s, 46 objects/s recovering 2023-09-23T21:04:47.691 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:47.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:47 smithi123 ceph-mon[193903]: pgmap v1170: 89 pgs: 2 activating+undersized+degraded+remapped, 2 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 4.1 MiB/s wr, 64 op/s; 37293/264552 objects degraded (14.097%); 11700/264552 objects misplaced (4.423%); 3.0 MiB/s, 46 objects/s recovering 2023-09-23T21:04:47.717 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.761 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:47.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:47.869 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:47.895 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:47.938 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:47.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:48.016 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 3] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:04:48.016 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:04:48.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:48.069 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.113 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:48.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:48.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:48.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 98 10 88186 88176 56.2143 0 - 0.0155853 2023-09-23T21:04:48.248 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.303 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:48.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:48.408 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:48.434 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.479 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:48.585 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:48.612 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.656 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:48.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:48.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:48 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 37293/264552 objects degraded (14.097%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:04:48.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:48 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 37293/264552 objects degraded (14.097%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:04:48.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:48 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 37293/264552 objects degraded (14.097%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:04:48.761 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:04:48.787 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:48.831 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:04:48.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:48.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:04:48.935 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:48.962 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:49.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 99 10 88186 88176 55.6466 0 - 0.0155853 2023-09-23T21:04:49.361 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:49.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:49.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:49.465 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:49.507 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:49.528 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:49.571 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:49.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:49.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:49.656 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:49.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:49 smithi121 ceph-mon[211526]: pgmap v1171: 89 pgs: 2 activating+undersized+degraded+remapped, 2 remapped+peering, 15 peering, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.1 MiB/s wr, 49 op/s; 33950/264552 objects degraded (12.833%); 12570/264552 objects misplaced (4.751%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T21:04:49.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:49 smithi121 ceph-mon[219709]: pgmap v1171: 89 pgs: 2 activating+undersized+degraded+remapped, 2 remapped+peering, 15 peering, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.1 MiB/s wr, 49 op/s; 33950/264552 objects degraded (12.833%); 12570/264552 objects misplaced (4.751%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T21:04:49.677 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:49.709 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:49.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:49 smithi123 ceph-mon[193903]: pgmap v1171: 89 pgs: 2 activating+undersized+degraded+remapped, 2 remapped+peering, 15 peering, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.1 MiB/s wr, 49 op/s; 33950/264552 objects degraded (12.833%); 12570/264552 objects misplaced (4.751%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T21:04:49.837 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:04:49.863 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:04:49.864 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:04:49.864 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:04:49.864 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:04:49.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:04:49.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:04:49.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:04:49.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:04:49.865 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:04:50.126 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:04:50.126 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:04:50.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:04:50.236086+0000 min lat: 0.00123114 max lat: 18.7578 avg lat: 0.0166307 2023-09-23T21:04:50.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:04:50.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 100 5 88186 88181 55.0934 0.078125 18.3328 0.0166307 2023-09-23T21:04:50.279 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:50.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:50.384 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:50.409 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:50.454 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:50.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:50.563 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:50.589 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:50.633 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:50.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:50.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:50.739 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:04:50.768 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:51.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 101 5 88186 88181 54.5481 0 - 0.0166307 2023-09-23T21:04:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:51 smithi121 ceph-mon[211526]: pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.7 MiB/s wr, 42 op/s; 38703/264618 objects degraded (14.626%); 36576/264618 objects misplaced (13.822%); 16 MiB/s, 254 objects/s recovering 2023-09-23T21:04:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:51 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T21:04:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:51 smithi121 ceph-mon[219709]: pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.7 MiB/s wr, 42 op/s; 38703/264618 objects degraded (14.626%); 36576/264618 objects misplaced (13.822%); 16 MiB/s, 254 objects/s recovering 2023-09-23T21:04:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:51 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T21:04:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:51 smithi123 ceph-mon[193903]: pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.7 MiB/s wr, 42 op/s; 38703/264618 objects degraded (14.626%); 36576/264618 objects misplaced (13.822%); 16 MiB/s, 254 objects/s recovering 2023-09-23T21:04:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:51 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T21:04:52.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 102 5 88186 88181 54.0134 0 - 0.0166307 2023-09-23T21:04:52.785 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:04:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:52.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:04:52.889 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:53.013 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:04:53.040 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:53.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:53.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:53.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:53 smithi121 ceph-mon[211526]: pgmap v1173: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.4 MiB/s wr, 38 op/s; 38703/264618 objects degraded (14.626%); 36576/264618 objects misplaced (13.822%); 14 MiB/s, 230 objects/s recovering 2023-09-23T21:04:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:53 smithi121 ceph-mon[219709]: pgmap v1173: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.4 MiB/s wr, 38 op/s; 38703/264618 objects degraded (14.626%); 36576/264618 objects misplaced (13.822%); 14 MiB/s, 230 objects/s recovering 2023-09-23T21:04:53.192 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:53.218 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:53.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 103 5 88186 88181 53.4892 0 - 0.0166307 2023-09-23T21:04:53.267 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:53.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:53.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:53.372 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:53.400 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:53 smithi123 ceph-mon[193903]: pgmap v1173: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.4 MiB/s wr, 38 op/s; 38703/264618 objects degraded (14.626%); 36576/264618 objects misplaced (13.822%); 14 MiB/s, 230 objects/s recovering 2023-09-23T21:04:54.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 104 5 88186 88181 52.975 0 - 0.0166307 2023-09-23T21:04:55.013 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:04:55.087 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:04:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:04:54] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-09-23T21:04:55.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 105 5 88186 88181 52.4706 0 - 0.0166307 2023-09-23T21:04:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:55 smithi121 ceph-mon[211526]: pgmap v1174: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s; 41646/264618 objects degraded (15.738%); 36106/264618 objects misplaced (13.645%); 13 MiB/s, 210 objects/s recovering 2023-09-23T21:04:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:55 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41646/264618 objects degraded (15.738%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:04:55.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:55 smithi121 ceph-mon[219709]: pgmap v1174: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s; 41646/264618 objects degraded (15.738%); 36106/264618 objects misplaced (13.645%); 13 MiB/s, 210 objects/s recovering 2023-09-23T21:04:55.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:55 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41646/264618 objects degraded (15.738%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:04:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:55 smithi123 ceph-mon[193903]: pgmap v1174: 89 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s; 41646/264618 objects degraded (15.738%); 36106/264618 objects misplaced (13.645%); 13 MiB/s, 210 objects/s recovering 2023-09-23T21:04:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:55 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41646/264618 objects degraded (15.738%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:04:55.511 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:55.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:55.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:55.615 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:55.641 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:55.684 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:55.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:55.789 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:04:55.796 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:55.834 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:55.861 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:04:55.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:04:55.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:55.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:04:56.050 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:04:56.076 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.127 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:04:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:04:56.231 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:04:56.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 106 5 88186 88181 51.9757 0 - 0.0166307 2023-09-23T21:04:56.274 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.371 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:04:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:04:56.480 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:04:56.506 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.554 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:04:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:04:56.661 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:56.688 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.732 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:56.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:56.837 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:04:56.864 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:56.909 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:04:56.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:04:56.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:04:57.015 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:04:57.075 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:04:57.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 107 5 88186 88181 51.4901 0 - 0.0166307 2023-09-23T21:04:57.439 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:04:57.441 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:04:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1095459922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:04:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3784373999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:57 smithi121 ceph-mon[211526]: pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s; 41646/264618 objects degraded (15.738%); 35662/264618 objects misplaced (13.477%); 14 MiB/s, 231 objects/s recovering 2023-09-23T21:04:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1095459922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:04:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3784373999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:57 smithi121 ceph-mon[219709]: pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s; 41646/264618 objects degraded (15.738%); 35662/264618 objects misplaced (13.477%); 14 MiB/s, 231 objects/s recovering 2023-09-23T21:04:57.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1095459922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:04:57.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3784373999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:04:57.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:57 smithi123 ceph-mon[193903]: pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 2.1 MiB/s wr, 34 op/s; 41646/264618 objects degraded (15.738%); 35662/264618 objects misplaced (13.477%); 14 MiB/s, 231 objects/s recovering 2023-09-23T21:04:58.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 108 5 88186 88181 51.0133 0 - 0.0166307 2023-09-23T21:04:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1095459922' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:04:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3784373999' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T21:04:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:58.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:58.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1095459922' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3784373999' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:58.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:04:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1095459922' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3784373999' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: osdmap e374: 8 total, 8 up, 7 in 2023-09-23T21:04:58.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: failed to encode map e374 with expected crc 2023-09-23T21:04:58.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:04:58.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:04:59.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 109 5 88186 88181 50.5454 0 - 0.0166307 2023-09-23T21:04:59.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.9 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5561 2023-09-23T21:04:59.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.b starting backfill to osd.0 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.1 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5452 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.e starting backfill to osd.0 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.8 continuing backfill to osd.3 from (349'1473,370'5531] 11:101db76c:::benchmark_data_smithi121_477989_object36331:head to 370'5531 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.b continuing backfill to osd.3 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.9 continuing backfill to osd.3 from (349'1505,370'5561] MIN to 370'5561 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.1 continuing backfill to osd.3 from (349'1394,370'5452] MIN to 370'5452 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: 11.e continuing backfill to osd.3 from (347'239,370'5599] MIN to 370'5599 2023-09-23T21:04:59.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[211526]: pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41647/264618 objects degraded (15.739%); 41193/264618 objects misplaced (15.567%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:04:59.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.9 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5561 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.b starting backfill to osd.0 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.1 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5452 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.e starting backfill to osd.0 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.8 continuing backfill to osd.3 from (349'1473,370'5531] 11:101db76c:::benchmark_data_smithi121_477989_object36331:head to 370'5531 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.b continuing backfill to osd.3 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:04:59.650 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.9 continuing backfill to osd.3 from (349'1505,370'5561] MIN to 370'5561 2023-09-23T21:04:59.651 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.1 continuing backfill to osd.3 from (349'1394,370'5452] MIN to 370'5452 2023-09-23T21:04:59.651 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: 11.e continuing backfill to osd.3 from (347'239,370'5599] MIN to 370'5599 2023-09-23T21:04:59.651 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:04:59 smithi121 ceph-mon[219709]: pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41647/264618 objects degraded (15.739%); 41193/264618 objects misplaced (15.567%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T21:04:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: osdmap e375: 8 total, 8 up, 7 in 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.9 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5561 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.b starting backfill to osd.0 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.1 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5452 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: failed to encode map e375 with expected crc 2023-09-23T21:04:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.e starting backfill to osd.0 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:04:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.8 continuing backfill to osd.3 from (349'1473,370'5531] 11:101db76c:::benchmark_data_smithi121_477989_object36331:head to 370'5531 2023-09-23T21:04:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.b continuing backfill to osd.3 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:04:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.9 continuing backfill to osd.3 from (349'1505,370'5561] MIN to 370'5561 2023-09-23T21:04:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.1 continuing backfill to osd.3 from (349'1394,370'5452] MIN to 370'5452 2023-09-23T21:04:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: 11.e continuing backfill to osd.3 from (347'239,370'5599] MIN to 370'5599 2023-09-23T21:04:59.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:04:59 smithi123 ceph-mon[193903]: pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 1 peering, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41647/264618 objects degraded (15.739%); 41193/264618 objects misplaced (15.567%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T21:05:00.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 110 5 88186 88181 50.086 0 - 0.0166307 2023-09-23T21:05:00.252 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:00.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:00.357 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:00.384 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:00.429 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:00.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:00.536 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:00.562 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:00.606 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:00.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:00.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:00.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[211526]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[211526]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T21:05:00.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[211526]: 11.6 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5399 2023-09-23T21:05:00.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[211526]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[211526]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[211526]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[219709]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[219709]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T21:05:00.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[219709]: 11.6 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5399 2023-09-23T21:05:00.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[219709]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[219709]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:00 smithi121 ceph-mon[219709]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.713 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:00 smithi123 ceph-mon[193903]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:00 smithi123 ceph-mon[193903]: osdmap e376: 8 total, 8 up, 7 in 2023-09-23T21:05:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:00 smithi123 ceph-mon[193903]: 11.6 starting backfill to osd.0 from (0'0,0'0] MAX to 370'5399 2023-09-23T21:05:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:00 smithi123 ceph-mon[193903]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:00 smithi123 ceph-mon[193903]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:00 smithi123 ceph-mon[193903]: failed to encode map e376 with expected crc 2023-09-23T21:05:00.776 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:00.867 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:05:01.114 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:01.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:01.199 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:05:01.219 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:01.220 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:05:01.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 111 5 88186 88181 49.6349 0 - 0.0166307 2023-09-23T21:05:01.239 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:05:01.248 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:01.259 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:05:01.283 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:05:01.301 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:01.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:01.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:01.305 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:05:01.391 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:91726 10.2 10.67783df2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:43.511418+0000", 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 437.884659613, 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73011041099999996, 2023-09-23T21:05:01.399 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 91726 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511418+0000", 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.400 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511418+0000", 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511416+0000", 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.401 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511422+0000", 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9599999999999997e-06 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511432+0000", 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8719999999999997e-06 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511441+0000", 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2909999999999996e-06 2023-09-23T21:05:01.402 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511493+0000", 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1400000000000003e-05 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511502+0000", 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.38e-06 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.403 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241515+0000", 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73001329000000004 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241528+0000", 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2694000000000001e-05 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.404 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140910 10.12 10:4837830a:::benchmark_data_smithi121_358375_object56165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.102455+0000", 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 205.29362221400001, 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1947788660000001, 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140910 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.405 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102455+0000", 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102455+0000", 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102453+0000", 2023-09-23T21:05:01.406 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102463+0000", 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8449999999999998e-06 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102481+0000", 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8437999999999999e-05 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.407 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102492+0000", 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047e-05 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102549+0000", 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7170999999999999e-05 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102559+0000", 2023-09-23T21:05:01.408 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0237999999999999e-05 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424192+0000", 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.321633053 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424199+0000", 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0870000000000001e-06 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.409 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.036174+0000", 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6119743660000001 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.036312+0000", 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013814200000000001 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.410 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.036596+0000", 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00028440600000000001 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.293913+0000", 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25731645600000003 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.293966+0000", 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3164999999999997e-05 2023-09-23T21:05:01.411 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.296611+0000", 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026455139999999999 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.413 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.296678+0000", 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6105999999999994e-05 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297234+0000", 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00055633400000000002 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140922 10.2 10:4783e976:::benchmark_data_smithi121_358375_object56177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:05:01.414 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.206018+0000", 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 205.190058808, 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0967715469999999, 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140922 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.415 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206018+0000", 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206018+0000", 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206017+0000", 2023-09-23T21:05:01.416 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206024+0000", 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5529999999999998e-06 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206037+0000", 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3502e-05 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.417 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206046+0000", 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8319999999999995e-06 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206112+0000", 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6548e-05 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.418 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206126+0000", 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3871e-05 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424107+0000", 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.21798082399999999 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.419 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424119+0000", 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2058e-05 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.285655+0000", 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8615357120000002 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.287673+0000", 2023-09-23T21:05:01.420 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020179690000000001 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.290434+0000", 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027612829999999998 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.421 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.296758+0000", 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063240839999999998 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297811+0000", 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0010529599999999999 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.298391+0000", 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057977500000000004 2023-09-23T21:05:01.422 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.299305+0000", 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00091387200000000003 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.302790+0000", 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00348504 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.423 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140925 10.2 10:46517284:::benchmark_data_smithi121_358375_object56180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.213680+0000", 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 205.18239706099999, 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1225043499999998, 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140925 2023-09-23T21:05:01.424 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213680+0000", 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.425 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213680+0000", 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213678+0000", 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.426 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213688+0000", 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6809999999999993e-06 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213703+0000", 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5082000000000001e-05 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.427 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213712+0000", 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1060000000000006e-06 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213775+0000", 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2755000000000005e-05 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.428 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213786+0000", 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1973999999999999e-05 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424140+0000", 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.210353283 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.429 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424147+0000", 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1380000000000001e-06 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.303011+0000", 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8788636099999998 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.430 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.305262+0000", 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022513820000000001 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.308610+0000", 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033481280000000001 2023-09-23T21:05:01.431 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.332116+0000", 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.023506422999999999 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.332590+0000", 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00047317300000000002 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.432 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.332729+0000", 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139398 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.333445+0000", 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00071641100000000002 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.433 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336184+0000", 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027390510000000002 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140934 10.2 10:4673e96c:::benchmark_data_smithi121_358375_object56189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.283977+0000", 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 205.11210061200001, 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0573845450000001, 2023-09-23T21:05:01.434 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140934 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283977+0000", 2023-09-23T21:05:01.435 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283977+0000", 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283975+0000", 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.436 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283981+0000", 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.913e-06 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283992+0000", 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0363999999999999e-05 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283997+0000", 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7910000000000001e-06 2023-09-23T21:05:01.437 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.284040+0000", 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3022000000000002e-05 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.284048+0000", 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2520000000000002e-06 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.438 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424166+0000", 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14011805399999999 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424173+0000", 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8689999999999998e-06 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.312215+0000", 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.888042714 2023-09-23T21:05:01.439 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.314573+0000", 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023574109999999998 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.318102+0000", 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003529435 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.440 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336333+0000", 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018230745 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336933+0000", 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00060003200000000002 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.337441+0000", 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050855500000000005 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.441 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.338135+0000", 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00069391699999999995 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.341361+0000", 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003225819 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.442 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140943 10.12 10:492afa11:::benchmark_data_smithi121_358375_object56198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.514244+0000", 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.881833667, 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.783543908, 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140943 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.443 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514244+0000", 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514244+0000", 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.444 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514242+0000", 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514252+0000", 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0256e-05 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.548140+0000", 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.033888505999999999 2023-09-23T21:05:01.445 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.548154+0000", 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3625000000000001e-05 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.037993+0000", 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.48983858 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.446 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.038159+0000", 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016624700000000001 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.038384+0000", 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00022518899999999999 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.293920+0000", 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25553648400000001 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.447 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297302+0000", 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003381472 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297361+0000", 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8845000000000003e-05 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297420+0000", 2023-09-23T21:05:01.448 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9021999999999998e-05 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297787+0000", 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00036763300000000002 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140955 10.2 10:435f7b81:::benchmark_data_smithi121_358375_object56210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.449 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.557460+0000", 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.83861676199999, 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.788443177, 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140955 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.450 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557460+0000", 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557460+0000", 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557458+0000", 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.451 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557467+0000", 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4740000000000005e-06 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.452 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557479+0000", 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1987999999999999e-05 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557489+0000", 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5889999999999992e-06 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.453 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.322214+0000", 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.764725766 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.324762+0000", 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025475789999999999 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.328397+0000", 2023-09-23T21:05:01.454 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036347319999999999 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336482+0000", 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080858570000000001 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.455 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.341956+0000", 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054735859999999999 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.342423+0000", 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00046714599999999999 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.343086+0000", 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00066331699999999996 2023-09-23T21:05:01.456 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.345904+0000", 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028171120000000001 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.457 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140956 10.12 10:4f74787e:::benchmark_data_smithi121_358375_object56211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.562575+0000", 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.83350227599999, 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.793809465, 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.458 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140956 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562575+0000", 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562575+0000", 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.459 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562573+0000", 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562582+0000", 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6419999999999996e-06 2023-09-23T21:05:01.460 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562596+0000", 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3827e-05 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562606+0000", 2023-09-23T21:05:01.461 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0251000000000001e-05 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.345960+0000", 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7833538820000001 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.462 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.346160+0000", 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020042000000000001 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.346599+0000", 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00043851799999999999 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.463 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.350847+0000", 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042478890000000004 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.350857+0000", 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0258000000000001e-05 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.355774+0000", 2023-09-23T21:05:01.464 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049169360000000002 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.355811+0000", 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7187000000000002e-05 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356384+0000", 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057330499999999997 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.465 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140957 10.2 10:42e2eb9f:::benchmark_data_smithi121_358375_object56212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.563491+0000", 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.83258663000001, 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8074938659999999, 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.466 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140957 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563491+0000", 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.467 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563491+0000", 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563489+0000", 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563496+0000", 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2979999999999997e-06 2023-09-23T21:05:01.468 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563508+0000", 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1928e-05 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563516+0000", 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2220000000000002e-06 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.469 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.346650+0000", 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.783134309 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.349101+0000", 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002450972 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.352487+0000", 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033854990000000001 2023-09-23T21:05:01.470 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366032+0000", 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013545171 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366538+0000", 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050632300000000004 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.471 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.367103+0000", 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00056445900000000001 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.367988+0000", 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00088564700000000004 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.370984+0000", 2023-09-23T21:05:01.472 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029961340000000001 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141012 10.12 10:4b4d14f7:::benchmark_data_smithi121_358375_object56267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.723211+0000", 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.672866693, 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6431093799999998, 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.473 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141012 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723211+0000", 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.474 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723211+0000", 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723208+0000", 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723219+0000", 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0429e-05 2023-09-23T21:05:01.475 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723233+0000", 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3854e-05 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723242+0000", 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3659999999999998e-06 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.476 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.348512+0000", 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.625270223 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.348662+0000", 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014937300000000001 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.349009+0000", 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00034730699999999999 2023-09-23T21:05:01.477 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356434+0000", 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074248750000000001 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.365465+0000", 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090311019999999992 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.478 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.365476+0000", 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1039999999999999e-05 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.365544+0000", 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7500000000000001e-05 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366320+0000", 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00077635300000000003 2023-09-23T21:05:01.479 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141018 10.2 10:4724d2b5:::benchmark_data_smithi121_358375_object56273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.732723+0000", 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.663353962, 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6436562540000001, 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.480 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141018 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732723+0000", 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732723+0000", 2023-09-23T21:05:01.481 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732721+0000", 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732729+0000", 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2729999999999997e-06 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.482 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732737+0000", 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8610000000000004e-06 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732744+0000", 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2650000000000002e-06 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.483 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.355973+0000", 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.623229349 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.358295+0000", 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002321427 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.484 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.361636+0000", 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003341157 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371580+0000", 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.009944652 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371760+0000", 2023-09-23T21:05:01.485 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000179514 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.372291+0000", 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00053071699999999999 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.486 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.373187+0000", 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089667000000000002 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376379+0000", 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031922399999999998 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.487 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141049 10.12 10:486305eb:::benchmark_data_smithi121_358375_object56304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.799440+0000", 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.596637676, 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5723892450000001, 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141049 2023-09-23T21:05:01.488 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799440+0000", 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799440+0000", 2023-09-23T21:05:01.489 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799438+0000", 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799448+0000", 2023-09-23T21:05:01.490 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.077e-05 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799466+0000", 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7416999999999999e-05 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799477+0000", 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.131e-05 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.491 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356459+0000", 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.556982026 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356634+0000", 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017506999999999999 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.492 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.357018+0000", 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00038430700000000002 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366425+0000", 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094066500000000008 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366550+0000", 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012518800000000001 2023-09-23T21:05:01.493 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371053+0000", 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004503155 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.494 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371122+0000", 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8137000000000002e-05 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371829+0000", 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00070721499999999997 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.495 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141098 10.12 10:4b556bc3:::benchmark_data_smithi121_358375_object56353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.986992+0000", 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.40908469799999, 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.42552641, 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.496 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141098 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986992+0000", 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.497 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986992+0000", 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986991+0000", 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.498 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986997+0000", 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3400000000000003e-06 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.987007+0000", 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6770000000000007e-06 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.499 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.987013+0000", 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.874e-06 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371859+0000", 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3848462750000001 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.372023+0000", 2023-09-23T21:05:01.500 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016354499999999999 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.372386+0000", 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00036363600000000003 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378916+0000", 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065301129999999997 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.501 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386328+0000", 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.007411829 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.406124+0000", 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019796022 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.502 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.406173+0000", 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8587000000000001e-05 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.412519+0000", 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063460089999999997 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141102 10.12 10:4b3897e2:::benchmark_data_smithi121_358375_object56357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.503 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.001005+0000", 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.395071874, 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4121252740000001, 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141102 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.504 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001005+0000", 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001005+0000", 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001004+0000", 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.505 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001012+0000", 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3110000000000006e-06 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001026+0000", 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4273e-05 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.506 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001036+0000", 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8900000000000002e-06 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.373843+0000", 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3728068849999999 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.373969+0000", 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012648600000000001 2023-09-23T21:05:01.507 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.374219+0000", 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024986799999999999 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378935+0000", 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047160769999999999 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.508 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386385+0000", 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074491829999999998 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.412569+0000", 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026184926000000001 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.412628+0000", 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9061000000000003e-05 2023-09-23T21:05:01.509 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413131+0000", 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050210599999999999 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141120 10.12 10:4b149f10:::benchmark_data_smithi121_358375_object56375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.063325+0000", 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.33275214700001, 2023-09-23T21:05:01.510 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.350347706, 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141120 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063325+0000", 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063325+0000", 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063323+0000", 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.512 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063332+0000", 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7700000000000007e-06 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063351+0000", 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9137999999999999e-05 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063361+0000", 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9329999999999996e-06 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.513 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376082+0000", 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3127212770000001 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376247+0000", 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016494200000000001 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.514 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376632+0000", 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00038421499999999998 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378953+0000", 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023215639999999999 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386392+0000", 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074385110000000001 2023-09-23T21:05:01.515 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413179+0000", 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026787261 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413234+0000", 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4830999999999997e-05 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.516 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413673+0000", 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00043899400000000001 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.517 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141122 10.2 10:400b253d:::benchmark_data_smithi121_358375_object56377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.070895+0000", 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.32518260399999, 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3548394610000001, 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141122 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.518 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070895+0000", 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070895+0000", 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.519 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070893+0000", 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070901+0000", 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9789999999999993e-06 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.520 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070916+0000", 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5024999999999999e-05 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070926+0000", 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.91e-06 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.521 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376577+0000", 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3056514889999999 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378571+0000", 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019931689999999999 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.522 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.382394+0000", 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0038230619999999999 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386569+0000", 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041757859999999999 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.523 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.396732+0000", 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010162642 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.421647+0000", 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024914828999999999 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.524 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.422506+0000", 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00085867199999999999 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.425734+0000", 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032284150000000001 2023-09-23T21:05:01.525 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141127 10.12 10:49d57e63:::benchmark_data_smithi121_358375_object56382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.103373+0000", 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 203.292704671, 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3186484979999999, 2023-09-23T21:05:01.526 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141127 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.527 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103373+0000", 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103373+0000", 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103371+0000", 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:01.528 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103379+0000", 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7620000000000001e-06 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103396+0000", 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7099999999999999e-05 2023-09-23T21:05:01.529 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103406+0000", 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5240000000000006e-06 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378966+0000", 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.27556041 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.530 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.379096+0000", 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012971799999999999 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.379492+0000", 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039616199999999998 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386370+0000", 2023-09-23T21:05:01.531 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068777669999999999 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386398+0000", 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8461000000000001e-05 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.421131+0000", 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.034733030999999998 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.532 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.421194+0000", 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2656000000000005e-05 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.422021+0000", 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00082691700000000004 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.533 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88124 11.c 11:386a4abc:::benchmark_data_smithi121_477989_object88123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 32.661014311999999, 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6441045829999998, 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88124 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.534 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.535 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735062+0000", 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735084+0000", 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.234e-05 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735099+0000", 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.508e-05 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.536 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735108+0000", 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3689999999999996e-06 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735157+0000", 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8273999999999999e-05 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.537 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735257+0000", 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000100432 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735419+0000", 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000162158 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.741925+0000", 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00650594 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.538 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.744875+0000", 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002949981 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757406+0000", 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012530777999999999 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.539 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757417+0000", 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0699e-05 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770169+0000", 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0127527590000001 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770180+0000", 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0428000000000001e-05 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.540 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.379055+0000", 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6088747780000001 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.379167+0000", 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112796 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.541 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88126 11.c 11:3f012f98:::benchmark_data_smithi121_477989_object88125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 32.659347263000001, 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6457891419999999, 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88126 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.542 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.543 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736728+0000", 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736758+0000", 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0073000000000002e-05 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736775+0000", 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6716000000000001e-05 2023-09-23T21:05:01.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736784+0000", 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1719999999999996e-06 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738664+0000", 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00187952 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738803+0000", 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013961799999999999 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738967+0000", 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016343 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.744883+0000", 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0059158769999999999 2023-09-23T21:05:01.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757481+0000", 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012598689 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757491+0000", 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.098e-06 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770209+0000", 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.012718877 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770217+0000", 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1679999999999998e-06 2023-09-23T21:05:01.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382403+0000", 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6121860369999998 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382519+0000", 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116488 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88142 11.c 11:3d9b819d:::benchmark_data_smithi121_477989_object88141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 32.639746107000001, 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6474134190000003, 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:01.550 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88142 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756330+0000", 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756356+0000", 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6440000000000001e-05 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756382+0000", 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5948999999999998e-05 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:01.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756392+0000", 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5999999999999996e-06 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757514+0000", 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001122088 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757521+0000", 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8099999999999998e-06 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770236+0000", 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0127141230000001 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770241+0000", 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9309999999999996e-06 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382568+0000", 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.612326293 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:01.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382717+0000", 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014953000000000001 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.383252+0000", 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00053481999999999998 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.388314+0000", 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050622560000000002 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.396573+0000", 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0082585120000000008 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.399933+0000", 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033602250000000001 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:01.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.399996+0000", 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2965000000000004e-05 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.403745+0000", 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037484340000000001 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:01.559 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:01.560 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:05:01.560 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:01.586 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:01.628 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:01.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:01.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:01.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:01 smithi121 ceph-mon[211526]: pgmap v1180: 89 pgs: 5 active+recovery_wait+degraded, 1 remapped+peering, 4 peering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47012/264618 objects degraded (17.766%); 47531/264618 objects misplaced (17.962%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:05:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1906436503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41647/264618 objects degraded (15.739%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:01.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:01 smithi121 ceph-mon[219709]: pgmap v1180: 89 pgs: 5 active+recovery_wait+degraded, 1 remapped+peering, 4 peering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47012/264618 objects degraded (17.766%); 47531/264618 objects misplaced (17.962%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:05:01.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1906436503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:01.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41647/264618 objects degraded (15.739%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:01.713 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:05:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:01 smithi123 ceph-mon[193903]: pgmap v1180: 89 pgs: 5 active+recovery_wait+degraded, 1 remapped+peering, 4 peering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 64 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47012/264618 objects degraded (17.766%); 47531/264618 objects misplaced (17.962%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:05:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1906436503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41647/264618 objects degraded (15.739%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:01.735 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:01.739 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:05:01.741 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.741 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.741 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.741 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.741 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.742 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.742 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.742 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:05:01.771 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:02.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 112 5 88186 88181 49.1918 0 - 0.0166307 2023-09-23T21:05:02.734 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:05:02.978 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:02 smithi123 conmon[131622]: 2023-09-23T21:05:02.656+0000 7fe79295f700 -1 osd.6 377 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:03.034 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:05:03.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 113 5 88186 88181 48.7566 0 - 0.0166307 2023-09-23T21:05:03.472 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:03.578 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:03.605 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:03.648 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:03.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:03.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:03 smithi123 conmon[131622]: 2023-09-23T21:05:03.621+0000 7fe79295f700 -1 osd.6 378 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:03 smithi123 ceph-mon[193903]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:03 smithi123 ceph-mon[193903]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:03 smithi123 ceph-mon[193903]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T21:05:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:03 smithi123 ceph-mon[193903]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:03 smithi123 ceph-mon[193903]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:03 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:03 smithi123 ceph-mon[193903]: pgmap v1182: 89 pgs: 1 remapped+peering, 8 active+recovery_wait+degraded, 3 peering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47021/264618 objects degraded (17.769%); 60877/264618 objects misplaced (23.006%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T21:05:03.753 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:03.778 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:03.822 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:03.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:03.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[219709]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[219709]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[219709]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T21:05:03.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[219709]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[219709]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:03.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[219709]: pgmap v1182: 89 pgs: 1 remapped+peering, 8 active+recovery_wait+degraded, 3 peering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47021/264618 objects degraded (17.769%); 60877/264618 objects misplaced (23.006%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T21:05:03.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[211526]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[211526]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[211526]: osdmap e377: 8 total, 8 up, 7 in 2023-09-23T21:05:03.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[211526]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[211526]: failed to encode map e377 with expected crc 2023-09-23T21:05:03.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:03.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:03 smithi121 ceph-mon[211526]: pgmap v1182: 89 pgs: 1 remapped+peering, 8 active+recovery_wait+degraded, 3 peering, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 62 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47021/264618 objects degraded (17.769%); 60877/264618 objects misplaced (23.006%); 2.9 MiB/s, 47 objects/s recovering 2023-09-23T21:05:03.929 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:03.958 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.213 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:04.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:04.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 114 5 88186 88181 48.329 0 - 0.0166307 2023-09-23T21:05:04.317 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:04.343 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.386 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:04.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:04.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:04.491 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:04.502 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:04.503 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:04.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[211526]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[211526]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[211526]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[211526]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T21:05:04.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[211526]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.525 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:04.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[219709]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[219709]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[219709]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[219709]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T21:05:04.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[219709]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:04 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:04.527 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:04.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:04 smithi123 conmon[131622]: 2023-09-23T21:05:04.571+0000 7fe79295f700 -1 osd.6 378 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:04 smithi123 ceph-mon[193903]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:04 smithi123 ceph-mon[193903]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:04 smithi123 ceph-mon[193903]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:04 smithi123 ceph-mon[193903]: osdmap e378: 8 total, 8 up, 7 in 2023-09-23T21:05:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:04 smithi123 ceph-mon[193903]: failed to encode map e378 with expected crc 2023-09-23T21:05:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:04 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:05.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:05:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:05:04] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-09-23T21:05:05.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 115 5 88186 88181 47.9088 0 - 0.0166307 2023-09-23T21:05:05.478 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:05:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:05 smithi123 ceph-mon[193903]: failed to encode map e378 with expected crc 2023-09-23T21:05:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:05 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:05 smithi123 ceph-mon[193903]: pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44074/264618 objects degraded (16.656%); 61062/264618 objects misplaced (23.076%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T21:05:05.729 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:05 smithi123 conmon[131622]: 2023-09-23T21:05:05.562+0000 7fe79295f700 -1 osd.6 378 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:05.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:05 smithi121 ceph-mon[211526]: failed to encode map e378 with expected crc 2023-09-23T21:05:05.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:05 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:05.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:05 smithi121 ceph-mon[211526]: pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44074/264618 objects degraded (16.656%); 61062/264618 objects misplaced (23.076%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T21:05:05.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:05 smithi121 ceph-mon[219709]: failed to encode map e378 with expected crc 2023-09-23T21:05:05.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:05 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:05.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:05 smithi121 ceph-mon[219709]: pgmap v1184: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44074/264618 objects degraded (16.656%); 61062/264618 objects misplaced (23.076%); 4.9 MiB/s, 79 objects/s recovering 2023-09-23T21:05:05.982 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:05.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:05.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:06.085 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:06.124 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:06.167 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:06.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:06.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:06.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 116 5 88186 88181 47.4959 0 - 0.0166307 2023-09-23T21:05:06.272 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:06.297 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:06.342 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:06.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:06.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:06.448 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:06.488 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:06.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:06 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 44074/264618 objects degraded (16.656%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:06 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 44074/264618 objects degraded (16.656%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:06.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:06 smithi123 conmon[131622]: 2023-09-23T21:05:06.584+0000 7fe79295f700 -1 osd.6 378 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:06.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:06 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:06.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 44074/264618 objects degraded (16.656%), 14 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:06.802 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:06.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:06.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:06.906 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:06.935 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:07.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 117 5 88186 88181 47.09 0 - 0.0166307 2023-09-23T21:05:07.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:07 smithi123 conmon[131622]: 2023-09-23T21:05:07.550+0000 7fe79295f700 -1 osd.6 378 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:07.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:07 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:07.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:07 smithi123 ceph-mon[193903]: pgmap v1185: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 66234/264618 objects misplaced (25.030%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:05:07.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:07 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:07.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:07 smithi121 ceph-mon[211526]: pgmap v1185: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 66234/264618 objects misplaced (25.030%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:05:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:07 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:07 smithi121 ceph-mon[219709]: pgmap v1185: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 66234/264618 objects misplaced (25.030%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:05:08.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 118 5 88186 88181 46.6911 0 - 0.0166307 2023-09-23T21:05:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:08 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:08.742 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:08 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:08.742 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:08 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:09.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 119 5 88186 88181 46.2988 0 - 0.0166307 2023-09-23T21:05:09.657 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:09.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:09 smithi123 ceph-mon[193903]: pgmap v1186: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65692/264618 objects misplaced (24.825%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:05:09.762 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:09.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:09 smithi121 ceph-mon[211526]: pgmap v1186: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65692/264618 objects misplaced (24.825%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:05:09.790 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:09 smithi121 ceph-mon[219709]: pgmap v1186: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65692/264618 objects misplaced (24.825%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:05:09.793 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:09.968 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:09.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:10.072 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:10.100 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:10.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:05:10.238362+0000 min lat: 0.00123114 max lat: 18.7578 avg lat: 0.0166307 2023-09-23T21:05:10.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:05:10.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 120 5 88186 88181 45.9131 0 - 0.0166307 2023-09-23T21:05:10.482 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:10.542 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:10.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:10.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:10.647 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:10.673 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:10.717 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:10.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:10.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:10.856 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:10.919 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:05:10.946 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1028891311' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:05:10.947 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:10 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:05:10.947 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1028891311' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:05:10.947 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:10 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:05:10.978 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:05:10.978 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:05:10.978 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:05:10.978 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:05:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1028891311' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:05:10.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:10 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:05:11.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 121 5 88186 88181 45.5337 0 - 0.0166307 2023-09-23T21:05:11.816 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:11.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:11.920 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:11.953 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:12.084 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: failed to encode map e379 with expected crc 2023-09-23T21:05:12.084 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: failed to encode map e379 with expected crc 2023-09-23T21:05:12.084 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: pgmap v1188: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65352/264618 objects misplaced (24.697%); 10 MiB/s, 164 objects/s recovering 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 38683/264618 objects degraded (14.618%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: failed to encode map e380 with expected crc 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[211526]: failed to encode map e380 with expected crc 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: failed to encode map e379 with expected crc 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: failed to encode map e379 with expected crc 2023-09-23T21:05:12.085 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:05:12.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T21:05:12.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: pgmap v1188: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65352/264618 objects misplaced (24.697%); 10 MiB/s, 164 objects/s recovering 2023-09-23T21:05:12.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 38683/264618 objects degraded (14.618%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:12.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: failed to encode map e380 with expected crc 2023-09-23T21:05:12.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T21:05:12.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:11 smithi121 ceph-mon[219709]: failed to encode map e380 with expected crc 2023-09-23T21:05:12.124 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:12.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:12.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:12.228 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: failed to encode map e379 with expected crc 2023-09-23T21:05:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: failed to encode map e379 with expected crc 2023-09-23T21:05:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:05:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: osdmap e379: 8 total, 8 up, 7 in 2023-09-23T21:05:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: pgmap v1188: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65352/264618 objects misplaced (24.697%); 10 MiB/s, 164 objects/s recovering 2023-09-23T21:05:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 38683/264618 objects degraded (14.618%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: failed to encode map e380 with expected crc 2023-09-23T21:05:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: osdmap e380: 8 total, 8 up, 7 in 2023-09-23T21:05:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:11 smithi123 ceph-mon[193903]: failed to encode map e380 with expected crc 2023-09-23T21:05:12.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 122 5 88186 88181 45.1606 0 - 0.0166307 2023-09-23T21:05:12.257 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:12.419 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2542219888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2542219888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:13.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: failed to encode map e380 with expected crc 2023-09-23T21:05:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2542219888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:05:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: osdmap e381: 8 total, 8 up, 7 in 2023-09-23T21:05:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:13.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 123 5 88186 88181 44.7935 0 - 0.0166307 2023-09-23T21:05:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:13 smithi123 ceph-mon[193903]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:13 smithi123 ceph-mon[193903]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:13 smithi123 ceph-mon[193903]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:13 smithi123 ceph-mon[193903]: 11.b continuing backfill to osd.0 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:05:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:13 smithi123 ceph-mon[193903]: 11.b continuing backfill to osd.3 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:05:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:13 smithi123 ceph-mon[193903]: pgmap v1191: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65352/264618 objects misplaced (24.697%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T21:05:14.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 124 5 88186 88181 44.4323 0 - 0.0166307 2023-09-23T21:05:14.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[219709]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[219709]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[219709]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[219709]: 11.b continuing backfill to osd.0 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:05:14.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[219709]: 11.b continuing backfill to osd.3 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:05:14.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[219709]: pgmap v1191: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65352/264618 objects misplaced (24.697%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T21:05:14.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[211526]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.345 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[211526]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[211526]: failed to encode map e381 with expected crc 2023-09-23T21:05:14.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[211526]: 11.b continuing backfill to osd.0 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:05:14.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[211526]: 11.b continuing backfill to osd.3 from (349'1457,370'5513] MIN to 370'5513 2023-09-23T21:05:14.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:13 smithi121 ceph-mon[211526]: pgmap v1191: 89 pgs: 2 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38683/264618 objects degraded (14.618%); 65352/264618 objects misplaced (24.697%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T21:05:15.091 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:05:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:05:14] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T21:05:15.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 125 5 88186 88181 44.0769 0 - 0.0166307 2023-09-23T21:05:15.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:15 smithi121 ceph-mon[211526]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40579/264618 objects degraded (15.335%); 66756/264618 objects misplaced (25.227%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T21:05:15.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:15 smithi121 ceph-mon[219709]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40579/264618 objects degraded (15.335%); 66756/264618 objects misplaced (25.227%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T21:05:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:15 smithi123 ceph-mon[193903]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 3 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 70 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40579/264618 objects degraded (15.335%); 66756/264618 objects misplaced (25.227%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T21:05:15.660 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:15.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:15.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:15.794 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:16.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 126 5 88186 88181 43.7272 0 - 0.0166307 2023-09-23T21:05:16.443 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:16 smithi123 conmon[131622]: 2023-09-23T21:05:16.144+0000 7fe79295f700 -1 osd.6 381 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:16.551 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:16.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:16.560 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:05:16.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:16.662 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:16.668 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:05:16.689 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:16.694 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:16 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:16.694 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 40579/264618 objects degraded (15.335%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:16.694 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:16 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:16.694 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 40579/264618 objects degraded (15.335%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:16 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 40579/264618 objects degraded (15.335%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:16.736 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:16.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:16.842 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:16.871 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:16.926 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:16.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:16.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:17.030 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:17.059 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.106 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:05:17.112 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:17.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:17.135 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:05:17.153 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:17.160 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:05:17.187 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:05:17.197 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:17.211 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:05:17.216 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:17.236 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:05:17.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 127 5 88186 88181 43.383 0 - 0.0166307 2023-09-23T21:05:17.247 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.257 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:05:17.281 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.283 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:05:17.393 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:17.449 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:17 smithi123 conmon[131622]: 2023-09-23T21:05:17.101+0000 7fe79295f700 -1 osd.6 381 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:17.500 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:17.527 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.574 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:17 smithi121 ceph-mon[219709]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 66374/264618 objects misplaced (25.083%); 1/88206 objects unfound (0.001%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T21:05:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3899422223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:17 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:17 smithi121 ceph-mon[211526]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 66374/264618 objects misplaced (25.083%); 1/88206 objects unfound (0.001%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T21:05:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3899422223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:17 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:17.679 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:17.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:17 smithi123 ceph-mon[193903]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 66374/264618 objects misplaced (25.083%); 1/88206 objects unfound (0.001%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T21:05:17.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3899422223' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:17.703 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:17 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:17.705 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:17.750 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:17.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:17.855 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:17.886 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:18.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 128 5 88186 88181 43.0441 0 - 0.0166307 2023-09-23T21:05:18.571 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:05:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:18 smithi123 ceph-mon[193903]: Health check failed: 1/88206 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-09-23T21:05:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:18 smithi123 ceph-mon[193903]: Health check failed: 5 slow ops, oldest one blocked for 36 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:18 smithi121 ceph-mon[211526]: Health check failed: 1/88206 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-09-23T21:05:19.043 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:18 smithi121 ceph-mon[211526]: Health check failed: 5 slow ops, oldest one blocked for 36 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:19.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:18 smithi121 ceph-mon[219709]: Health check failed: 1/88206 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-09-23T21:05:19.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:18 smithi121 ceph-mon[219709]: Health check failed: 5 slow ops, oldest one blocked for 36 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:19.045 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:19.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:19.146 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:19.191 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:19.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 129 5 88186 88181 42.7105 0 - 0.0166307 2023-09-23T21:05:19.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:19 smithi123 ceph-mon[193903]: pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 66374/264618 objects misplaced (25.083%); 1/88206 objects unfound (0.001%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:05:19.835 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:19 smithi121 ceph-mon[211526]: pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 66374/264618 objects misplaced (25.083%); 1/88206 objects unfound (0.001%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:05:19.836 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:19 smithi121 ceph-mon[219709]: pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 66374/264618 objects misplaced (25.083%); 1/88206 objects unfound (0.001%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:05:20.239 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 130 5 88186 88181 42.382 0 - 0.0166307 2023-09-23T21:05:21.241 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 131 5 88186 88181 42.0586 0 - 0.0166307 2023-09-23T21:05:21.310 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:21.406 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:21 smithi121 ceph-mon[219709]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T21:05:21.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:21 smithi121 ceph-mon[211526]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T21:05:21.413 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:21.437 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:21 smithi123 ceph-mon[193903]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T21:05:21.440 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.486 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:21.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:21.594 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:21.632 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.666 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:21.682 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:21.787 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:21.814 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:21.865 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:21.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:21.900 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:21.901 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:21.970 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:21.996 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:22.041 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:22.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:22.092 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 53157/264618 objects degraded (20.088%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:22.146 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:22.173 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:22.217 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:22.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:22.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:22.238 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 132 5 88186 88181 41.74 0 - 0.0166307 2023-09-23T21:05:22.323 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:22.343 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 53157/264618 objects degraded (20.088%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:22.347 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 53157/264618 objects degraded (20.088%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:22.350 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:22.395 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:22.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:22.500 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:22.526 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:22.574 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:22.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:22.679 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:22.706 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:22.753 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:23.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:23.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:23.226 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:05:23.240 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 133 5 88186 88181 41.4258 0 - 0.0166307 2023-09-23T21:05:23.308 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:23.339 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:23.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3366942966' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:05:23.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:23 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:05:23.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:23 smithi121 ceph-mon[211526]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T21:05:23.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3366942966' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:05:23.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:23 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:05:23.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:23 smithi121 ceph-mon[219709]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T21:05:23.457 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:05:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3366942966' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:05:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:23 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:05:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:23 smithi123 ceph-mon[193903]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T21:05:24.003 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:24.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:24.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:24.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:24.153 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:24.240 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 134 5 88186 88181 41.1167 0 - 0.0166307 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[211526]: failed to encode map e382 with expected crc 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[211526]: failed to encode map e382 with expected crc 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[211526]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[219709]: failed to encode map e382 with expected crc 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[219709]: failed to encode map e382 with expected crc 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:05:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:24 smithi121 ceph-mon[219709]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T21:05:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:24 smithi123 ceph-mon[193903]: failed to encode map e382 with expected crc 2023-09-23T21:05:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:24 smithi123 ceph-mon[193903]: failed to encode map e382 with expected crc 2023-09-23T21:05:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:24 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:05:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:24 smithi123 ceph-mon[193903]: osdmap e382: 8 total, 8 up, 7 in 2023-09-23T21:05:24.960 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:24.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:25.064 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:25.092 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:05:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:05:24] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-09-23T21:05:25.096 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.182 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:05:25.207 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:05:25.209 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:25.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:25.248 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 135 5 88186 88181 40.8097 0 - 0.0166307 2023-09-23T21:05:25.314 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:25.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: 11.e continuing backfill to osd.0 from (347'239,370'5599] MIN to 370'5599 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: 11.e starting backfill to osd.4 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[219709]: pgmap v1199: 89 pgs: 5 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 68 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T21:05:25.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: 11.e continuing backfill to osd.0 from (347'239,370'5599] MIN to 370'5599 2023-09-23T21:05:25.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: 11.e starting backfill to osd.4 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:05:25.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:25 smithi121 ceph-mon[211526]: pgmap v1199: 89 pgs: 5 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 68 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T21:05:25.356 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.424 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:25.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: osdmap e383: 8 total, 8 up, 7 in 2023-09-23T21:05:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: 11.e continuing backfill to osd.0 from (347'239,370'5599] MIN to 370'5599 2023-09-23T21:05:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: 11.e starting backfill to osd.4 from (0'0,0'0] MAX to 370'5599 2023-09-23T21:05:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: failed to encode map e383 with expected crc 2023-09-23T21:05:25.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:25 smithi123 ceph-mon[193903]: pgmap v1199: 89 pgs: 5 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 68 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53157/264618 objects degraded (20.088%); 65946/264618 objects misplaced (24.921%); 1/88206 objects unfound (0.001%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T21:05:25.522 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:25.528 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:25.555 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.602 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:25.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:25.708 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:25.736 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.785 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:25.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:25.891 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:25.918 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:25.964 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:25.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:26.037 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:26.070 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:26.088 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:26.096 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:26.129 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.129 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.129 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.129 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: 11.c starting backfill to osd.4 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:05:26.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T21:05:26.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3190509173' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:26.130 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:26.134 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:05:26.153 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:26.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: 11.c starting backfill to osd.4 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:05:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T21:05:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3190509173' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:26.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:26.248 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 136 5 88186 88181 40.5098 0 - 0.0166307 2023-09-23T21:05:26.260 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:26.291 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:26.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.384 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: 11.c starting backfill to osd.4 from (0'0,0'0] MAX to 370'5513 2023-09-23T21:05:26.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: osdmap e384: 8 total, 8 up, 7 in 2023-09-23T21:05:26.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: failed to encode map e384 with expected crc 2023-09-23T21:05:26.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3190509173' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:26.385 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:05:26.991 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:26.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:26.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:27.095 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:27.123 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:27.248 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 137 5 88186 88181 40.2141 0 - 0.0166307 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[211526]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[211526]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[211526]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[211526]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[211526]: pgmap v1202: 89 pgs: 1 activating+remapped, 5 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 68 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51293/264618 objects degraded (19.384%); 69245/264618 objects misplaced (26.168%); 1/88206 objects unfound (0.001%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:05:27.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[219709]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[219709]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T21:05:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[219709]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[219709]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:27 smithi121 ceph-mon[219709]: pgmap v1202: 89 pgs: 1 activating+remapped, 5 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 68 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51293/264618 objects degraded (19.384%); 69245/264618 objects misplaced (26.168%); 1/88206 objects unfound (0.001%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:05:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:27 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:27 smithi123 ceph-mon[193903]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:05:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:27 smithi123 ceph-mon[193903]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T21:05:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:27 smithi123 ceph-mon[193903]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:27 smithi123 ceph-mon[193903]: failed to encode map e385 with expected crc 2023-09-23T21:05:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:27 smithi123 ceph-mon[193903]: pgmap v1202: 89 pgs: 1 activating+remapped, 5 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 68 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51293/264618 objects degraded (19.384%); 69245/264618 objects misplaced (26.168%); 1/88206 objects unfound (0.001%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:05:28.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 138 5 88186 88181 39.9228 0 - 0.0166307 2023-09-23T21:05:28.302 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:28.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:28.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:28.405 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:28.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[211526]: failed to encode map e385 with expected crc 2023-09-23T21:05:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 51293/264618 objects degraded (19.384%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:28.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[219709]: failed to encode map e385 with expected crc 2023-09-23T21:05:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 51293/264618 objects degraded (19.384%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:28.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:28.431 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:28 smithi123 ceph-mon[193903]: failed to encode map e385 with expected crc 2023-09-23T21:05:28.431 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:28 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 51293/264618 objects degraded (19.384%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:05:28.432 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:28.432 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:28.433 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:28.479 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:28.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:28.591 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:28.622 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.112 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:29.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:29.217 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:29.245 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.248 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 139 5 88186 88181 39.6357 0 - 0.0166307 2023-09-23T21:05:29.293 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:29.400 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:29.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:29 smithi121 ceph-mon[219709]: pgmap v1203: 89 pgs: 1 activating+remapped, 2 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 71 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51268/264618 objects degraded (19.374%); 69270/264618 objects misplaced (26.177%); 1/88206 objects unfound (0.001%); 3.7 MiB/s, 1 keys/s, 60 objects/s recovering 2023-09-23T21:05:29.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:29 smithi121 ceph-mon[211526]: pgmap v1203: 89 pgs: 1 activating+remapped, 2 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 71 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51268/264618 objects degraded (19.374%); 69270/264618 objects misplaced (26.177%); 1/88206 objects unfound (0.001%); 3.7 MiB/s, 1 keys/s, 60 objects/s recovering 2023-09-23T21:05:29.426 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:29 smithi123 ceph-mon[193903]: pgmap v1203: 89 pgs: 1 activating+remapped, 2 peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 71 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51268/264618 objects degraded (19.374%); 69270/264618 objects misplaced (26.177%); 1/88206 objects unfound (0.001%); 3.7 MiB/s, 1 keys/s, 60 objects/s recovering 2023-09-23T21:05:29.429 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.479 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:29.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:29.585 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:29.612 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:29.657 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:29.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:29.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:29.762 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:29.793 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:30.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:05:30.249710+0000 min lat: 0.00123114 max lat: 18.7578 avg lat: 0.0166307 2023-09-23T21:05:30.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:05:30.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 140 5 88186 88181 39.3526 0 - 0.0166307 2023-09-23T21:05:30.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:30 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:05:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:30 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:05:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:30 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:05:30.547 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:30.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:30.651 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:30.681 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:30.815 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:05:31.148 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:31 smithi121 ceph-mon[211526]: pgmap v1204: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49366/264618 objects degraded (18.656%); 65097/264618 objects misplaced (24.600%); 16 MiB/s, 1 keys/s, 262 objects/s recovering 2023-09-23T21:05:31.148 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:31 smithi121 ceph-mon[219709]: pgmap v1204: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49366/264618 objects degraded (18.656%); 65097/264618 objects misplaced (24.600%); 16 MiB/s, 1 keys/s, 262 objects/s recovering 2023-09-23T21:05:31.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 141 5 88186 88181 39.0736 0 - 0.0166307 2023-09-23T21:05:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:31 smithi123 ceph-mon[193903]: pgmap v1204: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49366/264618 objects degraded (18.656%); 65097/264618 objects misplaced (24.600%); 16 MiB/s, 1 keys/s, 262 objects/s recovering 2023-09-23T21:05:31.570 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:31.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:31.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:31.675 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:31.708 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:31.727 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:05:31.752 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:05:31.777 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:05:31.803 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:05:31.828 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:05:31.854 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:05:31.874 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:05:31.895 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:05:31.896 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:31.900 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:05:31.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:31.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:32.002 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:32.042 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:32.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 142 5 88186 88181 38.7985 0 - 0.0166307 2023-09-23T21:05:32.349 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:32.351 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:32 smithi121 ceph-mon[211526]: Health check cleared: OBJECT_UNFOUND (was: 1/88206 objects unfound (0.001%)) 2023-09-23T21:05:32.351 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:32 smithi121 ceph-mon[211526]: Health check cleared: SLOW_OPS (was: 5 slow ops, oldest one blocked for 36 sec, osd.6 has slow ops) 2023-09-23T21:05:32.351 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/887037338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:32.352 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:32 smithi121 ceph-mon[219709]: Health check cleared: OBJECT_UNFOUND (was: 1/88206 objects unfound (0.001%)) 2023-09-23T21:05:32.352 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:32 smithi121 ceph-mon[219709]: Health check cleared: SLOW_OPS (was: 5 slow ops, oldest one blocked for 36 sec, osd.6 has slow ops) 2023-09-23T21:05:32.352 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/887037338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:32.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:32.449 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:32 smithi123 ceph-mon[193903]: Health check cleared: OBJECT_UNFOUND (was: 1/88206 objects unfound (0.001%)) 2023-09-23T21:05:32.449 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:32 smithi123 ceph-mon[193903]: Health check cleared: SLOW_OPS (was: 5 slow ops, oldest one blocked for 36 sec, osd.6 has slow ops) 2023-09-23T21:05:32.450 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/887037338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:32.454 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:32.483 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:32.532 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:32.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:32.581 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82131 10.6 10:6abfd253:::benchmark_data_smithi121_358375_object82130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.473119+0000", 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 577.73386127799995, 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0092071929999999, 2023-09-23T21:05:32.582 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82131 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473119+0000", 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473119+0000", 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473118+0000", 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473164+0000", 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6202999999999998e-05 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473179+0000", 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5135e-05 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473190+0000", 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0943e-05 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473323+0000", 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013271499999999999 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473334+0000", 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1313e-05 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.472285+0000", 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.998951276 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.474038+0000", 2023-09-23T21:05:32.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001752408 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.474147+0000", 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000109298 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482179+0000", 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080320080000000002 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482192+0000", 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2642e-05 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482220+0000", 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8218e-05 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482285+0000", 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4950999999999995e-05 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482326+0000", 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1459999999999999e-05 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82133 10.6 10:63e8454a:::benchmark_data_smithi121_358375_object82132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 577.73377408600004, 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.01243694, 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82133 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.591 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473206+0000", 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473255+0000", 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9543999999999999e-05 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473263+0000", 2023-09-23T21:05:32.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.106e-06 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473278+0000", 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4651000000000001e-05 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473362+0000", 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4215999999999995e-05 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.473371+0000", 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9430000000000006e-06 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.476809+0000", 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0034375790000001 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.476935+0000", 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012644800000000001 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.477029+0000", 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3652000000000004e-05 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482363+0000", 2023-09-23T21:05:32.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053339640000000001 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482387+0000", 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.421e-05 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485541+0000", 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031537000000000002 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485602+0000", 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0899000000000001e-05 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485643+0000", 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1616000000000003e-05 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.598 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82167 10.6 10:628536db:::benchmark_data_smithi121_358375_object82166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.616057+0000", 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 577.59092305800004, 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.87047633700000004, 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.599 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82167 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616057+0000", 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616057+0000", 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616056+0000", 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616105+0000", 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.867e-05 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616120+0000", 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5238e-05 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616128+0000", 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5939999999999996e-06 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616192+0000", 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3879000000000003e-05 2023-09-23T21:05:32.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.616202+0000", 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0848e-05 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479337+0000", 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86313474099999998 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479507+0000", 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016978800000000001 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.479677+0000", 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000169586 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485520+0000", 2023-09-23T21:05:32.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0058433160000000003 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485651+0000", 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000131278 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486447+0000", 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00079564700000000002 2023-09-23T21:05:32.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486498+0000", 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1010000000000001e-05 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486534+0000", 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5933999999999998e-05 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.606 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:82175 10.6 10:6b1c2f60:::benchmark_data_smithi121_358375_object82174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:55:54.626165+0000", 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 577.58081534600001, 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.86051544000000002, 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.607 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82175 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626165+0000", 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626165+0000", 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626164+0000", 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626188+0000", 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3451999999999998e-05 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626197+0000", 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7440000000000002e-06 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626204+0000", 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.781e-06 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626249+0000", 2023-09-23T21:05:32.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4608999999999999e-05 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:54.626256+0000", 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5560000000000002e-06 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482405+0000", 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.85614919499999997 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482547+0000", 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000141285 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.482686+0000", 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013936499999999999 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485656+0000", 2023-09-23T21:05:32.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029699399999999999 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.485728+0000", 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2163999999999996e-05 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486608+0000", 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00087970500000000005 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486650+0000", 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1770999999999997e-05 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:55:55.486681+0000", 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0961999999999999e-05 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90547 10.c 10:38f673f1:::benchmark_data_smithi121_358375_object5802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.927985+0000", 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 494.27899513, 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24451520700000001, 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90547 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.615 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927985+0000", 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927985+0000", 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927984+0000", 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927989+0000", 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3669999999999996e-06 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.927998+0000", 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8799999999999997e-06 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928004+0000", 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7620000000000001e-06 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928042+0000", 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7648999999999998e-05 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928130+0000", 2023-09-23T21:05:32.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8039000000000002e-05 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.928220+0000", 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0078000000000001e-05 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.930554+0000", 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023342699999999998 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.930699+0000", 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014521900000000001 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932382+0000", 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016824559999999999 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932413+0000", 2023-09-23T21:05:32.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0815999999999998e-05 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172501+0000", 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24008792700000001 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.45470.0:90547 10.c e268/265 ondisk, result = 0)", 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.931963+0000", 2023-09-23T21:05:32.621 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 494.27501772400001, 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24055167899999999, 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931963+0000", 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931963+0000", 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931962+0000", 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931965+0000", 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9299999999999999e-06 2023-09-23T21:05:32.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931970+0000", 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8409999999999999e-06 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.931974+0000", 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2890000000000002e-06 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932374+0000", 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000400739 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932378+0000", 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1690000000000002e-06 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172514+0000", 2023-09-23T21:05:32.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24013602100000001 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:90548 10.4 10:202f5be9:::benchmark_data_smithi121_358375_object5803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 494.27445924, 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243222459, 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.626 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 90548 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932521+0000", 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:17.932525+0000", 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9750000000000001e-06 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172121+0000", 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23959696899999999 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172147+0000", 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5612000000000001e-05 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172546+0000", 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039850399999999999 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172644+0000", 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8318000000000003e-05 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.172716+0000", 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1630000000000004e-05 2023-09-23T21:05:32.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175320+0000", 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026049089999999999 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175444+0000", 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012345800000000001 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175635+0000", 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000191528 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175695+0000", 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9590999999999999e-05 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:18.175744+0000", 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8677000000000002e-05 2023-09-23T21:05:32.632 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:110359 10.16 10:6ce1aa55:::benchmark_data_smithi121_358375_object25614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 321.374296657, 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7173976529999999, 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 110359 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832682+0000", 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832691+0000", 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0270000000000001e-06 2023-09-23T21:05:32.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832711+0000", 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.999e-05 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832721+0000", 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0033e-05 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832788+0000", 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7268999999999996e-05 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832802+0000", 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3319000000000001e-05 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:11.299916+0000", 2023-09-23T21:05:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.467114749 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:11.299930+0000", 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3368e-05 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.542785+0000", 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2428551299999999 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.542973+0000", 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018853699999999999 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.543153+0000", 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000180065 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548645+0000", 2023-09-23T21:05:32.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054919060000000004 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548699+0000", 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4026000000000001e-05 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548751+0000", 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2011000000000001e-05 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548821+0000", 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.97e-05 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.550081+0000", 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001260401 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151000 10.16 10:6fb6daa8:::benchmark_data_smithi121_358375_object66255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.79730961499999, 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.019180497, 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151000 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409669+0000", 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409680+0000", 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1647e-05 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409699+0000", 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8763e-05 2023-09-23T21:05:32.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409711+0000", 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409930+0000", 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021934400000000001 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410095+0000", 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016493700000000001 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410262+0000", 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016674600000000001 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.415289+0000", 2023-09-23T21:05:32.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050267660000000002 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428299+0000", 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013010763 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428313+0000", 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4064e-05 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428836+0000", 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0005229959999999 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428851+0000", 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4960000000000001e-05 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151024 10.6 10.8ded8286 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.708048894, 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.930075294, 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151024 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.649 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498930+0000", 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498938+0000", 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3890000000000005e-06 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498955+0000", 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6996000000000001e-05 2023-09-23T21:05:32.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498965+0000", 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0629999999999995e-06 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499015+0000", 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0868000000000002e-05 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499026+0000", 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0883999999999999e-05 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428989+0000", 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92996320600000004 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429007+0000", 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7419e-05 2023-09-23T21:05:32.653 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151064 10.16 10.14ca33f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.571283131, 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79318550200000004, 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.654 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151064 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635696+0000", 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635703+0000", 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8220000000000001e-06 2023-09-23T21:05:32.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635720+0000", 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7116999999999999e-05 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635729+0000", 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1929999999999999e-06 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635796+0000", 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7163999999999996e-05 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635810+0000", 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4469e-05 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428874+0000", 2023-09-23T21:05:32.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79306313900000003 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428883+0000", 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2390000000000008e-06 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151070 10.6 10.e7155d86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:05:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.52059742899999, 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74265465799999997, 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151070 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686381+0000", 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686389+0000", 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.204e-06 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686407+0000", 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8130000000000001e-05 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686416+0000", 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2369999999999998e-06 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686477+0000", 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1432000000000004e-05 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686488+0000", 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1026000000000001e-05 2023-09-23T21:05:32.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429028+0000", 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74253917700000005 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429038+0000", 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.007e-05 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.664 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151071 10.16 10.6fc596f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.51336148799999, 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73529290700000005, 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.665 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151071 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:05:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693617+0000", 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693627+0000", 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4120000000000007e-06 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693644+0000", 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7669999999999999e-05 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693654+0000", 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1980000000000007e-06 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693712+0000", 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8579999999999998e-05 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693726+0000", 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3847e-05 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428903+0000", 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73517710400000003 2023-09-23T21:05:32.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428912+0000", 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7469999999999994e-06 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151082 10.16 10.dd3bf356 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:05:32.670 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.49334339500001, 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71530539400000004, 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151082 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713635+0000", 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713645+0000", 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0113e-05 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713670+0000", 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4507000000000001e-05 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713679+0000", 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1710000000000008e-06 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713749+0000", 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0607999999999999e-05 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713761+0000", 2023-09-23T21:05:32.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1841999999999999e-05 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428934+0000", 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71517288000000001 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428942+0000", 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3040000000000005e-06 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.675 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151084 10.6 10.dfef7e86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.48659545999999, 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70867917800000002, 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151084 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720383+0000", 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720392+0000", 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1699999999999997e-06 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720411+0000", 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9681e-05 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720422+0000", 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0587999999999999e-05 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720488+0000", 2023-09-23T21:05:32.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.648e-05 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720504+0000", 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6172999999999999e-05 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429056+0000", 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70855175100000001 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429064+0000", 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0150000000000003e-06 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151096 10.16 10.c63710d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.41652528200001, 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63851427199999999, 2023-09-23T21:05:32.681 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151096 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790453+0000", 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790471+0000", 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7876999999999998e-05 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790488+0000", 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6642000000000001e-05 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790498+0000", 2023-09-23T21:05:32.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6569999999999992e-06 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790553+0000", 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5707e-05 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790563+0000", 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6520000000000001e-06 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428961+0000", 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63839818400000004 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428969+0000", 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3799999999999994e-06 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151153 10.6 10.b41dbb46 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 212.16545293999999, 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.387565296, 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151153 2023-09-23T21:05:32.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041526+0000", 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041533+0000", 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1250000000000004e-06 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041546+0000", 2023-09-23T21:05:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2289e-05 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041552+0000", 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041613+0000", 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0640000000000002e-05 2023-09-23T21:05:32.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041623+0000", 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0309000000000001e-05 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429084+0000", 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.38746082799999998 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429093+0000", 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6910000000000003e-06 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:74980 11.2 e365/355 11:45510e08:::benchmark_data_smithi121_477989_object74979:head v 365'4674, mlcod=347'301)", 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:10.932351+0000", 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.274629809000004, 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.030042619, 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.932351+0000", 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.932351+0000", 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.932350+0000", 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.932375+0000", 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4669999999999999e-05 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.932390+0000", 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4803e-05 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.932397+0000", 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4660000000000002e-06 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.934984+0000", 2023-09-23T21:05:32.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025869679999999998 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.935011+0000", 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6886999999999999e-05 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.962337+0000", 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.027326434 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:10.962393+0000", 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6045999999999998e-05 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.698 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:86696 11.2 e367/355 11:4e156f2f:::benchmark_data_smithi121_477989_object86695:head v 367'5396, mlcod=347'1300)", 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:26.995908+0000", 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 65.211072400999996, 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.033091593000000002, 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:26.995908+0000", 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:26.995908+0000", 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:26.995907+0000", 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:26.995936+0000", 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8322000000000001e-05 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:26.995951+0000", 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5838999999999999e-05 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:26.995966+0000", 2023-09-23T21:05:32.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.468e-05 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:27.002014+0000", 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0060481249999999997 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:27.002046+0000", 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1569000000000002e-05 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:27.028955+0000", 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026909197999999999 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:27.029000+0000", 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4625e-05 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(11.7 385/374 [PushOp(11:ebab1812:::benchmark_data_smithi121_477989_object7848:head, version: 347'548, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:ebab1812:::benchmark_data_smithi121_477989_object7848:head@347'548, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:05:32.050512+0000", 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.15646817700000001, 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030916400000000001, 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:05:32.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050512+0000", 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050512+0000", 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050511+0000", 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:05:32.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050546+0000", 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5413000000000001e-05 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050571+0000", 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4941000000000001e-05 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:05:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050591+0000", 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0327999999999999e-05 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050692+0000", 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010014900000000001 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.050712+0000", 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0095999999999999e-05 2023-09-23T21:05:32.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:05:32.053604+0000", 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028922119999999999 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:05:32.708 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:05:32.708 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:32.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:32 smithi123 conmon[131622]: 2023-09-23T21:05:32.449+0000 7fe79295f700 -1 osd.6 385 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:32.736 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:32.785 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:32.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:32.891 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:32.919 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:32.965 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:32.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:32.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:33.072 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:33.099 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:33.144 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:33.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:33.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:33.176 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:05:33.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 143 5 88186 88181 38.5273 0 - 0.0166307 2023-09-23T21:05:33.249 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:33.277 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:33.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2793648293' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:05:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:33 smithi121 ceph-mon[219709]: pgmap v1205: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49366/264618 objects degraded (18.656%); 65097/264618 objects misplaced (24.600%); 14 MiB/s, 1 keys/s, 226 objects/s recovering 2023-09-23T21:05:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2793648293' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:05:33.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:33 smithi121 ceph-mon[211526]: pgmap v1205: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49366/264618 objects degraded (18.656%); 65097/264618 objects misplaced (24.600%); 14 MiB/s, 1 keys/s, 226 objects/s recovering 2023-09-23T21:05:33.457 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2793648293' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:05:33.457 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:33 smithi123 ceph-mon[193903]: pgmap v1205: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49366/264618 objects degraded (18.656%); 65097/264618 objects misplaced (24.600%); 14 MiB/s, 1 keys/s, 226 objects/s recovering 2023-09-23T21:05:33.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:33 smithi123 conmon[131622]: 2023-09-23T21:05:33.457+0000 7fe79295f700 -1 osd.6 386 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:34.045 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:34.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:34.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:34.185 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.248 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:34.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 144 5 88186 88181 38.2598 0 - 0.0166307 2023-09-23T21:05:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:34.353 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:34.382 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2793648293' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[219709]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2793648293' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:34 smithi121 ceph-mon[211526]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.426 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:34.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:34.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2793648293' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:05:34.430 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T21:05:34.431 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.431 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:34 smithi123 ceph-mon[193903]: failed to encode map e386 with expected crc 2023-09-23T21:05:34.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:34.533 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:34.560 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.605 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:34.713 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:34.741 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:34.789 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:34.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:34.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:34.895 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:34.924 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:35.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:05:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:05:34] "GET /metrics HTTP/1.1" 200 36710 "" "Prometheus/2.43.0" 2023-09-23T21:05:35.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 145 5 88186 88181 37.996 0 - 0.0166307 2023-09-23T21:05:35.426 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: 11.8 continuing backfill to osd.3 from (349'1473,370'5531] 11:101db76c:::benchmark_data_smithi121_477989_object36331:head to 370'5531 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: 11.8 starting backfill to osd.5 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:05:35.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:35 smithi123 ceph-mon[193903]: pgmap v1208: 89 pgs: 1 remapped+peering, 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47495/264618 objects degraded (17.949%); 52226/264618 objects misplaced (19.736%); 12 MiB/s, 1 keys/s, 197 objects/s recovering 2023-09-23T21:05:35.455 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:35.455 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: 11.8 continuing backfill to osd.3 from (349'1473,370'5531] 11:101db76c:::benchmark_data_smithi121_477989_object36331:head to 370'5531 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: 11.8 starting backfill to osd.5 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[211526]: pgmap v1208: 89 pgs: 1 remapped+peering, 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47495/264618 objects degraded (17.949%); 52226/264618 objects misplaced (19.736%); 12 MiB/s, 1 keys/s, 197 objects/s recovering 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: 11.8 continuing backfill to osd.3 from (349'1473,370'5531] 11:101db76c:::benchmark_data_smithi121_477989_object36331:head to 370'5531 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: 11.8 starting backfill to osd.5 from (0'0,0'0] MAX to 370'5531 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: failed to encode map e387 with expected crc 2023-09-23T21:05:35.458 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:35 smithi121 ceph-mon[219709]: pgmap v1208: 89 pgs: 1 remapped+peering, 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47495/264618 objects degraded (17.949%); 52226/264618 objects misplaced (19.736%); 12 MiB/s, 1 keys/s, 197 objects/s recovering 2023-09-23T21:05:35.458 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:05:35.633 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:35.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:35.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:35.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:35 smithi123 conmon[131622]: 2023-09-23T21:05:35.441+0000 7fe79295f700 -1 osd.6 388 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:35.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:35.764 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:35.767 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:05:35.811 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:35.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:35.916 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:35.949 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.004 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:36.108 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:36.136 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.144 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:36.181 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:36.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:36.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 146 5 88186 88181 37.7358 0 - 0.0166307 2023-09-23T21:05:36.286 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 47495/264618 objects degraded (17.949%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.312 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: 11.a continuing backfill to osd.3 from (349'1380,370'5437] MIN to 370'5437 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[219709]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 47495/264618 objects degraded (17.949%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.313 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: 11.a continuing backfill to osd.3 from (349'1380,370'5437] MIN to 370'5437 2023-09-23T21:05:36.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.314 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:36 smithi121 ceph-mon[211526]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.315 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.366 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 47495/264618 objects degraded (17.949%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.469 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.470 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: 11.a continuing backfill to osd.3 from (349'1380,370'5437] MIN to 370'5437 2023-09-23T21:05:36.470 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.470 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:36 smithi123 ceph-mon[193903]: failed to encode map e388 with expected crc 2023-09-23T21:05:36.470 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:36.497 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.511 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:05:36.541 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:36.647 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:36.674 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:36.723 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:36.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:36.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:36 smithi123 conmon[131622]: 2023-09-23T21:05:36.469+0000 7fe79295f700 -1 osd.6 388 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:36.827 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:36.853 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:37.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 147 5 88186 88181 37.4792 0 - 0.0166307 2023-09-23T21:05:37.385 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:37.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:37.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:37 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:37 smithi123 ceph-mon[193903]: pgmap v1210: 89 pgs: 1 remapped+peering, 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47678/264618 objects degraded (18.018%); 52400/264618 objects misplaced (19.802%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:05:37.489 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:37.513 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:37 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:37.513 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:37 smithi121 ceph-mon[211526]: pgmap v1210: 89 pgs: 1 remapped+peering, 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47678/264618 objects degraded (18.018%); 52400/264618 objects misplaced (19.802%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:05:37.513 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:37 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:37.513 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:37 smithi121 ceph-mon[219709]: pgmap v1210: 89 pgs: 1 remapped+peering, 3 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47678/264618 objects degraded (18.018%); 52400/264618 objects misplaced (19.802%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:05:37.521 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:37.621 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:37.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:37.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:37.725 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:37.756 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:38.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 148 5 88186 88181 37.226 0 - 0.0166307 2023-09-23T21:05:38.257 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:38.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:38.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:38.362 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:38.388 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:38.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:38 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:38.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:38 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:38.436 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:38.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:38 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:38.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:38.545 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:38.574 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:38.619 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:38.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:38.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:38.725 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:38.756 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:38.852 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:38.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:38.955 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:38.984 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:39.098 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:39.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:39.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:39.202 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:39.231 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:39.249 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 149 5 88186 88181 36.9762 0 - 0.0166307 2023-09-23T21:05:39.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:39 smithi121 ceph-mon[211526]: pgmap v1211: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49418/264618 objects degraded (18.675%); 65402/264618 objects misplaced (24.716%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:05:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:39 smithi121 ceph-mon[219709]: pgmap v1211: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49418/264618 objects degraded (18.675%); 65402/264618 objects misplaced (24.716%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:05:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:39 smithi123 ceph-mon[193903]: pgmap v1211: 89 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49418/264618 objects degraded (18.675%); 65402/264618 objects misplaced (24.716%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:05:39.523 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:39.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:39.610 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:39.628 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:39.664 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:39.666 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:39.750 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:39.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:39.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:39.853 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:39.885 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:39.960 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:39.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:39.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:40.065 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:40.092 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:40.139 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:40.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:40.225 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:05:40.244 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:40.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 150 5 88186 88181 36.7297 0 - 0.0166307 2023-09-23T21:05:40.272 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:40.316 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:40.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:40.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2073197924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2073197924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:40.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:40.457 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2073197924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:05:41.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 151 5 88186 88181 36.4866 0 - 0.0166307 2023-09-23T21:05:41.475 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[211526]: failed to encode map e389 with expected crc 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[211526]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[211526]: pgmap v1213: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 4 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264615 objects degraded (12.437%); 74092/264615 objects misplaced (28.000%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[219709]: failed to encode map e389 with expected crc 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:41.476 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[219709]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T21:05:41.477 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:41 smithi121 ceph-mon[219709]: pgmap v1213: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 4 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264615 objects degraded (12.437%); 74092/264615 objects misplaced (28.000%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:05:41.478 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:41 smithi123 conmon[131622]: 2023-09-23T21:05:41.067+0000 7fe79295f700 -1 osd.6 389 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:41 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:41 smithi123 ceph-mon[193903]: failed to encode map e389 with expected crc 2023-09-23T21:05:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:05:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:41 smithi123 ceph-mon[193903]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T21:05:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:41 smithi123 ceph-mon[193903]: pgmap v1213: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 4 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264615 objects degraded (12.437%); 74092/264615 objects misplaced (28.000%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:05:42.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 152 5 88186 88181 36.2466 0 - 0.0166307 2023-09-23T21:05:42.307 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:42.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:42.411 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:42.436 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:42 smithi123 conmon[131622]: 2023-09-23T21:05:42.092+0000 7fe79295f700 -1 osd.6 390 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:42.436 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:42.436 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:05:42.436 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 32909/264615 objects degraded (12.437%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:42.436 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: Health check failed: 5 slow ops, oldest one blocked for 65 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:42.437 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.437 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.437 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T21:05:42.437 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.437 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:05:42.439 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:42.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:42.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:05:42.456 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 32909/264615 objects degraded (12.437%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: Health check failed: 5 slow ops, oldest one blocked for 65 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 32909/264615 objects degraded (12.437%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: Health check failed: 5 slow ops, oldest one blocked for 65 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:42.457 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.458 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.458 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T21:05:42.458 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: failed to encode map e390 with expected crc 2023-09-23T21:05:42.458 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:05:42.492 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:42.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:42.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:42.630 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:42.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:42.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:42.954 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:05:42.954 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:05:42.955 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:05:42.955 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:05:43.050 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:43.084 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:43.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 153 5 88186 88181 36.0097 0 - 0.0166307 2023-09-23T21:05:43.331 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:05:43.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:43.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:43.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:43.334 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[211526]: pgmap v1215: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 4 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264615 objects degraded (12.437%); 74092/264615 objects misplaced (28.000%); 15 MiB/s, 233 objects/s recovering 2023-09-23T21:05:43.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:43.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:43.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:43.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:43 smithi121 ceph-mon[219709]: pgmap v1215: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 4 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264615 objects degraded (12.437%); 74092/264615 objects misplaced (28.000%); 15 MiB/s, 233 objects/s recovering 2023-09-23T21:05:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:43.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:05:43.437 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:43.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:43 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:43.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:43.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:43.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:43 smithi123 ceph-mon[193903]: pgmap v1215: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 4 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264615 objects degraded (12.437%); 74092/264615 objects misplaced (28.000%); 15 MiB/s, 233 objects/s recovering 2023-09-23T21:05:43.468 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:44.177 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:44.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:44.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 154 5 88186 88181 35.776 0 - 0.0166307 2023-09-23T21:05:44.284 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:44.312 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:44.441 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:05:44 smithi123 conmon[131622]: 2023-09-23T21:05:44.131+0000 7fe79295f700 -1 osd.6 390 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:05:44.447 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:05:44.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2643126983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:05:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:44 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2643126983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:05:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:44 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2643126983' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:05:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:44 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:05:45.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:05:45.114 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:05:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:05:44] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:05:45.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 155 5 88186 88181 35.5452 0 - 0.0166307 2023-09-23T21:05:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:45 smithi123 ceph-mon[193903]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:45 smithi123 ceph-mon[193903]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2643126983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:05:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:45 smithi123 ceph-mon[193903]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T21:05:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:45 smithi123 ceph-mon[193903]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:45 smithi123 ceph-mon[193903]: pgmap v1217: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 74092/264618 objects misplaced (28.000%); 16 MiB/s, 252 objects/s recovering 2023-09-23T21:05:45.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[211526]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[211526]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2643126983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[211526]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[211526]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[211526]: pgmap v1217: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 74092/264618 objects misplaced (28.000%); 16 MiB/s, 252 objects/s recovering 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[219709]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[219709]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2643126983' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:05:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[219709]: osdmap e391: 8 total, 8 up, 7 in 2023-09-23T21:05:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[219709]: failed to encode map e391 with expected crc 2023-09-23T21:05:45.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:45 smithi121 ceph-mon[219709]: pgmap v1217: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 74092/264618 objects misplaced (28.000%); 16 MiB/s, 252 objects/s recovering 2023-09-23T21:05:46.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 156 5 88186 88181 35.3174 0 - 0.0166307 2023-09-23T21:05:46.480 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:46.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:46.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:46.584 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:46.586 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:05:46.616 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:46.672 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:46.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:46.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:46.696 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:46 smithi121 ceph-mon[211526]: failed to encode map e391 with expected crc 2023-09-23T21:05:46.696 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 32909/264618 objects degraded (12.436%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:05:46.696 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:46 smithi121 ceph-mon[219709]: failed to encode map e391 with expected crc 2023-09-23T21:05:46.696 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 32909/264618 objects degraded (12.436%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:05:46.697 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:05:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:46 smithi123 ceph-mon[193903]: failed to encode map e391 with expected crc 2023-09-23T21:05:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 32909/264618 objects degraded (12.436%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:05:46.778 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:46.807 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:46.855 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:46.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:46.962 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:47.006 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.062 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:47.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:47.146 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:47.147 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:05:47.168 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:47.180 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:05:47.196 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.215 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:05:47.239 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:05:47.244 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:47.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:47.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 157 5 88186 88181 35.0925 0 - 0.0166307 2023-09-23T21:05:47.258 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:05:47.278 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:05:47.350 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:47.379 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.388 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:47.560 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:47.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:47.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:47.621 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:47.665 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:47.694 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.802 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:47.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:47.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:47.908 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:47.938 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:47.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:47 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:47.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:47 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:47.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:47 smithi123 ceph-mon[193903]: pgmap v1218: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 150 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73714/264618 objects misplaced (27.857%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T21:05:47.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3491897038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:47.984 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:47.984 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:47.984 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[211526]: pgmap v1218: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 150 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73714/264618 objects misplaced (27.857%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T21:05:47.984 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3491897038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:47.985 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:47.985 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:05:47.985 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[219709]: pgmap v1218: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 150 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73714/264618 objects misplaced (27.857%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T21:05:47.985 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3491897038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:05:48.088 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:48.192 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:48.219 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:48.250 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 158 5 88186 88181 34.8704 0 - 0.0166307 2023-09-23T21:05:48.268 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:48.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:48.375 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:48.402 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:48.452 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:48.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:48.557 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:48.583 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:48.627 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:48.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:48.732 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:48.762 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:49.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 159 5 88186 88181 34.6512 0 - 0.0166307 2023-09-23T21:05:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:49 smithi121 ceph-mon[211526]: pgmap v1219: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73714/264618 objects misplaced (27.857%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T21:05:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:49 smithi121 ceph-mon[219709]: pgmap v1219: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73714/264618 objects misplaced (27.857%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T21:05:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:49 smithi123 ceph-mon[193903]: pgmap v1219: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73714/264618 objects misplaced (27.857%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T21:05:49.712 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:49.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:49.815 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:49.842 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:50.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:05:50.251753+0000 min lat: 0.00123114 max lat: 18.7578 avg lat: 0.0166307 2023-09-23T21:05:50.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:05:50.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 160 5 88186 88181 34.4346 0 - 0.0166307 2023-09-23T21:05:50.711 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:50.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:50.815 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:50.843 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:50.890 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:50.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:50.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:50.998 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:51.026 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:51.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 161 5 88186 88181 34.2208 0 - 0.0166307 2023-09-23T21:05:51.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:51 smithi121 ceph-mon[211526]: pgmap v1220: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 104 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:05:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:51 smithi121 ceph-mon[219709]: pgmap v1220: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 104 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:05:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:51 smithi123 ceph-mon[193903]: pgmap v1220: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 104 B/s wr, 0 op/s; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:05:51.501 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:51.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:51.605 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:51.635 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:51.798 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:51.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:51.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:51.907 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:51.936 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:51.986 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:51.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:52.098 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:52.133 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:52.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 162 5 88186 88181 34.0096 0 - 0.0166307 2023-09-23T21:05:52.487 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:52.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:52.549 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:05:52.592 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:52.630 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:52.655 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:05:52.655 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:05:52.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/80883583' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:05:52.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/80883583' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:05:52.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/80883583' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:05:53.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 163 5 88186 88181 33.801 0 - 0.0166307 2023-09-23T21:05:53.559 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/80883583' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2424485682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[211526]: pgmap v1222: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/80883583' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2424485682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:53 smithi121 ceph-mon[219709]: pgmap v1222: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:05:53.939 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:53.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/80883583' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:05:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: osdmap e392: 8 total, 8 up, 7 in 2023-09-23T21:05:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: failed to encode map e392 with expected crc 2023-09-23T21:05:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2424485682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:05:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:53 smithi123 ceph-mon[193903]: pgmap v1222: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:05:54.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:54.071 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:54.119 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:54.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:54.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:54.225 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:54.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 164 5 88186 88181 33.5949 0 - 0.0166307 2023-09-23T21:05:54.253 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:54.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:54.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:54.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[211526]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T21:05:54.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[211526]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[211526]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[211526]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[211526]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:54.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:54.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[219709]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T21:05:54.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[219709]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[219709]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[219709]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:54 smithi121 ceph-mon[219709]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.906 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:05:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:05:54] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:05:54.915 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:54.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:54 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:05:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:05:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:54 smithi123 ceph-mon[193903]: osdmap e393: 8 total, 8 up, 7 in 2023-09-23T21:05:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:54 smithi123 ceph-mon[193903]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:54 smithi123 ceph-mon[193903]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:54 smithi123 ceph-mon[193903]: failed to encode map e393 with expected crc 2023-09-23T21:05:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:54 smithi123 ceph-mon[193903]: failed to encode map e393 with expected crc 2023-09-23T21:05:55.020 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:55.032 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:05:55.048 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:55.052 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:05:55.053 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.053 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.054 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.054 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.054 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.054 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.054 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.054 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:05:55.095 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:55.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:55.203 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:55.230 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:55.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 165 5 88186 88181 33.3914 0 - 0.0166307 2023-09-23T21:05:55.276 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:55.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:55.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:55.381 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:05:55.411 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:55.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:55 smithi121 ceph-mon[211526]: failed to encode map e393 with expected crc 2023-09-23T21:05:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:55 smithi121 ceph-mon[211526]: failed to encode map e393 with expected crc 2023-09-23T21:05:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:55 smithi121 ceph-mon[211526]: pgmap v1224: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T21:05:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:55 smithi121 ceph-mon[219709]: failed to encode map e393 with expected crc 2023-09-23T21:05:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:55 smithi121 ceph-mon[219709]: failed to encode map e393 with expected crc 2023-09-23T21:05:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:55 smithi121 ceph-mon[219709]: pgmap v1224: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T21:05:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:55 smithi123 ceph-mon[193903]: failed to encode map e393 with expected crc 2023-09-23T21:05:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:55 smithi123 ceph-mon[193903]: failed to encode map e393 with expected crc 2023-09-23T21:05:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:55 smithi123 ceph-mon[193903]: pgmap v1224: 89 pgs: 1 remapped+peering, 2 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 32909/264618 objects degraded (12.436%); 73326/264618 objects misplaced (27.710%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T21:05:56.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 166 5 88186 88181 33.1903 0 - 0.0166307 2023-09-23T21:05:56.392 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:05:56.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:56 smithi121 ceph-mon[211526]: failed to encode map e393 with expected crc 2023-09-23T21:05:56.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:56 smithi121 ceph-mon[219709]: failed to encode map e393 with expected crc 2023-09-23T21:05:56.667 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:05:56.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:56.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:05:56.775 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:56.800 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:56 smithi123 ceph-mon[193903]: failed to encode map e393 with expected crc 2023-09-23T21:05:56.803 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:56.852 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:56.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:56.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:56.960 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:56.985 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:57.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 167 5 88186 88181 32.9916 0 - 0.0166307 2023-09-23T21:05:57.279 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:05:57.619 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:57.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:57.727 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:05:57.755 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:57.804 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:05:57.807 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:57 smithi123 ceph-mon[193903]: pgmap v1225: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71885/264618 objects misplaced (27.166%); 1/88206 objects unfound (0.001%); 14 MiB/s, 228 objects/s recovering 2023-09-23T21:05:57.807 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:57.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:57.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:05:57.822 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:57.908 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:05:57.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:57 smithi121 ceph-mon[211526]: pgmap v1225: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71885/264618 objects misplaced (27.166%); 1/88206 objects unfound (0.001%); 14 MiB/s, 228 objects/s recovering 2023-09-23T21:05:57.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:57 smithi121 ceph-mon[219709]: pgmap v1225: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71885/264618 objects misplaced (27.166%); 1/88206 objects unfound (0.001%); 14 MiB/s, 228 objects/s recovering 2023-09-23T21:05:57.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:05:57.938 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.114 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:05:58.238 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:05:58.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:05:58.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 168 5 88186 88181 32.7952 0 - 0.0166307 2023-09-23T21:05:58.342 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:58.369 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.417 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:58.524 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:05:58.551 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.601 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:05:58.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:05:58.708 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:05:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:58 smithi123 ceph-mon[193903]: Health check failed: 1/88206 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-09-23T21:05:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:58 smithi123 ceph-mon[193903]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:05:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:58 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:05:58.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:58 smithi123 ceph-mon[193903]: Health check update: 0 slow ops, oldest one blocked for 73 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:58.734 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:58.781 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:05:58.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:58.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:05:58.886 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:05:58.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[211526]: Health check failed: 1/88206 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-09-23T21:05:58.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[211526]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:05:58.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:05:58.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[211526]: Health check update: 0 slow ops, oldest one blocked for 73 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:58.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[219709]: Health check failed: 1/88206 objects unfound (0.001%) (OBJECT_UNFOUND) 2023-09-23T21:05:58.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[219709]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T21:05:58.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T21:05:58.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:58 smithi121 ceph-mon[219709]: Health check update: 0 slow ops, oldest one blocked for 73 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:05:58.917 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:59.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 169 5 88186 88181 32.6012 0 - 0.0166307 2023-09-23T21:05:59.582 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:05:59.617 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:05:59.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:59.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:05:59.641 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:05:59.721 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:05:59.747 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:59.796 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:05:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:05:59.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:05:59.902 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:05:59.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:05:59 smithi121 ceph-mon[219709]: pgmap v1226: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71885/264618 objects misplaced (27.166%); 1/88206 objects unfound (0.001%); 11 MiB/s, 180 objects/s recovering 2023-09-23T21:05:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:05:59 smithi121 ceph-mon[211526]: pgmap v1226: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71885/264618 objects misplaced (27.166%); 1/88206 objects unfound (0.001%); 11 MiB/s, 180 objects/s recovering 2023-09-23T21:05:59.931 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:05:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:05:59 smithi123 ceph-mon[193903]: pgmap v1226: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71885/264618 objects misplaced (27.166%); 1/88206 objects unfound (0.001%); 11 MiB/s, 180 objects/s recovering 2023-09-23T21:06:00.251 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 170 5 88186 88181 32.4095 0 - 0.0166307 2023-09-23T21:06:00.778 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:00.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:00.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:00.881 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:00.914 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:01.125 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:01.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:01.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:01.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:01 smithi121 ceph-mon[219709]: pgmap v1227: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 14 MiB/s, 222 objects/s recovering 2023-09-23T21:06:01.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:01 smithi121 ceph-mon[211526]: pgmap v1227: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 14 MiB/s, 222 objects/s recovering 2023-09-23T21:06:01.229 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:01.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 171 5 88186 88181 32.22 0 - 0.0166307 2023-09-23T21:06:01.266 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:01.332 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:01.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:01.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:01.437 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:01.467 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:01.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:01 smithi123 ceph-mon[193903]: pgmap v1227: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 14 MiB/s, 222 objects/s recovering 2023-09-23T21:06:01.518 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:01.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:01.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:01.623 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:01.669 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:01.738 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:01.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:01.743 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:01.766 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:01.842 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:01.869 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:01.910 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:01.910 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:01.919 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:01.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:01.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:02.024 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:02.059 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:02.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 172 5 88186 88181 32.0327 0 - 0.0166307 2023-09-23T21:06:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:06:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:06:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1146424603' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:06:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:06:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:06:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:06:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1146424603' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:06:02.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:02 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:06:02.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:02 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:06:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:02 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:06:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1146424603' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:06:02.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:02 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:06:02.922 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:06:03.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 173 5 88186 88181 31.8476 0 - 0.0166307 2023-09-23T21:06:04.065 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:04.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[211526]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[211526]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[211526]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[211526]: pgmap v1229: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 12 MiB/s, 198 objects/s recovering 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[211526]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[211526]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[219709]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[219709]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:06:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[219709]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T21:06:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[219709]: pgmap v1229: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 12 MiB/s, 198 objects/s recovering 2023-09-23T21:06:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[219709]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:03 smithi121 ceph-mon[219709]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.169 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:04.202 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:03 smithi123 ceph-mon[193903]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:03 smithi123 ceph-mon[193903]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:06:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:03 smithi123 ceph-mon[193903]: osdmap e394: 8 total, 8 up, 7 in 2023-09-23T21:06:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:03 smithi123 ceph-mon[193903]: pgmap v1229: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 12 MiB/s, 198 objects/s recovering 2023-09-23T21:06:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:03 smithi123 ceph-mon[193903]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:03 smithi123 ceph-mon[193903]: failed to encode map e394 with expected crc 2023-09-23T21:06:04.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 174 5 88186 88181 31.6646 0 - 0.0166307 2023-09-23T21:06:04.353 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:04.460 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:04.487 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.534 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:04.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:04.640 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:04.668 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:04.816 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:04.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:04.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:06:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:06:04] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.43.0" 2023-09-23T21:06:04.920 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:04.945 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:05.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 175 5 88186 88181 31.4837 0 - 0.0166307 2023-09-23T21:06:05.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:05 smithi121 ceph-mon[211526]: pgmap v1230: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:06:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:05 smithi121 ceph-mon[219709]: pgmap v1230: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:06:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:05 smithi123 ceph-mon[193903]: pgmap v1230: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 71443/264618 objects misplaced (26.999%); 1/88206 objects unfound (0.001%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:06:05.673 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:06:05.674 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:06:05.746 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:06:05.896 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:05.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:06.001 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:06.031 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:06.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 176 5 88186 88181 31.3049 0 - 0.0166307 2023-09-23T21:06:06.723 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:06.723 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:06.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:06.776 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:06.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:06.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:06.880 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:06.907 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:06.923 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:06.954 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:06.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:06.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:07.001 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:06:07.023 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:06:07.045 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:06:07.056 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:07.061 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:07.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:06:07.096 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:06:07.101 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:07.122 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:06:07.155 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:07.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:07.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 177 5 88186 88181 31.1281 0 - 0.0166307 2023-09-23T21:06:07.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:07.293 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:07.344 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:07.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:07.448 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:07.484 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:07.701 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:07.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:07.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:07.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2632875406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:07.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:07 smithi123 ceph-mon[193903]: pgmap v1231: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70956/264618 objects misplaced (26.815%); 1/88206 objects unfound (0.001%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:07.770 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2632875406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:07.770 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:07 smithi121 ceph-mon[211526]: pgmap v1231: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70956/264618 objects misplaced (26.815%); 1/88206 objects unfound (0.001%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:07.771 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2632875406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:07.771 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:07 smithi121 ceph-mon[219709]: pgmap v1231: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70956/264618 objects misplaced (26.815%); 1/88206 objects unfound (0.001%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:07.805 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:07.832 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:07.879 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:07.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:07.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:07.984 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:08.012 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.059 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:08.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:08.165 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:08.192 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.243 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:08.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:08.252 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 178 5 88186 88181 30.9532 0 - 0.0166307 2023-09-23T21:06:08.349 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:08.380 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:08.463 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3119510406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:08.463 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:08 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:08.480 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:06:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3119510406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:08 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:08.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3119510406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:08 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:09.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 179 5 88186 88181 30.7803 0 - 0.0166307 2023-09-23T21:06:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:09 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:09 smithi123 ceph-mon[193903]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:09 smithi123 ceph-mon[193903]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:09 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:09 smithi123 ceph-mon[193903]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T21:06:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:09 smithi123 ceph-mon[193903]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:09 smithi123 ceph-mon[193903]: pgmap v1233: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70956/264618 objects misplaced (26.815%); 1/88206 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:06:09.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:09.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[211526]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[211526]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:09.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[211526]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T21:06:09.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[211526]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.792 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[211526]: pgmap v1233: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70956/264618 objects misplaced (26.815%); 1/88206 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:06:09.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:09.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[219709]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[219709]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:09.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[219709]: osdmap e395: 8 total, 8 up, 7 in 2023-09-23T21:06:09.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[219709]: failed to encode map e395 with expected crc 2023-09-23T21:06:09.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:09 smithi121 ceph-mon[219709]: pgmap v1233: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70956/264618 objects misplaced (26.815%); 1/88206 objects unfound (0.001%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T21:06:10.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:06:10.253507+0000 min lat: 0.00123114 max lat: 18.7578 avg lat: 0.0166307 2023-09-23T21:06:10.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:06:10.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 180 5 88186 88181 30.6094 0 - 0.0166307 2023-09-23T21:06:10.956 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:10.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:11.062 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:11.098 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:11.200 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:11.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:11.209 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:06:11.230 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:11 smithi121 ceph-mon[211526]: pgmap v1234: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70390/264618 objects misplaced (26.601%); 1/88206 objects unfound (0.001%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T21:06:11.230 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:11 smithi121 ceph-mon[219709]: pgmap v1234: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70390/264618 objects misplaced (26.601%); 1/88206 objects unfound (0.001%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T21:06:11.234 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:06:11.235 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.236 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:06:11.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 181 5 88186 88181 30.4403 0 - 0.0166307 2023-09-23T21:06:11.304 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:11.333 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:11.385 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:11.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:11 smithi123 ceph-mon[193903]: pgmap v1234: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 70390/264618 objects misplaced (26.601%); 1/88206 objects unfound (0.001%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T21:06:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:11.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:11.410 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:06:11.410 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:06:11.411 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:06:11.411 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:06:11.491 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:11.518 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:11.565 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:11.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:11.614 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:06:11.615 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:06:11.673 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:11.700 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:11.747 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:11.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:11.852 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:12.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 182 5 88186 88181 30.2731 0 - 0.0166307 2023-09-23T21:06:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:12 smithi123 ceph-mon[193903]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:12 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:12.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:06:12 smithi123 conmon[131622]: 2023-09-23T21:06:12.178+0000 7fe79295f700 -1 osd.6 395 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:06:12.586 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:12.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:12.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:12.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:12 smithi121 ceph-mon[211526]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:12 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:12.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:12 smithi121 ceph-mon[219709]: from='client.46385 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:12.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:12 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:12.690 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:12.716 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:12.982 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:06:13.231 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:13.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:13.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 183 5 88186 88181 30.1077 0 - 0.0166307 2023-09-23T21:06:13.319 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:06:13.339 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:13.366 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:13.419 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:13.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:13.478 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:06:13 smithi123 conmon[131622]: 2023-09-23T21:06:13.218+0000 7fe79295f700 -1 osd.6 395 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:06:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:13 smithi123 ceph-mon[193903]: pgmap v1235: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68951/264618 objects misplaced (26.057%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:06:13.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:13 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:13.524 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:13.552 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:13.574 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:13.574 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:13.574 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[219709]: pgmap v1235: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68951/264618 objects misplaced (26.057%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:06:13.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:13.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:13.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:13.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[211526]: pgmap v1235: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68951/264618 objects misplaced (26.057%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:06:13.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:13 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:13.598 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:13.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:13.706 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:13.733 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:13.781 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:13.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:13.889 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:13.916 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:14.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 184 5 88186 88181 29.9441 0 - 0.0166307 2023-09-23T21:06:14.273 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:14.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:14.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:14.358 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:14.377 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:14.384 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1234,"stamp":"2023-09-23T21:06:11.013628+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5779691424,"num_objects":88206,"num_object_clones":0,"num_object_copies":264618,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":43840,"num_objects_misplaced":70390,"num_objects_unfound":1,"num_objects_dirty":88206,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":88482,"num_write_kb":5646093,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18941,"num_bytes_recovered":1236638988,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76336,"ondisk_log_size":76336,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18052600,"kb_used_data":14326464,"kb_used_omap":21,"kb_used_meta":3726058,"kb_avail":638163464,"statfs":{"total":671965249536,"available":653479387136,"internally_reserved":0,"allocated":14670299136,"data_stored":14659219407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3815483875},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"upper_bound":262144},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1053,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1053,"num_bytes_recovered":69009408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.104585"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":165,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.909150+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:06:02.909150+0000","last_peered":"2023-09-23T21:06:02.909150+0000","last_clean":"2023-09-23T21:06:02.909150+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:06:02.909150+0000","last_undegraded":"2023-09-23T21:06:02.909150+0000","last_fullsized":"2023-09-23T21:06:02.909150+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":34,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906404+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:06:02.906404+0000","last_peered":"2023-09-23T21:06:02.906404+0000","last_clean":"2023-09-23T21:06:02.906404+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:06:02.906404+0000","last_undegraded":"2023-09-23T21:06:02.906404+0000","last_fullsized":"2023-09-23T21:06:02.906404+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":307,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.909523+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:06:02.909523+0000","last_peered":"2023-09-23T21:06:02.909523+0000","last_clean":"2023-09-23T21:06:02.909523+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:06:02.909523+0000","last_undegraded":"2023-09-23T21:06:02.909523+0000","last_fullsized":"2023-09-23T21:06:02.909523+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":31,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.915463+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:06:02.915463+0000","last_peered":"2023-09-23T21:06:02.915463+0000","last_clean":"2023-09-23T21:06:02.915463+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:06:02.915463+0000","last_undegraded":"2023-09-23T21:06:02.915463+0000","last_fullsized":"2023-09-23T21:06:02.915463+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":52,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.905860+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:06:02.905860+0000","last_peered":"2023-09-23T21:06:02.905860+0000","last_clean":"2023-09-23T21:06:02.905860+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:06:02.905860+0000","last_undegraded":"2023-09-23T21:06:02.905860+0000","last_fullsized":"2023-09-23T21:06:02.905860+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":259,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.535751+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:05:52.535751+0000","last_peered":"2023-09-23T21:05:52.535751+0000","last_clean":"2023-09-23T21:05:52.535751+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:05:52.535751+0000","last_undegraded":"2023-09-23T21:05:52.535751+0000","last_fullsized":"2023-09-23T21:05:52.535751+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":31,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914607+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:06:02.914607+0000","last_peered":"2023-09-23T21:06:02.914607+0000","last_clean":"2023-09-23T21:06:02.914607+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:06:02.914607+0000","last_undegraded":"2023-09-23T21:06:02.914607+0000","last_fullsized":"2023-09-23T21:06:02.914607+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":34,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.905797+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:06:02.905797+0000","last_peered":"2023-09-23T21:06:02.905797+0000","last_clean":"2023-09-23T21:06:02.905797+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:06:02.905797+0000","last_undegraded":"2023-09-23T21:06:02.905797+0000","last_fullsized":"2023-09-23T21:06:02.905797+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":17,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.183718+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:05:54.183718+0000","last_peered":"2023-09-23T21:05:54.183718+0000","last_clean":"2023-09-23T21:05:54.183718+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:05:54.183718+0000","last_undegraded":"2023-09-23T21:05:54.183718+0000","last_fullsized":"2023-09-23T21:05:54.183718+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":34,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.905987+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:06:02.905987+0000","last_peered":"2023-09-23T21:06:02.905987+0000","last_clean":"2023-09-23T21:06:02.905987+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:06:02.905987+0000","last_undegraded":"2023-09-23T21:06:02.905987+0000","last_fullsized":"2023-09-23T21:06:02.905987+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":50,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906446+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:06:02.906446+0000","last_peered":"2023-09-23T21:06:02.906446+0000","last_clean":"2023-09-23T21:06:02.906446+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:06:02.906446+0000","last_undegraded":"2023-09-23T21:06:02.906446+0000","last_fullsized":"2023-09-23T21:06:02.906446+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.908765+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:06:02.908765+0000","last_peered":"2023-09-23T21:06:02.908765+0000","last_clean":"2023-09-23T21:06:02.908765+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:06:02.908765+0000","last_undegraded":"2023-09-23T21:06:02.908765+0000","last_fullsized":"2023-09-23T21:06:02.908765+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":339,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.537458+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:05:52.537458+0000","last_peered":"2023-09-23T21:05:52.537458+0000","last_clean":"2023-09-23T21:05:52.537458+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:05:52.537458+0000","last_undegraded":"2023-09-23T21:05:52.537458+0000","last_fullsized":"2023-09-23T21:05:52.537458+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":312,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.373369+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:05:54.373369+0000","last_peered":"2023-09-23T21:05:54.373369+0000","last_clean":"2023-09-23T21:05:54.373369+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:05:54.373369+0000","last_undegraded":"2023-09-23T21:05:54.373369+0000","last_fullsized":"2023-09-23T21:05:54.373369+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":84,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.919096+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:06:02.919096+0000","last_peered":"2023-09-23T21:06:02.919096+0000","last_clean":"2023-09-23T21:06:02.919096+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:06:02.919096+0000","last_undegraded":"2023-09-23T21:06:02.919096+0000","last_fullsized":"2023-09-23T21:06:02.919096+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906317+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:06:02.906317+0000","last_peered":"2023-09-23T21:06:02.906317+0000","last_clean":"2023-09-23T21:06:02.906317+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:06:02.906317+0000","last_undegraded":"2023-09-23T21:06:02.906317+0000","last_fullsized":"2023-09-23T21:06:02.906317+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906045+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:06:02.906045+0000","last_peered":"2023-09-23T21:06:02.906045+0000","last_clean":"2023-09-23T21:06:02.906045+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:06:02.906045+0000","last_undegraded":"2023-09-23T21:06:02.906045+0000","last_fullsized":"2023-09-23T21:06:02.906045+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":52,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906486+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:06:02.906486+0000","last_peered":"2023-09-23T21:06:02.906486+0000","last_clean":"2023-09-23T21:06:02.906486+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:06:02.906486+0000","last_undegraded":"2023-09-23T21:06:02.906486+0000","last_fullsized":"2023-09-23T21:06:02.906486+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":194,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.373270+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:05:54.373270+0000","last_peered":"2023-09-23T21:05:54.373270+0000","last_clean":"2023-09-23T21:05:54.373270+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:05:54.373270+0000","last_undegraded":"2023-09-23T21:05:54.373270+0000","last_fullsized":"2023-09-23T21:05:54.373270+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":31,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.915531+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:06:02.915531+0000","last_peered":"2023-09-23T21:06:02.915531+0000","last_clean":"2023-09-23T21:06:02.915531+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:06:02.915531+0000","last_undegraded":"2023-09-23T21:06:02.915531+0000","last_fullsized":"2023-09-23T21:06:02.915531+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":389,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.535219+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:05:52.535219+0000","last_peered":"2023-09-23T21:05:52.535219+0000","last_clean":"2023-09-23T21:05:52.535219+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:05:52.535219+0000","last_undegraded":"2023-09-23T21:05:52.535219+0000","last_fullsized":"2023-09-23T21:05:52.535219+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":34,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906531+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:06:02.906531+0000","last_peered":"2023-09-23T21:06:02.906531+0000","last_clean":"2023-09-23T21:06:02.906531+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:06:02.906531+0000","last_undegraded":"2023-09-23T21:06:02.906531+0000","last_fullsized":"2023-09-23T21:06:02.906531+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"373'5615","reported_seq":6966,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:02.916113+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:06:02.916113+0000","last_peered":"2023-09-23T21:06:02.916113+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:06:02.916113+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367984640,"num_objects":5615,"num_object_clones":0,"num_object_copies":16845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5615,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":635},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":4979}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":98,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.373535+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:05:54.373535+0000","last_peered":"2023-09-23T21:05:54.373535+0000","last_clean":"2023-09-23T21:05:54.373535+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:05:54.373535+0000","last_undegraded":"2023-09-23T21:05:54.373535+0000","last_fullsized":"2023-09-23T21:05:54.373535+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.183364+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:05:54.183364+0000","last_peered":"2023-09-23T21:05:54.183364+0000","last_clean":"2023-09-23T21:05:54.183364+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:05:54.183364+0000","last_undegraded":"2023-09-23T21:05:54.183364+0000","last_fullsized":"2023-09-23T21:05:54.183364+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"370'5399","reported_seq":5463,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:02.919075+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:06:02.919075+0000","last_peered":"2023-09-23T21:06:02.919075+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:06:02.919075+0000","last_undegraded":"2023-09-23T21:06:02.919075+0000","last_fullsized":"2023-09-23T21:06:02.919075+0000","mapping_epoch":375,"log_start":"347'397","ondisk_log_start":"347'397","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5002,"log_dups_size":0,"ondisk_log_size":5002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353828864,"num_objects":5399,"num_object_clones":0,"num_object_copies":16197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5399,"num_objects_unfound":0,"num_objects_dirty":5399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5399,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":5399}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":92,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.919231+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:06:02.919231+0000","last_peered":"2023-09-23T21:06:02.919231+0000","last_clean":"2023-09-23T21:06:02.919231+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:06:02.919231+0000","last_undegraded":"2023-09-23T21:06:02.919231+0000","last_fullsized":"2023-09-23T21:06:02.919231+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":354,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.909449+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:06:02.909449+0000","last_peered":"2023-09-23T21:06:02.909449+0000","last_clean":"2023-09-23T21:06:02.909449+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:06:02.909449+0000","last_undegraded":"2023-09-23T21:06:02.909449+0000","last_fullsized":"2023-09-23T21:06:02.909449+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"367'5465","reported_seq":7681,"reported_epoch":390,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-09-23T21:05:54.927099+0000","last_change":"2023-09-23T21:05:54.989856+0000","last_active":"2023-09-23T21:04:28.774586+0000","last_peered":"2023-09-23T21:04:28.746162+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:03:39.498174+0000","last_became_peered":"2023-09-23T21:03:39.498174+0000","last_unstale":"2023-09-23T21:05:54.927099+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5465,"log_dups_size":0,"ondisk_log_size":5465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":10931,"num_objects_misplaced":0,"num_objects_unfound":1,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5465,"num_write_kb":349760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1072,"num_bytes_recovered":70254592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["0","2","4","7"],"object_location_counts":[{"shards":"3,6","objects":1},{"shards":"6","objects":5464}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":21,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.183767+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:05:54.183767+0000","last_peered":"2023-09-23T21:05:54.183767+0000","last_clean":"2023-09-23T21:05:54.183767+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:05:54.183767+0000","last_undegraded":"2023-09-23T21:05:54.183767+0000","last_fullsized":"2023-09-23T21:05:54.183767+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":281,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.535591+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:05:52.535591+0000","last_peered":"2023-09-23T21:05:52.535591+0000","last_clean":"2023-09-23T21:05:52.535591+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:05:52.535591+0000","last_undegraded":"2023-09-23T21:05:52.535591+0000","last_fullsized":"2023-09-23T21:05:52.535591+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"370'5525","reported_seq":6135,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:05:52.539022+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:05:52.539022+0000","last_peered":"2023-09-23T21:05:52.539022+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:05:52.539022+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"349'1468","ondisk_log_start":"349'1468","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362086400,"num_objects":5525,"num_object_clones":0,"num_object_copies":16575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5525,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":272},{"shards":"2,6,7","objects":5253}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":34,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906342+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:06:02.906342+0000","last_peered":"2023-09-23T21:06:02.906342+0000","last_clean":"2023-09-23T21:06:02.906342+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:06:02.906342+0000","last_undegraded":"2023-09-23T21:06:02.906342+0000","last_fullsized":"2023-09-23T21:06:02.906342+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":31,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.915005+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:06:02.915005+0000","last_peered":"2023-09-23T21:06:02.915005+0000","last_clean":"2023-09-23T21:06:02.915005+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:06:02.915005+0000","last_undegraded":"2023-09-23T21:06:02.915005+0000","last_fullsized":"2023-09-23T21:06:02.915005+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"370'5654","reported_seq":8716,"reported_epoch":393,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:05:55.000753+0000","last_change":"2023-09-23T21:05:34.257054+0000","last_active":"2023-09-23T21:05:55.000753+0000","last_peered":"2023-09-23T21:05:55.000753+0000","last_clean":"2023-09-23T21:03:37.385653+0000","last_became_active":"2023-09-23T21:05:12.775986+0000","last_became_peered":"2023-09-23T21:05:12.775986+0000","last_unstale":"2023-09-23T21:05:55.000753+0000","last_undegraded":"2023-09-23T21:05:55.000753+0000","last_fullsized":"2023-09-23T21:05:12.621268+0000","mapping_epoch":380,"log_start":"349'1599","ondisk_log_start":"349'1599","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4055,"log_dups_size":0,"ondisk_log_size":4055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:18.107688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4204,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1450,"num_bytes_recovered":95027200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6","objects":1450},{"shards":"1,6,7","objects":3127},{"shards":"2,5,6","objects":1077}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":51,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906098+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:06:02.906098+0000","last_peered":"2023-09-23T21:06:02.906098+0000","last_clean":"2023-09-23T21:06:02.906098+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:06:02.906098+0000","last_undegraded":"2023-09-23T21:06:02.906098+0000","last_fullsized":"2023-09-23T21:06:02.906098+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":34,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906260+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:06:02.906260+0000","last_peered":"2023-09-23T21:06:02.906260+0000","last_clean":"2023-09-23T21:06:02.906260+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:06:02.906260+0000","last_undegraded":"2023-09-23T21:06:02.906260+0000","last_fullsized":"2023-09-23T21:06:02.906260+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"370'5491","reported_seq":13091,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:02.920564+0000","last_change":"2023-09-23T21:05:34.191496+0000","last_active":"2023-09-23T21:06:02.920564+0000","last_peered":"2023-09-23T21:06:02.920564+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:04:42.572134+0000","last_became_peered":"2023-09-23T21:04:42.572134+0000","last_unstale":"2023-09-23T21:06:02.920564+0000","last_undegraded":"2023-09-23T21:05:34.190372+0000","last_fullsized":"2023-09-23T21:04:42.399994+0000","mapping_epoch":372,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:37.804028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359858176,"num_objects":5491,"num_object_clones":0,"num_object_copies":16473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1740,"num_objects_misplaced":4743,"num_objects_unfound":0,"num_objects_dirty":5491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5491,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3747,"num_bytes_recovered":245563392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,1],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":748},{"shards":"1,2","objects":3751},{"shards":"1,2,7","objects":992}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":319,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.535647+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:05:52.535647+0000","last_peered":"2023-09-23T21:05:52.535647+0000","last_clean":"2023-09-23T21:05:52.535647+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:05:52.535647+0000","last_undegraded":"2023-09-23T21:05:52.535647+0000","last_fullsized":"2023-09-23T21:05:52.535647+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":405,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.919280+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:06:02.919280+0000","last_peered":"2023-09-23T21:06:02.919280+0000","last_clean":"2023-09-23T21:06:02.919280+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:06:02.919280+0000","last_undegraded":"2023-09-23T21:06:02.919280+0000","last_fullsized":"2023-09-23T21:06:02.919280+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"370'5452","reported_seq":5531,"reported_epoch":392,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:05:52.535973+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:05:52.535973+0000","last_peered":"2023-09-23T21:05:52.535973+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:05:52.535973+0000","last_undegraded":"2023-09-23T21:05:52.535973+0000","last_fullsized":"2023-09-23T21:05:52.535973+0000","mapping_epoch":374,"log_start":"349'1394","ondisk_log_start":"349'1394","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4058,"log_dups_size":0,"ondisk_log_size":4058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357302272,"num_objects":5452,"num_object_clones":0,"num_object_copies":16356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10904,"num_objects_unfound":0,"num_objects_dirty":5452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5452,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5452}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.373108+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:05:54.373108+0000","last_peered":"2023-09-23T21:05:54.373108+0000","last_clean":"2023-09-23T21:05:54.373108+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:05:54.373108+0000","last_undegraded":"2023-09-23T21:05:54.373108+0000","last_fullsized":"2023-09-23T21:05:54.373108+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":432,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.538494+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:05:52.538494+0000","last_peered":"2023-09-23T21:05:52.538494+0000","last_clean":"2023-09-23T21:05:52.538494+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:05:52.538494+0000","last_undegraded":"2023-09-23T21:05:52.538494+0000","last_fullsized":"2023-09-23T21:05:52.538494+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"367'5481","reported_seq":5648,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:02.913804+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:06:02.913804+0000","last_peered":"2023-09-23T21:06:02.913804+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:06:02.913804+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5481,"log_dups_size":0,"ondisk_log_size":5481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359202816,"num_objects":5481,"num_object_clones":0,"num_object_copies":16443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5481,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":42},{"shards":"1,6,7","objects":5439}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.905928+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:06:02.905928+0000","last_peered":"2023-09-23T21:06:02.905928+0000","last_clean":"2023-09-23T21:06:02.905928+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:06:02.905928+0000","last_undegraded":"2023-09-23T21:06:02.905928+0000","last_fullsized":"2023-09-23T21:06:02.905928+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":182,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.916985+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:06:02.916985+0000","last_peered":"2023-09-23T21:06:02.916985+0000","last_clean":"2023-09-23T21:06:02.916985+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:06:02.916985+0000","last_undegraded":"2023-09-23T21:06:02.916985+0000","last_fullsized":"2023-09-23T21:06:02.916985+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"370'5514","reported_seq":10992,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.919418+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:06:02.919418+0000","last_peered":"2023-09-23T21:06:02.919418+0000","last_clean":"2023-09-23T21:06:02.919418+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:06:02.919418+0000","last_undegraded":"2023-09-23T21:06:02.919418+0000","last_fullsized":"2023-09-23T21:06:02.919418+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361365504,"num_objects":5514,"num_object_clones":0,"num_object_copies":16542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5514,"num_write_kb":352896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":289,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:05.277385+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:06:05.277385+0000","last_peered":"2023-09-23T21:06:05.277385+0000","last_clean":"2023-09-23T21:06:05.277385+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:06:05.277385+0000","last_undegraded":"2023-09-23T21:06:05.277385+0000","last_fullsized":"2023-09-23T21:06:05.277385+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":59,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.183216+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:05:54.183216+0000","last_peered":"2023-09-23T21:05:54.183216+0000","last_clean":"2023-09-23T21:05:54.183216+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:05:54.183216+0000","last_undegraded":"2023-09-23T21:05:54.183216+0000","last_fullsized":"2023-09-23T21:05:54.183216+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.908850+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:06:02.908850+0000","last_peered":"2023-09-23T21:06:02.908850+0000","last_clean":"2023-09-23T21:06:02.908850+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:06:02.908850+0000","last_undegraded":"2023-09-23T21:06:02.908850+0000","last_fullsized":"2023-09-23T21:06:02.908850+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"370'5513","reported_seq":5626,"reported_epoch":392,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:05:52.543369+0000","last_change":"2023-09-23T21:05:12.897699+0000","last_active":"2023-09-23T21:05:52.543369+0000","last_peered":"2023-09-23T21:05:52.543369+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:05:52.543369+0000","last_undegraded":"2023-09-23T21:05:12.617976+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"349'1457","ondisk_log_start":"349'1457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4056,"log_dups_size":0,"ondisk_log_size":4056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5513,"num_objects_misplaced":7846,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":3180},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":370,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:05.277350+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:06:05.277350+0000","last_peered":"2023-09-23T21:06:05.277350+0000","last_clean":"2023-09-23T21:06:05.277350+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:06:05.277350+0000","last_undegraded":"2023-09-23T21:06:05.277350+0000","last_fullsized":"2023-09-23T21:06:05.277350+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":343,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.543288+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:05:52.543288+0000","last_peered":"2023-09-23T21:05:52.543288+0000","last_clean":"2023-09-23T21:05:52.543288+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:05:52.543288+0000","last_undegraded":"2023-09-23T21:05:52.543288+0000","last_fullsized":"2023-09-23T21:05:52.543288+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":26,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.909079+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:06:02.909079+0000","last_peered":"2023-09-23T21:06:02.909079+0000","last_clean":"2023-09-23T21:06:02.909079+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:06:02.909079+0000","last_undegraded":"2023-09-23T21:06:02.909079+0000","last_fullsized":"2023-09-23T21:06:02.909079+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"370'5531","reported_seq":5693,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:02.918446+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:06:02.918446+0000","last_peered":"2023-09-23T21:06:02.918446+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:06:02.918446+0000","last_undegraded":"2023-09-23T21:06:02.918446+0000","last_fullsized":"2023-09-23T21:06:02.918446+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4058,"log_dups_size":0,"ondisk_log_size":4058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362479616,"num_objects":5531,"num_object_clones":0,"num_object_copies":16593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11027,"num_objects_unfound":0,"num_objects_dirty":5531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":5531}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":371,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.918815+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:06:02.918815+0000","last_peered":"2023-09-23T21:06:02.918815+0000","last_clean":"2023-09-23T21:06:02.918815+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:06:02.918815+0000","last_undegraded":"2023-09-23T21:06:02.918815+0000","last_fullsized":"2023-09-23T21:06:02.918815+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":289,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.911792+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:06:02.911792+0000","last_peered":"2023-09-23T21:06:02.911792+0000","last_clean":"2023-09-23T21:06:02.911792+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:06:02.911792+0000","last_undegraded":"2023-09-23T21:06:02.911792+0000","last_fullsized":"2023-09-23T21:06:02.911792+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":26,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.916845+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:06:02.916845+0000","last_peered":"2023-09-23T21:06:02.916845+0000","last_clean":"2023-09-23T21:06:02.916845+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:06:02.916845+0000","last_undegraded":"2023-09-23T21:06:02.916845+0000","last_fullsized":"2023-09-23T21:06:02.916845+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"370'5437","reported_seq":11676,"reported_epoch":392,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:05:52.535419+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:05:52.535419+0000","last_peered":"2023-09-23T21:05:52.535419+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:05:52.535419+0000","last_undegraded":"2023-09-23T21:05:52.535419+0000","last_fullsized":"2023-09-23T21:05:52.535419+0000","mapping_epoch":387,"log_start":"349'1380","ondisk_log_start":"349'1380","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356319232,"num_objects":5437,"num_object_clones":0,"num_object_copies":16311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5437,"num_objects_unfound":0,"num_objects_dirty":5437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5437,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5437}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":209,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.919176+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:06:02.919176+0000","last_peered":"2023-09-23T21:06:02.919176+0000","last_clean":"2023-09-23T21:06:02.919176+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:06:02.919176+0000","last_undegraded":"2023-09-23T21:06:02.919176+0000","last_fullsized":"2023-09-23T21:06:02.919176+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":691,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:02.908837+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:06:02.908837+0000","last_peered":"2023-09-23T21:06:02.908837+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:06:02.908837+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906038+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:06:02.906038+0000","last_peered":"2023-09-23T21:06:02.906038+0000","last_clean":"2023-09-23T21:06:02.906038+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:06:02.906038+0000","last_undegraded":"2023-09-23T21:06:02.906038+0000","last_fullsized":"2023-09-23T21:06:02.906038+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":287,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T21:05:40.114338+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:05:40.114338+0000","last_peered":"2023-09-23T21:05:40.114338+0000","last_clean":"2023-09-23T21:05:40.114338+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:05:40.114338+0000","last_undegraded":"2023-09-23T21:05:40.114338+0000","last_fullsized":"2023-09-23T21:05:40.114338+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"370'5561","reported_seq":5656,"reported_epoch":392,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:05:52.535194+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:05:52.535194+0000","last_peered":"2023-09-23T21:05:52.535194+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:05:52.535194+0000","last_undegraded":"2023-09-23T21:05:52.535194+0000","last_fullsized":"2023-09-23T21:05:52.535194+0000","mapping_epoch":374,"log_start":"349'1505","ondisk_log_start":"349'1505","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4056,"log_dups_size":0,"ondisk_log_size":4056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364445696,"num_objects":5561,"num_object_clones":0,"num_object_copies":16683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11122,"num_objects_unfound":0,"num_objects_dirty":5561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5561,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":5561}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.908734+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:06:02.908734+0000","last_peered":"2023-09-23T21:06:02.908734+0000","last_clean":"2023-09-23T21:06:02.908734+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:06:02.908734+0000","last_undegraded":"2023-09-23T21:06:02.908734+0000","last_fullsized":"2023-09-23T21:06:02.908734+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":40,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.915481+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:06:02.915481+0000","last_peered":"2023-09-23T21:06:02.915481+0000","last_clean":"2023-09-23T21:06:02.915481+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:06:02.915481+0000","last_undegraded":"2023-09-23T21:06:02.915481+0000","last_fullsized":"2023-09-23T21:06:02.915481+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.915133+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:06:02.915133+0000","last_peered":"2023-09-23T21:06:02.915133+0000","last_clean":"2023-09-23T21:06:02.915133+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:06:02.915133+0000","last_undegraded":"2023-09-23T21:06:02.915133+0000","last_fullsized":"2023-09-23T21:06:02.915133+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"370'5513","reported_seq":9116,"reported_epoch":394,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:02.912849+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:06:02.912849+0000","last_peered":"2023-09-23T21:06:02.912849+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:06:02.912849+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:06:02.912849+0000","mapping_epoch":383,"log_start":"347'153","ondisk_log_start":"347'153","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":1762},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":31,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914218+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:06:02.914218+0000","last_peered":"2023-09-23T21:06:02.914218+0000","last_clean":"2023-09-23T21:06:02.914218+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:06:02.914218+0000","last_undegraded":"2023-09-23T21:06:02.914218+0000","last_fullsized":"2023-09-23T21:06:02.914218+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":291,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:05.277410+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:06:05.277410+0000","last_peered":"2023-09-23T21:06:05.277410+0000","last_clean":"2023-09-23T21:06:05.277410+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:06:05.277410+0000","last_undegraded":"2023-09-23T21:06:05.277410+0000","last_fullsized":"2023-09-23T21:06:05.277410+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":69,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.183650+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:05:54.183650+0000","last_peered":"2023-09-23T21:05:54.183650+0000","last_clean":"2023-09-23T21:05:54.183650+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:05:54.183650+0000","last_undegraded":"2023-09-23T21:05:54.183650+0000","last_fullsized":"2023-09-23T21:05:54.183650+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"373'5432","reported_seq":13607,"reported_epoch":395,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:06:10.276208+0000","last_change":"2023-09-23T21:05:34.382714+0000","last_active":"2023-09-23T21:06:10.276208+0000","last_peered":"2023-09-23T21:06:10.276208+0000","last_clean":"2023-09-23T21:04:28.738326+0000","last_became_active":"2023-09-23T21:04:58.713307+0000","last_became_peered":"2023-09-23T21:04:58.713307+0000","last_unstale":"2023-09-23T21:06:10.276208+0000","last_undegraded":"2023-09-23T21:06:10.276208+0000","last_fullsized":"2023-09-23T21:04:58.419619+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:14.294697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355991552,"num_objects":5432,"num_object_clones":0,"num_object_copies":16296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2344,"num_objects_unfound":0,"num_objects_dirty":5432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5432,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4048,"num_bytes_recovered":265289728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":3088},{"shards":"1,2,4","objects":3},{"shards":"1,2,4,7","objects":2},{"shards":"1,2,7","objects":2339}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":51,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906619+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:06:02.906619+0000","last_peered":"2023-09-23T21:06:02.906619+0000","last_clean":"2023-09-23T21:06:02.906619+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:06:02.906619+0000","last_undegraded":"2023-09-23T21:06:02.906619+0000","last_fullsized":"2023-09-23T21:06:02.906619+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.918825+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:06:02.918825+0000","last_peered":"2023-09-23T21:06:02.918825+0000","last_clean":"2023-09-23T21:06:02.918825+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:06:02.918825+0000","last_undegraded":"2023-09-23T21:06:02.918825+0000","last_fullsized":"2023-09-23T21:06:02.918825+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.905715+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:06:02.905715+0000","last_peered":"2023-09-23T21:06:02.905715+0000","last_clean":"2023-09-23T21:06:02.905715+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:06:02.905715+0000","last_undegraded":"2023-09-23T21:06:02.905715+0000","last_fullsized":"2023-09-23T21:06:02.905715+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"370'5599","reported_seq":5699,"reported_epoch":394,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:02.914761+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:06:02.914761+0000","last_peered":"2023-09-23T21:06:02.914761+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:06:02.914761+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:06:02.914761+0000","mapping_epoch":382,"log_start":"347'239","ondisk_log_start":"347'239","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366936064,"num_objects":5599,"num_object_clones":0,"num_object_copies":16797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5596,"num_objects_misplaced":5602,"num_objects_unfound":0,"num_objects_dirty":5599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5599,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":5596}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":289,"reported_epoch":392,"state":"active+clean","last_fresh":"2023-09-23T21:05:52.535780+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:05:52.535780+0000","last_peered":"2023-09-23T21:05:52.535780+0000","last_clean":"2023-09-23T21:05:52.535780+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:05:52.535780+0000","last_undegraded":"2023-09-23T21:05:52.535780+0000","last_fullsized":"2023-09-23T21:05:52.535780+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":35,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.915272+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:06:02.915272+0000","last_peered":"2023-09-23T21:06:02.915272+0000","last_clean":"2023-09-23T21:06:02.915272+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:06:02.915272+0000","last_undegraded":"2023-09-23T21:06:02.915272+0000","last_fullsized":"2023-09-23T21:06:02.915272+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":79,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.183332+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:05:54.183332+0000","last_peered":"2023-09-23T21:05:54.183332+0000","last_clean":"2023-09-23T21:05:54.183332+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:05:54.183332+0000","last_undegraded":"2023-09-23T21:05:54.183332+0000","last_fullsized":"2023-09-23T21:05:54.183332+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914937+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:06:02.914937+0000","last_peered":"2023-09-23T21:06:02.914937+0000","last_clean":"2023-09-23T21:06:02.914937+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:06:02.914937+0000","last_undegraded":"2023-09-23T21:06:02.914937+0000","last_fullsized":"2023-09-23T21:06:02.914937+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":417,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.918989+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:06:02.918989+0000","last_peered":"2023-09-23T21:06:02.918989+0000","last_clean":"2023-09-23T21:06:02.918989+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:06:02.918989+0000","last_undegraded":"2023-09-23T21:06:02.918989+0000","last_fullsized":"2023-09-23T21:06:02.918989+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":56,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.373214+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:05:54.373214+0000","last_peered":"2023-09-23T21:05:54.373214+0000","last_clean":"2023-09-23T21:05:54.373214+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:05:54.373214+0000","last_undegraded":"2023-09-23T21:05:54.373214+0000","last_fullsized":"2023-09-23T21:05:54.373214+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":60,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906180+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:06:02.906180+0000","last_peered":"2023-09-23T21:06:02.906180+0000","last_clean":"2023-09-23T21:06:02.906180+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:06:02.906180+0000","last_undegraded":"2023-09-23T21:06:02.906180+0000","last_fullsized":"2023-09-23T21:06:02.906180+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":280,"reported_epoch":388,"state":"active+clean","last_fresh":"2023-09-23T21:05:40.037965+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:05:40.037965+0000","last_peered":"2023-09-23T21:05:40.037965+0000","last_clean":"2023-09-23T21:05:40.037965+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:05:40.037965+0000","last_undegraded":"2023-09-23T21:05:40.037965+0000","last_fullsized":"2023-09-23T21:05:40.037965+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":52,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906108+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:06:02.906108+0000","last_peered":"2023-09-23T21:06:02.906108+0000","last_clean":"2023-09-23T21:06:02.906108+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:06:02.906108+0000","last_undegraded":"2023-09-23T21:06:02.906108+0000","last_fullsized":"2023-09-23T21:06:02.906108+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":34,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.906529+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:06:02.906529+0000","last_peered":"2023-09-23T21:06:02.906529+0000","last_clean":"2023-09-23T21:06:02.906529+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:06:02.906529+0000","last_undegraded":"2023-09-23T21:06:02.906529+0000","last_fullsized":"2023-09-23T21:06:02.906529+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":43,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.905888+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:06:02.905888+0000","last_peered":"2023-09-23T21:06:02.905888+0000","last_clean":"2023-09-23T21:06:02.905888+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:06:02.905888+0000","last_undegraded":"2023-09-23T21:06:02.905888+0000","last_fullsized":"2023-09-23T21:06:02.905888+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":37,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914487+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:06:02.914487+0000","last_peered":"2023-09-23T21:06:02.914487+0000","last_clean":"2023-09-23T21:06:02.914487+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:06:02.914487+0000","last_undegraded":"2023-09-23T21:06:02.914487+0000","last_fullsized":"2023-09-23T21:06:02.914487+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":393,"state":"active+clean","last_fresh":"2023-09-23T21:05:54.372921+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:05:54.372921+0000","last_peered":"2023-09-23T21:05:54.372921+0000","last_clean":"2023-09-23T21:05:54.372921+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:05:54.372921+0000","last_undegraded":"2023-09-23T21:05:54.372921+0000","last_fullsized":"2023-09-23T21:05:54.372921+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5779095552,"num_objects":88182,"num_object_clones":0,"num_object_copies":264546,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":43838,"num_objects_misplaced":70390,"num_objects_unfound":1,"num_objects_dirty":88182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88182,"num_write_kb":5643648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18824,"num_bytes_recovered":1233649664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18464636928,"data_stored":18464636928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76120,"ondisk_log_size":76120,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201729,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463352,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4380772,"kb_used_data":3891420,"kb_used_omap":0,"kb_used_meta":489344,"kb_avail":89364380,"statfs":{"total":95995035648,"available":91509125120,"internally_reserved":0,"allocated":3984814080,"data_stored":3983689972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":501088256},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"upper_bound":262144},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.1240000000000001,"15min":2.1589999999999998},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.65000000000000002,"5min":7.2480000000000002,"15min":102.242},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":1.1539999999999999,"15min":1.861},"min":{"1min":0.26700000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60799999999999998,"5min":9.7110000000000003,"15min":102.502},"last":0.59399999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:05:05 2023","interfaces":[{"interface":"back","average":{"1min":7.6369999999999996,"5min":4.0129999999999999,"15min":2.456},"min":{"1min":0.30099999999999999,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":119.077,"5min":119.077,"15min":119.077},"last":0.48799999999999999},{"interface":"front","average":{"1min":6.96,"5min":3.9540000000000002,"15min":2.5579999999999998},"min":{"1min":0.308,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":119.313,"5min":119.313,"15min":119.313},"last":0.53400000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":6.3570000000000002,"15min":3.089},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.4830000000000001,"5min":176.45400000000001,"15min":176.45400000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":6.4630000000000001,"15min":3.2429999999999999},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":2.419,"5min":176.46799999999999,"15min":176.46799999999999},"last":0.57399999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":3.7490000000000001,"15min":4.4400000000000004},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.26900000000000002},"max":{"1min":0.59299999999999997,"5min":178.40000000000001,"15min":250.804},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":3.8010000000000002,"15min":4.3899999999999997},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":2.1429999999999998,"5min":178.26599999999999,"15min":250.446},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":1.1759999999999999,"15min":0.97299999999999998},"min":{"1min":0.307,"5min":0.27000000000000002,"15min":0.24299999999999999},"max":{"1min":0.65100000000000002,"5min":13.1,"15min":13.1},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.439,"5min":0.92100000000000004,"15min":0.999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":0.61599999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.47599999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.1160000000000001,"15min":1.6639999999999999},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65500000000000003,"5min":11.281000000000001,"15min":105.121},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.46600000000000003,"5min":1.1919999999999999,"15min":1.7070000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":0.68700000000000006,"5min":8.6229999999999993,"15min":106.30500000000001},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:05:05 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.173,"15min":1.5149999999999999},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.28199999999999997},"max":{"1min":7.093,"5min":7.093,"15min":101.04600000000001},"last":0.441},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.2070000000000001,"15min":1.5189999999999999},"min":{"1min":0.37,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":7.1849999999999996,"5min":7.1849999999999996,"15min":101.081},"last":0.56299999999999994}]}]},{"osd":1,"up_from":276,"seq":1185410973796,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4406784,"kb_used_data":4105016,"kb_used_omap":5,"kb_used_meta":301754,"kb_avail":89338368,"statfs":{"total":95995035648,"available":91482488832,"internally_reserved":0,"allocated":4203536384,"data_stored":4201603711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":308996835},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.30599999999999999,"5min":0.70399999999999996,"15min":0.48899999999999999},"min":{"1min":0.20100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.47899999999999998,"5min":11.959,"15min":11.959},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.29999999999999999,"5min":1.1319999999999999,"15min":0.76700000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.49099999999999999,"5min":18.047999999999998,"15min":18.047999999999998},"last":0.29899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":2.2770000000000001,"15min":1.089},"min":{"1min":0.23400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57299999999999995,"5min":78.409000000000006,"15min":78.409000000000006},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.35799999999999998,"5min":1.835,"15min":0.90400000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.22},"max":{"1min":0.56000000000000005,"5min":76.245000000000005,"15min":76.245000000000005},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":3.621,"15min":2.4300000000000002},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":0.71499999999999997,"5min":77.75,"15min":172.44399999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.35999999999999999,"5min":4.4020000000000001,"15min":2.5960000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":0.74399999999999999,"5min":77.849000000000004,"15min":172.58500000000001},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.6499999999999999,"15min":0.88600000000000001},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66000000000000003,"5min":17.006,"15min":17.006},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.86699999999999999,"15min":0.67500000000000004},"min":{"1min":0.379,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68200000000000005,"5min":11.916,"15min":11.916},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":1.383,"15min":0.83299999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72599999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.45700000000000002,"5min":1.7869999999999999,"15min":0.95499999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.68799999999999994,"5min":17.076000000000001,"15min":17.076000000000001},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":2.8460000000000001,"15min":1.339},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.69499999999999995,"5min":127.995,"15min":127.995},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":3.1909999999999998,"15min":1.4239999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68100000000000005,"5min":126.836,"15min":126.836},"last":0.502}]},{"osd":7,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":3.069,"15min":1.4239999999999999},"min":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71299999999999997,"5min":151.791,"15min":151.791},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.238,"15min":1.171},"min":{"1min":0.42699999999999999,"5min":0.40400000000000003,"15min":0.38},"max":{"1min":0.73799999999999999,"5min":151.809,"15min":151.809},"last":0.49299999999999999}]}]},{"osd":0,"up_from":92,"seq":395136991536,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1233740,"kb_used_data":521156,"kb_used_omap":5,"kb_used_meta":712570,"kb_avail":92511412,"statfs":{"total":95995035648,"available":94731685888,"internally_reserved":0,"allocated":533663744,"data_stored":531833495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":729672314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.6749999999999998,"15min":1.2669999999999999},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.184},"max":{"1min":52.590000000000003,"5min":61.341999999999999,"15min":61.341999999999999},"last":0.247},{"interface":"front","average":{"1min":3.165,"5min":2.5579999999999998,"15min":1.266},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":52.276000000000003,"5min":61.299999999999997,"15min":61.299999999999997},"last":0.442}]},{"osd":2,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":2.778,"15min":2.0939999999999999},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.161},"max":{"1min":0.61399999999999999,"5min":183.77099999999999,"15min":183.77099999999999},"last":0.374},{"interface":"front","average":{"1min":0.38400000000000001,"5min":3.258,"15min":2.3039999999999998},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.59799999999999998,"5min":183.78700000000001,"15min":183.78700000000001},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":3.7370000000000001,"15min":3.9489999999999998},"min":{"1min":0.311,"5min":0.25,"15min":0.183},"max":{"1min":0.64700000000000002,"5min":186.143,"15min":186.143},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":4.0069999999999997,"15min":4.0380000000000003},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.23000000000000001},"max":{"1min":0.623,"5min":186.97399999999999,"15min":186.97399999999999},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2969999999999999,"15min":1.379},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.247},"max":{"1min":1.4350000000000001,"5min":9.7080000000000002,"15min":10.525},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.80900000000000005,"15min":1.022},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.26200000000000001},"max":{"1min":0.67600000000000005,"5min":6.0490000000000004,"15min":12.461},"last":0.41899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.4830000000000001,"15min":2.5819999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28000000000000003},"max":{"1min":1.411,"5min":9.7439999999999998,"15min":154.30600000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.82299999999999995,"15min":1.96},"min":{"1min":0.42699999999999999,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":0.67000000000000004,"5min":5.9950000000000001,"15min":154.42400000000001},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":5.8390000000000004,"15min":2.7610000000000001},"min":{"1min":0.35899999999999999,"5min":0.34599999999999997,"15min":0.31},"max":{"1min":0.64200000000000002,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":6.1609999999999996,"15min":2.8929999999999998},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":0.60799999999999998,"5min":169.767,"15min":169.767},"last":0.502}]},{"osd":7,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.97799999999999998,"15min":2.52},"min":{"1min":0.40799999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69599999999999995,"5min":6.1130000000000004,"15min":164.26400000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.503,"5min":1.2490000000000001,"15min":2.6099999999999999},"min":{"1min":0.38700000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":0.63900000000000001,"5min":6.5410000000000004,"15min":164.23099999999999},"last":0.53000000000000003}]}]},{"osd":2,"up_from":129,"seq":554050781427,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4277432,"kb_used_data":3692464,"kb_used_omap":7,"kb_used_meta":584952,"kb_avail":89467720,"statfs":{"total":95995035648,"available":91614945280,"internally_reserved":0,"allocated":3781083136,"data_stored":3779206902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":598990929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":1.2949999999999999,"15min":1.7210000000000001},"min":{"1min":0.21299999999999999,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":0.96699999999999997,"5min":15.667,"15min":89.644999999999996},"last":0.217},{"interface":"front","average":{"1min":0.36599999999999999,"5min":1.7190000000000001,"15min":1.881},"min":{"1min":0.251,"5min":0.23599999999999999,"15min":0.222},"max":{"1min":0.76600000000000001,"5min":12.445,"15min":88.599999999999994},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:05:55 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":5.5549999999999997,"15min":2.343},"min":{"1min":0.22500000000000001,"5min":0.19,"15min":0.188},"max":{"1min":1.9419999999999999,"5min":212.12,"15min":212.12},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":5.5369999999999999,"15min":2.1349999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.79100000000000004,"5min":213.02799999999999,"15min":213.02799999999999},"last":0.375}]},{"osd":3,"last update":"Sat Sep 23 21:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":3.0609999999999999,"15min":3.8239999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":0.47799999999999998,"5min":66.085999999999999,"15min":151.43799999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.7919999999999998,"15min":3.8559999999999999},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.215},"max":{"1min":0.50900000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":0.307}]},{"osd":4,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":1.167,"15min":1.167},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.0049999999999999,"5min":12.586,"15min":12.586},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.421,"15min":1.216},"min":{"1min":0.371,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3300000000000001,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.498,"15min":2.6110000000000002},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.2749999999999999,"5min":9.9909999999999997,"15min":173.34},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":1.2370000000000001,"15min":2.4470000000000001},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.29399999999999998},"max":{"1min":1.2,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.41999999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":3.4500000000000002,"15min":1.9319999999999999},"min":{"1min":0.35299999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.149,"5min":175.72499999999999,"15min":175.72499999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.51500000000000001,"5min":3.3610000000000002,"15min":2.016},"min":{"1min":0.38700000000000001,"5min":0.36399999999999999,"15min":0.32300000000000001},"max":{"1min":1.228,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":1.498,"15min":2.7429999999999999},"min":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.32700000000000001},"max":{"1min":2.504,"5min":12.609,"15min":177.30799999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.5780000000000001,"15min":2.673},"min":{"1min":0.39500000000000002,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.3720000000000001,"5min":10.016,"15min":177.40899999999999},"last":0.47699999999999998}]}]},{"osd":3,"up_from":148,"seq":635655160021,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":526840,"kb_used_data":10032,"kb_used_omap":2,"kb_used_meta":516797,"kb_avail":93218312,"statfs":{"total":95995035648,"available":95455551488,"internally_reserved":0,"allocated":10272768,"data_stored":8223871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":529200181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.6739999999999999,"15min":1.8320000000000001},"min":{"1min":0.20999999999999999,"5min":0.20200000000000001,"15min":0.16700000000000001},"max":{"1min":44.963000000000001,"5min":44.963000000000001,"15min":95.507999999999996},"last":0.22500000000000001},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.101,"15min":1.823},"min":{"1min":0.20300000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":9.7370000000000001,"5min":9.7759999999999998,"15min":95.259},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.339,"15min":1.0129999999999999},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.7639999999999993,"5min":12.035,"15min":19.786999999999999},"last":0.307},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.6160000000000001,"15min":1.2110000000000001},"min":{"1min":0.253,"5min":0.182,"15min":0.182},"max":{"1min":2.3159999999999998,"5min":15.763999999999999,"15min":15.763999999999999},"last":0.38200000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.571,"15min":1.3129999999999999},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.182},"max":{"1min":3.2749999999999999,"5min":12.763999999999999,"15min":12.859},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.3380000000000001,"15min":1.4019999999999999},"min":{"1min":0.28000000000000003,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":9.7140000000000004,"5min":12.096,"15min":19.754999999999999},"last":0.437}]},{"osd":4,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.6629999999999998,"15min":2.194},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.222},"max":{"1min":3.5249999999999999,"5min":70.728999999999999,"15min":111.36},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":2.3439999999999999,"15min":1.9990000000000001},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":3.2490000000000001,"5min":70.712000000000003,"15min":112.473},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.157,"15min":1.6579999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":9.6359999999999992,"5min":9.6359999999999992,"15min":64.528000000000006},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.78,"15min":1.6839999999999999},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.249},"max":{"1min":3.5649999999999999,"5min":10.617000000000001,"15min":14.054},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.855,"15min":1.7310000000000001},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":3.5910000000000002,"5min":10.52,"15min":19.568000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.655,"15min":1.575},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":14.356,"5min":14.356,"15min":15.573},"last":0.44800000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.286,"15min":2.7269999999999999},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":9.6859999999999999,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.8199999999999998,"15min":2.8490000000000002},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.1419999999999999,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.64700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986639,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2019156,"kb_used_data":1477452,"kb_used_omap":0,"kb_used_meta":541696,"kb_avail":91725996,"statfs":{"total":95995035648,"available":93927419904,"internally_reserved":0,"allocated":1512910848,"data_stored":1511771000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":554696704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":1.349,"15min":3.085},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.48499999999999999,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.495},{"interface":"front","average":{"1min":0.436,"5min":1.1919999999999999,"15min":3.0739999999999998},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.63,"5min":8.5220000000000002,"15min":144.77000000000001},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:06:07 2023","interfaces":[{"interface":"back","average":{"1min":6.4459999999999997,"5min":6.3929999999999998,"15min":2.6269999999999998},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":118.411,"5min":118.411,"15min":118.411},"last":0.33500000000000002},{"interface":"front","average":{"1min":6.274,"5min":6.1340000000000003,"15min":2.4740000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":118.075,"5min":118.075,"15min":118.075},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":7.4100000000000001,"5min":5.8289999999999997,"15min":3.2269999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":110.193,"5min":164.517,"15min":164.517},"last":0.48499999999999999},{"interface":"front","average":{"1min":7.3220000000000001,"5min":5.7889999999999997,"15min":3.3170000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":110.30200000000001,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.53900000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:06:06 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.2709999999999999,"15min":2.2610000000000001},"min":{"1min":0.32500000000000001,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.8420000000000001,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.2829999999999999,"15min":2.3889999999999998},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.82799999999999996,"5min":7.5579999999999998,"15min":142.34700000000001},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:05:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":0.80400000000000005,"15min":1.0660000000000001},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":4.1779999999999999,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.38},{"interface":"front","average":{"1min":1.01,"5min":0.86399999999999999,"15min":1.131},"min":{"1min":0.311,"5min":0.28000000000000003,"15min":0.21099999999999999},"max":{"1min":4.9740000000000002,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.52100000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.998,"15min":0.97599999999999998},"min":{"1min":0.28299999999999997,"5min":0.27500000000000002,"15min":0.26100000000000001},"max":{"1min":0.65800000000000003,"5min":11.800000000000001,"15min":11.887},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.1120000000000001,"15min":0.93100000000000005},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.871,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:05:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.9929999999999999,"15min":1.8340000000000001},"min":{"1min":0.35799999999999998,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":13.667999999999999,"5min":145.721,"15min":145.721},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.0979999999999999,"5min":3.274,"15min":1.998},"min":{"1min":0.40999999999999998,"5min":0.30499999999999999,"15min":0.28000000000000003},"max":{"1min":9.2439999999999998,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.46400000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725003,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1207876,"kb_used_data":628924,"kb_used_omap":0,"kb_used_meta":578944,"kb_avail":92537276,"statfs":{"total":95995035648,"available":94758170624,"internally_reserved":0,"allocated":644018176,"data_stored":642890456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":592838656},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.78000000000000003,"15min":1.857},"min":{"1min":0.25600000000000001,"5min":0.24199999999999999,"15min":0.19600000000000001},"max":{"1min":0.59299999999999997,"5min":23.600000000000001,"15min":124.873},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.1259999999999999,"15min":1.9930000000000001},"min":{"1min":0.24199999999999999,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":0.61099999999999999,"5min":23.532,"15min":123.554},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:05:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":4.141,"15min":1.669},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.254},"max":{"1min":7.8200000000000003,"5min":137.143,"15min":137.143},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.997,"5min":3.8410000000000002,"15min":1.5760000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":5.4450000000000003,"5min":139.73699999999999,"15min":139.73699999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.5720000000000001,"15min":1.6060000000000001},"min":{"1min":0.28799999999999998,"5min":0.28000000000000003,"15min":0.22500000000000001},"max":{"1min":0.61599999999999999,"5min":103.63800000000001,"15min":103.63800000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.44,"5min":2.8330000000000002,"15min":1.6910000000000001},"min":{"1min":0.307,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":0.64800000000000002,"5min":103.664,"15min":103.664},"last":0.33600000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.609,"15min":4.0869999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.573,"5min":97.349999999999994,"15min":198.33099999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":2.4750000000000001,"15min":4.0540000000000003},"min":{"1min":0.40799999999999997,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":97.316999999999993,"15min":198.47300000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.69199999999999995,"15min":0.83099999999999996},"min":{"1min":0.33300000000000002,"5min":0.254,"15min":0.247},"max":{"1min":2.5070000000000001,"5min":12.739000000000001,"15min":12.739000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.696,"5min":0.90000000000000002,"15min":1.0049999999999999},"min":{"1min":0.34799999999999998,"5min":0.27500000000000002,"15min":0.247},"max":{"1min":8.3740000000000006,"5min":10.792,"15min":12.539},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":1.702,"15min":1.2150000000000001},"min":{"1min":0.32800000000000001,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":0.57899999999999996,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.501},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.835,"15min":1.3400000000000001},"min":{"1min":0.38900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.73099999999999998,"5min":70.266000000000005,"15min":70.266000000000005},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:05:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.0129999999999999,"15min":1.2509999999999999},"min":{"1min":0.39700000000000002,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":8.4109999999999996,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.66900000000000004,"15min":1.0880000000000001},"min":{"1min":0.44800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.536,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.61699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":530317312,"data_stored":530317312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4197908480,"data_stored":4197908480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3775528960,"data_stored":3775528960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":5111808,"data_stored":5111808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1508769792,"data_stored":1508769792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":640745472,"data_stored":640745472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3980132352,"data_stored":3980132352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3826122752,"data_stored":3826122752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:06:14.392 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:06:14.404 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:14.456 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:14.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:14.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:14.474 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:14.478 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:06:14 smithi123 conmon[131622]: 2023-09-23T21:06:14.225+0000 7fe79295f700 -1 osd.6 395 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:06:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:14 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:14 smithi123 ceph-mon[193903]: Health check cleared: OBJECT_UNFOUND (was: 1/88206 objects unfound (0.001%)) 2023-09-23T21:06:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:14 smithi123 ceph-mon[193903]: Health check cleared: SLOW_OPS (was: 0 slow ops, oldest one blocked for 73 sec, osd.6 has slow ops) 2023-09-23T21:06:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:14 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:14.564 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:14.589 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:14.589 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[219709]: Health check cleared: OBJECT_UNFOUND (was: 1/88206 objects unfound (0.001%)) 2023-09-23T21:06:14.589 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[219709]: Health check cleared: SLOW_OPS (was: 0 slow ops, oldest one blocked for 73 sec, osd.6 has slow ops) 2023-09-23T21:06:14.589 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:14.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:14.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[211526]: Health check cleared: OBJECT_UNFOUND (was: 1/88206 objects unfound (0.001%)) 2023-09-23T21:06:14.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[211526]: Health check cleared: SLOW_OPS (was: 0 slow ops, oldest one blocked for 73 sec, osd.6 has slow ops) 2023-09-23T21:06:14.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:14 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:14.592 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:14.645 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:14.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:14.749 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:14.780 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:14.842 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:06:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:06:14] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2023-09-23T21:06:14.928 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:14.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:15.031 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:15.065 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:15.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 185 5 88186 88181 29.7822 0 - 0.0166307 2023-09-23T21:06:15.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:06:15.445 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:15 smithi121 ceph-mon[211526]: pgmap v1236: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68951/264618 objects misplaced (26.057%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:06:15.446 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:15 smithi121 ceph-mon[211526]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:15.565 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:15.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:15.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:15.669 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:15.694 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:15 smithi121 ceph-mon[219709]: pgmap v1236: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68951/264618 objects misplaced (26.057%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:06:15.694 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:15 smithi121 ceph-mon[219709]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:15.696 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:15.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:15 smithi123 ceph-mon[193903]: pgmap v1236: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68951/264618 objects misplaced (26.057%); 16 MiB/s, 249 objects/s recovering 2023-09-23T21:06:15.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:15 smithi123 ceph-mon[193903]: 5 slow requests (by type [ 'delayed' : 5 ] most affected pool [ 'unique_pool_6' : 5 ]) 2023-09-23T21:06:15.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:06:15 smithi123 conmon[131622]: 2023-09-23T21:06:15.274+0000 7fe79295f700 -1 osd.6 395 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:06:16.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 186 5 88186 88181 29.6222 0 - 0.0166307 2023-09-23T21:06:16.728 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:06:16 smithi123 conmon[131622]: 2023-09-23T21:06:16.240+0000 7fe79295f700 -1 osd.6 395 get_health_metrics reporting 3 slow ops, oldest is osd_op(client.36195.0:88133 11.5 11.736d1745 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e369) 2023-09-23T21:06:16.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:16 smithi123 ceph-mon[193903]: 3 slow requests (by type [ 'delayed' : 3 ] most affected pool [ 'unique_pool_6' : 3 ]) 2023-09-23T21:06:16.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:16 smithi121 ceph-mon[211526]: 3 slow requests (by type [ 'delayed' : 3 ] most affected pool [ 'unique_pool_6' : 3 ]) 2023-09-23T21:06:16.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:16 smithi121 ceph-mon[219709]: 3 slow requests (by type [ 'delayed' : 3 ] most affected pool [ 'unique_pool_6' : 3 ]) 2023-09-23T21:06:17.200 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:17.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:17.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 187 5 88186 88181 29.4638 0 - 0.0166307 2023-09-23T21:06:17.304 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:17.337 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:17.403 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:17.452 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:17 smithi121 ceph-mon[211526]: pgmap v1237: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68304/264618 objects misplaced (25.812%); 17 MiB/s, 265 objects/s recovering 2023-09-23T21:06:17.453 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:17 smithi121 ceph-mon[219709]: pgmap v1237: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68304/264618 objects misplaced (25.812%); 17 MiB/s, 265 objects/s recovering 2023-09-23T21:06:17.643 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:17.644 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:17.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:17.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:17.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:17 smithi123 ceph-mon[193903]: pgmap v1237: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68304/264618 objects misplaced (25.812%); 17 MiB/s, 265 objects/s recovering 2023-09-23T21:06:17.747 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:17.775 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:17.828 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:17.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:17.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:17.934 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:17.969 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:18.032 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:18.033 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:06:18.034 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:06:18.034 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:06:18.034 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:06:18.034 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:06:18.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:18.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:18.136 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:18.167 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:18.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 188 5 88186 88181 29.3071 0 - 0.0166307 2023-09-23T21:06:18.479 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:06:18.716 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1049154023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.716 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.716 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1049154023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.716 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1049154023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:19.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 189 5 88186 88181 29.1521 0 - 0.0166307 2023-09-23T21:06:19.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:19 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:19.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:19 smithi123 ceph-mon[193903]: failed to encode map e396 with expected crc 2023-09-23T21:06:19.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:19 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:19 smithi123 ceph-mon[193903]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T21:06:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:19 smithi123 ceph-mon[193903]: failed to encode map e396 with expected crc 2023-09-23T21:06:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:19 smithi123 ceph-mon[193903]: pgmap v1239: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68304/264618 objects misplaced (25.812%); 17 MiB/s, 265 objects/s recovering 2023-09-23T21:06:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[211526]: failed to encode map e396 with expected crc 2023-09-23T21:06:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[211526]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T21:06:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[211526]: failed to encode map e396 with expected crc 2023-09-23T21:06:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[211526]: pgmap v1239: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68304/264618 objects misplaced (25.812%); 17 MiB/s, 265 objects/s recovering 2023-09-23T21:06:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:19.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[219709]: failed to encode map e396 with expected crc 2023-09-23T21:06:19.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:19.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[219709]: osdmap e396: 8 total, 8 up, 7 in 2023-09-23T21:06:19.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[219709]: failed to encode map e396 with expected crc 2023-09-23T21:06:19.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:19 smithi121 ceph-mon[219709]: pgmap v1239: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 68304/264618 objects misplaced (25.812%); 17 MiB/s, 265 objects/s recovering 2023-09-23T21:06:20.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 190 5 88186 88181 28.9987 0 - 0.0166307 2023-09-23T21:06:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:20 smithi123 ceph-mon[193903]: failed to encode map e396 with expected crc 2023-09-23T21:06:20.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:20 smithi121 ceph-mon[211526]: failed to encode map e396 with expected crc 2023-09-23T21:06:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:20 smithi121 ceph-mon[219709]: failed to encode map e396 with expected crc 2023-09-23T21:06:21.021 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:06:21.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 191 5 88186 88181 28.8469 0 - 0.0166307 2023-09-23T21:06:21.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:21 smithi123 ceph-mon[193903]: pgmap v1240: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 19 MiB/s, 308 objects/s recovering 2023-09-23T21:06:21.744 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:21.746 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:21 smithi121 ceph-mon[219709]: pgmap v1240: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 19 MiB/s, 308 objects/s recovering 2023-09-23T21:06:21.746 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:21 smithi121 ceph-mon[211526]: pgmap v1240: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 19 MiB/s, 308 objects/s recovering 2023-09-23T21:06:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:21.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:21.848 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:21.883 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:21.979 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:21.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:21.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:21.998 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:22.083 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:22.114 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:22.209 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:22.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:22.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:22.253 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 192 5 88186 88181 28.6966 0 - 0.0166307 2023-09-23T21:06:22.313 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:22.346 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:22.504 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:22.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:22.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:22.537 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:06:22.539 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:06:22.610 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:22.614 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:06:22.638 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:06:22.642 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:22.668 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:06:22.699 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:06:22.723 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:06:22.744 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:06:22.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:22 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1286998794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:22 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1286998794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:22.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:22 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1286998794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:23.206 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:06:23.254 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 193 5 88186 88181 28.548 0 - 0.0166307 2023-09-23T21:06:23.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:23 smithi121 ceph-mon[211526]: from='client.36426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:23.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:23 smithi121 ceph-mon[211526]: pgmap v1241: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 10 MiB/s, 164 objects/s recovering 2023-09-23T21:06:23.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:23 smithi121 ceph-mon[219709]: from='client.36426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:23.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:23 smithi121 ceph-mon[219709]: pgmap v1241: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 10 MiB/s, 164 objects/s recovering 2023-09-23T21:06:23.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:23 smithi123 ceph-mon[193903]: from='client.36426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:23 smithi123 ceph-mon[193903]: pgmap v1241: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 10 MiB/s, 164 objects/s recovering 2023-09-23T21:06:24.110 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1240,"stamp":"2023-09-23T21:06:21.016211+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5779691424,"num_objects":88206,"num_object_clones":0,"num_object_copies":264618,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":43840,"num_objects_misplaced":67307,"num_objects_unfound":0,"num_objects_dirty":88206,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":88482,"num_write_kb":5646093,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22024,"num_bytes_recovered":1438686476,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76336,"ondisk_log_size":76336,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18233732,"kb_used_data":14497292,"kb_used_omap":21,"kb_used_meta":3736362,"kb_avail":637982332,"statfs":{"total":671965249536,"available":653293907968,"internally_reserved":0,"allocated":14845227008,"data_stored":14834090209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3826035171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"upper_bound":262144},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-3083,"num_objects_unfound":-1,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3083,"num_bytes_recovered":202047488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002582"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":166,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.472867+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:06:08.472867+0000","last_peered":"2023-09-23T21:06:08.472867+0000","last_clean":"2023-09-23T21:06:08.472867+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:06:08.472867+0000","last_undegraded":"2023-09-23T21:06:08.472867+0000","last_fullsized":"2023-09-23T21:06:08.472867+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":35,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470635+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:06:08.470635+0000","last_peered":"2023-09-23T21:06:08.470635+0000","last_clean":"2023-09-23T21:06:08.470635+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:06:08.470635+0000","last_undegraded":"2023-09-23T21:06:08.470635+0000","last_fullsized":"2023-09-23T21:06:08.470635+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.472401+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:06:08.472401+0000","last_peered":"2023-09-23T21:06:08.472401+0000","last_clean":"2023-09-23T21:06:08.472401+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:06:08.472401+0000","last_undegraded":"2023-09-23T21:06:08.472401+0000","last_fullsized":"2023-09-23T21:06:08.472401+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":32,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.605258+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:06:08.605258+0000","last_peered":"2023-09-23T21:06:08.605258+0000","last_clean":"2023-09-23T21:06:08.605258+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:06:08.605258+0000","last_undegraded":"2023-09-23T21:06:08.605258+0000","last_fullsized":"2023-09-23T21:06:08.605258+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":53,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469044+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:06:08.469044+0000","last_peered":"2023-09-23T21:06:08.469044+0000","last_clean":"2023-09-23T21:06:08.469044+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:06:08.469044+0000","last_undegraded":"2023-09-23T21:06:08.469044+0000","last_fullsized":"2023-09-23T21:06:08.469044+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":261,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914573+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:06:02.914573+0000","last_peered":"2023-09-23T21:06:02.914573+0000","last_clean":"2023-09-23T21:06:02.914573+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:06:02.914573+0000","last_undegraded":"2023-09-23T21:06:02.914573+0000","last_fullsized":"2023-09-23T21:06:02.914573+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":32,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.604917+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:06:08.604917+0000","last_peered":"2023-09-23T21:06:08.604917+0000","last_clean":"2023-09-23T21:06:08.604917+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:06:08.604917+0000","last_undegraded":"2023-09-23T21:06:08.604917+0000","last_fullsized":"2023-09-23T21:06:08.604917+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":35,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469006+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:06:08.469006+0000","last_peered":"2023-09-23T21:06:08.469006+0000","last_clean":"2023-09-23T21:06:08.469006+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:06:08.469006+0000","last_undegraded":"2023-09-23T21:06:08.469006+0000","last_fullsized":"2023-09-23T21:06:08.469006+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":18,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018153+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:06:03.018153+0000","last_peered":"2023-09-23T21:06:03.018153+0000","last_clean":"2023-09-23T21:06:03.018153+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:06:03.018153+0000","last_undegraded":"2023-09-23T21:06:03.018153+0000","last_fullsized":"2023-09-23T21:06:03.018153+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":35,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469321+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:06:08.469321+0000","last_peered":"2023-09-23T21:06:08.469321+0000","last_clean":"2023-09-23T21:06:08.469321+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:06:08.469321+0000","last_undegraded":"2023-09-23T21:06:08.469321+0000","last_fullsized":"2023-09-23T21:06:08.469321+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":51,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470718+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:06:08.470718+0000","last_peered":"2023-09-23T21:06:08.470718+0000","last_clean":"2023-09-23T21:06:08.470718+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:06:08.470718+0000","last_undegraded":"2023-09-23T21:06:08.470718+0000","last_fullsized":"2023-09-23T21:06:08.470718+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.472470+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:06:08.472470+0000","last_peered":"2023-09-23T21:06:08.472470+0000","last_clean":"2023-09-23T21:06:08.472470+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:06:08.472470+0000","last_undegraded":"2023-09-23T21:06:08.472470+0000","last_fullsized":"2023-09-23T21:06:08.472470+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":341,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914001+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:06:02.914001+0000","last_peered":"2023-09-23T21:06:02.914001+0000","last_clean":"2023-09-23T21:06:02.914001+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:06:02.914001+0000","last_undegraded":"2023-09-23T21:06:02.914001+0000","last_fullsized":"2023-09-23T21:06:02.914001+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":313,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.019163+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:06:03.019163+0000","last_peered":"2023-09-23T21:06:03.019163+0000","last_clean":"2023-09-23T21:06:03.019163+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:06:03.019163+0000","last_undegraded":"2023-09-23T21:06:03.019163+0000","last_fullsized":"2023-09-23T21:06:03.019163+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":85,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.603535+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:06:08.603535+0000","last_peered":"2023-09-23T21:06:08.603535+0000","last_clean":"2023-09-23T21:06:08.603535+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:06:08.603535+0000","last_undegraded":"2023-09-23T21:06:08.603535+0000","last_fullsized":"2023-09-23T21:06:08.603535+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":44,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470003+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:06:08.470003+0000","last_peered":"2023-09-23T21:06:08.470003+0000","last_clean":"2023-09-23T21:06:08.470003+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:06:08.470003+0000","last_undegraded":"2023-09-23T21:06:08.470003+0000","last_fullsized":"2023-09-23T21:06:08.470003+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469389+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:06:08.469389+0000","last_peered":"2023-09-23T21:06:08.469389+0000","last_clean":"2023-09-23T21:06:08.469389+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:06:08.469389+0000","last_undegraded":"2023-09-23T21:06:08.469389+0000","last_fullsized":"2023-09-23T21:06:08.469389+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":53,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470789+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:06:08.470789+0000","last_peered":"2023-09-23T21:06:08.470789+0000","last_clean":"2023-09-23T21:06:08.470789+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:06:08.470789+0000","last_undegraded":"2023-09-23T21:06:08.470789+0000","last_fullsized":"2023-09-23T21:06:08.470789+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":195,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018894+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:06:03.018894+0000","last_peered":"2023-09-23T21:06:03.018894+0000","last_clean":"2023-09-23T21:06:03.018894+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:06:03.018894+0000","last_undegraded":"2023-09-23T21:06:03.018894+0000","last_fullsized":"2023-09-23T21:06:03.018894+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":32,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.605309+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:06:08.605309+0000","last_peered":"2023-09-23T21:06:08.605309+0000","last_clean":"2023-09-23T21:06:08.605309+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:06:08.605309+0000","last_undegraded":"2023-09-23T21:06:08.605309+0000","last_fullsized":"2023-09-23T21:06:08.605309+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":391,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.913756+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:06:02.913756+0000","last_peered":"2023-09-23T21:06:02.913756+0000","last_clean":"2023-09-23T21:06:02.913756+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:06:02.913756+0000","last_undegraded":"2023-09-23T21:06:02.913756+0000","last_fullsized":"2023-09-23T21:06:02.913756+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":35,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470505+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:06:08.470505+0000","last_peered":"2023-09-23T21:06:08.470505+0000","last_clean":"2023-09-23T21:06:08.470505+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:06:08.470505+0000","last_undegraded":"2023-09-23T21:06:08.470505+0000","last_fullsized":"2023-09-23T21:06:08.470505+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"373'5615","reported_seq":6967,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:08.477284+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:06:08.477284+0000","last_peered":"2023-09-23T21:06:08.477284+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:06:08.477284+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367984640,"num_objects":5615,"num_object_clones":0,"num_object_copies":16845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5615,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":635},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":4979}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":99,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018901+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:06:03.018901+0000","last_peered":"2023-09-23T21:06:03.018901+0000","last_clean":"2023-09-23T21:06:03.018901+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:06:03.018901+0000","last_undegraded":"2023-09-23T21:06:03.018901+0000","last_fullsized":"2023-09-23T21:06:03.018901+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.017923+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:06:03.017923+0000","last_peered":"2023-09-23T21:06:03.017923+0000","last_clean":"2023-09-23T21:06:03.017923+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:06:03.017923+0000","last_undegraded":"2023-09-23T21:06:03.017923+0000","last_fullsized":"2023-09-23T21:06:03.017923+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"370'5399","reported_seq":5464,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:08.478619+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:06:08.478619+0000","last_peered":"2023-09-23T21:06:08.478619+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:06:08.478619+0000","last_undegraded":"2023-09-23T21:06:08.478619+0000","last_fullsized":"2023-09-23T21:06:08.478619+0000","mapping_epoch":375,"log_start":"347'397","ondisk_log_start":"347'397","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5002,"log_dups_size":0,"ondisk_log_size":5002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353828864,"num_objects":5399,"num_object_clones":0,"num_object_copies":16197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5399,"num_objects_unfound":0,"num_objects_dirty":5399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5399,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":5399}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":93,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.478799+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:06:08.478799+0000","last_peered":"2023-09-23T21:06:08.478799+0000","last_clean":"2023-09-23T21:06:08.478799+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:06:08.478799+0000","last_undegraded":"2023-09-23T21:06:08.478799+0000","last_fullsized":"2023-09-23T21:06:08.478799+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":355,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.472323+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:06:08.472323+0000","last_peered":"2023-09-23T21:06:08.472323+0000","last_clean":"2023-09-23T21:06:08.472323+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:06:08.472323+0000","last_undegraded":"2023-09-23T21:06:08.472323+0000","last_fullsized":"2023-09-23T21:06:08.472323+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"367'5465","reported_seq":7685,"reported_epoch":393,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:04.539216+0000","last_change":"2023-09-23T21:05:54.989856+0000","last_active":"2023-09-23T21:04:28.774586+0000","last_peered":"2023-09-23T21:04:28.746162+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:03:39.498174+0000","last_became_peered":"2023-09-23T21:03:39.498174+0000","last_unstale":"2023-09-23T21:06:04.539216+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5465,"log_dups_size":0,"ondisk_log_size":5465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":10931,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5465,"num_write_kb":349760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1072,"num_bytes_recovered":70254592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["0","1","2","4","7"],"object_location_counts":[{"shards":"1,6,7","objects":3083},{"shards":"6,7","objects":2381},{"shards":"7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":22,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018204+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:06:03.018204+0000","last_peered":"2023-09-23T21:06:03.018204+0000","last_clean":"2023-09-23T21:06:03.018204+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:06:03.018204+0000","last_undegraded":"2023-09-23T21:06:03.018204+0000","last_fullsized":"2023-09-23T21:06:03.018204+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":283,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914312+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:06:02.914312+0000","last_peered":"2023-09-23T21:06:02.914312+0000","last_clean":"2023-09-23T21:06:02.914312+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:06:02.914312+0000","last_undegraded":"2023-09-23T21:06:02.914312+0000","last_fullsized":"2023-09-23T21:06:02.914312+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"370'5525","reported_seq":6137,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:02.916848+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:06:02.916848+0000","last_peered":"2023-09-23T21:06:02.916848+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:06:02.916848+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"349'1468","ondisk_log_start":"349'1468","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362086400,"num_objects":5525,"num_object_clones":0,"num_object_copies":16575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5525,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":272},{"shards":"2,6,7","objects":5253}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":35,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470077+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:06:08.470077+0000","last_peered":"2023-09-23T21:06:08.470077+0000","last_clean":"2023-09-23T21:06:08.470077+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:06:08.470077+0000","last_undegraded":"2023-09-23T21:06:08.470077+0000","last_fullsized":"2023-09-23T21:06:08.470077+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":32,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.604829+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:06:08.604829+0000","last_peered":"2023-09-23T21:06:08.604829+0000","last_clean":"2023-09-23T21:06:08.604829+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:06:08.604829+0000","last_undegraded":"2023-09-23T21:06:08.604829+0000","last_fullsized":"2023-09-23T21:06:08.604829+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"370'5654","reported_seq":11596,"reported_epoch":395,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:06:11.178514+0000","last_change":"2023-09-23T21:05:34.257054+0000","last_active":"2023-09-23T21:06:11.178514+0000","last_peered":"2023-09-23T21:06:11.178514+0000","last_clean":"2023-09-23T21:03:37.385653+0000","last_became_active":"2023-09-23T21:05:12.775986+0000","last_became_peered":"2023-09-23T21:05:12.775986+0000","last_unstale":"2023-09-23T21:06:11.178514+0000","last_undegraded":"2023-09-23T21:06:11.178514+0000","last_fullsized":"2023-09-23T21:05:12.621268+0000","mapping_epoch":380,"log_start":"349'1599","ondisk_log_start":"349'1599","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4055,"log_dups_size":0,"ondisk_log_size":4055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:18.107688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2765,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2889,"num_bytes_recovered":189333504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6","objects":2889},{"shards":"1,6,7","objects":2765}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":52,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469668+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:06:08.469668+0000","last_peered":"2023-09-23T21:06:08.469668+0000","last_clean":"2023-09-23T21:06:08.469668+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:06:08.469668+0000","last_undegraded":"2023-09-23T21:06:08.469668+0000","last_fullsized":"2023-09-23T21:06:08.469668+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":35,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469932+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:06:08.469932+0000","last_peered":"2023-09-23T21:06:08.469932+0000","last_clean":"2023-09-23T21:06:08.469932+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:06:08.469932+0000","last_undegraded":"2023-09-23T21:06:08.469932+0000","last_fullsized":"2023-09-23T21:06:08.469932+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"370'5491","reported_seq":13092,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:08.604710+0000","last_change":"2023-09-23T21:05:34.191496+0000","last_active":"2023-09-23T21:06:08.604710+0000","last_peered":"2023-09-23T21:06:08.604710+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:04:42.572134+0000","last_became_peered":"2023-09-23T21:04:42.572134+0000","last_unstale":"2023-09-23T21:06:08.604710+0000","last_undegraded":"2023-09-23T21:05:34.190372+0000","last_fullsized":"2023-09-23T21:04:42.399994+0000","mapping_epoch":372,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:37.804028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359858176,"num_objects":5491,"num_object_clones":0,"num_object_copies":16473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1740,"num_objects_misplaced":4743,"num_objects_unfound":0,"num_objects_dirty":5491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5491,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3747,"num_bytes_recovered":245563392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,1],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":748},{"shards":"1,2","objects":3751},{"shards":"1,2,7","objects":992}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":321,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914383+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:06:02.914383+0000","last_peered":"2023-09-23T21:06:02.914383+0000","last_clean":"2023-09-23T21:06:02.914383+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:06:02.914383+0000","last_undegraded":"2023-09-23T21:06:02.914383+0000","last_fullsized":"2023-09-23T21:06:02.914383+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":406,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.603595+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:06:08.603595+0000","last_peered":"2023-09-23T21:06:08.603595+0000","last_clean":"2023-09-23T21:06:08.603595+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:06:08.603595+0000","last_undegraded":"2023-09-23T21:06:08.603595+0000","last_fullsized":"2023-09-23T21:06:08.603595+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"370'5452","reported_seq":5533,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:02.913709+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:06:02.913709+0000","last_peered":"2023-09-23T21:06:02.913709+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:06:02.913709+0000","last_undegraded":"2023-09-23T21:06:02.913709+0000","last_fullsized":"2023-09-23T21:06:02.913709+0000","mapping_epoch":374,"log_start":"349'1394","ondisk_log_start":"349'1394","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4058,"log_dups_size":0,"ondisk_log_size":4058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357302272,"num_objects":5452,"num_object_clones":0,"num_object_copies":16356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10904,"num_objects_unfound":0,"num_objects_dirty":5452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5452,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5452}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":22,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018795+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:06:03.018795+0000","last_peered":"2023-09-23T21:06:03.018795+0000","last_clean":"2023-09-23T21:06:03.018795+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:06:03.018795+0000","last_undegraded":"2023-09-23T21:06:03.018795+0000","last_fullsized":"2023-09-23T21:06:03.018795+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":434,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914286+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:06:02.914286+0000","last_peered":"2023-09-23T21:06:02.914286+0000","last_clean":"2023-09-23T21:06:02.914286+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:06:02.914286+0000","last_undegraded":"2023-09-23T21:06:02.914286+0000","last_fullsized":"2023-09-23T21:06:02.914286+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"367'5481","reported_seq":5649,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:08.476288+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:06:08.476288+0000","last_peered":"2023-09-23T21:06:08.476288+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:06:08.476288+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5481,"log_dups_size":0,"ondisk_log_size":5481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359202816,"num_objects":5481,"num_object_clones":0,"num_object_copies":16443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5481,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":42},{"shards":"1,6,7","objects":5439}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":44,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469127+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:06:08.469127+0000","last_peered":"2023-09-23T21:06:08.469127+0000","last_clean":"2023-09-23T21:06:08.469127+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:06:08.469127+0000","last_undegraded":"2023-09-23T21:06:08.469127+0000","last_fullsized":"2023-09-23T21:06:08.469127+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":183,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.476523+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:06:08.476523+0000","last_peered":"2023-09-23T21:06:08.476523+0000","last_clean":"2023-09-23T21:06:08.476523+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:06:08.476523+0000","last_undegraded":"2023-09-23T21:06:08.476523+0000","last_fullsized":"2023-09-23T21:06:08.476523+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"370'5514","reported_seq":10993,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.604222+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:06:08.604222+0000","last_peered":"2023-09-23T21:06:08.604222+0000","last_clean":"2023-09-23T21:06:08.604222+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:06:08.604222+0000","last_undegraded":"2023-09-23T21:06:08.604222+0000","last_fullsized":"2023-09-23T21:06:08.604222+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361365504,"num_objects":5514,"num_object_clones":0,"num_object_copies":16542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5514,"num_write_kb":352896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":290,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.473404+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:06:08.473404+0000","last_peered":"2023-09-23T21:06:08.473404+0000","last_clean":"2023-09-23T21:06:08.473404+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:06:08.473404+0000","last_undegraded":"2023-09-23T21:06:08.473404+0000","last_fullsized":"2023-09-23T21:06:08.473404+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":60,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018246+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:06:03.018246+0000","last_peered":"2023-09-23T21:06:03.018246+0000","last_clean":"2023-09-23T21:06:03.018246+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:06:03.018246+0000","last_undegraded":"2023-09-23T21:06:03.018246+0000","last_fullsized":"2023-09-23T21:06:03.018246+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.472554+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:06:08.472554+0000","last_peered":"2023-09-23T21:06:08.472554+0000","last_clean":"2023-09-23T21:06:08.472554+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:06:08.472554+0000","last_undegraded":"2023-09-23T21:06:08.472554+0000","last_fullsized":"2023-09-23T21:06:08.472554+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"370'5513","reported_seq":5628,"reported_epoch":394,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:02.919427+0000","last_change":"2023-09-23T21:05:12.897699+0000","last_active":"2023-09-23T21:06:02.919427+0000","last_peered":"2023-09-23T21:06:02.919427+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:06:02.919427+0000","last_undegraded":"2023-09-23T21:05:12.617976+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"349'1457","ondisk_log_start":"349'1457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4056,"log_dups_size":0,"ondisk_log_size":4056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5513,"num_objects_misplaced":7846,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":3180},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":371,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.473112+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:06:08.473112+0000","last_peered":"2023-09-23T21:06:08.473112+0000","last_clean":"2023-09-23T21:06:08.473112+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:06:08.473112+0000","last_undegraded":"2023-09-23T21:06:08.473112+0000","last_fullsized":"2023-09-23T21:06:08.473112+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":345,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.919687+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:06:02.919687+0000","last_peered":"2023-09-23T21:06:02.919687+0000","last_clean":"2023-09-23T21:06:02.919687+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:06:02.919687+0000","last_undegraded":"2023-09-23T21:06:02.919687+0000","last_fullsized":"2023-09-23T21:06:02.919687+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":27,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.473252+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:06:08.473252+0000","last_peered":"2023-09-23T21:06:08.473252+0000","last_clean":"2023-09-23T21:06:08.473252+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:06:08.473252+0000","last_undegraded":"2023-09-23T21:06:08.473252+0000","last_fullsized":"2023-09-23T21:06:08.473252+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"370'5531","reported_seq":5694,"reported_epoch":395,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:08.603226+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:06:08.603226+0000","last_peered":"2023-09-23T21:06:08.603226+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:06:08.603226+0000","last_undegraded":"2023-09-23T21:06:08.603226+0000","last_fullsized":"2023-09-23T21:06:08.603226+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4058,"log_dups_size":0,"ondisk_log_size":4058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362479616,"num_objects":5531,"num_object_clones":0,"num_object_copies":16593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11027,"num_objects_unfound":0,"num_objects_dirty":5531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":5531}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":372,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.603319+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:06:08.603319+0000","last_peered":"2023-09-23T21:06:08.603319+0000","last_clean":"2023-09-23T21:06:08.603319+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:06:08.603319+0000","last_undegraded":"2023-09-23T21:06:08.603319+0000","last_fullsized":"2023-09-23T21:06:08.603319+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":290,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.472484+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:06:08.472484+0000","last_peered":"2023-09-23T21:06:08.472484+0000","last_clean":"2023-09-23T21:06:08.472484+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:06:08.472484+0000","last_undegraded":"2023-09-23T21:06:08.472484+0000","last_fullsized":"2023-09-23T21:06:08.472484+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":27,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.476393+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:06:08.476393+0000","last_peered":"2023-09-23T21:06:08.476393+0000","last_clean":"2023-09-23T21:06:08.476393+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:06:08.476393+0000","last_undegraded":"2023-09-23T21:06:08.476393+0000","last_fullsized":"2023-09-23T21:06:08.476393+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"370'5437","reported_seq":11678,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:02.913432+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:06:02.913432+0000","last_peered":"2023-09-23T21:06:02.913432+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:06:02.913432+0000","last_undegraded":"2023-09-23T21:06:02.913432+0000","last_fullsized":"2023-09-23T21:06:02.913432+0000","mapping_epoch":387,"log_start":"349'1380","ondisk_log_start":"349'1380","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356319232,"num_objects":5437,"num_object_clones":0,"num_object_copies":16311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5437,"num_objects_unfound":0,"num_objects_dirty":5437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5437,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5437}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":210,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.603543+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:06:08.603543+0000","last_peered":"2023-09-23T21:06:08.603543+0000","last_clean":"2023-09-23T21:06:08.603543+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:06:08.603543+0000","last_undegraded":"2023-09-23T21:06:08.603543+0000","last_fullsized":"2023-09-23T21:06:08.603543+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":692,"reported_epoch":395,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:08.472182+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:06:08.472182+0000","last_peered":"2023-09-23T21:06:08.472182+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:06:08.472182+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469312+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:06:08.469312+0000","last_peered":"2023-09-23T21:06:08.469312+0000","last_clean":"2023-09-23T21:06:08.469312+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:06:08.469312+0000","last_undegraded":"2023-09-23T21:06:08.469312+0000","last_fullsized":"2023-09-23T21:06:08.469312+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":288,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T21:05:40.210998+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:05:40.210998+0000","last_peered":"2023-09-23T21:05:40.210998+0000","last_clean":"2023-09-23T21:05:40.210998+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:05:40.210998+0000","last_undegraded":"2023-09-23T21:05:40.210998+0000","last_fullsized":"2023-09-23T21:05:40.210998+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"370'5561","reported_seq":5658,"reported_epoch":394,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:02.913635+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:06:02.913635+0000","last_peered":"2023-09-23T21:06:02.913635+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:06:02.913635+0000","last_undegraded":"2023-09-23T21:06:02.913635+0000","last_fullsized":"2023-09-23T21:06:02.913635+0000","mapping_epoch":374,"log_start":"349'1505","ondisk_log_start":"349'1505","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4056,"log_dups_size":0,"ondisk_log_size":4056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364445696,"num_objects":5561,"num_object_clones":0,"num_object_copies":16683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11122,"num_objects_unfound":0,"num_objects_dirty":5561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5561,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":5561}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.472110+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:06:08.472110+0000","last_peered":"2023-09-23T21:06:08.472110+0000","last_clean":"2023-09-23T21:06:08.472110+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:06:08.472110+0000","last_undegraded":"2023-09-23T21:06:08.472110+0000","last_fullsized":"2023-09-23T21:06:08.472110+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":41,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.604670+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:06:08.604670+0000","last_peered":"2023-09-23T21:06:08.604670+0000","last_clean":"2023-09-23T21:06:08.604670+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:06:08.604670+0000","last_undegraded":"2023-09-23T21:06:08.604670+0000","last_fullsized":"2023-09-23T21:06:08.604670+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":32,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.605035+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:06:08.605035+0000","last_peered":"2023-09-23T21:06:08.605035+0000","last_clean":"2023-09-23T21:06:08.605035+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:06:08.605035+0000","last_undegraded":"2023-09-23T21:06:08.605035+0000","last_fullsized":"2023-09-23T21:06:08.605035+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"370'5513","reported_seq":9117,"reported_epoch":395,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:08.476056+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:06:08.476056+0000","last_peered":"2023-09-23T21:06:08.476056+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:06:08.476056+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:06:08.476056+0000","mapping_epoch":383,"log_start":"347'153","ondisk_log_start":"347'153","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":1762},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":32,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.604983+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:06:08.604983+0000","last_peered":"2023-09-23T21:06:08.604983+0000","last_clean":"2023-09-23T21:06:08.604983+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:06:08.604983+0000","last_undegraded":"2023-09-23T21:06:08.604983+0000","last_fullsized":"2023-09-23T21:06:08.604983+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":292,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.474619+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:06:08.474619+0000","last_peered":"2023-09-23T21:06:08.474619+0000","last_clean":"2023-09-23T21:06:08.474619+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:06:08.474619+0000","last_undegraded":"2023-09-23T21:06:08.474619+0000","last_fullsized":"2023-09-23T21:06:08.474619+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":70,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018354+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:06:03.018354+0000","last_peered":"2023-09-23T21:06:03.018354+0000","last_clean":"2023-09-23T21:06:03.018354+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:06:03.018354+0000","last_undegraded":"2023-09-23T21:06:03.018354+0000","last_fullsized":"2023-09-23T21:06:03.018354+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"373'5432","reported_seq":16896,"reported_epoch":396,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:06:20.280640+0000","last_change":"2023-09-23T21:05:34.382714+0000","last_active":"2023-09-23T21:06:20.280640+0000","last_peered":"2023-09-23T21:06:20.280640+0000","last_clean":"2023-09-23T21:04:28.738326+0000","last_became_active":"2023-09-23T21:04:58.713307+0000","last_became_peered":"2023-09-23T21:04:58.713307+0000","last_unstale":"2023-09-23T21:06:20.280640+0000","last_undegraded":"2023-09-23T21:06:20.280640+0000","last_fullsized":"2023-09-23T21:04:58.419619+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:14.294697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355991552,"num_objects":5432,"num_object_clones":0,"num_object_copies":16296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":700,"num_objects_unfound":0,"num_objects_dirty":5432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5432,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5692,"num_bytes_recovered":373030912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":4732},{"shards":"1,2,4","objects":3},{"shards":"1,2,4,7","objects":2},{"shards":"1,2,7","objects":695}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":52,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470597+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:06:08.470597+0000","last_peered":"2023-09-23T21:06:08.470597+0000","last_clean":"2023-09-23T21:06:08.470597+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:06:08.470597+0000","last_undegraded":"2023-09-23T21:06:08.470597+0000","last_fullsized":"2023-09-23T21:06:08.470597+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":27,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.478335+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:06:08.478335+0000","last_peered":"2023-09-23T21:06:08.478335+0000","last_clean":"2023-09-23T21:06:08.478335+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:06:08.478335+0000","last_undegraded":"2023-09-23T21:06:08.478335+0000","last_fullsized":"2023-09-23T21:06:08.478335+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":44,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.468914+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:06:08.468914+0000","last_peered":"2023-09-23T21:06:08.468914+0000","last_clean":"2023-09-23T21:06:08.468914+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:06:08.468914+0000","last_undegraded":"2023-09-23T21:06:08.468914+0000","last_fullsized":"2023-09-23T21:06:08.468914+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"370'5599","reported_seq":5700,"reported_epoch":395,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:08.475132+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:06:08.475132+0000","last_peered":"2023-09-23T21:06:08.475132+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:06:08.475132+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:06:08.475132+0000","mapping_epoch":382,"log_start":"347'239","ondisk_log_start":"347'239","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366936064,"num_objects":5599,"num_object_clones":0,"num_object_copies":16797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5596,"num_objects_misplaced":5602,"num_objects_unfound":0,"num_objects_dirty":5599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5599,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":5596}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":291,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:02.914641+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:06:02.914641+0000","last_peered":"2023-09-23T21:06:02.914641+0000","last_clean":"2023-09-23T21:06:02.914641+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:06:02.914641+0000","last_undegraded":"2023-09-23T21:06:02.914641+0000","last_fullsized":"2023-09-23T21:06:02.914641+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":36,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.605142+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:06:08.605142+0000","last_peered":"2023-09-23T21:06:08.605142+0000","last_clean":"2023-09-23T21:06:08.605142+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:06:08.605142+0000","last_undegraded":"2023-09-23T21:06:08.605142+0000","last_fullsized":"2023-09-23T21:06:08.605142+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":80,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.017861+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:06:03.017861+0000","last_peered":"2023-09-23T21:06:03.017861+0000","last_clean":"2023-09-23T21:06:03.017861+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:06:03.017861+0000","last_undegraded":"2023-09-23T21:06:03.017861+0000","last_fullsized":"2023-09-23T21:06:03.017861+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":41,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.604842+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:06:08.604842+0000","last_peered":"2023-09-23T21:06:08.604842+0000","last_clean":"2023-09-23T21:06:08.604842+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:06:08.604842+0000","last_undegraded":"2023-09-23T21:06:08.604842+0000","last_fullsized":"2023-09-23T21:06:08.604842+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":418,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.603653+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:06:08.603653+0000","last_peered":"2023-09-23T21:06:08.603653+0000","last_clean":"2023-09-23T21:06:08.603653+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:06:08.603653+0000","last_undegraded":"2023-09-23T21:06:08.603653+0000","last_fullsized":"2023-09-23T21:06:08.603653+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":57,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018559+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:06:03.018559+0000","last_peered":"2023-09-23T21:06:03.018559+0000","last_clean":"2023-09-23T21:06:03.018559+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:06:03.018559+0000","last_undegraded":"2023-09-23T21:06:03.018559+0000","last_fullsized":"2023-09-23T21:06:03.018559+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":61,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469749+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:06:08.469749+0000","last_peered":"2023-09-23T21:06:08.469749+0000","last_clean":"2023-09-23T21:06:08.469749+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:06:08.469749+0000","last_undegraded":"2023-09-23T21:06:08.469749+0000","last_fullsized":"2023-09-23T21:06:08.469749+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":281,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T21:05:40.211090+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:05:40.211090+0000","last_peered":"2023-09-23T21:05:40.211090+0000","last_clean":"2023-09-23T21:05:40.211090+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:05:40.211090+0000","last_undegraded":"2023-09-23T21:05:40.211090+0000","last_fullsized":"2023-09-23T21:05:40.211090+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":53,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469454+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:06:08.469454+0000","last_peered":"2023-09-23T21:06:08.469454+0000","last_clean":"2023-09-23T21:06:08.469454+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:06:08.469454+0000","last_undegraded":"2023-09-23T21:06:08.469454+0000","last_fullsized":"2023-09-23T21:06:08.469454+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":35,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.470472+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:06:08.470472+0000","last_peered":"2023-09-23T21:06:08.470472+0000","last_clean":"2023-09-23T21:06:08.470472+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:06:08.470472+0000","last_undegraded":"2023-09-23T21:06:08.470472+0000","last_fullsized":"2023-09-23T21:06:08.470472+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":44,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.469179+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:06:08.469179+0000","last_peered":"2023-09-23T21:06:08.469179+0000","last_clean":"2023-09-23T21:06:08.469179+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:06:08.469179+0000","last_undegraded":"2023-09-23T21:06:08.469179+0000","last_fullsized":"2023-09-23T21:06:08.469179+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":38,"reported_epoch":395,"state":"active+clean","last_fresh":"2023-09-23T21:06:08.604844+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:06:08.604844+0000","last_peered":"2023-09-23T21:06:08.604844+0000","last_clean":"2023-09-23T21:06:08.604844+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:06:08.604844+0000","last_undegraded":"2023-09-23T21:06:08.604844+0000","last_fullsized":"2023-09-23T21:06:08.604844+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":394,"state":"active+clean","last_fresh":"2023-09-23T21:06:03.018200+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:06:03.018200+0000","last_peered":"2023-09-23T21:06:03.018200+0000","last_clean":"2023-09-23T21:06:03.018200+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:06:03.018200+0000","last_undegraded":"2023-09-23T21:06:03.018200+0000","last_fullsized":"2023-09-23T21:06:03.018200+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5779095552,"num_objects":88182,"num_object_clones":0,"num_object_copies":264546,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":43838,"num_objects_misplaced":67307,"num_objects_unfound":0,"num_objects_dirty":88182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88182,"num_write_kb":5643648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21907,"num_bytes_recovered":1435697152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18640338944,"data_stored":18640338944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76120,"ondisk_log_size":76120,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201731,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463353,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4380872,"kb_used_data":3891456,"kb_used_omap":0,"kb_used_meta":489408,"kb_avail":89364280,"statfs":{"total":95995035648,"available":91509022720,"internally_reserved":0,"allocated":3984850944,"data_stored":3983715493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":501153792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"upper_bound":262144},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.1240000000000001,"15min":2.1589999999999998},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.65000000000000002,"5min":7.2480000000000002,"15min":102.242},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.41399999999999998,"5min":1.1539999999999999,"15min":1.861},"min":{"1min":0.26700000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60799999999999998,"5min":9.7110000000000003,"15min":102.502},"last":0.39400000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":3.907,"15min":2.3660000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":0.55200000000000005,"5min":119.077,"15min":119.077},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":3.8039999999999998,"15min":2.4590000000000001},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":2.8740000000000001,"5min":119.313,"15min":119.313},"last":0.28399999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":6.3570000000000002,"15min":3.089},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.4830000000000001,"5min":176.45400000000001,"15min":176.45400000000001},"last":0.5},{"interface":"front","average":{"1min":0.56999999999999995,"5min":6.4630000000000001,"15min":3.2429999999999999},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":2.419,"5min":176.46799999999999,"15min":176.46799999999999},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":3.7490000000000001,"15min":4.4400000000000004},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.26900000000000002},"max":{"1min":0.59299999999999997,"5min":178.40000000000001,"15min":250.804},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":3.8010000000000002,"15min":4.3899999999999997},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":2.1429999999999998,"5min":178.26599999999999,"15min":250.446},"last":0.58199999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":1.1759999999999999,"15min":0.97299999999999998},"min":{"1min":0.307,"5min":0.27000000000000002,"15min":0.24299999999999999},"max":{"1min":0.65100000000000002,"5min":13.1,"15min":13.1},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.439,"5min":0.92100000000000004,"15min":0.999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":0.61599999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.47099999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.1160000000000001,"15min":1.6639999999999999},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65500000000000003,"5min":11.281000000000001,"15min":105.121},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":1.1919999999999999,"15min":1.7070000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":0.68700000000000006,"5min":8.6229999999999993,"15min":106.30500000000001},"last":0.377}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.123,"15min":1.5089999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":0.70899999999999996,"5min":7.093,"15min":101.04600000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.1890000000000001,"15min":1.516},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":0.66800000000000004,"5min":7.1849999999999996,"15min":101.081},"last":0.52200000000000002}]}]},{"osd":1,"up_from":276,"seq":1185410973798,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4406800,"kb_used_data":4105032,"kb_used_omap":5,"kb_used_meta":301754,"kb_avail":89338352,"statfs":{"total":95995035648,"available":91482472448,"internally_reserved":0,"allocated":4203552768,"data_stored":4201612462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":308996835},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.30599999999999999,"5min":0.70399999999999996,"15min":0.48899999999999999},"min":{"1min":0.20100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.47899999999999998,"5min":11.959,"15min":11.959},"last":0.215},{"interface":"front","average":{"1min":0.29999999999999999,"5min":1.1319999999999999,"15min":0.76700000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.49099999999999999,"5min":18.047999999999998,"15min":18.047999999999998},"last":0.30399999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":2.2770000000000001,"15min":1.089},"min":{"1min":0.23400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57299999999999995,"5min":78.409000000000006,"15min":78.409000000000006},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.35799999999999998,"5min":1.835,"15min":0.90400000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.22},"max":{"1min":0.56000000000000005,"5min":76.245000000000005,"15min":76.245000000000005},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":3.621,"15min":2.4300000000000002},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":0.71499999999999997,"5min":77.75,"15min":172.44399999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.35999999999999999,"5min":4.4020000000000001,"15min":2.5960000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":0.74399999999999999,"5min":77.849000000000004,"15min":172.58500000000001},"last":0.56399999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.6499999999999999,"15min":0.88600000000000001},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66000000000000003,"5min":17.006,"15min":17.006},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.86699999999999999,"15min":0.67500000000000004},"min":{"1min":0.379,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68200000000000005,"5min":11.916,"15min":11.916},"last":0.35699999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":1.383,"15min":0.83299999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72599999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.45700000000000002,"5min":1.7869999999999999,"15min":0.95499999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.68799999999999994,"5min":17.076000000000001,"15min":17.076000000000001},"last":0.68999999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":2.8460000000000001,"15min":1.339},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.69499999999999995,"5min":127.995,"15min":127.995},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.47599999999999998,"5min":3.1909999999999998,"15min":1.4239999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68100000000000005,"5min":126.836,"15min":126.836},"last":0.64800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":3.069,"15min":1.4239999999999999},"min":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71299999999999997,"5min":151.791,"15min":151.791},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.238,"15min":1.171},"min":{"1min":0.42699999999999999,"5min":0.40400000000000003,"15min":0.38},"max":{"1min":0.73799999999999999,"5min":151.809,"15min":151.809},"last":0.51000000000000001}]}]},{"osd":0,"up_from":92,"seq":395136991538,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332828,"kb_used_data":616148,"kb_used_omap":5,"kb_used_meta":716666,"kb_avail":92412324,"statfs":{"total":95995035648,"available":94630219776,"internally_reserved":0,"allocated":630935552,"data_stored":629097670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":733866618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.6749999999999998,"15min":1.2669999999999999},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.184},"max":{"1min":52.590000000000003,"5min":61.341999999999999,"15min":61.341999999999999},"last":0.17799999999999999},{"interface":"front","average":{"1min":3.165,"5min":2.5579999999999998,"15min":1.266},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":52.276000000000003,"5min":61.299999999999997,"15min":61.299999999999997},"last":0.31900000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":2.778,"15min":2.0939999999999999},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.161},"max":{"1min":0.61399999999999999,"5min":183.77099999999999,"15min":183.77099999999999},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":3.258,"15min":2.3039999999999998},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.59799999999999998,"5min":183.78700000000001,"15min":183.78700000000001},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":3.7370000000000001,"15min":3.9489999999999998},"min":{"1min":0.311,"5min":0.25,"15min":0.183},"max":{"1min":0.64700000000000002,"5min":186.143,"15min":186.143},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":4.0069999999999997,"15min":4.0380000000000003},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.23000000000000001},"max":{"1min":0.623,"5min":186.97399999999999,"15min":186.97399999999999},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2969999999999999,"15min":1.379},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.247},"max":{"1min":1.4350000000000001,"5min":9.7080000000000002,"15min":10.525},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.80900000000000005,"15min":1.022},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.26200000000000001},"max":{"1min":0.67600000000000005,"5min":6.0490000000000004,"15min":12.461},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.4830000000000001,"15min":2.5819999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28000000000000003},"max":{"1min":1.411,"5min":9.7439999999999998,"15min":154.30600000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.82299999999999995,"15min":1.96},"min":{"1min":0.42699999999999999,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":0.67000000000000004,"5min":5.9950000000000001,"15min":154.42400000000001},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":5.8390000000000004,"15min":2.7610000000000001},"min":{"1min":0.35899999999999999,"5min":0.34599999999999997,"15min":0.31},"max":{"1min":0.64200000000000002,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.47199999999999998,"5min":6.1609999999999996,"15min":2.8929999999999998},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":0.60799999999999998,"5min":169.767,"15min":169.767},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.97799999999999998,"15min":2.52},"min":{"1min":0.40799999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69599999999999995,"5min":6.1130000000000004,"15min":164.26400000000001},"last":0.435},{"interface":"front","average":{"1min":0.503,"5min":1.2490000000000001,"15min":2.6099999999999999},"min":{"1min":0.38700000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":0.63900000000000001,"5min":6.5410000000000004,"15min":164.23099999999999},"last":0.45500000000000002}]}]},{"osd":2,"up_from":129,"seq":554050781429,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4277464,"kb_used_data":3692496,"kb_used_omap":7,"kb_used_meta":584952,"kb_avail":89467688,"statfs":{"total":95995035648,"available":91614912512,"internally_reserved":0,"allocated":3781115904,"data_stored":3779224404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":598990929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":1.2949999999999999,"15min":1.7210000000000001},"min":{"1min":0.21299999999999999,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":0.96699999999999997,"5min":15.667,"15min":89.644999999999996},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.36599999999999999,"5min":1.7190000000000001,"15min":1.881},"min":{"1min":0.251,"5min":0.23599999999999999,"15min":0.222},"max":{"1min":0.76600000000000001,"5min":12.445,"15min":88.599999999999994},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:05:55 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":5.5549999999999997,"15min":2.343},"min":{"1min":0.22500000000000001,"5min":0.19,"15min":0.188},"max":{"1min":1.9419999999999999,"5min":212.12,"15min":212.12},"last":0.371},{"interface":"front","average":{"1min":0.35399999999999998,"5min":5.5369999999999999,"15min":2.1349999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.79100000000000004,"5min":213.02799999999999,"15min":213.02799999999999},"last":0.26400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":3.0609999999999999,"15min":3.8239999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":0.47799999999999998,"5min":66.085999999999999,"15min":151.43799999999999},"last":0.311},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.7919999999999998,"15min":3.8559999999999999},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.215},"max":{"1min":0.50900000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":1.167,"15min":1.167},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.0049999999999999,"5min":12.586,"15min":12.586},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.421,"15min":1.216},"min":{"1min":0.371,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3300000000000001,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":0.41899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.498,"15min":2.6110000000000002},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.2749999999999999,"5min":9.9909999999999997,"15min":173.34},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":1.2370000000000001,"15min":2.4470000000000001},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.29399999999999998},"max":{"1min":1.2,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.432}]},{"osd":6,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":3.4500000000000002,"15min":1.9319999999999999},"min":{"1min":0.35299999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.149,"5min":175.72499999999999,"15min":175.72499999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":3.3610000000000002,"15min":2.016},"min":{"1min":0.38700000000000001,"5min":0.36399999999999999,"15min":0.32300000000000001},"max":{"1min":1.228,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":1.498,"15min":2.7429999999999999},"min":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.32700000000000001},"max":{"1min":2.504,"5min":12.609,"15min":177.30799999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.5780000000000001,"15min":2.673},"min":{"1min":0.39500000000000002,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.3720000000000001,"5min":10.016,"15min":177.40899999999999},"last":0.38900000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160023,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":526856,"kb_used_data":10048,"kb_used_omap":2,"kb_used_meta":516797,"kb_avail":93218296,"statfs":{"total":95995035648,"available":95455535104,"internally_reserved":0,"allocated":10289152,"data_stored":8232622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":529200181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.6739999999999999,"15min":1.8320000000000001},"min":{"1min":0.20999999999999999,"5min":0.20200000000000001,"15min":0.16700000000000001},"max":{"1min":44.963000000000001,"5min":44.963000000000001,"15min":95.507999999999996},"last":0.221},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.101,"15min":1.823},"min":{"1min":0.20300000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":9.7370000000000001,"5min":9.7759999999999998,"15min":95.259},"last":0.39700000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.339,"15min":1.0129999999999999},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.7639999999999993,"5min":12.035,"15min":19.786999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.6160000000000001,"15min":1.2110000000000001},"min":{"1min":0.253,"5min":0.182,"15min":0.182},"max":{"1min":2.3159999999999998,"5min":15.763999999999999,"15min":15.763999999999999},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.571,"15min":1.3129999999999999},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.182},"max":{"1min":3.2749999999999999,"5min":12.763999999999999,"15min":12.859},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.3380000000000001,"15min":1.4019999999999999},"min":{"1min":0.28000000000000003,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":9.7140000000000004,"5min":12.096,"15min":19.754999999999999},"last":0.437}]},{"osd":4,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.6629999999999998,"15min":2.194},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.222},"max":{"1min":3.5249999999999999,"5min":70.728999999999999,"15min":111.36},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":2.3439999999999999,"15min":1.9990000000000001},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":3.2490000000000001,"5min":70.712000000000003,"15min":112.473},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.157,"15min":1.6579999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":9.6359999999999992,"5min":9.6359999999999992,"15min":64.528000000000006},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.78,"15min":1.6839999999999999},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.249},"max":{"1min":3.5649999999999999,"5min":10.617000000000001,"15min":14.054},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.855,"15min":1.7310000000000001},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":3.5910000000000002,"5min":10.52,"15min":19.568000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.655,"15min":1.575},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":14.356,"5min":14.356,"15min":15.573},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.286,"15min":2.7269999999999999},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":9.6859999999999999,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.8199999999999998,"15min":2.8490000000000002},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.1419999999999999,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.53000000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986641,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2019168,"kb_used_data":1477464,"kb_used_omap":0,"kb_used_meta":541696,"kb_avail":91725984,"statfs":{"total":95995035648,"available":93927407616,"internally_reserved":0,"allocated":1512923136,"data_stored":1511779507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":554696704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":1.349,"15min":3.085},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.48499999999999999,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.308},{"interface":"front","average":{"1min":0.436,"5min":1.1919999999999999,"15min":3.0739999999999998},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.63,"5min":8.5220000000000002,"15min":144.77000000000001},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:06:07 2023","interfaces":[{"interface":"back","average":{"1min":6.4459999999999997,"5min":6.3929999999999998,"15min":2.6269999999999998},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":118.411,"5min":118.411,"15min":118.411},"last":0.34899999999999998},{"interface":"front","average":{"1min":6.274,"5min":6.1340000000000003,"15min":2.4740000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":118.075,"5min":118.075,"15min":118.075},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":7.4100000000000001,"5min":5.8289999999999997,"15min":3.2269999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":110.193,"5min":164.517,"15min":164.517},"last":1.3360000000000001},{"interface":"front","average":{"1min":7.3220000000000001,"5min":5.7889999999999997,"15min":3.3170000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":110.30200000000001,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.439}]},{"osd":3,"last update":"Sat Sep 23 21:06:06 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.2709999999999999,"15min":2.2610000000000001},"min":{"1min":0.32500000000000001,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.8420000000000001,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.2829999999999999,"15min":2.3889999999999998},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.82799999999999996,"5min":7.5579999999999998,"15min":142.34700000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.82499999999999996,"15min":1.0609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.24099999999999999},"max":{"1min":2.9049999999999998,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.371},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.875,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29899999999999999,"15min":0.21099999999999999},"max":{"1min":0.58299999999999996,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.495}]},{"osd":6,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.998,"15min":0.97599999999999998},"min":{"1min":0.28299999999999997,"5min":0.27500000000000002,"15min":0.26100000000000001},"max":{"1min":0.65800000000000003,"5min":11.800000000000001,"15min":11.887},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.1120000000000001,"15min":0.93100000000000005},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.871,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":3.0110000000000001,"15min":1.8280000000000001},"min":{"1min":0.42799999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":0.72799999999999998,"5min":145.721,"15min":145.721},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":3.282,"15min":1.992},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":0.78100000000000003,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.65000000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725005,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1289744,"kb_used_data":704648,"kb_used_omap":0,"kb_used_meta":585088,"kb_avail":92455408,"statfs":{"total":95995035648,"available":94674337792,"internally_reserved":0,"allocated":721559552,"data_stored":720428051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599130112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.78000000000000003,"15min":1.857},"min":{"1min":0.25600000000000001,"5min":0.24199999999999999,"15min":0.19600000000000001},"max":{"1min":0.59299999999999997,"5min":23.600000000000001,"15min":124.873},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.1259999999999999,"15min":1.9930000000000001},"min":{"1min":0.24199999999999999,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":0.61099999999999999,"5min":23.532,"15min":123.554},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":4.1200000000000001,"15min":1.6699999999999999},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":0.67200000000000004,"5min":137.143,"15min":137.143},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":3.8199999999999998,"15min":1.5800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.247},"max":{"1min":0.66200000000000003,"5min":139.73699999999999,"15min":139.73699999999999},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.5720000000000001,"15min":1.6060000000000001},"min":{"1min":0.28799999999999998,"5min":0.28000000000000003,"15min":0.22500000000000001},"max":{"1min":0.61599999999999999,"5min":103.63800000000001,"15min":103.63800000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.44,"5min":2.8330000000000002,"15min":1.6910000000000001},"min":{"1min":0.307,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":0.64800000000000002,"5min":103.664,"15min":103.664},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.609,"15min":4.0869999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.573,"5min":97.349999999999994,"15min":198.33099999999999},"last":0.505},{"interface":"front","average":{"1min":0.50900000000000001,"5min":2.4750000000000001,"15min":4.0540000000000003},"min":{"1min":0.40799999999999997,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":97.316999999999993,"15min":198.47300000000001},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.69799999999999995,"15min":0.81699999999999995},"min":{"1min":0.33100000000000002,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":0.69899999999999995,"5min":12.739000000000001,"15min":12.739000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.88,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":0.59599999999999997,"5min":10.792,"15min":12.539},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":1.702,"15min":1.2150000000000001},"min":{"1min":0.32800000000000001,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":0.57899999999999996,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.835,"15min":1.3400000000000001},"min":{"1min":0.38900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.73099999999999998,"5min":70.266000000000005,"15min":70.266000000000005},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.014,"15min":1.2230000000000001},"min":{"1min":0.40799999999999997,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.65200000000000002,"15min":1.0780000000000001},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.47999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":627310592,"data_stored":627310592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4197908480,"data_stored":4197908480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3775528960,"data_stored":3775528960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":5111808,"data_stored":5111808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1508769792,"data_stored":1508769792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":719454208,"data_stored":719454208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3980132352,"data_stored":3980132352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3826122752,"data_stored":3826122752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:06:24.119 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:06:24.160 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:24.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:24.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:24.254 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 194 5 88186 88181 28.4009 0 - 0.0166307 2023-09-23T21:06:24.254 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:24.265 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:24.291 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:24.343 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:24.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:24.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:24.452 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:24.479 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:24.529 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:24.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:24.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:24.633 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:24.653 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:24 smithi123 ceph-mon[193903]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.653 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:24 smithi123 ceph-mon[193903]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T21:06:24.653 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:24 smithi123 ceph-mon[193903]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.653 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:24 smithi123 ceph-mon[193903]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.663 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[211526]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[211526]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T21:06:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[211526]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[211526]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[219709]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[219709]: osdmap e397: 8 total, 8 up, 7 in 2023-09-23T21:06:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[219709]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:24 smithi121 ceph-mon[219709]: failed to encode map e397 with expected crc 2023-09-23T21:06:24.672 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:24.672 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:24.716 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:24.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:24.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:24.822 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:25.450 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:06:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:06:24] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2023-09-23T21:06:25.451 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 195 5 88186 88181 28.2552 0 - 0.0166307 2023-09-23T21:06:25.478 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:25.527 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:25.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:25.633 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:25.668 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:25.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:25 smithi123 ceph-mon[193903]: failed to encode map e398 with expected crc 2023-09-23T21:06:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:25 smithi123 ceph-mon[193903]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T21:06:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:25 smithi123 ceph-mon[193903]: failed to encode map e398 with expected crc 2023-09-23T21:06:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:25 smithi123 ceph-mon[193903]: failed to encode map e398 with expected crc 2023-09-23T21:06:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:25 smithi123 ceph-mon[193903]: pgmap v1244: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T21:06:25.743 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:06:25.799 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:25.815 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:25.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:25.919 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:25.949 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:26.004 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:26.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:26.012 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[211526]: failed to encode map e398 with expected crc 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[211526]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[211526]: failed to encode map e398 with expected crc 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[211526]: failed to encode map e398 with expected crc 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[211526]: pgmap v1244: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[219709]: failed to encode map e398 with expected crc 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[219709]: osdmap e398: 8 total, 8 up, 7 in 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[219709]: failed to encode map e398 with expected crc 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[219709]: failed to encode map e398 with expected crc 2023-09-23T21:06:26.013 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:25 smithi121 ceph-mon[219709]: pgmap v1244: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 67307/264618 objects misplaced (25.436%); 9.5 MiB/s, 152 objects/s recovering 2023-09-23T21:06:26.111 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:26.141 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:26.153 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:26.195 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:26.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:26.254 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 196 5 88186 88181 28.1111 0 - 0.0166307 2023-09-23T21:06:26.299 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:26.327 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:26.876 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:26.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:26.980 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:27.008 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:27.055 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:27.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:27.159 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:27.188 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:27.254 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 197 5 88186 88181 27.9684 0 - 0.0166307 2023-09-23T21:06:27.425 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:06:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2947564141' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:27 smithi123 ceph-mon[193903]: pgmap v1245: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 66607/264618 objects misplaced (25.171%); 13 MiB/s, 212 objects/s recovering 2023-09-23T21:06:27.637 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:27.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:27.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2947564141' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:27.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:27 smithi121 ceph-mon[211526]: pgmap v1245: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 66607/264618 objects misplaced (25.171%); 13 MiB/s, 212 objects/s recovering 2023-09-23T21:06:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2947564141' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:27 smithi121 ceph-mon[219709]: pgmap v1245: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 43840/264618 objects degraded (16.567%); 66607/264618 objects misplaced (25.171%); 13 MiB/s, 212 objects/s recovering 2023-09-23T21:06:27.742 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:27.772 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:27.918 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:27.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:27.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:27.933 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:06:27.933 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 1 pg inactive","count":1},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 6 pgs undersized","count":15},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1644,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":399,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":15},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":74},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+remapped+backfill_wait","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"activating+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":88206,"data_bytes":5779691424,"bytes_used":18786496512,"bytes_avail":653178753024,"bytes_total":671965249536,"inactive_pgs_ratio":0.01123595517128706,"degraded_objects":43840,"degraded_total":264618,"degraded_ratio":0.16567278114111661,"misplaced_objects":66607,"misplaced_total":264618,"misplaced_ratio":0.25171001216848438,"recovering_objects_per_sec":212,"recovering_bytes_per_sec":13898173,"recovering_keys_per_sec":0,"num_objects_recovered":1697,"num_bytes_recovered":111214592,"num_keys_recovered":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":26,"modified":"2023-09-23T20:58:42.835119+0000","services":{}},"progress_events":{"2639f328-dcb0-4a01-b6c7-21332e7147ac":{"message":"Global Recovery Event (2m)\n [======================......] (remaining: 36s)","progress":0.82022470235824585,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:06:28.022 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:28.055 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:28.254 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 198 5 88186 88181 27.8272 0 - 0.0166307 2023-09-23T21:06:28.447 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:28.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:28.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:28.551 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:28.582 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:28.630 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:28.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: failed to encode map e399 with expected crc 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3364344695' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: failed to encode map e399 with expected crc 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3364344695' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:06:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 43840/264618 objects degraded (16.567%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: failed to encode map e399 with expected crc 2023-09-23T21:06:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: osdmap e399: 8 total, 8 up, 7 in 2023-09-23T21:06:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3364344695' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:06:28.734 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:28.764 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:28.810 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:28.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:28.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:28.914 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:28.942 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:29.053 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:29.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:29.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:29.112 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:06:29.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:06:29.157 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:29.183 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:29.226 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:29.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:29.254 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 199 5 88186 88181 27.6874 0 - 0.0166307 2023-09-23T21:06:29.329 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:29.358 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:29.465 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:29.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:29.490 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:29.569 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:29.597 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:29.644 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:29.646 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[219709]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.646 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[219709]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[219709]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T21:06:29.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[219709]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[219709]: pgmap v1248: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 64377/264618 objects misplaced (24.328%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:06:29.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[211526]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[211526]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[211526]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T21:06:29.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[211526]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:29 smithi121 ceph-mon[211526]: pgmap v1248: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 64377/264618 objects misplaced (24.328%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:06:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:29.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:29 smithi123 ceph-mon[193903]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:29 smithi123 ceph-mon[193903]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:29 smithi123 ceph-mon[193903]: osdmap e400: 8 total, 8 up, 7 in 2023-09-23T21:06:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:29 smithi123 ceph-mon[193903]: failed to encode map e400 with expected crc 2023-09-23T21:06:29.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:29 smithi123 ceph-mon[193903]: pgmap v1248: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 64377/264618 objects misplaced (24.328%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:06:29.749 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:29.779 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:29.826 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:29.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:29.929 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:29.956 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:30.002 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:30.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:30.106 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:30.136 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:30.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:30.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:30.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:06:30.255704+0000 min lat: 0.00123114 max lat: 18.7578 avg lat: 0.0166307 2023-09-23T21:06:30.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:06:30.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 200 5 88186 88181 27.5489 0 - 0.0166307 2023-09-23T21:06:30.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:30.314 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:30.354 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:06:30.357 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:30.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:30.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:30.461 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:30.488 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:30.543 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:30.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:30.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:30.630 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:06:30.647 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:30.651 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:06:30.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.653 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.654 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.654 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.654 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:06:30.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:30 smithi121 ceph-mon[211526]: failed to encode map e401 with expected crc 2023-09-23T21:06:30.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:30 smithi121 ceph-mon[211526]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T21:06:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:30 smithi121 ceph-mon[211526]: failed to encode map e401 with expected crc 2023-09-23T21:06:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:30 smithi121 ceph-mon[219709]: failed to encode map e401 with expected crc 2023-09-23T21:06:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:30 smithi121 ceph-mon[219709]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T21:06:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:30 smithi121 ceph-mon[219709]: failed to encode map e401 with expected crc 2023-09-23T21:06:30.676 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:30 smithi123 ceph-mon[193903]: failed to encode map e401 with expected crc 2023-09-23T21:06:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:30 smithi123 ceph-mon[193903]: osdmap e401: 8 total, 8 up, 7 in 2023-09-23T21:06:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:30 smithi123 ceph-mon[193903]: failed to encode map e401 with expected crc 2023-09-23T21:06:31.094 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:31.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:31.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:31.198 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:31.225 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 200.78 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 88186 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 27.451 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 45.8818 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 267.688 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 439 2023-09-23T21:06:31.255 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 734.114 2023-09-23T21:06:31.256 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 4283 2023-09-23T21:06:31.256 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T21:06:31.256 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0235038 2023-09-23T21:06:31.256 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.934063 2023-09-23T21:06:31.256 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 122.235 2023-09-23T21:06:31.256 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00123114 2023-09-23T21:06:31.259 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:06:31.315 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:31.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:31.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:31.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:31.455 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:31.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:31 smithi123 ceph-mon[193903]: failed to encode map e401 with expected crc 2023-09-23T21:06:31.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:31 smithi123 ceph-mon[193903]: failed to encode map e401 with expected crc 2023-09-23T21:06:31.615 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:31 smithi123 ceph-mon[193903]: pgmap v1250: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 61612/264618 objects misplaced (23.283%) 2023-09-23T21:06:31.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:31 smithi121 ceph-mon[219709]: failed to encode map e401 with expected crc 2023-09-23T21:06:31.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:31 smithi121 ceph-mon[219709]: failed to encode map e401 with expected crc 2023-09-23T21:06:31.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:31 smithi121 ceph-mon[219709]: pgmap v1250: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 61612/264618 objects misplaced (23.283%) 2023-09-23T21:06:31.702 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:31 smithi121 ceph-mon[211526]: failed to encode map e401 with expected crc 2023-09-23T21:06:31.702 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:31 smithi121 ceph-mon[211526]: failed to encode map e401 with expected crc 2023-09-23T21:06:31.702 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:31 smithi121 ceph-mon[211526]: pgmap v1250: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 61612/264618 objects misplaced (23.283%) 2023-09-23T21:06:31.884 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:31.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:31.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:31.928 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:31.928 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:31.991 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:32.033 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:32.200 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:32.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:32.307 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:32.347 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:32.406 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:32.447 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:32.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:32.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:32 smithi123 ceph-mon[193903]: Health check failed: 5 slow ops, oldest one blocked for 105 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:06:32.552 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:32.596 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:32.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:32 smithi121 ceph-mon[211526]: Health check failed: 5 slow ops, oldest one blocked for 105 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:06:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:32 smithi121 ceph-mon[219709]: Health check failed: 5 slow ops, oldest one blocked for 105 sec, osd.6 has slow ops (SLOW_OPS) 2023-09-23T21:06:32.807 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:32.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:32.874 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:06:32.913 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:06:32.914 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:32.930 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:06:32.992 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:33.382 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:33.610 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:33.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:33.620 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:33 smithi121 ceph-mon[211526]: from='client.46418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:33.621 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:33 smithi121 ceph-mon[211526]: pgmap v1251: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 61612/264618 objects misplaced (23.283%) 2023-09-23T21:06:33.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:33 smithi121 ceph-mon[219709]: from='client.46418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:33.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:33 smithi121 ceph-mon[219709]: pgmap v1251: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 61612/264618 objects misplaced (23.283%) 2023-09-23T21:06:33.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:33.722 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:33 smithi123 ceph-mon[193903]: from='client.46418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:33 smithi123 ceph-mon[193903]: pgmap v1251: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42100/264618 objects degraded (15.910%); 61612/264618 objects misplaced (23.283%) 2023-09-23T21:06:33.759 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:33.875 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:33.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:33.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:33.981 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:34.008 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:34.024 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:06:34.052 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:34.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:34.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:34.160 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:34.187 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:34.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:34.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:34.331 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:34.361 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:34.675 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:34.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:34.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:34.785 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:34.899 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1250,"stamp":"2023-09-23T21:06:31.018783+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5779691424,"num_objects":88206,"num_object_clones":0,"num_object_copies":264618,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":42100,"num_objects_misplaced":61612,"num_objects_unfound":0,"num_objects_dirty":88206,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":88482,"num_write_kb":5646093,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20130,"num_bytes_recovered":1314561292,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76336,"ondisk_log_size":76336,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18421356,"kb_used_data":14697460,"kb_used_omap":21,"kb_used_meta":3723818,"kb_avail":637794708,"statfs":{"total":671965249536,"available":653101780992,"internally_reserved":0,"allocated":15050199040,"data_stored":15038909475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3813190115},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"upper_bound":262144},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1740,"num_objects_misplaced":-5695,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1894,"num_bytes_recovered":-124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001468"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":171,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440034+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:06:28.440034+0000","last_peered":"2023-09-23T21:06:28.440034+0000","last_clean":"2023-09-23T21:06:28.440034+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:06:28.440034+0000","last_undegraded":"2023-09-23T21:06:28.440034+0000","last_fullsized":"2023-09-23T21:06:28.440034+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":40,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438517+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:06:28.438517+0000","last_peered":"2023-09-23T21:06:28.438517+0000","last_clean":"2023-09-23T21:06:28.438517+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:06:28.438517+0000","last_undegraded":"2023-09-23T21:06:28.438517+0000","last_fullsized":"2023-09-23T21:06:28.438517+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":313,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.439739+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:06:28.439739+0000","last_peered":"2023-09-23T21:06:28.439739+0000","last_clean":"2023-09-23T21:06:28.439739+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:06:28.439739+0000","last_undegraded":"2023-09-23T21:06:28.439739+0000","last_fullsized":"2023-09-23T21:06:28.439739+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":36,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604780+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:06:27.604780+0000","last_peered":"2023-09-23T21:06:27.604780+0000","last_clean":"2023-09-23T21:06:27.604780+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:06:27.604780+0000","last_undegraded":"2023-09-23T21:06:27.604780+0000","last_fullsized":"2023-09-23T21:06:27.604780+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":58,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.437847+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:06:28.437847+0000","last_peered":"2023-09-23T21:06:28.437847+0000","last_clean":"2023-09-23T21:06:28.437847+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:06:28.437847+0000","last_undegraded":"2023-09-23T21:06:28.437847+0000","last_fullsized":"2023-09-23T21:06:28.437847+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":267,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440716+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:06:28.440716+0000","last_peered":"2023-09-23T21:06:28.440716+0000","last_clean":"2023-09-23T21:06:28.440716+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:06:28.440716+0000","last_undegraded":"2023-09-23T21:06:28.440716+0000","last_fullsized":"2023-09-23T21:06:28.440716+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":36,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604927+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:06:27.604927+0000","last_peered":"2023-09-23T21:06:27.604927+0000","last_clean":"2023-09-23T21:06:27.604927+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:06:27.604927+0000","last_undegraded":"2023-09-23T21:06:27.604927+0000","last_fullsized":"2023-09-23T21:06:27.604927+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":40,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.437800+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:06:28.437800+0000","last_peered":"2023-09-23T21:06:28.437800+0000","last_clean":"2023-09-23T21:06:28.437800+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:06:28.437800+0000","last_undegraded":"2023-09-23T21:06:28.437800+0000","last_fullsized":"2023-09-23T21:06:28.437800+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":22,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.686648+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:06:24.686648+0000","last_peered":"2023-09-23T21:06:24.686648+0000","last_clean":"2023-09-23T21:06:24.686648+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:06:24.686648+0000","last_undegraded":"2023-09-23T21:06:24.686648+0000","last_fullsized":"2023-09-23T21:06:24.686648+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":40,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438499+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:06:28.438499+0000","last_peered":"2023-09-23T21:06:28.438499+0000","last_clean":"2023-09-23T21:06:28.438499+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:06:28.438499+0000","last_undegraded":"2023-09-23T21:06:28.438499+0000","last_fullsized":"2023-09-23T21:06:28.438499+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":56,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438594+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:06:28.438594+0000","last_peered":"2023-09-23T21:06:28.438594+0000","last_clean":"2023-09-23T21:06:28.438594+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:06:28.438594+0000","last_undegraded":"2023-09-23T21:06:28.438594+0000","last_fullsized":"2023-09-23T21:06:28.438594+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":32,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.439650+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:06:28.439650+0000","last_peered":"2023-09-23T21:06:28.439650+0000","last_clean":"2023-09-23T21:06:28.439650+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:06:28.439650+0000","last_undegraded":"2023-09-23T21:06:28.439650+0000","last_fullsized":"2023-09-23T21:06:28.439650+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":347,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440457+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:06:28.440457+0000","last_peered":"2023-09-23T21:06:28.440457+0000","last_clean":"2023-09-23T21:06:28.440457+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:06:28.440457+0000","last_undegraded":"2023-09-23T21:06:28.440457+0000","last_fullsized":"2023-09-23T21:06:28.440457+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":317,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.672318+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:06:24.672318+0000","last_peered":"2023-09-23T21:06:24.672318+0000","last_clean":"2023-09-23T21:06:24.672318+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:06:24.672318+0000","last_undegraded":"2023-09-23T21:06:24.672318+0000","last_fullsized":"2023-09-23T21:06:24.672318+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":89,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.417002+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:06:27.417002+0000","last_peered":"2023-09-23T21:06:27.417002+0000","last_clean":"2023-09-23T21:06:27.417002+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:06:27.417002+0000","last_undegraded":"2023-09-23T21:06:27.417002+0000","last_fullsized":"2023-09-23T21:06:27.417002+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":49,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438227+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:06:28.438227+0000","last_peered":"2023-09-23T21:06:28.438227+0000","last_clean":"2023-09-23T21:06:28.438227+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:06:28.438227+0000","last_undegraded":"2023-09-23T21:06:28.438227+0000","last_fullsized":"2023-09-23T21:06:28.438227+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438066+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:06:28.438066+0000","last_peered":"2023-09-23T21:06:28.438066+0000","last_clean":"2023-09-23T21:06:28.438066+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:06:28.438066+0000","last_undegraded":"2023-09-23T21:06:28.438066+0000","last_fullsized":"2023-09-23T21:06:28.438066+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438644+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:06:28.438644+0000","last_peered":"2023-09-23T21:06:28.438644+0000","last_clean":"2023-09-23T21:06:28.438644+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:06:28.438644+0000","last_undegraded":"2023-09-23T21:06:28.438644+0000","last_fullsized":"2023-09-23T21:06:28.438644+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":199,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.671935+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:06:24.671935+0000","last_peered":"2023-09-23T21:06:24.671935+0000","last_clean":"2023-09-23T21:06:24.671935+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:06:24.671935+0000","last_undegraded":"2023-09-23T21:06:24.671935+0000","last_fullsized":"2023-09-23T21:06:24.671935+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":36,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604856+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:06:27.604856+0000","last_peered":"2023-09-23T21:06:27.604856+0000","last_clean":"2023-09-23T21:06:27.604856+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:06:27.604856+0000","last_undegraded":"2023-09-23T21:06:27.604856+0000","last_fullsized":"2023-09-23T21:06:27.604856+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":397,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.439958+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:06:28.439958+0000","last_peered":"2023-09-23T21:06:28.439958+0000","last_clean":"2023-09-23T21:06:28.439958+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:06:28.439958+0000","last_undegraded":"2023-09-23T21:06:28.439958+0000","last_fullsized":"2023-09-23T21:06:28.439958+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":40,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438825+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:06:28.438825+0000","last_peered":"2023-09-23T21:06:28.438825+0000","last_clean":"2023-09-23T21:06:28.438825+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:06:28.438825+0000","last_undegraded":"2023-09-23T21:06:28.438825+0000","last_fullsized":"2023-09-23T21:06:28.438825+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"373'5615","reported_seq":6972,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:28.450984+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:06:28.450984+0000","last_peered":"2023-09-23T21:06:28.450984+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:06:28.450984+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367984640,"num_objects":5615,"num_object_clones":0,"num_object_copies":16845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5615,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":635},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":4979}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":103,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.672982+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:06:24.672982+0000","last_peered":"2023-09-23T21:06:24.672982+0000","last_clean":"2023-09-23T21:06:24.672982+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:06:24.672982+0000","last_undegraded":"2023-09-23T21:06:24.672982+0000","last_fullsized":"2023-09-23T21:06:24.672982+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":22,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.685644+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:06:24.685644+0000","last_peered":"2023-09-23T21:06:24.685644+0000","last_clean":"2023-09-23T21:06:24.685644+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:06:24.685644+0000","last_undegraded":"2023-09-23T21:06:24.685644+0000","last_fullsized":"2023-09-23T21:06:24.685644+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"370'5399","reported_seq":5469,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:28.446861+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:06:28.446861+0000","last_peered":"2023-09-23T21:06:28.446861+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:06:28.446861+0000","last_undegraded":"2023-09-23T21:06:28.446861+0000","last_fullsized":"2023-09-23T21:06:28.446861+0000","mapping_epoch":375,"log_start":"347'397","ondisk_log_start":"347'397","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5002,"log_dups_size":0,"ondisk_log_size":5002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353828864,"num_objects":5399,"num_object_clones":0,"num_object_copies":16197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5399,"num_objects_unfound":0,"num_objects_dirty":5399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5399,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":5399}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":98,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.447027+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:06:28.447027+0000","last_peered":"2023-09-23T21:06:28.447027+0000","last_clean":"2023-09-23T21:06:28.447027+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:06:28.447027+0000","last_undegraded":"2023-09-23T21:06:28.447027+0000","last_fullsized":"2023-09-23T21:06:28.447027+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":361,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.439653+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:06:28.439653+0000","last_peered":"2023-09-23T21:06:28.439653+0000","last_clean":"2023-09-23T21:06:28.439653+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:06:28.439653+0000","last_undegraded":"2023-09-23T21:06:28.439653+0000","last_fullsized":"2023-09-23T21:06:28.439653+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"367'5465","reported_seq":7687,"reported_epoch":395,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:11.196439+0000","last_change":"2023-09-23T21:05:54.989856+0000","last_active":"2023-09-23T21:04:28.774586+0000","last_peered":"2023-09-23T21:04:28.746162+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:03:39.498174+0000","last_became_peered":"2023-09-23T21:03:39.498174+0000","last_unstale":"2023-09-23T21:06:11.196439+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5465,"log_dups_size":0,"ondisk_log_size":5465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":10931,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5465,"num_write_kb":349760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1072,"num_bytes_recovered":70254592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":1072},{"shards":"1,6,7","objects":2011},{"shards":"2,6,7","objects":2381},{"shards":"7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":26,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.686410+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:06:24.686410+0000","last_peered":"2023-09-23T21:06:24.686410+0000","last_clean":"2023-09-23T21:06:24.686410+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:06:24.686410+0000","last_undegraded":"2023-09-23T21:06:24.686410+0000","last_fullsized":"2023-09-23T21:06:24.686410+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":289,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440321+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:06:28.440321+0000","last_peered":"2023-09-23T21:06:28.440321+0000","last_clean":"2023-09-23T21:06:28.440321+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:06:28.440321+0000","last_undegraded":"2023-09-23T21:06:28.440321+0000","last_fullsized":"2023-09-23T21:06:28.440321+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"370'5525","reported_seq":6143,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:28.443829+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:06:28.443829+0000","last_peered":"2023-09-23T21:06:28.443829+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:06:28.443829+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"349'1468","ondisk_log_start":"349'1468","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362086400,"num_objects":5525,"num_object_clones":0,"num_object_copies":16575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5525,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":272},{"shards":"2,6,7","objects":5253}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":40,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438320+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:06:28.438320+0000","last_peered":"2023-09-23T21:06:28.438320+0000","last_clean":"2023-09-23T21:06:28.438320+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:06:28.438320+0000","last_undegraded":"2023-09-23T21:06:28.438320+0000","last_fullsized":"2023-09-23T21:06:28.438320+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":36,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.605007+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:06:27.605007+0000","last_peered":"2023-09-23T21:06:27.605007+0000","last_clean":"2023-09-23T21:06:27.605007+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:06:27.605007+0000","last_undegraded":"2023-09-23T21:06:27.605007+0000","last_fullsized":"2023-09-23T21:06:27.605007+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"370'5654","reported_seq":17144,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.463527+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:06:29.463527+0000","last_peered":"2023-09-23T21:06:29.463527+0000","last_clean":"2023-09-23T21:06:29.463527+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:06:29.463527+0000","last_undegraded":"2023-09-23T21:06:29.463527+0000","last_fullsized":"2023-09-23T21:06:29.463527+0000","mapping_epoch":400,"log_start":"349'1599","ondisk_log_start":"349'1599","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4055,"log_dups_size":0,"ondisk_log_size":4055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":57,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438008+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:06:28.438008+0000","last_peered":"2023-09-23T21:06:28.438008+0000","last_clean":"2023-09-23T21:06:28.438008+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:06:28.438008+0000","last_undegraded":"2023-09-23T21:06:28.438008+0000","last_fullsized":"2023-09-23T21:06:28.438008+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":40,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438170+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:06:28.438170+0000","last_peered":"2023-09-23T21:06:28.438170+0000","last_clean":"2023-09-23T21:06:28.438170+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:06:28.438170+0000","last_undegraded":"2023-09-23T21:06:28.438170+0000","last_fullsized":"2023-09-23T21:06:28.438170+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"370'5491","reported_seq":13535,"reported_epoch":400,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:06:28.975020+0000","last_change":"2023-09-23T21:06:27.596301+0000","last_active":"2023-09-23T21:06:28.975020+0000","last_peered":"2023-09-23T21:06:28.975020+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:04:42.572134+0000","last_became_peered":"2023-09-23T21:04:42.572134+0000","last_unstale":"2023-09-23T21:06:28.975020+0000","last_undegraded":"2023-09-23T21:06:28.975020+0000","last_fullsized":"2023-09-23T21:04:42.399994+0000","mapping_epoch":372,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:37.804028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359858176,"num_objects":5491,"num_object_clones":0,"num_object_copies":16473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2513,"num_objects_unfound":0,"num_objects_dirty":5491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5491,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3966,"num_bytes_recovered":259915776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,1],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":529},{"shards":"1,2","objects":3970},{"shards":"1,2,7","objects":992}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":327,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440358+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:06:28.440358+0000","last_peered":"2023-09-23T21:06:28.440358+0000","last_clean":"2023-09-23T21:06:28.440358+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:06:28.440358+0000","last_undegraded":"2023-09-23T21:06:28.440358+0000","last_fullsized":"2023-09-23T21:06:28.440358+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":410,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.416614+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:06:27.416614+0000","last_peered":"2023-09-23T21:06:27.416614+0000","last_clean":"2023-09-23T21:06:27.416614+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:06:27.416614+0000","last_undegraded":"2023-09-23T21:06:27.416614+0000","last_fullsized":"2023-09-23T21:06:27.416614+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"370'5452","reported_seq":5539,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:28.439882+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:06:28.439882+0000","last_peered":"2023-09-23T21:06:28.439882+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:06:28.439882+0000","last_undegraded":"2023-09-23T21:06:28.439882+0000","last_fullsized":"2023-09-23T21:06:28.439882+0000","mapping_epoch":374,"log_start":"349'1394","ondisk_log_start":"349'1394","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4058,"log_dups_size":0,"ondisk_log_size":4058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357302272,"num_objects":5452,"num_object_clones":0,"num_object_copies":16356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10904,"num_objects_unfound":0,"num_objects_dirty":5452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5452,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5452}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":26,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.671848+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:06:24.671848+0000","last_peered":"2023-09-23T21:06:24.671848+0000","last_clean":"2023-09-23T21:06:24.671848+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:06:24.671848+0000","last_undegraded":"2023-09-23T21:06:24.671848+0000","last_fullsized":"2023-09-23T21:06:24.671848+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":440,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440788+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:06:28.440788+0000","last_peered":"2023-09-23T21:06:28.440788+0000","last_clean":"2023-09-23T21:06:28.440788+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:06:28.440788+0000","last_undegraded":"2023-09-23T21:06:28.440788+0000","last_fullsized":"2023-09-23T21:06:28.440788+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"367'5481","reported_seq":5654,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:28.443576+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:06:28.443576+0000","last_peered":"2023-09-23T21:06:28.443576+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:06:28.443576+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5481,"log_dups_size":0,"ondisk_log_size":5481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359202816,"num_objects":5481,"num_object_clones":0,"num_object_copies":16443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5481,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":42},{"shards":"1,6,7","objects":5439}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":49,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.437929+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:06:28.437929+0000","last_peered":"2023-09-23T21:06:28.437929+0000","last_clean":"2023-09-23T21:06:28.437929+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:06:28.437929+0000","last_undegraded":"2023-09-23T21:06:28.437929+0000","last_fullsized":"2023-09-23T21:06:28.437929+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":188,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.448774+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:06:28.448774+0000","last_peered":"2023-09-23T21:06:28.448774+0000","last_clean":"2023-09-23T21:06:28.448774+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:06:28.448774+0000","last_undegraded":"2023-09-23T21:06:28.448774+0000","last_fullsized":"2023-09-23T21:06:28.448774+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"370'5514","reported_seq":10997,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.416750+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:06:27.416750+0000","last_peered":"2023-09-23T21:06:27.416750+0000","last_clean":"2023-09-23T21:06:27.416750+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:06:27.416750+0000","last_undegraded":"2023-09-23T21:06:27.416750+0000","last_fullsized":"2023-09-23T21:06:27.416750+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361365504,"num_objects":5514,"num_object_clones":0,"num_object_copies":16542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5514,"num_write_kb":352896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":295,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440465+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:06:28.440465+0000","last_peered":"2023-09-23T21:06:28.440465+0000","last_clean":"2023-09-23T21:06:28.440465+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:06:28.440465+0000","last_undegraded":"2023-09-23T21:06:28.440465+0000","last_fullsized":"2023-09-23T21:06:28.440465+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":64,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.686493+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:06:24.686493+0000","last_peered":"2023-09-23T21:06:24.686493+0000","last_clean":"2023-09-23T21:06:24.686493+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:06:24.686493+0000","last_undegraded":"2023-09-23T21:06:24.686493+0000","last_fullsized":"2023-09-23T21:06:24.686493+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":50,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.439733+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:06:28.439733+0000","last_peered":"2023-09-23T21:06:28.439733+0000","last_clean":"2023-09-23T21:06:28.439733+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:06:28.439733+0000","last_undegraded":"2023-09-23T21:06:28.439733+0000","last_fullsized":"2023-09-23T21:06:28.439733+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"370'5513","reported_seq":5634,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:28.446196+0000","last_change":"2023-09-23T21:05:12.897699+0000","last_active":"2023-09-23T21:06:28.446196+0000","last_peered":"2023-09-23T21:06:28.446196+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:06:28.446196+0000","last_undegraded":"2023-09-23T21:05:12.617976+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"349'1457","ondisk_log_start":"349'1457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4056,"log_dups_size":0,"ondisk_log_size":4056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5513,"num_objects_misplaced":7846,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":3180},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":376,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440260+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:06:28.440260+0000","last_peered":"2023-09-23T21:06:28.440260+0000","last_clean":"2023-09-23T21:06:28.440260+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:06:28.440260+0000","last_undegraded":"2023-09-23T21:06:28.440260+0000","last_fullsized":"2023-09-23T21:06:28.440260+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":351,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.441859+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:06:28.441859+0000","last_peered":"2023-09-23T21:06:28.441859+0000","last_clean":"2023-09-23T21:06:28.441859+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:06:28.441859+0000","last_undegraded":"2023-09-23T21:06:28.441859+0000","last_fullsized":"2023-09-23T21:06:28.441859+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":32,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440369+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:06:28.440369+0000","last_peered":"2023-09-23T21:06:28.440369+0000","last_clean":"2023-09-23T21:06:28.440369+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:06:28.440369+0000","last_undegraded":"2023-09-23T21:06:28.440369+0000","last_fullsized":"2023-09-23T21:06:28.440369+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"370'5531","reported_seq":5698,"reported_epoch":399,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:27.415581+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:06:27.415581+0000","last_peered":"2023-09-23T21:06:27.415581+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:06:27.415581+0000","last_undegraded":"2023-09-23T21:06:27.415581+0000","last_fullsized":"2023-09-23T21:06:27.415581+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4058,"log_dups_size":0,"ondisk_log_size":4058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362479616,"num_objects":5531,"num_object_clones":0,"num_object_copies":16593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11027,"num_objects_unfound":0,"num_objects_dirty":5531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":5531}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":376,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.415826+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:06:27.415826+0000","last_peered":"2023-09-23T21:06:27.415826+0000","last_clean":"2023-09-23T21:06:27.415826+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:06:27.415826+0000","last_undegraded":"2023-09-23T21:06:27.415826+0000","last_fullsized":"2023-09-23T21:06:27.415826+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":295,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.439930+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:06:28.439930+0000","last_peered":"2023-09-23T21:06:28.439930+0000","last_clean":"2023-09-23T21:06:28.439930+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:06:28.439930+0000","last_undegraded":"2023-09-23T21:06:28.439930+0000","last_fullsized":"2023-09-23T21:06:28.439930+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":32,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.448636+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:06:28.448636+0000","last_peered":"2023-09-23T21:06:28.448636+0000","last_clean":"2023-09-23T21:06:28.448636+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:06:28.448636+0000","last_undegraded":"2023-09-23T21:06:28.448636+0000","last_fullsized":"2023-09-23T21:06:28.448636+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"370'5437","reported_seq":11684,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:28.440188+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:06:28.440188+0000","last_peered":"2023-09-23T21:06:28.440188+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:06:28.440188+0000","last_undegraded":"2023-09-23T21:06:28.440188+0000","last_fullsized":"2023-09-23T21:06:28.440188+0000","mapping_epoch":387,"log_start":"349'1380","ondisk_log_start":"349'1380","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356319232,"num_objects":5437,"num_object_clones":0,"num_object_copies":16311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5437,"num_objects_unfound":0,"num_objects_dirty":5437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5437,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5437}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":214,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.417080+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:06:27.417080+0000","last_peered":"2023-09-23T21:06:27.417080+0000","last_clean":"2023-09-23T21:06:27.417080+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:06:27.417080+0000","last_undegraded":"2023-09-23T21:06:27.417080+0000","last_fullsized":"2023-09-23T21:06:27.417080+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":697,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:28.439600+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:06:28.439600+0000","last_peered":"2023-09-23T21:06:28.439600+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:06:28.439600+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":49,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.437970+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:06:28.437970+0000","last_peered":"2023-09-23T21:06:28.437970+0000","last_clean":"2023-09-23T21:06:28.437970+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:06:28.437970+0000","last_undegraded":"2023-09-23T21:06:28.437970+0000","last_fullsized":"2023-09-23T21:06:28.437970+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":288,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T21:05:40.210998+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:05:40.210998+0000","last_peered":"2023-09-23T21:05:40.210998+0000","last_clean":"2023-09-23T21:05:40.210998+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:05:40.210998+0000","last_undegraded":"2023-09-23T21:05:40.210998+0000","last_fullsized":"2023-09-23T21:05:40.210998+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"370'5561","reported_seq":5664,"reported_epoch":400,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:28.439589+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:06:28.439589+0000","last_peered":"2023-09-23T21:06:28.439589+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:06:28.439589+0000","last_undegraded":"2023-09-23T21:06:28.439589+0000","last_fullsized":"2023-09-23T21:06:28.439589+0000","mapping_epoch":374,"log_start":"349'1505","ondisk_log_start":"349'1505","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4056,"log_dups_size":0,"ondisk_log_size":4056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364445696,"num_objects":5561,"num_object_clones":0,"num_object_copies":16683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11122,"num_objects_unfound":0,"num_objects_dirty":5561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5561,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":5561}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":32,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.439522+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:06:28.439522+0000","last_peered":"2023-09-23T21:06:28.439522+0000","last_clean":"2023-09-23T21:06:28.439522+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:06:28.439522+0000","last_undegraded":"2023-09-23T21:06:28.439522+0000","last_fullsized":"2023-09-23T21:06:28.439522+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604497+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:06:27.604497+0000","last_peered":"2023-09-23T21:06:27.604497+0000","last_clean":"2023-09-23T21:06:27.604497+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:06:27.604497+0000","last_undegraded":"2023-09-23T21:06:27.604497+0000","last_fullsized":"2023-09-23T21:06:27.604497+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":36,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604553+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:06:27.604553+0000","last_peered":"2023-09-23T21:06:27.604553+0000","last_clean":"2023-09-23T21:06:27.604553+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:06:27.604553+0000","last_undegraded":"2023-09-23T21:06:27.604553+0000","last_fullsized":"2023-09-23T21:06:27.604553+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"370'5513","reported_seq":9122,"reported_epoch":400,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:28.443050+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:06:28.443050+0000","last_peered":"2023-09-23T21:06:28.443050+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:06:28.443050+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:06:28.443050+0000","mapping_epoch":383,"log_start":"347'153","ondisk_log_start":"347'153","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":1762},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":36,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604788+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:06:27.604788+0000","last_peered":"2023-09-23T21:06:27.604788+0000","last_clean":"2023-09-23T21:06:27.604788+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:06:27.604788+0000","last_undegraded":"2023-09-23T21:06:27.604788+0000","last_fullsized":"2023-09-23T21:06:27.604788+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":297,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.445659+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:06:28.445659+0000","last_peered":"2023-09-23T21:06:28.445659+0000","last_clean":"2023-09-23T21:06:28.445659+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:06:28.445659+0000","last_undegraded":"2023-09-23T21:06:28.445659+0000","last_fullsized":"2023-09-23T21:06:28.445659+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":74,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.685854+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:06:24.685854+0000","last_peered":"2023-09-23T21:06:24.685854+0000","last_clean":"2023-09-23T21:06:24.685854+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:06:24.685854+0000","last_undegraded":"2023-09-23T21:06:24.685854+0000","last_fullsized":"2023-09-23T21:06:24.685854+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"373'5432","reported_seq":18311,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.447599+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:06:28.447599+0000","last_peered":"2023-09-23T21:06:28.447599+0000","last_clean":"2023-09-23T21:06:28.447599+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:06:28.447599+0000","last_undegraded":"2023-09-23T21:06:28.447599+0000","last_fullsized":"2023-09-23T21:06:28.447599+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355991552,"num_objects":5432,"num_object_clones":0,"num_object_copies":16296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5432,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":57,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438834+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:06:28.438834+0000","last_peered":"2023-09-23T21:06:28.438834+0000","last_clean":"2023-09-23T21:06:28.438834+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:06:28.438834+0000","last_undegraded":"2023-09-23T21:06:28.438834+0000","last_fullsized":"2023-09-23T21:06:28.438834+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":32,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.446613+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:06:28.446613+0000","last_peered":"2023-09-23T21:06:28.446613+0000","last_clean":"2023-09-23T21:06:28.446613+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:06:28.446613+0000","last_undegraded":"2023-09-23T21:06:28.446613+0000","last_fullsized":"2023-09-23T21:06:28.446613+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":49,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.437705+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:06:28.437705+0000","last_peered":"2023-09-23T21:06:28.437705+0000","last_clean":"2023-09-23T21:06:28.437705+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:06:28.437705+0000","last_undegraded":"2023-09-23T21:06:28.437705+0000","last_fullsized":"2023-09-23T21:06:28.437705+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"370'5599","reported_seq":5705,"reported_epoch":400,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:28.442998+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:06:28.442998+0000","last_peered":"2023-09-23T21:06:28.442998+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:06:28.442998+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:06:28.442998+0000","mapping_epoch":382,"log_start":"347'239","ondisk_log_start":"347'239","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366936064,"num_objects":5599,"num_object_clones":0,"num_object_copies":16797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5596,"num_objects_misplaced":5602,"num_objects_unfound":0,"num_objects_dirty":5599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5599,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":5596}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":297,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.440839+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:06:28.440839+0000","last_peered":"2023-09-23T21:06:28.440839+0000","last_clean":"2023-09-23T21:06:28.440839+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:06:28.440839+0000","last_undegraded":"2023-09-23T21:06:28.440839+0000","last_fullsized":"2023-09-23T21:06:28.440839+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":40,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604680+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:06:27.604680+0000","last_peered":"2023-09-23T21:06:27.604680+0000","last_clean":"2023-09-23T21:06:27.604680+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:06:27.604680+0000","last_undegraded":"2023-09-23T21:06:27.604680+0000","last_fullsized":"2023-09-23T21:06:27.604680+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":84,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.686893+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:06:24.686893+0000","last_peered":"2023-09-23T21:06:24.686893+0000","last_clean":"2023-09-23T21:06:24.686893+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:06:24.686893+0000","last_undegraded":"2023-09-23T21:06:24.686893+0000","last_fullsized":"2023-09-23T21:06:24.686893+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":45,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604958+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:06:27.604958+0000","last_peered":"2023-09-23T21:06:27.604958+0000","last_clean":"2023-09-23T21:06:27.604958+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:06:27.604958+0000","last_undegraded":"2023-09-23T21:06:27.604958+0000","last_fullsized":"2023-09-23T21:06:27.604958+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":422,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.416284+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:06:27.416284+0000","last_peered":"2023-09-23T21:06:27.416284+0000","last_clean":"2023-09-23T21:06:27.416284+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:06:27.416284+0000","last_undegraded":"2023-09-23T21:06:27.416284+0000","last_fullsized":"2023-09-23T21:06:27.416284+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":61,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.672272+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:06:24.672272+0000","last_peered":"2023-09-23T21:06:24.672272+0000","last_clean":"2023-09-23T21:06:24.672272+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:06:24.672272+0000","last_undegraded":"2023-09-23T21:06:24.672272+0000","last_fullsized":"2023-09-23T21:06:24.672272+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":66,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438144+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:06:28.438144+0000","last_peered":"2023-09-23T21:06:28.438144+0000","last_clean":"2023-09-23T21:06:28.438144+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:06:28.438144+0000","last_undegraded":"2023-09-23T21:06:28.438144+0000","last_fullsized":"2023-09-23T21:06:28.438144+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":281,"reported_epoch":389,"state":"active+clean","last_fresh":"2023-09-23T21:05:40.211090+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:05:40.211090+0000","last_peered":"2023-09-23T21:05:40.211090+0000","last_clean":"2023-09-23T21:05:40.211090+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:05:40.211090+0000","last_undegraded":"2023-09-23T21:05:40.211090+0000","last_fullsized":"2023-09-23T21:05:40.211090+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":58,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438104+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:06:28.438104+0000","last_peered":"2023-09-23T21:06:28.438104+0000","last_clean":"2023-09-23T21:06:28.438104+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:06:28.438104+0000","last_undegraded":"2023-09-23T21:06:28.438104+0000","last_fullsized":"2023-09-23T21:06:28.438104+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":40,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438352+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:06:28.438352+0000","last_peered":"2023-09-23T21:06:28.438352+0000","last_clean":"2023-09-23T21:06:28.438352+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:06:28.438352+0000","last_undegraded":"2023-09-23T21:06:28.438352+0000","last_fullsized":"2023-09-23T21:06:28.438352+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":49,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T21:06:28.438384+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:06:28.438384+0000","last_peered":"2023-09-23T21:06:28.438384+0000","last_clean":"2023-09-23T21:06:28.438384+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:06:28.438384+0000","last_undegraded":"2023-09-23T21:06:28.438384+0000","last_fullsized":"2023-09-23T21:06:28.438384+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":42,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T21:06:27.604789+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:06:27.604789+0000","last_peered":"2023-09-23T21:06:27.604789+0000","last_clean":"2023-09-23T21:06:27.604789+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:06:27.604789+0000","last_undegraded":"2023-09-23T21:06:27.604789+0000","last_fullsized":"2023-09-23T21:06:27.604789+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":26,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-09-23T21:06:24.671674+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:06:24.671674+0000","last_peered":"2023-09-23T21:06:24.671674+0000","last_clean":"2023-09-23T21:06:24.671674+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:06:24.671674+0000","last_undegraded":"2023-09-23T21:06:24.671674+0000","last_fullsized":"2023-09-23T21:06:24.671674+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5779095552,"num_objects":88182,"num_object_clones":0,"num_object_copies":264546,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":42098,"num_objects_misplaced":61612,"num_objects_unfound":0,"num_objects_dirty":88182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88182,"num_write_kb":5643648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20013,"num_bytes_recovered":1311571968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18841731072,"data_stored":18841731072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76120,"ondisk_log_size":76120,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201733,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463354,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4380896,"kb_used_data":3891480,"kb_used_omap":0,"kb_used_meta":489408,"kb_avail":89364256,"statfs":{"total":95995035648,"available":91508998144,"internally_reserved":0,"allocated":3984875520,"data_stored":3983732507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":501153792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"upper_bound":262144},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":1.1240000000000001,"15min":2.1589999999999998},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.65000000000000002,"5min":7.2480000000000002,"15min":102.242},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.41399999999999998,"5min":1.1539999999999999,"15min":1.861},"min":{"1min":0.26700000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60799999999999998,"5min":9.7110000000000003,"15min":102.502},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":3.907,"15min":2.3660000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":0.55200000000000005,"5min":119.077,"15min":119.077},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":3.8039999999999998,"15min":2.4590000000000001},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":2.8740000000000001,"5min":119.313,"15min":119.313},"last":0.44500000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":6.3570000000000002,"15min":3.089},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.4830000000000001,"5min":176.45400000000001,"15min":176.45400000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":6.4630000000000001,"15min":3.2429999999999999},"min":{"1min":0.33100000000000002,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":2.419,"5min":176.46799999999999,"15min":176.46799999999999},"last":0.29499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":3.7490000000000001,"15min":4.4400000000000004},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.26900000000000002},"max":{"1min":0.59299999999999997,"5min":178.40000000000001,"15min":250.804},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":3.8010000000000002,"15min":4.3899999999999997},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":2.1429999999999998,"5min":178.26599999999999,"15min":250.446},"last":0.307}]},{"osd":4,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":1.1759999999999999,"15min":0.97299999999999998},"min":{"1min":0.307,"5min":0.27000000000000002,"15min":0.24299999999999999},"max":{"1min":0.65100000000000002,"5min":13.1,"15min":13.1},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.439,"5min":0.92100000000000004,"15min":0.999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":0.61599999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.1160000000000001,"15min":1.6639999999999999},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65500000000000003,"5min":11.281000000000001,"15min":105.121},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":1.1919999999999999,"15min":1.7070000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":0.68700000000000006,"5min":8.6229999999999993,"15min":106.30500000000001},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.123,"15min":1.5089999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":0.70899999999999996,"5min":7.093,"15min":101.04600000000001},"last":0.375},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.1890000000000001,"15min":1.516},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":0.66800000000000004,"5min":7.1849999999999996,"15min":101.081},"last":0.49299999999999999}]}]},{"osd":1,"up_from":276,"seq":1185410973800,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4429584,"kb_used_data":4123720,"kb_used_omap":5,"kb_used_meta":305850,"kb_avail":89315568,"statfs":{"total":95995035648,"available":91459141632,"internally_reserved":0,"allocated":4222689280,"data_stored":4220718363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":313191139},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.30599999999999999,"5min":0.70399999999999996,"15min":0.48899999999999999},"min":{"1min":0.20100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.47899999999999998,"5min":11.959,"15min":11.959},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.29999999999999999,"5min":1.1319999999999999,"15min":0.76700000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.49099999999999999,"5min":18.047999999999998,"15min":18.047999999999998},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":2.2770000000000001,"15min":1.089},"min":{"1min":0.23400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57299999999999995,"5min":78.409000000000006,"15min":78.409000000000006},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.35799999999999998,"5min":1.835,"15min":0.90400000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.22},"max":{"1min":0.56000000000000005,"5min":76.245000000000005,"15min":76.245000000000005},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":3.621,"15min":2.4300000000000002},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":0.71499999999999997,"5min":77.75,"15min":172.44399999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.35999999999999999,"5min":4.4020000000000001,"15min":2.5960000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":0.74399999999999999,"5min":77.849000000000004,"15min":172.58500000000001},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.6499999999999999,"15min":0.88600000000000001},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66000000000000003,"5min":17.006,"15min":17.006},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.86699999999999999,"15min":0.67500000000000004},"min":{"1min":0.379,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68200000000000005,"5min":11.916,"15min":11.916},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":1.383,"15min":0.83299999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72599999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":1.7869999999999999,"15min":0.95499999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.68799999999999994,"5min":17.076000000000001,"15min":17.076000000000001},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":2.8460000000000001,"15min":1.339},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.69499999999999995,"5min":127.995,"15min":127.995},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":3.1909999999999998,"15min":1.4239999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68100000000000005,"5min":126.836,"15min":126.836},"last":0.441}]},{"osd":7,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":3.069,"15min":1.4239999999999999},"min":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71299999999999997,"5min":151.791,"15min":151.791},"last":0.501},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.238,"15min":1.171},"min":{"1min":0.42699999999999999,"5min":0.40400000000000003,"15min":0.38},"max":{"1min":0.73799999999999999,"5min":151.809,"15min":151.809},"last":0.56899999999999995}]}]},{"osd":0,"up_from":92,"seq":395136991540,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1385496,"kb_used_data":681104,"kb_used_omap":5,"kb_used_meta":704378,"kb_avail":92359656,"statfs":{"total":95995035648,"available":94576287744,"internally_reserved":0,"allocated":697450496,"data_stored":695585855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":721283706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.6749999999999998,"15min":1.2669999999999999},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.184},"max":{"1min":52.590000000000003,"5min":61.341999999999999,"15min":61.341999999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":3.165,"5min":2.5579999999999998,"15min":1.266},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":52.276000000000003,"5min":61.299999999999997,"15min":61.299999999999997},"last":0.23400000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":2.778,"15min":2.0939999999999999},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.161},"max":{"1min":0.61399999999999999,"5min":183.77099999999999,"15min":183.77099999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":3.258,"15min":2.3039999999999998},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.59799999999999998,"5min":183.78700000000001,"15min":183.78700000000001},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":3.7370000000000001,"15min":3.9489999999999998},"min":{"1min":0.311,"5min":0.25,"15min":0.183},"max":{"1min":0.64700000000000002,"5min":186.143,"15min":186.143},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":4.0069999999999997,"15min":4.0380000000000003},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.23000000000000001},"max":{"1min":0.623,"5min":186.97399999999999,"15min":186.97399999999999},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2969999999999999,"15min":1.379},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.247},"max":{"1min":1.4350000000000001,"5min":9.7080000000000002,"15min":10.525},"last":0.372},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.80900000000000005,"15min":1.022},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.26200000000000001},"max":{"1min":0.67600000000000005,"5min":6.0490000000000004,"15min":12.461},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.4830000000000001,"15min":2.5819999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28000000000000003},"max":{"1min":1.411,"5min":9.7439999999999998,"15min":154.30600000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.82299999999999995,"15min":1.96},"min":{"1min":0.42699999999999999,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":0.67000000000000004,"5min":5.9950000000000001,"15min":154.42400000000001},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":5.8390000000000004,"15min":2.7610000000000001},"min":{"1min":0.35899999999999999,"5min":0.34599999999999997,"15min":0.31},"max":{"1min":0.64200000000000002,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.47199999999999998,"5min":6.1609999999999996,"15min":2.8929999999999998},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":0.60799999999999998,"5min":169.767,"15min":169.767},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.97799999999999998,"15min":2.52},"min":{"1min":0.40799999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69599999999999995,"5min":6.1130000000000004,"15min":164.26400000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.503,"5min":1.2490000000000001,"15min":2.6099999999999999},"min":{"1min":0.38700000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":0.63900000000000001,"5min":6.5410000000000004,"15min":164.23099999999999},"last":0.48799999999999999}]}]},{"osd":2,"up_from":129,"seq":554050781431,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4277528,"kb_used_data":3692560,"kb_used_omap":7,"kb_used_meta":584952,"kb_avail":89467624,"statfs":{"total":95995035648,"available":91614846976,"internally_reserved":0,"allocated":3781181440,"data_stored":3779259329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":598990929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":1.2949999999999999,"15min":1.7210000000000001},"min":{"1min":0.21299999999999999,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":0.96699999999999997,"5min":15.667,"15min":89.644999999999996},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.36599999999999999,"5min":1.7190000000000001,"15min":1.881},"min":{"1min":0.251,"5min":0.23599999999999999,"15min":0.222},"max":{"1min":0.76600000000000001,"5min":12.445,"15min":88.599999999999994},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:05:55 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":5.5549999999999997,"15min":2.343},"min":{"1min":0.22500000000000001,"5min":0.19,"15min":0.188},"max":{"1min":1.9419999999999999,"5min":212.12,"15min":212.12},"last":0.435},{"interface":"front","average":{"1min":0.35399999999999998,"5min":5.5369999999999999,"15min":2.1349999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.79100000000000004,"5min":213.02799999999999,"15min":213.02799999999999},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":3.0609999999999999,"15min":3.8239999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":0.47799999999999998,"5min":66.085999999999999,"15min":151.43799999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.7919999999999998,"15min":3.8559999999999999},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.215},"max":{"1min":0.50900000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":0.42199999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":1.167,"15min":1.167},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.0049999999999999,"5min":12.586,"15min":12.586},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.421,"15min":1.216},"min":{"1min":0.371,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3300000000000001,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.498,"15min":2.6110000000000002},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.2749999999999999,"5min":9.9909999999999997,"15min":173.34},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":1.2370000000000001,"15min":2.4470000000000001},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.29399999999999998},"max":{"1min":1.2,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":3.4500000000000002,"15min":1.9319999999999999},"min":{"1min":0.35299999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.149,"5min":175.72499999999999,"15min":175.72499999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":3.3610000000000002,"15min":2.016},"min":{"1min":0.38700000000000001,"5min":0.36399999999999999,"15min":0.32300000000000001},"max":{"1min":1.228,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":1.498,"15min":2.7429999999999999},"min":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.32700000000000001},"max":{"1min":2.504,"5min":12.609,"15min":177.30799999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.5780000000000001,"15min":2.673},"min":{"1min":0.39500000000000002,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.3720000000000001,"5min":10.016,"15min":177.40899999999999},"last":0.53300000000000003}]}]},{"osd":3,"up_from":148,"seq":635655160025,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":526920,"kb_used_data":10112,"kb_used_omap":2,"kb_used_meta":516797,"kb_avail":93218232,"statfs":{"total":95995035648,"available":95455469568,"internally_reserved":0,"allocated":10354688,"data_stored":8267547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":529200181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.6739999999999999,"15min":1.8320000000000001},"min":{"1min":0.20999999999999999,"5min":0.20200000000000001,"15min":0.16700000000000001},"max":{"1min":44.963000000000001,"5min":44.963000000000001,"15min":95.507999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.101,"15min":1.823},"min":{"1min":0.20300000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":9.7370000000000001,"5min":9.7759999999999998,"15min":95.259},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.339,"15min":1.0129999999999999},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.7639999999999993,"5min":12.035,"15min":19.786999999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.6160000000000001,"15min":1.2110000000000001},"min":{"1min":0.253,"5min":0.182,"15min":0.182},"max":{"1min":2.3159999999999998,"5min":15.763999999999999,"15min":15.763999999999999},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.571,"15min":1.3129999999999999},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.182},"max":{"1min":3.2749999999999999,"5min":12.763999999999999,"15min":12.859},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.3380000000000001,"15min":1.4019999999999999},"min":{"1min":0.28000000000000003,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":9.7140000000000004,"5min":12.096,"15min":19.754999999999999},"last":0.309}]},{"osd":4,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.6629999999999998,"15min":2.194},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.222},"max":{"1min":3.5249999999999999,"5min":70.728999999999999,"15min":111.36},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":2.3439999999999999,"15min":1.9990000000000001},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":3.2490000000000001,"5min":70.712000000000003,"15min":112.473},"last":0.66000000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.157,"15min":1.6579999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":9.6359999999999992,"5min":9.6359999999999992,"15min":64.528000000000006},"last":0.438},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.78,"15min":1.6839999999999999},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.249},"max":{"1min":3.5649999999999999,"5min":10.617000000000001,"15min":14.054},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.855,"15min":1.7310000000000001},"min":{"1min":0.318,"5min":0.312,"15min":0.312},"max":{"1min":3.5910000000000002,"5min":10.52,"15min":19.568000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.655,"15min":1.575},"min":{"1min":0.34300000000000003,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":14.356,"5min":14.356,"15min":15.573},"last":0.626}]},{"osd":7,"last update":"Sat Sep 23 21:05:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":2.286,"15min":2.7269999999999999},"min":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":9.6859999999999999,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.8199999999999998,"15min":2.8490000000000002},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.1419999999999999,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.53700000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986643,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2014020,"kb_used_data":1471804,"kb_used_omap":0,"kb_used_meta":542208,"kb_avail":91731132,"statfs":{"total":95995035648,"available":93932679168,"internally_reserved":0,"allocated":1507127296,"data_stored":1505972266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":555220992},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":1.349,"15min":3.085},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.48499999999999999,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.436,"5min":1.1919999999999999,"15min":3.0739999999999998},"min":{"1min":0.28100000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.63,"5min":8.5220000000000002,"15min":144.77000000000001},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 21:06:07 2023","interfaces":[{"interface":"back","average":{"1min":6.4459999999999997,"5min":6.3929999999999998,"15min":2.6269999999999998},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":118.411,"5min":118.411,"15min":118.411},"last":0.46999999999999997},{"interface":"front","average":{"1min":6.274,"5min":6.1340000000000003,"15min":2.4740000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":118.075,"5min":118.075,"15min":118.075},"last":0.47999999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":7.4100000000000001,"5min":5.8289999999999997,"15min":3.2269999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":110.193,"5min":164.517,"15min":164.517},"last":0.33300000000000002},{"interface":"front","average":{"1min":7.3220000000000001,"5min":5.7889999999999997,"15min":3.3170000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":110.30200000000001,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.55700000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:06:06 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.2709999999999999,"15min":2.2610000000000001},"min":{"1min":0.32500000000000001,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.8420000000000001,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.2829999999999999,"15min":2.3889999999999998},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.82799999999999996,"5min":7.5579999999999998,"15min":142.34700000000001},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.82499999999999996,"15min":1.0609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.24099999999999999},"max":{"1min":2.9049999999999998,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.875,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29899999999999999,"15min":0.21099999999999999},"max":{"1min":0.58299999999999996,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.998,"15min":0.97599999999999998},"min":{"1min":0.28299999999999997,"5min":0.27500000000000002,"15min":0.26100000000000001},"max":{"1min":0.65800000000000003,"5min":11.800000000000001,"15min":11.887},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.1120000000000001,"15min":0.93100000000000005},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.26200000000000001},"max":{"1min":2.871,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":3.0110000000000001,"15min":1.8280000000000001},"min":{"1min":0.42799999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":0.72799999999999998,"5min":145.721,"15min":145.721},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.53600000000000003,"5min":3.282,"15min":1.992},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":0.78100000000000003,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.56299999999999994}]}]},{"osd":5,"up_from":39,"seq":167503725007,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1406912,"kb_used_data":826680,"kb_used_omap":0,"kb_used_meta":580224,"kb_avail":92338240,"statfs":{"total":95995035648,"available":94554357760,"internally_reserved":0,"allocated":846520320,"data_stored":845373608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":594149376},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.78000000000000003,"15min":1.857},"min":{"1min":0.25600000000000001,"5min":0.24199999999999999,"15min":0.19600000000000001},"max":{"1min":0.59299999999999997,"5min":23.600000000000001,"15min":124.873},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.1259999999999999,"15min":1.9930000000000001},"min":{"1min":0.24199999999999999,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":0.61099999999999999,"5min":23.532,"15min":123.554},"last":0.38400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":4.1200000000000001,"15min":1.6699999999999999},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":0.67200000000000004,"5min":137.143,"15min":137.143},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.47199999999999998,"5min":3.8199999999999998,"15min":1.5800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.247},"max":{"1min":0.66200000000000003,"5min":139.73699999999999,"15min":139.73699999999999},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.5720000000000001,"15min":1.6060000000000001},"min":{"1min":0.28799999999999998,"5min":0.28000000000000003,"15min":0.22500000000000001},"max":{"1min":0.61599999999999999,"5min":103.63800000000001,"15min":103.63800000000001},"last":0.432},{"interface":"front","average":{"1min":0.44,"5min":2.8330000000000002,"15min":1.6910000000000001},"min":{"1min":0.307,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":0.64800000000000002,"5min":103.664,"15min":103.664},"last":0.442}]},{"osd":3,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.609,"15min":4.0869999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.573,"5min":97.349999999999994,"15min":198.33099999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":2.4750000000000001,"15min":4.0540000000000003},"min":{"1min":0.40799999999999997,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":97.316999999999993,"15min":198.47300000000001},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.69799999999999995,"15min":0.81699999999999995},"min":{"1min":0.33100000000000002,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":0.69899999999999995,"5min":12.739000000000001,"15min":12.739000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.88,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":0.59599999999999997,"5min":10.792,"15min":12.539},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":1.702,"15min":1.2150000000000001},"min":{"1min":0.32800000000000001,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":0.57899999999999996,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.835,"15min":1.3400000000000001},"min":{"1min":0.38900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.73099999999999998,"5min":70.266000000000005,"15min":70.266000000000005},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.014,"15min":1.2230000000000001},"min":{"1min":0.40799999999999997,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.65200000000000002,"15min":1.0780000000000001},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":692453376,"data_stored":692453376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4226875392,"data_stored":4226875392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3775528960,"data_stored":3775528960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":5111808,"data_stored":5111808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1500971008,"data_stored":1500971008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":842334208,"data_stored":842334208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3980132352,"data_stored":3980132352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3818323968,"data_stored":3818323968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:06:34.913 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:35.057 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:35.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:35.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:35.071 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:06:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:06:34] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.43.0" 2023-09-23T21:06:35.171 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:35.171 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:35.230 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:35.312 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:35.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:35 smithi123 ceph-mon[193903]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 134 B/s rd, 134 B/s wr, 30 op/s; 42100/263931 objects degraded (15.951%); 60829/263931 objects misplaced (23.047%) 2023-09-23T21:06:35.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:35 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 42100/263931 objects degraded (15.951%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:35.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:35.340 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:35.340 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:35.372 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:35 smithi121 ceph-mon[211526]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 134 B/s rd, 134 B/s wr, 30 op/s; 42100/263931 objects degraded (15.951%); 60829/263931 objects misplaced (23.047%) 2023-09-23T21:06:35.372 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:35 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 42100/263931 objects degraded (15.951%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:35.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:35 smithi121 ceph-mon[219709]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 134 B/s rd, 134 B/s wr, 30 op/s; 42100/263931 objects degraded (15.951%); 60829/263931 objects misplaced (23.047%) 2023-09-23T21:06:35.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:35 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 42100/263931 objects degraded (15.951%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:35.416 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:35.489 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:35.650 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:35.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:35.776 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:35.827 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:36.087 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:36.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:36.159 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:06:36.189 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:06:36.192 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:36.282 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:36.380 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3375262173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:36.389 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:36 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:36.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:36.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3375262173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:36.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:36 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:36.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3375262173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:36.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:36 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:06:36.491 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:36.544 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:36.611 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:36.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:36.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:36.727 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:36.745 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:36.758 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:37.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:37.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:06:37.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T21:06:37.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[211526]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 32 KiB/s wr, 220 op/s; 41617/259350 objects degraded (16.047%); 59791/259350 objects misplaced (23.054%) 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:37 smithi121 ceph-mon[219709]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 32 KiB/s wr, 220 op/s; 41617/259350 objects degraded (16.047%); 59791/259350 objects misplaced (23.054%) 2023-09-23T21:06:37.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:37.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:06:37.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: osdmap e402: 8 total, 8 up, 7 in 2023-09-23T21:06:37.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: failed to encode map e402 with expected crc 2023-09-23T21:06:37.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:37 smithi123 ceph-mon[193903]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 32 KiB/s wr, 220 op/s; 41617/259350 objects degraded (16.047%); 59791/259350 objects misplaced (23.054%) 2023-09-23T21:06:37.736 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:37.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:37.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:37.848 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:37.906 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:06:37.951 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:38.155 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:38.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:38.267 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:38.314 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:38 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T21:06:38.315 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:38 smithi123 ceph-mon[193903]: Health check cleared: SLOW_OPS (was: 5 slow ops, oldest one blocked for 105 sec, osd.6 has slow ops) 2023-09-23T21:06:38.318 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:38.407 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:38 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T21:06:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:38 smithi121 ceph-mon[211526]: Health check cleared: SLOW_OPS (was: 5 slow ops, oldest one blocked for 105 sec, osd.6 has slow ops) 2023-09-23T21:06:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:38.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:38 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-09-23T21:06:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:38 smithi121 ceph-mon[219709]: Health check cleared: SLOW_OPS (was: 5 slow ops, oldest one blocked for 105 sec, osd.6 has slow ops) 2023-09-23T21:06:38.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:38.515 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:38.608 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:38.608 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:38.625 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:38.733 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:38.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:38.831 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:38.855 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:38.890 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:06:38.890 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:38.933 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:06:38.966 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:38.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:38.972 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:06:39.007 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:06:39.051 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:06:39.072 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:39.084 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:06:39.115 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:06:39.151 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:39.158 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:06:39.185 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.185 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.186 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.186 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.186 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.186 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.186 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.186 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:06:39.342 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:39.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2570997163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:39 smithi121 ceph-mon[211526]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 107 B/s rd, 27 KiB/s wr, 184 op/s; 41617/259350 objects degraded (16.047%); 59791/259350 objects misplaced (23.054%) 2023-09-23T21:06:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2570997163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:39 smithi121 ceph-mon[219709]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 107 B/s rd, 27 KiB/s wr, 184 op/s; 41617/259350 objects degraded (16.047%); 59791/259350 objects misplaced (23.054%) 2023-09-23T21:06:39.431 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2570997163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:39.431 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:39 smithi123 ceph-mon[193903]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 107 B/s rd, 27 KiB/s wr, 184 op/s; 41617/259350 objects degraded (16.047%); 59791/259350 objects misplaced (23.054%) 2023-09-23T21:06:39.449 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:39.452 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:06:39.521 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:39.950 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:39.956 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:39.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:39.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:40.062 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:40.097 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:40.158 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:06:40.250 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:40.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:40.354 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:40.413 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:40.585 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:40.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:40.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:40.691 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:40.758 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:41.173 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:41.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:41.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:41.281 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:41.386 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:41 smithi121 ceph-mon[211526]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 26 KiB/s wr, 413 op/s; 41059/252231 objects degraded (16.278%); 57271/252231 objects misplaced (22.706%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:06:41.386 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41059/252231 objects degraded (16.278%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:41.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:41 smithi121 ceph-mon[219709]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 26 KiB/s wr, 413 op/s; 41059/252231 objects degraded (16.278%); 57271/252231 objects misplaced (22.706%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:06:41.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41059/252231 objects degraded (16.278%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:41.405 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:41 smithi123 ceph-mon[193903]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 26 KiB/s wr, 413 op/s; 41059/252231 objects degraded (16.278%); 57271/252231 objects misplaced (22.706%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:06:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41059/252231 objects degraded (16.278%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:41.504 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:41.566 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:41.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:41.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:41.684 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:41.782 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:41.897 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:06:41.898 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:06:41.898 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:06:41.898 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:06:42.009 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:42.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:42.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:42.219 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:42.360 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:42.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:42.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:42.409 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:06:42.410 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:06:42.465 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:42.515 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:42.630 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:42.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:42.735 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:42.781 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:42.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:06:42.825 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:42.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:42.929 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:42.958 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:43.031 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:43.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:43.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:43.146 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:43.195 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:43.261 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:43.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:43.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:43.278 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:06:43.297 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:43.365 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:43.412 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[219709]: from='client.46433 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[219709]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 26 KiB/s wr, 413 op/s; 41059/252231 objects degraded (16.278%); 57271/252231 objects misplaced (22.706%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:06:43.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[211526]: from='client.46433 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:43.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:43.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:43.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:43 smithi121 ceph-mon[211526]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 26 KiB/s wr, 413 op/s; 41059/252231 objects degraded (16.278%); 57271/252231 objects misplaced (22.706%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:06:43.725 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:43 smithi123 ceph-mon[193903]: from='client.46433 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:43.725 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:06:43.725 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:43.725 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:43 smithi123 ceph-mon[193903]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 26 KiB/s wr, 413 op/s; 41059/252231 objects degraded (16.278%); 57271/252231 objects misplaced (22.706%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:06:44.008 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:44.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:44.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:44.031 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1256,"stamp":"2023-09-23T21:06:41.021566+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5509027768,"num_objects":84077,"num_object_clones":0,"num_object_copies":252231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":41059,"num_objects_misplaced":57271,"num_objects_unfound":0,"num_objects_dirty":84077,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":92621,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21913,"num_bytes_recovered":1431411980,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72336,"ondisk_log_size":72336,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17917656,"kb_used_data":14149024,"kb_used_omap":21,"kb_used_meta":3768362,"kb_avail":638298408,"statfs":{"total":671965249536,"available":653617569792,"internally_reserved":0,"allocated":14488600576,"data_stored":14477191431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3858803171},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,7,6,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-270663656,"num_objects":-4129,"num_object_clones":0,"num_object_copies":-12387,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-1041,"num_objects_misplaced":-4341,"num_objects_unfound":0,"num_objects_dirty":-4129,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4139,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1783,"num_bytes_recovered":116850688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002783"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":172,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.452069+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:06:29.452069+0000","last_peered":"2023-09-23T21:06:29.452069+0000","last_clean":"2023-09-23T21:06:29.452069+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:06:29.452069+0000","last_undegraded":"2023-09-23T21:06:29.452069+0000","last_fullsized":"2023-09-23T21:06:29.452069+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":41,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.444013+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:06:29.444013+0000","last_peered":"2023-09-23T21:06:29.444013+0000","last_clean":"2023-09-23T21:06:29.444013+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:06:29.444013+0000","last_undegraded":"2023-09-23T21:06:29.444013+0000","last_fullsized":"2023-09-23T21:06:29.444013+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":314,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.450568+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:06:29.450568+0000","last_peered":"2023-09-23T21:06:29.450568+0000","last_clean":"2023-09-23T21:06:29.450568+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:06:29.450568+0000","last_undegraded":"2023-09-23T21:06:29.450568+0000","last_fullsized":"2023-09-23T21:06:29.450568+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":38,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026941+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:06:30.026941+0000","last_peered":"2023-09-23T21:06:30.026941+0000","last_clean":"2023-09-23T21:06:30.026941+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:06:30.026941+0000","last_undegraded":"2023-09-23T21:06:30.026941+0000","last_fullsized":"2023-09-23T21:06:30.026941+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":59,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442249+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:06:29.442249+0000","last_peered":"2023-09-23T21:06:29.442249+0000","last_clean":"2023-09-23T21:06:29.442249+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:06:29.442249+0000","last_undegraded":"2023-09-23T21:06:29.442249+0000","last_fullsized":"2023-09-23T21:06:29.442249+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":268,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451849+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:06:29.451849+0000","last_peered":"2023-09-23T21:06:29.451849+0000","last_clean":"2023-09-23T21:06:29.451849+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:06:29.451849+0000","last_undegraded":"2023-09-23T21:06:29.451849+0000","last_fullsized":"2023-09-23T21:06:29.451849+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":38,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026948+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:06:30.026948+0000","last_peered":"2023-09-23T21:06:30.026948+0000","last_clean":"2023-09-23T21:06:30.026948+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:06:30.026948+0000","last_undegraded":"2023-09-23T21:06:30.026948+0000","last_fullsized":"2023-09-23T21:06:30.026948+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":41,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442246+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:06:29.442246+0000","last_peered":"2023-09-23T21:06:29.442246+0000","last_clean":"2023-09-23T21:06:29.442246+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:06:29.442246+0000","last_undegraded":"2023-09-23T21:06:29.442246+0000","last_fullsized":"2023-09-23T21:06:29.442246+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":25,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.449910+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:06:29.449910+0000","last_peered":"2023-09-23T21:06:29.449910+0000","last_clean":"2023-09-23T21:06:29.449910+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:06:29.449910+0000","last_undegraded":"2023-09-23T21:06:29.449910+0000","last_fullsized":"2023-09-23T21:06:29.449910+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":41,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443598+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:06:29.443598+0000","last_peered":"2023-09-23T21:06:29.443598+0000","last_clean":"2023-09-23T21:06:29.443598+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:06:29.443598+0000","last_undegraded":"2023-09-23T21:06:29.443598+0000","last_fullsized":"2023-09-23T21:06:29.443598+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":57,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.444085+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:06:29.444085+0000","last_peered":"2023-09-23T21:06:29.444085+0000","last_clean":"2023-09-23T21:06:29.444085+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:06:29.444085+0000","last_undegraded":"2023-09-23T21:06:29.444085+0000","last_fullsized":"2023-09-23T21:06:29.444085+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":33,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451730+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:06:29.451730+0000","last_peered":"2023-09-23T21:06:29.451730+0000","last_clean":"2023-09-23T21:06:29.451730+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:06:29.451730+0000","last_undegraded":"2023-09-23T21:06:29.451730+0000","last_fullsized":"2023-09-23T21:06:29.451730+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":348,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451497+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:06:29.451497+0000","last_peered":"2023-09-23T21:06:29.451497+0000","last_clean":"2023-09-23T21:06:29.451497+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:06:29.451497+0000","last_undegraded":"2023-09-23T21:06:29.451497+0000","last_fullsized":"2023-09-23T21:06:29.451497+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":320,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.410011+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:06:30.410011+0000","last_peered":"2023-09-23T21:06:30.410011+0000","last_clean":"2023-09-23T21:06:30.410011+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:06:30.410011+0000","last_undegraded":"2023-09-23T21:06:30.410011+0000","last_fullsized":"2023-09-23T21:06:30.410011+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":91,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443184+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:06:29.443184+0000","last_peered":"2023-09-23T21:06:29.443184+0000","last_clean":"2023-09-23T21:06:29.443184+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:06:29.443184+0000","last_undegraded":"2023-09-23T21:06:29.443184+0000","last_fullsized":"2023-09-23T21:06:29.443184+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":50,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443491+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:06:29.443491+0000","last_peered":"2023-09-23T21:06:29.443491+0000","last_clean":"2023-09-23T21:06:29.443491+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:06:29.443491+0000","last_undegraded":"2023-09-23T21:06:29.443491+0000","last_fullsized":"2023-09-23T21:06:29.443491+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":59,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442301+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:06:29.442301+0000","last_peered":"2023-09-23T21:06:29.442301+0000","last_clean":"2023-09-23T21:06:29.442301+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:06:29.442301+0000","last_undegraded":"2023-09-23T21:06:29.442301+0000","last_fullsized":"2023-09-23T21:06:29.442301+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":59,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.444152+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:06:29.444152+0000","last_peered":"2023-09-23T21:06:29.444152+0000","last_clean":"2023-09-23T21:06:29.444152+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:06:29.444152+0000","last_undegraded":"2023-09-23T21:06:29.444152+0000","last_fullsized":"2023-09-23T21:06:29.444152+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":202,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.409821+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:06:30.409821+0000","last_peered":"2023-09-23T21:06:30.409821+0000","last_clean":"2023-09-23T21:06:30.409821+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:06:30.409821+0000","last_undegraded":"2023-09-23T21:06:30.409821+0000","last_fullsized":"2023-09-23T21:06:30.409821+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":38,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.027014+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:06:30.027014+0000","last_peered":"2023-09-23T21:06:30.027014+0000","last_clean":"2023-09-23T21:06:30.027014+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:06:30.027014+0000","last_undegraded":"2023-09-23T21:06:30.027014+0000","last_fullsized":"2023-09-23T21:06:30.027014+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":398,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451215+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:06:29.451215+0000","last_peered":"2023-09-23T21:06:29.451215+0000","last_clean":"2023-09-23T21:06:29.451215+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:06:29.451215+0000","last_undegraded":"2023-09-23T21:06:29.451215+0000","last_fullsized":"2023-09-23T21:06:29.451215+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":41,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442660+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:06:29.442660+0000","last_peered":"2023-09-23T21:06:29.442660+0000","last_clean":"2023-09-23T21:06:29.442660+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:06:29.442660+0000","last_undegraded":"2023-09-23T21:06:29.442660+0000","last_fullsized":"2023-09-23T21:06:29.442660+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"402'5898","reported_seq":7256,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:40.045412+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:06:40.045412+0000","last_peered":"2023-09-23T21:06:40.045412+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:06:40.045412+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"347'1128","ondisk_log_start":"347'1128","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349437952,"num_objects":5332,"num_object_clones":0,"num_object_copies":15996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5898,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"0,1,4,7","objects":352},{"shards":"1,4","objects":-283},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":4979}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":106,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.410330+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:06:30.410330+0000","last_peered":"2023-09-23T21:06:30.410330+0000","last_clean":"2023-09-23T21:06:30.410330+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:06:30.410330+0000","last_undegraded":"2023-09-23T21:06:30.410330+0000","last_fullsized":"2023-09-23T21:06:30.410330+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":25,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.449656+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:06:29.449656+0000","last_peered":"2023-09-23T21:06:29.449656+0000","last_clean":"2023-09-23T21:06:29.449656+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:06:29.449656+0000","last_undegraded":"2023-09-23T21:06:29.449656+0000","last_fullsized":"2023-09-23T21:06:29.449656+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"402'5646","reported_seq":5717,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:40.222869+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:06:40.222869+0000","last_peered":"2023-09-23T21:06:40.222869+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:06:40.222869+0000","last_undegraded":"2023-09-23T21:06:40.222869+0000","last_fullsized":"2023-09-23T21:06:40.222869+0000","mapping_epoch":375,"log_start":"347'912","ondisk_log_start":"347'912","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4734,"log_dups_size":0,"ondisk_log_size":4734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337641472,"num_objects":5152,"num_object_clones":0,"num_object_copies":15456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5152,"num_objects_unfound":0,"num_objects_dirty":5152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5646,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":5152}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":99,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.452473+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:06:29.452473+0000","last_peered":"2023-09-23T21:06:29.452473+0000","last_clean":"2023-09-23T21:06:29.452473+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:06:29.452473+0000","last_undegraded":"2023-09-23T21:06:29.452473+0000","last_fullsized":"2023-09-23T21:06:29.452473+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":362,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.450484+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:06:29.450484+0000","last_peered":"2023-09-23T21:06:29.450484+0000","last_clean":"2023-09-23T21:06:29.450484+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:06:29.450484+0000","last_undegraded":"2023-09-23T21:06:29.450484+0000","last_fullsized":"2023-09-23T21:06:29.450484+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"402'5736","reported_seq":8770,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:40.578481+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:06:40.578481+0000","last_peered":"2023-09-23T21:06:40.578481+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:06:40.578481+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"349'1611","ondisk_log_start":"349'1611","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4125,"log_dups_size":0,"ondisk_log_size":4125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340918272,"num_objects":5202,"num_object_clones":0,"num_object_copies":15606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10394,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5736,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1340,"num_bytes_recovered":87818240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":805},{"shards":"1,6,7","objects":2011},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":29,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.450034+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:06:29.450034+0000","last_peered":"2023-09-23T21:06:29.450034+0000","last_clean":"2023-09-23T21:06:29.450034+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:06:29.450034+0000","last_undegraded":"2023-09-23T21:06:29.450034+0000","last_fullsized":"2023-09-23T21:06:29.450034+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":290,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451589+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:06:29.451589+0000","last_peered":"2023-09-23T21:06:29.451589+0000","last_clean":"2023-09-23T21:06:29.451589+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:06:29.451589+0000","last_undegraded":"2023-09-23T21:06:29.451589+0000","last_fullsized":"2023-09-23T21:06:29.451589+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"402'5803","reported_seq":6422,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:40.540407+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:06:40.540407+0000","last_peered":"2023-09-23T21:06:40.540407+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:06:40.540407+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"349'1668","ondisk_log_start":"349'1668","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4135,"log_dups_size":0,"ondisk_log_size":4135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343867392,"num_objects":5247,"num_object_clones":0,"num_object_copies":15741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5803,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-6},{"shards":"2,6,7","objects":5244}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":41,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443574+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:06:29.443574+0000","last_peered":"2023-09-23T21:06:29.443574+0000","last_clean":"2023-09-23T21:06:29.443574+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:06:29.443574+0000","last_undegraded":"2023-09-23T21:06:29.443574+0000","last_fullsized":"2023-09-23T21:06:29.443574+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":38,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026546+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:06:30.026546+0000","last_peered":"2023-09-23T21:06:30.026546+0000","last_clean":"2023-09-23T21:06:30.026546+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:06:30.026546+0000","last_undegraded":"2023-09-23T21:06:30.026546+0000","last_fullsized":"2023-09-23T21:06:30.026546+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"402'5941","reported_seq":17432,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T21:06:40.484951+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:06:40.484951+0000","last_peered":"2023-09-23T21:06:40.484951+0000","last_clean":"2023-09-23T21:06:40.484951+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:06:40.484951+0000","last_undegraded":"2023-09-23T21:06:40.484951+0000","last_fullsized":"2023-09-23T21:06:40.484951+0000","mapping_epoch":400,"log_start":"349'1799","ondisk_log_start":"349'1799","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4142,"log_dups_size":0,"ondisk_log_size":4142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351731712,"num_objects":5367,"num_object_clones":0,"num_object_copies":16101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5941,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":58,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442476+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:06:29.442476+0000","last_peered":"2023-09-23T21:06:29.442476+0000","last_clean":"2023-09-23T21:06:29.442476+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:06:29.442476+0000","last_undegraded":"2023-09-23T21:06:29.442476+0000","last_fullsized":"2023-09-23T21:06:29.442476+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":41,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443422+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:06:29.443422+0000","last_peered":"2023-09-23T21:06:29.443422+0000","last_clean":"2023-09-23T21:06:29.443422+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:06:29.443422+0000","last_undegraded":"2023-09-23T21:06:29.443422+0000","last_fullsized":"2023-09-23T21:06:29.443422+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"402'5725","reported_seq":15793,"reported_epoch":402,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:06:39.129533+0000","last_change":"2023-09-23T21:06:27.596301+0000","last_active":"2023-09-23T21:06:39.129533+0000","last_peered":"2023-09-23T21:06:39.129533+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:04:42.572134+0000","last_became_peered":"2023-09-23T21:04:42.572134+0000","last_unstale":"2023-09-23T21:06:39.129533+0000","last_undegraded":"2023-09-23T21:06:39.129533+0000","last_fullsized":"2023-09-23T21:04:42.399994+0000","mapping_epoch":372,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5237,"log_dups_size":0,"ondisk_log_size":5237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:37.804028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344522752,"num_objects":5257,"num_object_clones":0,"num_object_copies":15771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1020,"num_objects_unfound":0,"num_objects_dirty":5257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5725,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4977,"num_bytes_recovered":326172672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,1],"avail_no_missing":["2","1","7"],"object_location_counts":[{"shards":"1,2","objects":4747},{"shards":"1,2,7","objects":510}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":328,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451600+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:06:29.451600+0000","last_peered":"2023-09-23T21:06:29.451600+0000","last_clean":"2023-09-23T21:06:29.451600+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:06:29.451600+0000","last_undegraded":"2023-09-23T21:06:29.451600+0000","last_fullsized":"2023-09-23T21:06:29.451600+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":412,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443377+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:06:29.443377+0000","last_peered":"2023-09-23T21:06:29.443377+0000","last_clean":"2023-09-23T21:06:29.443377+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:06:29.443377+0000","last_undegraded":"2023-09-23T21:06:29.443377+0000","last_fullsized":"2023-09-23T21:06:29.443377+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"402'5706","reported_seq":5794,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:40.579826+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:06:40.579826+0000","last_peered":"2023-09-23T21:06:40.579826+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:06:40.579826+0000","last_undegraded":"2023-09-23T21:06:40.579826+0000","last_fullsized":"2023-09-23T21:06:40.579826+0000","mapping_epoch":374,"log_start":"349'1594","ondisk_log_start":"349'1594","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4112,"log_dups_size":0,"ondisk_log_size":4112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340656128,"num_objects":5198,"num_object_clones":0,"num_object_copies":15594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10396,"num_objects_unfound":0,"num_objects_dirty":5198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5706,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":5198}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":29,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.409736+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:06:30.409736+0000","last_peered":"2023-09-23T21:06:30.409736+0000","last_clean":"2023-09-23T21:06:30.409736+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:06:30.409736+0000","last_undegraded":"2023-09-23T21:06:30.409736+0000","last_fullsized":"2023-09-23T21:06:30.409736+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":441,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451794+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:06:29.451794+0000","last_peered":"2023-09-23T21:06:29.451794+0000","last_clean":"2023-09-23T21:06:29.451794+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:06:29.451794+0000","last_undegraded":"2023-09-23T21:06:29.451794+0000","last_fullsized":"2023-09-23T21:06:29.451794+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"402'5729","reported_seq":5903,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:40.254309+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:06:40.254309+0000","last_peered":"2023-09-23T21:06:40.254309+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:06:40.254309+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"347'994","ondisk_log_start":"347'994","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4735,"log_dups_size":0,"ondisk_log_size":4735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342949888,"num_objects":5233,"num_object_clones":0,"num_object_copies":15699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5729,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-206},{"shards":"1,6,7","objects":5233}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":50,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442381+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:06:29.442381+0000","last_peered":"2023-09-23T21:06:29.442381+0000","last_clean":"2023-09-23T21:06:29.442381+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:06:29.442381+0000","last_undegraded":"2023-09-23T21:06:29.442381+0000","last_fullsized":"2023-09-23T21:06:29.442381+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":189,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.454588+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:06:29.454588+0000","last_peered":"2023-09-23T21:06:29.454588+0000","last_clean":"2023-09-23T21:06:29.454588+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:06:29.454588+0000","last_undegraded":"2023-09-23T21:06:29.454588+0000","last_fullsized":"2023-09-23T21:06:29.454588+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"402'5723","reported_seq":11209,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T21:06:38.942464+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:06:38.942464+0000","last_peered":"2023-09-23T21:06:38.942464+0000","last_clean":"2023-09-23T21:06:38.942464+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:06:38.942464+0000","last_undegraded":"2023-09-23T21:06:38.942464+0000","last_fullsized":"2023-09-23T21:06:38.942464+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5214,"log_dups_size":0,"ondisk_log_size":5214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347734040,"num_objects":5307,"num_object_clones":0,"num_object_copies":15921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5307,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5723,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":296,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.450990+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:06:29.450990+0000","last_peered":"2023-09-23T21:06:29.450990+0000","last_clean":"2023-09-23T21:06:29.450990+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:06:29.450990+0000","last_undegraded":"2023-09-23T21:06:29.450990+0000","last_fullsized":"2023-09-23T21:06:29.450990+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":67,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.450114+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:06:29.450114+0000","last_peered":"2023-09-23T21:06:29.450114+0000","last_clean":"2023-09-23T21:06:29.450114+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:06:29.450114+0000","last_undegraded":"2023-09-23T21:06:29.450114+0000","last_fullsized":"2023-09-23T21:06:29.450114+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":51,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451811+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:06:29.451811+0000","last_peered":"2023-09-23T21:06:29.451811+0000","last_clean":"2023-09-23T21:06:29.451811+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:06:29.451811+0000","last_undegraded":"2023-09-23T21:06:29.451811+0000","last_fullsized":"2023-09-23T21:06:29.451811+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"402'5769","reported_seq":6405,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:40.628061+0000","last_change":"2023-09-23T21:05:12.897699+0000","last_active":"2023-09-23T21:06:40.628061+0000","last_peered":"2023-09-23T21:06:40.628061+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:06:40.628061+0000","last_undegraded":"2023-09-23T21:05:12.617976+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"349'1657","ondisk_log_start":"349'1657","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4112,"log_dups_size":0,"ondisk_log_size":4112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344522752,"num_objects":5257,"num_object_clones":0,"num_object_copies":15771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5256,"num_objects_misplaced":7591,"num_objects_unfound":0,"num_objects_dirty":5257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":257,"num_bytes_recovered":16842752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":2923},{"shards":"2,6","objects":1},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":377,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.452511+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:06:29.452511+0000","last_peered":"2023-09-23T21:06:29.452511+0000","last_clean":"2023-09-23T21:06:29.452511+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:06:29.452511+0000","last_undegraded":"2023-09-23T21:06:29.452511+0000","last_fullsized":"2023-09-23T21:06:29.452511+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":352,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.452157+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:06:29.452157+0000","last_peered":"2023-09-23T21:06:29.452157+0000","last_clean":"2023-09-23T21:06:29.452157+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:06:29.452157+0000","last_undegraded":"2023-09-23T21:06:29.452157+0000","last_fullsized":"2023-09-23T21:06:29.452157+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":33,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.450915+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:06:29.450915+0000","last_peered":"2023-09-23T21:06:29.450915+0000","last_clean":"2023-09-23T21:06:29.450915+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:06:29.450915+0000","last_undegraded":"2023-09-23T21:06:29.450915+0000","last_fullsized":"2023-09-23T21:06:29.450915+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"402'5789","reported_seq":5958,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:39.112952+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:06:39.112952+0000","last_peered":"2023-09-23T21:06:39.112952+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:06:39.112952+0000","last_undegraded":"2023-09-23T21:06:39.112952+0000","last_fullsized":"2023-09-23T21:06:39.112952+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4316,"log_dups_size":0,"ondisk_log_size":4316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345571328,"num_objects":5273,"num_object_clones":0,"num_object_copies":15819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10511,"num_objects_unfound":0,"num_objects_dirty":5273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5789,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":5273}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":378,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442960+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:06:29.442960+0000","last_peered":"2023-09-23T21:06:29.442960+0000","last_clean":"2023-09-23T21:06:29.442960+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:06:29.442960+0000","last_undegraded":"2023-09-23T21:06:29.442960+0000","last_fullsized":"2023-09-23T21:06:29.442960+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":296,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.447550+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:06:29.447550+0000","last_peered":"2023-09-23T21:06:29.447550+0000","last_clean":"2023-09-23T21:06:29.447550+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:06:29.447550+0000","last_undegraded":"2023-09-23T21:06:29.447550+0000","last_fullsized":"2023-09-23T21:06:29.447550+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.454477+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:06:29.454477+0000","last_peered":"2023-09-23T21:06:29.454477+0000","last_clean":"2023-09-23T21:06:29.454477+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:06:29.454477+0000","last_undegraded":"2023-09-23T21:06:29.454477+0000","last_fullsized":"2023-09-23T21:06:29.454477+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"402'5694","reported_seq":11942,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:40.571413+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:06:40.571413+0000","last_peered":"2023-09-23T21:06:40.571413+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:06:40.571413+0000","last_undegraded":"2023-09-23T21:06:40.571413+0000","last_fullsized":"2023-09-23T21:06:40.571413+0000","mapping_epoch":387,"log_start":"349'1580","ondisk_log_start":"349'1580","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4114,"log_dups_size":0,"ondisk_log_size":4114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339476480,"num_objects":5180,"num_object_clones":0,"num_object_copies":15540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5180,"num_objects_unfound":0,"num_objects_dirty":5180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5694,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":5180}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":216,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443254+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:06:29.443254+0000","last_peered":"2023-09-23T21:06:29.443254+0000","last_clean":"2023-09-23T21:06:29.443254+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:06:29.443254+0000","last_undegraded":"2023-09-23T21:06:29.443254+0000","last_fullsized":"2023-09-23T21:06:29.443254+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":698,"reported_epoch":401,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:29.445157+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:06:29.445157+0000","last_peered":"2023-09-23T21:06:29.445157+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:06:29.445157+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":50,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442249+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:06:29.442249+0000","last_peered":"2023-09-23T21:06:29.442249+0000","last_clean":"2023-09-23T21:06:29.442249+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:06:29.442249+0000","last_undegraded":"2023-09-23T21:06:29.442249+0000","last_fullsized":"2023-09-23T21:06:29.442249+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":290,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.727049+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:06:30.727049+0000","last_peered":"2023-09-23T21:06:30.727049+0000","last_clean":"2023-09-23T21:06:30.727049+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:06:30.727049+0000","last_undegraded":"2023-09-23T21:06:30.727049+0000","last_fullsized":"2023-09-23T21:06:30.727049+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"402'5854","reported_seq":5958,"reported_epoch":402,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:40.621366+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:06:40.621366+0000","last_peered":"2023-09-23T21:06:40.621366+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:06:40.621366+0000","last_undegraded":"2023-09-23T21:06:40.621366+0000","last_fullsized":"2023-09-23T21:06:40.621366+0000","mapping_epoch":374,"log_start":"349'1705","ondisk_log_start":"349'1705","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4149,"log_dups_size":0,"ondisk_log_size":4149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10536,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5854,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":5268}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":33,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.445024+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:06:29.445024+0000","last_peered":"2023-09-23T21:06:29.445024+0000","last_clean":"2023-09-23T21:06:29.445024+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:06:29.445024+0000","last_undegraded":"2023-09-23T21:06:29.445024+0000","last_fullsized":"2023-09-23T21:06:29.445024+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":47,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026616+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:06:30.026616+0000","last_peered":"2023-09-23T21:06:30.026616+0000","last_clean":"2023-09-23T21:06:30.026616+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:06:30.026616+0000","last_undegraded":"2023-09-23T21:06:30.026616+0000","last_fullsized":"2023-09-23T21:06:30.026616+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":38,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026768+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:06:30.026768+0000","last_peered":"2023-09-23T21:06:30.026768+0000","last_clean":"2023-09-23T21:06:30.026768+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:06:30.026768+0000","last_undegraded":"2023-09-23T21:06:30.026768+0000","last_fullsized":"2023-09-23T21:06:30.026768+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"402'5745","reported_seq":9354,"reported_epoch":402,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:40.250929+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:06:40.250929+0000","last_peered":"2023-09-23T21:06:40.250929+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:06:40.250929+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:06:40.250929+0000","mapping_epoch":383,"log_start":"347'1026","ondisk_log_start":"347'1026","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346095616,"num_objects":5281,"num_object_clones":0,"num_object_copies":15843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":1530,"num_objects_unfound":0,"num_objects_dirty":5281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5745,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":1530},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":38,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026155+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:06:30.026155+0000","last_peered":"2023-09-23T21:06:30.026155+0000","last_clean":"2023-09-23T21:06:30.026155+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:06:30.026155+0000","last_undegraded":"2023-09-23T21:06:30.026155+0000","last_fullsized":"2023-09-23T21:06:30.026155+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":298,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451592+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:06:29.451592+0000","last_peered":"2023-09-23T21:06:29.451592+0000","last_clean":"2023-09-23T21:06:29.451592+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:06:29.451592+0000","last_undegraded":"2023-09-23T21:06:29.451592+0000","last_fullsized":"2023-09-23T21:06:29.451592+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":77,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.449900+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:06:29.449900+0000","last_peered":"2023-09-23T21:06:29.449900+0000","last_clean":"2023-09-23T21:06:29.449900+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:06:29.449900+0000","last_undegraded":"2023-09-23T21:06:29.449900+0000","last_fullsized":"2023-09-23T21:06:29.449900+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"402'5717","reported_seq":18597,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T21:06:40.279392+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:06:40.279392+0000","last_peered":"2023-09-23T21:06:40.279392+0000","last_clean":"2023-09-23T21:06:40.279392+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:06:40.279392+0000","last_undegraded":"2023-09-23T21:06:40.279392+0000","last_fullsized":"2023-09-23T21:06:40.279392+0000","mapping_epoch":397,"log_start":"347'945","ondisk_log_start":"347'945","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4772,"log_dups_size":0,"ondisk_log_size":4772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337313792,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5717,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":58,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442744+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:06:29.442744+0000","last_peered":"2023-09-23T21:06:29.442744+0000","last_clean":"2023-09-23T21:06:29.442744+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:06:29.442744+0000","last_undegraded":"2023-09-23T21:06:29.442744+0000","last_fullsized":"2023-09-23T21:06:29.442744+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.452076+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:06:29.452076+0000","last_peered":"2023-09-23T21:06:29.452076+0000","last_clean":"2023-09-23T21:06:29.452076+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:06:29.452076+0000","last_undegraded":"2023-09-23T21:06:29.452076+0000","last_fullsized":"2023-09-23T21:06:29.452076+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":50,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442143+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:06:29.442143+0000","last_peered":"2023-09-23T21:06:29.442143+0000","last_clean":"2023-09-23T21:06:29.442143+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:06:29.442143+0000","last_undegraded":"2023-09-23T21:06:29.442143+0000","last_fullsized":"2023-09-23T21:06:29.442143+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"402'5846","reported_seq":6447,"reported_epoch":402,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:40.302739+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:06:40.302739+0000","last_peered":"2023-09-23T21:06:40.302739+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:06:40.302739+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:06:40.302739+0000","mapping_epoch":382,"log_start":"347'1112","ondisk_log_start":"347'1112","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4734,"log_dups_size":0,"ondisk_log_size":4734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350748672,"num_objects":5352,"num_object_clones":0,"num_object_copies":16056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5349,"num_objects_misplaced":5355,"num_objects_unfound":0,"num_objects_dirty":5352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5846,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":247,"num_bytes_recovered":16187392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":5349}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":298,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.451849+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:06:29.451849+0000","last_peered":"2023-09-23T21:06:29.451849+0000","last_clean":"2023-09-23T21:06:29.451849+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:06:29.451849+0000","last_undegraded":"2023-09-23T21:06:29.451849+0000","last_fullsized":"2023-09-23T21:06:29.451849+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":42,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026854+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:06:30.026854+0000","last_peered":"2023-09-23T21:06:30.026854+0000","last_clean":"2023-09-23T21:06:30.026854+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:06:30.026854+0000","last_undegraded":"2023-09-23T21:06:30.026854+0000","last_fullsized":"2023-09-23T21:06:30.026854+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.449611+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:06:29.449611+0000","last_peered":"2023-09-23T21:06:29.449611+0000","last_clean":"2023-09-23T21:06:29.449611+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:06:29.449611+0000","last_undegraded":"2023-09-23T21:06:29.449611+0000","last_fullsized":"2023-09-23T21:06:29.449611+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":47,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026536+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:06:30.026536+0000","last_peered":"2023-09-23T21:06:30.026536+0000","last_clean":"2023-09-23T21:06:30.026536+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:06:30.026536+0000","last_undegraded":"2023-09-23T21:06:30.026536+0000","last_fullsized":"2023-09-23T21:06:30.026536+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":424,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443151+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:06:29.443151+0000","last_peered":"2023-09-23T21:06:29.443151+0000","last_clean":"2023-09-23T21:06:29.443151+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:06:29.443151+0000","last_undegraded":"2023-09-23T21:06:29.443151+0000","last_fullsized":"2023-09-23T21:06:29.443151+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":64,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.410042+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:06:30.410042+0000","last_peered":"2023-09-23T21:06:30.410042+0000","last_clean":"2023-09-23T21:06:30.410042+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:06:30.410042+0000","last_undegraded":"2023-09-23T21:06:30.410042+0000","last_fullsized":"2023-09-23T21:06:30.410042+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":67,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442515+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:06:29.442515+0000","last_peered":"2023-09-23T21:06:29.442515+0000","last_clean":"2023-09-23T21:06:29.442515+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:06:29.442515+0000","last_undegraded":"2023-09-23T21:06:29.442515+0000","last_fullsized":"2023-09-23T21:06:29.442515+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":283,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.726905+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:06:30.726905+0000","last_peered":"2023-09-23T21:06:30.726905+0000","last_clean":"2023-09-23T21:06:30.726905+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:06:30.726905+0000","last_undegraded":"2023-09-23T21:06:30.726905+0000","last_fullsized":"2023-09-23T21:06:30.726905+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":59,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.442337+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:06:29.442337+0000","last_peered":"2023-09-23T21:06:29.442337+0000","last_clean":"2023-09-23T21:06:29.442337+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:06:29.442337+0000","last_undegraded":"2023-09-23T21:06:29.442337+0000","last_fullsized":"2023-09-23T21:06:29.442337+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":41,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443838+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:06:29.443838+0000","last_peered":"2023-09-23T21:06:29.443838+0000","last_clean":"2023-09-23T21:06:29.443838+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:06:29.443838+0000","last_undegraded":"2023-09-23T21:06:29.443838+0000","last_fullsized":"2023-09-23T21:06:29.443838+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":50,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:29.443397+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:06:29.443397+0000","last_peered":"2023-09-23T21:06:29.443397+0000","last_clean":"2023-09-23T21:06:29.443397+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:06:29.443397+0000","last_undegraded":"2023-09-23T21:06:29.443397+0000","last_fullsized":"2023-09-23T21:06:29.443397+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":44,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.026935+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:06:30.026935+0000","last_peered":"2023-09-23T21:06:30.026935+0000","last_clean":"2023-09-23T21:06:30.026935+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:06:30.026935+0000","last_undegraded":"2023-09-23T21:06:30.026935+0000","last_fullsized":"2023-09-23T21:06:30.026935+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":29,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T21:06:30.409591+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:06:30.409591+0000","last_peered":"2023-09-23T21:06:30.409591+0000","last_clean":"2023-09-23T21:06:30.409591+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:06:30.409591+0000","last_undegraded":"2023-09-23T21:06:30.409591+0000","last_fullsized":"2023-09-23T21:06:30.409591+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5508431896,"num_objects":84053,"num_object_clones":0,"num_object_copies":252159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":41057,"num_objects_misplaced":57271,"num_objects_unfound":0,"num_objects_dirty":84053,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":92321,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21796,"num_bytes_recovered":1428422656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18165477376,"data_stored":18165465160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72120,"ondisk_log_size":72120,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201735,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463356,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4204024,"kb_used_data":3719344,"kb_used_omap":0,"kb_used_meta":484672,"kb_avail":89541128,"statfs":{"total":95995035648,"available":91690115072,"internally_reserved":0,"allocated":3808608256,"data_stored":3807434443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":496304128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4,6,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.131,"15min":2.1230000000000002},"min":{"1min":0.254,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6689999999999996,"5min":7.2480000000000002,"15min":102.242},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.107,"15min":1.8440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":0.89300000000000002,"5min":9.7110000000000003,"15min":102.502},"last":0.84299999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":3.907,"15min":2.3660000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":0.55200000000000005,"5min":119.077,"15min":119.077},"last":1.121},{"interface":"front","average":{"1min":0.52900000000000003,"5min":3.8039999999999998,"15min":2.4590000000000001},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":2.8740000000000001,"5min":119.313,"15min":119.313},"last":3.4220000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":4.1219999999999999,"15min":2.9380000000000002},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.5910000000000002,"5min":123.283,"15min":176.45400000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":4.1680000000000001,"15min":3.1080000000000001},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":1.3009999999999999,"5min":123.051,"15min":176.46799999999999},"last":1.2}]},{"osd":3,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.456,"15min":4.4130000000000003},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.26900000000000002},"max":{"1min":3.173,"5min":35.078000000000003,"15min":250.804},"last":3.173},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.5409999999999999,"15min":4.3600000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":2.1850000000000001,"5min":35.133000000000003,"15min":250.446},"last":2.1850000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.1579999999999999,"15min":0.93600000000000005},"min":{"1min":0.32000000000000001,"5min":0.27800000000000002,"15min":0.24299999999999999},"max":{"1min":0.878,"5min":13.1,"15min":13.1},"last":0.878},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.91100000000000003,"15min":0.96299999999999997},"min":{"1min":0.36699999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":2.9079999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.50600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.0980000000000001,"15min":1.5289999999999999},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.90700000000000003,"5min":11.281000000000001,"15min":105.121},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.208,"15min":1.585},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":8.6229999999999993,"15min":106.30500000000001},"last":2.9489999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.123,"15min":1.5089999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":0.70899999999999996,"5min":7.093,"15min":101.04600000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.1890000000000001,"15min":1.516},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":0.66800000000000004,"5min":7.1849999999999996,"15min":101.081},"last":0.39200000000000002}]}]},{"osd":1,"up_from":276,"seq":1185410973802,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4303728,"kb_used_data":3969896,"kb_used_omap":5,"kb_used_meta":333754,"kb_avail":89441424,"statfs":{"total":95995035648,"available":91588018176,"internally_reserved":0,"allocated":4065173504,"data_stored":4063187213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":341764835},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.30599999999999999,"5min":0.70399999999999996,"15min":0.48899999999999999},"min":{"1min":0.20100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.47899999999999998,"5min":11.959,"15min":11.959},"last":9.9469999999999992},{"interface":"front","average":{"1min":0.29999999999999999,"5min":1.1319999999999999,"15min":0.76700000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.49099999999999999,"5min":18.047999999999998,"15min":18.047999999999998},"last":3.7679999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":2.2770000000000001,"15min":1.089},"min":{"1min":0.23400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57299999999999995,"5min":78.409000000000006,"15min":78.409000000000006},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.35799999999999998,"5min":1.835,"15min":0.90400000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.22},"max":{"1min":0.56000000000000005,"5min":76.245000000000005,"15min":76.245000000000005},"last":4.8010000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":3.621,"15min":2.4300000000000002},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":0.71499999999999997,"5min":77.75,"15min":172.44399999999999},"last":7.3099999999999996},{"interface":"front","average":{"1min":0.35999999999999999,"5min":4.4020000000000001,"15min":2.5960000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":0.74399999999999999,"5min":77.849000000000004,"15min":172.58500000000001},"last":1.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.6499999999999999,"15min":0.88600000000000001},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66000000000000003,"5min":17.006,"15min":17.006},"last":4.7069999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.86699999999999999,"15min":0.67500000000000004},"min":{"1min":0.379,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68200000000000005,"5min":11.916,"15min":11.916},"last":4.7690000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":1.383,"15min":0.83299999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72599999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":3.827},{"interface":"front","average":{"1min":0.45700000000000002,"5min":1.7869999999999999,"15min":0.95499999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.68799999999999994,"5min":17.076000000000001,"15min":17.076000000000001},"last":1.9510000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":2.8460000000000001,"15min":1.339},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.69499999999999995,"5min":127.995,"15min":127.995},"last":3.8029999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":3.1909999999999998,"15min":1.4239999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68100000000000005,"5min":126.836,"15min":126.836},"last":1.925}]},{"osd":7,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":3.069,"15min":1.4239999999999999},"min":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71299999999999997,"5min":151.791,"15min":151.791},"last":7.048},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.238,"15min":1.171},"min":{"1min":0.42699999999999999,"5min":0.40400000000000003,"15min":0.38},"max":{"1min":0.73799999999999999,"5min":151.809,"15min":151.809},"last":7.3739999999999997}]}]},{"osd":0,"up_from":92,"seq":395136991542,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1393588,"kb_used_data":681004,"kb_used_omap":5,"kb_used_meta":712570,"kb_avail":92351564,"statfs":{"total":95995035648,"available":94568001536,"internally_reserved":0,"allocated":697348096,"data_stored":695471933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":729672314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.6749999999999998,"15min":1.2669999999999999},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.184},"max":{"1min":52.590000000000003,"5min":61.341999999999999,"15min":61.341999999999999},"last":2.9860000000000002},{"interface":"front","average":{"1min":3.165,"5min":2.5579999999999998,"15min":1.266},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":52.276000000000003,"5min":61.299999999999997,"15min":61.299999999999997},"last":3.0600000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":2.778,"15min":2.0939999999999999},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.161},"max":{"1min":0.61399999999999999,"5min":183.77099999999999,"15min":183.77099999999999},"last":23.234000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":3.258,"15min":2.3039999999999998},"min":{"1min":0.254,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.59799999999999998,"5min":183.78700000000001,"15min":183.78700000000001},"last":23.370999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":3.7370000000000001,"15min":3.9489999999999998},"min":{"1min":0.311,"5min":0.25,"15min":0.183},"max":{"1min":0.64700000000000002,"5min":186.143,"15min":186.143},"last":0.23499999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":4.0069999999999997,"15min":4.0380000000000003},"min":{"1min":0.28999999999999998,"5min":0.27800000000000002,"15min":0.23000000000000001},"max":{"1min":0.623,"5min":186.97399999999999,"15min":186.97399999999999},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2969999999999999,"15min":1.379},"min":{"1min":0.34699999999999998,"5min":0.29799999999999999,"15min":0.247},"max":{"1min":1.4350000000000001,"5min":9.7080000000000002,"15min":10.525},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.80900000000000005,"15min":1.022},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.26200000000000001},"max":{"1min":0.67600000000000005,"5min":6.0490000000000004,"15min":12.461},"last":0.31}]},{"osd":5,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.4830000000000001,"15min":2.5819999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28000000000000003},"max":{"1min":1.411,"5min":9.7439999999999998,"15min":154.30600000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.82299999999999995,"15min":1.96},"min":{"1min":0.42699999999999999,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":0.67000000000000004,"5min":5.9950000000000001,"15min":154.42400000000001},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":5.8390000000000004,"15min":2.7610000000000001},"min":{"1min":0.35899999999999999,"5min":0.34599999999999997,"15min":0.31},"max":{"1min":0.64200000000000002,"5min":170.12700000000001,"15min":170.12700000000001},"last":136.404},{"interface":"front","average":{"1min":0.47199999999999998,"5min":6.1609999999999996,"15min":2.8929999999999998},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":0.60799999999999998,"5min":169.767,"15min":169.767},"last":136.44999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:05:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.97799999999999998,"15min":2.52},"min":{"1min":0.40799999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69599999999999995,"5min":6.1130000000000004,"15min":164.26400000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.503,"5min":1.2490000000000001,"15min":2.6099999999999999},"min":{"1min":0.38700000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":0.63900000000000001,"5min":6.5410000000000004,"15min":164.23099999999999},"last":0.39100000000000001}]}]},{"osd":2,"up_from":129,"seq":554050781433,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4150328,"kb_used_data":3573424,"kb_used_omap":7,"kb_used_meta":576760,"kb_avail":89594824,"statfs":{"total":95995035648,"available":91745099776,"internally_reserved":0,"allocated":3659186176,"data_stored":3657248471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":590602321},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.302,"15min":1.3919999999999999},"min":{"1min":0.19400000000000001,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":1.7470000000000001,"5min":15.667,"15min":26.356000000000002},"last":3.8079999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.4419999999999999,"15min":1.556},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.663,"5min":12.445,"15min":26.341000000000001},"last":5.0010000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:05:55 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":5.5549999999999997,"15min":2.343},"min":{"1min":0.22500000000000001,"5min":0.19,"15min":0.188},"max":{"1min":1.9419999999999999,"5min":212.12,"15min":212.12},"last":2.431},{"interface":"front","average":{"1min":0.35399999999999998,"5min":5.5369999999999999,"15min":2.1349999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.79100000000000004,"5min":213.02799999999999,"15min":213.02799999999999},"last":0.88800000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":3.0609999999999999,"15min":3.8239999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":0.47799999999999998,"5min":66.085999999999999,"15min":151.43799999999999},"last":2.0600000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":2.7919999999999998,"15min":3.8559999999999999},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.215},"max":{"1min":0.50900000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":1.4390000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.101,"15min":1.1419999999999999},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.4570000000000001,"5min":12.586,"15min":12.586},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.1870000000000001,"15min":1.1910000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.575,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":4.9329999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.29,"15min":2.5760000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.6379999999999999,"5min":9.9909999999999997,"15min":173.34},"last":4.8860000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.079,"15min":2.3809999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":2.1949999999999998,"5min":13.701000000000001,"15min":173.15000000000001},"last":1.9870000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":3.2869999999999999,"15min":1.883},"min":{"1min":0.34200000000000003,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":2.1789999999999998,"5min":175.72499999999999,"15min":175.72499999999999},"last":2.0209999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":3.3250000000000002,"15min":1.984},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":1.5329999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.6060000000000001,"15min":2.7330000000000001},"min":{"1min":0.41399999999999998,"5min":0.40799999999999997,"15min":0.32700000000000001},"max":{"1min":9.6910000000000007,"5min":12.609,"15min":177.30799999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.355,"15min":2.6520000000000001},"min":{"1min":0.38900000000000001,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.615,"5min":10.016,"15min":177.40899999999999},"last":4.9619999999999997}]}]},{"osd":3,"up_from":148,"seq":635655160027,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":534632,"kb_used_data":7968,"kb_used_omap":2,"kb_used_meta":526653,"kb_avail":93210520,"statfs":{"total":95995035648,"available":95447572480,"internally_reserved":0,"allocated":8159232,"data_stored":6056717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":539292725},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.5580000000000001,"15min":1.7350000000000001},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.18099999999999999},"max":{"1min":3.8570000000000002,"5min":44.963000000000001,"15min":95.507999999999996},"last":3.3999999999999999},{"interface":"front","average":{"1min":0.438,"5min":1.02,"15min":1.698},"min":{"1min":0.219,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":1.2909999999999999,"5min":9.7759999999999998,"15min":95.259},"last":1.2909999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.339,"15min":1.0129999999999999},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.7639999999999993,"5min":12.035,"15min":19.786999999999999},"last":1.667},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.6160000000000001,"15min":1.2110000000000001},"min":{"1min":0.253,"5min":0.182,"15min":0.182},"max":{"1min":2.3159999999999998,"5min":15.763999999999999,"15min":15.763999999999999},"last":3.0960000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.464,"15min":1.2290000000000001},"min":{"1min":0.29699999999999999,"5min":0.223,"15min":0.182},"max":{"1min":13.944000000000001,"5min":13.944000000000001,"15min":13.944000000000001},"last":0.313},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.2470000000000001,"15min":1.2909999999999999},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":4.6349999999999998,"5min":12.096,"15min":19.754999999999999},"last":1.3680000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":2.645,"15min":2.1930000000000001},"min":{"1min":0.34799999999999998,"5min":0.307,"15min":0.222},"max":{"1min":8.1750000000000007,"5min":70.728999999999999,"15min":111.36},"last":3.1499999999999999},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1389999999999998,"15min":1.9019999999999999},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":4.1100000000000003,"5min":70.712000000000003,"15min":112.473},"last":0.35099999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.0740000000000001,"15min":1.5129999999999999},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":1.325,"5min":9.6359999999999992,"15min":64.528000000000006},"last":1.325},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.79,"15min":1.6220000000000001},"min":{"1min":0.42099999999999999,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":8.2490000000000006,"5min":10.617000000000001,"15min":14.054},"last":3.2149999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.857,"15min":1.6919999999999999},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.312},"max":{"1min":8.2780000000000005,"5min":10.52,"15min":19.568000000000001},"last":3.1869999999999998},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.4650000000000001,"15min":1.4750000000000001},"min":{"1min":0.39100000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":4.3920000000000003,"5min":14.356,"15min":15.573},"last":0.39100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":2.1930000000000001,"15min":2.6099999999999999},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":1.3480000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":1.3480000000000001},{"interface":"front","average":{"1min":1.341,"5min":2.8090000000000002,"15min":2.8490000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":8.218,"5min":73.364999999999995,"15min":222.11699999999999},"last":3.2440000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986645,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1954424,"kb_used_data":1406832,"kb_used_omap":0,"kb_used_meta":547584,"kb_avail":91790728,"statfs":{"total":95995035648,"available":93993705472,"internally_reserved":0,"allocated":1440595968,"data_stored":1439421539,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":560726016},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.2869999999999999,"15min":2.782},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.53200000000000003,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.119,"15min":2.8029999999999999},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.223},"max":{"1min":3.8959999999999999,"5min":8.5220000000000002,"15min":144.77000000000001},"last":3.927}]},{"osd":1,"last update":"Sat Sep 23 21:06:07 2023","interfaces":[{"interface":"back","average":{"1min":6.4459999999999997,"5min":6.3929999999999998,"15min":2.6269999999999998},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":118.411,"5min":118.411,"15min":118.411},"last":3.8439999999999999},{"interface":"front","average":{"1min":6.274,"5min":6.1340000000000003,"15min":2.4740000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":118.075,"5min":118.075,"15min":118.075},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":5.8410000000000002,"15min":3.2130000000000001},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":3.452,"5min":164.517,"15min":164.517},"last":2.4590000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":5.665,"15min":3.274},"min":{"1min":0.35599999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":0.56999999999999995,"5min":164.46799999999999,"15min":164.46799999999999},"last":1.0229999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:06:06 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.2709999999999999,"15min":2.2610000000000001},"min":{"1min":0.32500000000000001,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.8420000000000001,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.62},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.2829999999999999,"15min":2.3889999999999998},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.82799999999999996,"5min":7.5579999999999998,"15min":142.34700000000001},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.82499999999999996,"15min":1.0609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.24099999999999999},"max":{"1min":2.9049999999999998,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.875,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29899999999999999,"15min":0.21099999999999999},"max":{"1min":0.58299999999999996,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.77600000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.004,"15min":0.96699999999999997},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":0.70599999999999996,"5min":11.800000000000001,"15min":11.887},"last":2.0790000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.111,"15min":0.92100000000000004},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":2.3620000000000001,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.51500000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":3.0110000000000001,"15min":1.8280000000000001},"min":{"1min":0.42799999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":0.72799999999999998,"5min":145.721,"15min":145.721},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.53600000000000003,"5min":3.282,"15min":1.992},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":0.78100000000000003,"5min":148.45699999999999,"15min":148.45699999999999},"last":3.8879999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725009,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1376932,"kb_used_data":790556,"kb_used_omap":0,"kb_used_meta":586368,"kb_avail":92368220,"statfs":{"total":95995035648,"available":94585057280,"internally_reserved":0,"allocated":809529344,"data_stored":808371115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":600440832},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.78000000000000003,"15min":1.857},"min":{"1min":0.25600000000000001,"5min":0.24199999999999999,"15min":0.19600000000000001},"max":{"1min":0.59299999999999997,"5min":23.600000000000001,"15min":124.873},"last":10.016999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":1.1259999999999999,"15min":1.9930000000000001},"min":{"1min":0.24199999999999999,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":0.61099999999999999,"5min":23.532,"15min":123.554},"last":1.129}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":4.1200000000000001,"15min":1.6699999999999999},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":0.67200000000000004,"5min":137.143,"15min":137.143},"last":5.8879999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":3.8199999999999998,"15min":1.5800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.247},"max":{"1min":0.66200000000000003,"5min":139.73699999999999,"15min":139.73699999999999},"last":5.915}]},{"osd":2,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.5720000000000001,"15min":1.6060000000000001},"min":{"1min":0.28799999999999998,"5min":0.28000000000000003,"15min":0.22500000000000001},"max":{"1min":0.61599999999999999,"5min":103.63800000000001,"15min":103.63800000000001},"last":8.3770000000000007},{"interface":"front","average":{"1min":0.44,"5min":2.8330000000000002,"15min":1.6910000000000001},"min":{"1min":0.307,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":0.64800000000000002,"5min":103.664,"15min":103.664},"last":6.7640000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.609,"15min":4.0869999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.573,"5min":97.349999999999994,"15min":198.33099999999999},"last":8.0860000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":2.4750000000000001,"15min":4.0540000000000003},"min":{"1min":0.40799999999999997,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":97.316999999999993,"15min":198.47300000000001},"last":4.6609999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.69799999999999995,"15min":0.81699999999999995},"min":{"1min":0.33100000000000002,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":0.69899999999999995,"5min":12.739000000000001,"15min":12.739000000000001},"last":1.1719999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.88,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":0.59599999999999997,"5min":10.792,"15min":12.539},"last":1.633}]},{"osd":6,"last update":"Sat Sep 23 21:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":1.702,"15min":1.2150000000000001},"min":{"1min":0.32800000000000001,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":0.57899999999999996,"5min":71.230000000000004,"15min":71.230000000000004},"last":1.276},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.835,"15min":1.3400000000000001},"min":{"1min":0.38900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.73099999999999998,"5min":70.266000000000005,"15min":70.266000000000005},"last":0.377}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.014,"15min":1.2230000000000001},"min":{"1min":0.40799999999999997,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":8.4109999999999996,"15min":31.091999999999999},"last":5.0119999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.65200000000000002,"15min":1.0780000000000001},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":1.8919999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":692125696,"data_stored":692125696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4041015296,"data_stored":4041015296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3644461056,"data_stored":3644456984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2949120,"data_stored":2949120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1428557824,"data_stored":1428553752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":804913152,"data_stored":804913152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3784839168,"data_stored":3784835096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3766616064,"data_stored":3766616064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:06:44.037 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:06:44.130 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:44.247 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:44.257 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:44.404 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:44.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:44.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:44.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:44.559 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:44.646 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:44.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:44.754 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:44.796 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:44.919 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:44.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:44.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:45.025 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:45.063 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:45.133 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:45.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:45.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:06:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:06:44] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.43.0" 2023-09-23T21:06:45.173 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:06:45.173 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:06:45.173 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:06:45.173 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:06:45.173 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:06:45.237 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:45.273 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:45.417 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:45.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:45.487 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:06:45.519 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:45.574 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:45.674 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[211526]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.674 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[211526]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[211526]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[211526]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[211526]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[211526]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 41059/250857 objects degraded (16.367%); 55926/250857 objects misplaced (22.294%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/991038914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[219709]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[219709]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[219709]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[219709]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.675 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[219709]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[219709]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 41059/250857 objects degraded (16.367%); 55926/250857 objects misplaced (22.294%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T21:06:45.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:45 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/991038914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:45.968 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:45 smithi123 ceph-mon[193903]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:45 smithi123 ceph-mon[193903]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T21:06:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:45 smithi123 ceph-mon[193903]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:45 smithi123 ceph-mon[193903]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:45 smithi123 ceph-mon[193903]: failed to encode map e403 with expected crc 2023-09-23T21:06:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:45 smithi123 ceph-mon[193903]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 318 op/s; 41059/250857 objects degraded (16.367%); 55926/250857 objects misplaced (22.294%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T21:06:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:45 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/991038914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:06:46.055 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:46.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:46.163 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:46.190 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:46.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:06:46.495 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:46.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:46.604 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:46.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/991038914' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: 11.2 continuing backfill to osd.3 from (347'488,402'5861] MIN to 402'5861 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.657 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41059/250857 objects degraded (16.367%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:46.660 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:46.759 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:46.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:46.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:46.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:46.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/991038914' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:46.816 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: 11.2 continuing backfill to osd.3 from (347'488,402'5861] MIN to 402'5861 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41059/250857 objects degraded (16.367%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.817 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/991038914' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:06:46.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T21:06:46.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: 11.2 continuing backfill to osd.3 from (347'488,402'5861] MIN to 402'5861 2023-09-23T21:06:46.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: failed to encode map e404 with expected crc 2023-09-23T21:06:46.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41059/250857 objects degraded (16.367%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:46.827 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:46.868 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:46.914 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:47.007 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:47.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:47.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:47.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:47.280 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:47.340 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:06:47.382 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:06:47.406 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:47.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:47.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:47.441 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:06:47.472 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:06:47.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:06:47.521 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:06:47.558 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:47.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:06:47.616 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:06:47.661 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:06:47.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:47 smithi121 ceph-mon[211526]: pgmap v1261: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 35380/245544 objects degraded (14.409%); 59835/245544 objects misplaced (24.368%); 14 MiB/s, 223 objects/s recovering 2023-09-23T21:06:47.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3485285322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:47.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:47 smithi121 ceph-mon[219709]: pgmap v1261: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 35380/245544 objects degraded (14.409%); 59835/245544 objects misplaced (24.368%); 14 MiB/s, 223 objects/s recovering 2023-09-23T21:06:47.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3485285322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:47.673 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.673 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.674 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:06:47.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:47 smithi123 ceph-mon[193903]: pgmap v1261: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 35380/245544 objects degraded (14.409%); 59835/245544 objects misplaced (24.368%); 14 MiB/s, 223 objects/s recovering 2023-09-23T21:06:47.883 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3485285322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:47.990 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:06:47.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:47.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:06:48.095 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:48.122 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.163 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:48.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:48.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:48.266 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:48.291 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.339 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:48.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:48.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:48.442 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:48.467 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.509 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:48.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:48.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:48.612 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:48.662 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:06:48.769 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:48.907 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:48.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:49.019 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:49.074 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:49.212 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:49 smithi123 ceph-mon[193903]: pgmap v1262: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 35380/245544 objects degraded (14.409%); 59835/245544 objects misplaced (24.368%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:06:49.252 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:49.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:49.359 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:49.409 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:49.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:49 smithi121 ceph-mon[219709]: pgmap v1262: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 35380/245544 objects degraded (14.409%); 59835/245544 objects misplaced (24.368%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:06:49.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:49 smithi121 ceph-mon[211526]: pgmap v1262: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 35380/245544 objects degraded (14.409%); 59835/245544 objects misplaced (24.368%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:06:49.414 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:49.480 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:49.560 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:49.560 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:49.588 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:49.664 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:49.775 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:49.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:49.857 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:06:49.857 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 35380/245544 objects degraded (14.409%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1666,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":404,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+remapped+backfill_wait","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":2}],"num_pgs":89,"num_pools":5,"num_objects":81848,"data_bytes":5362948024,"bytes_used":17952051200,"bytes_avail":654013198336,"bytes_total":671965249536,"degraded_objects":35380,"degraded_total":245544,"degraded_ratio":0.14408822858632261,"misplaced_objects":59835,"misplaced_total":245544,"misplaced_ratio":0.24368341315609424,"recovering_objects_per_sec":116,"recovering_bytes_per_sec":7647762,"recovering_keys_per_sec":0,"num_objects_recovered":934,"num_bytes_recovered":61210624,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":278},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":27,"modified":"2023-09-23T21:06:45.024243+0000","services":{"osd":{"daemons":{"summary":"","6":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"2639f328-dcb0-4a01-b6c7-21332e7147ac":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 35s)","progress":0.84269660711288452,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:06:49.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:50.009 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:50.173 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:50.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:50.277 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:50.324 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2799302948' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:06:50.325 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2799302948' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:06:50.335 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:50.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2799302948' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:06:50.655 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:06:50.680 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:06:51.147 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:51.400 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:51 smithi123 ceph-mon[193903]: pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 34916/238527 objects degraded (14.638%); 63089/238527 objects misplaced (26.449%); 11 MiB/s, 173 objects/s recovering 2023-09-23T21:06:51.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:51 smithi121 ceph-mon[219709]: pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 34916/238527 objects degraded (14.638%); 63089/238527 objects misplaced (26.449%); 11 MiB/s, 173 objects/s recovering 2023-09-23T21:06:51.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:51 smithi121 ceph-mon[211526]: pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 34916/238527 objects degraded (14.638%); 63089/238527 objects misplaced (26.449%); 11 MiB/s, 173 objects/s recovering 2023-09-23T21:06:51.806 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:06:51.806 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:06:51.855 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:51.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:51.976 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:52.104 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:52.139 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:52.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 34916/238527 objects degraded (14.638%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 34916/238527 objects degraded (14.638%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 34916/238527 objects degraded (14.638%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:52.503 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:52.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:52.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:52.614 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:52.666 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:52.888 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:52.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:52.996 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:53.040 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:53.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/809719905' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:53 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:53 smithi121 ceph-mon[211526]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 34916/238527 objects degraded (14.638%); 63089/238527 objects misplaced (26.449%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T21:06:53.193 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:06:53.224 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:53.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:53.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:53.332 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:53.386 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/809719905' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.386 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:53 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.386 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:53 smithi123 ceph-mon[193903]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 34916/238527 objects degraded (14.638%); 63089/238527 objects misplaced (26.449%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T21:06:53.389 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:53.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/809719905' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:53 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:06:53.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:53 smithi121 ceph-mon[219709]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.9 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 34916/238527 objects degraded (14.638%); 63089/238527 objects misplaced (26.449%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T21:06:53.482 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:53.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:53.592 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:53.627 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:06:53.628 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:06:53.664 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:54.362 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:54.371 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:54.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:54.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:54.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[211526]: failed to encode map e405 with expected crc 2023-09-23T21:06:54.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[211526]: failed to encode map e405 with expected crc 2023-09-23T21:06:54.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:54.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[211526]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T21:06:54.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:54.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[219709]: failed to encode map e405 with expected crc 2023-09-23T21:06:54.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[219709]: failed to encode map e405 with expected crc 2023-09-23T21:06:54.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:54.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:54 smithi121 ceph-mon[219709]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T21:06:54.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:54 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:06:54.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:54 smithi123 ceph-mon[193903]: failed to encode map e405 with expected crc 2023-09-23T21:06:54.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:54 smithi123 ceph-mon[193903]: failed to encode map e405 with expected crc 2023-09-23T21:06:54.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:06:54.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:54 smithi123 ceph-mon[193903]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T21:06:54.492 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:54.537 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:54.769 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:54.776 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:06:54.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:54.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:54.879 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:06:54.950 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:06:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:06:54] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.43.0" 2023-09-23T21:06:54.953 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:54.994 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:06:55.035 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:06:55.095 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:06:55.137 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:06:55.172 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:06:55.183 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4140582804' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:55.183 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:55 smithi123 ceph-mon[193903]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 34916/237183 objects degraded (14.721%); 62659/237183 objects misplaced (26.418%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:06:55.208 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4140582804' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:55.208 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:55 smithi121 ceph-mon[211526]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 34916/237183 objects degraded (14.721%); 62659/237183 objects misplaced (26.418%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:06:55.209 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4140582804' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:06:55.209 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:55 smithi121 ceph-mon[219709]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 34916/237183 objects degraded (14.721%); 62659/237183 objects misplaced (26.418%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:06:55.211 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:06:55.251 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:06:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:55.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:06:55.256 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:06:55.279 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:55.285 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:06:55.287 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.288 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.288 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.288 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.288 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.288 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.288 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.288 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:06:55.356 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:55.429 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:55.538 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:55.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:55.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:55.645 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:55.691 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:55.803 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:55.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:55.907 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:06:55.969 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:56.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:06:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:56.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:06:56.215 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:56.249 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:56.326 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:56.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:56.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:56.411 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:06:56.430 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:56.548 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:56.712 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:56.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 34916/237183 objects degraded (14.721%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:56.730 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 34916/237183 objects degraded (14.721%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:56.730 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 34916/237183 objects degraded (14.721%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:06:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:56.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:56.832 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:56.917 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:57.017 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:57.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:57.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:57.127 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:06:57.167 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:57.249 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:06:57.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:57.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:06:57.354 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:06:57.388 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:57.398 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:57.475 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:06:57.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:57.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:06:57.535 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:06:57.535 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:06:57.535 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:06:57.535 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:06:57.535 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:06:57.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:06:57.682 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:06:57.687 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:06:57.689 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:57.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:57 smithi123 ceph-mon[193903]: pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34446/231156 objects degraded (14.902%); 61127/231156 objects misplaced (26.444%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:57.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:57.762 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:57 smithi121 ceph-mon[211526]: pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34446/231156 objects degraded (14.902%); 61127/231156 objects misplaced (26.444%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:57.762 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:57.762 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:57 smithi121 ceph-mon[219709]: pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34446/231156 objects degraded (14.902%); 61127/231156 objects misplaced (26.444%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:57.763 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:06:58.587 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:06:58.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:06:58.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:58 smithi121 ceph-mon[219709]: from='client.46448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:58.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:58 smithi121 ceph-mon[211526]: from='client.46448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:58.926 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:06:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:58.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:06:58.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:58 smithi123 ceph-mon[193903]: from='client.46448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:06:59.035 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:06:59.079 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:59.114 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:06:59.389 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1267,"stamp":"2023-09-23T21:06:57.027863+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5048637368,"num_objects":77052,"num_object_clones":0,"num_object_copies":231156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34446,"num_objects_misplaced":61127,"num_objects_unfound":0,"num_objects_dirty":77052,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":99646,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23772,"num_bytes_recovered":1553243404,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73461,"ondisk_log_size":73461,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":333,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16864096,"kb_used_data":13020264,"kb_used_omap":21,"kb_used_meta":3843498,"kb_avail":639351968,"statfs":{"total":671965249536,"available":654696415232,"internally_reserved":0,"allocated":13332750336,"data_stored":13321233854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3935742435},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-314310656,"num_objects":-4796,"num_object_clones":0,"num_object_copies":-14388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-934,"num_objects_misplaced":1292,"num_objects_unfound":0,"num_objects_dirty":-4796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4796,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":925,"num_bytes_recovered":60620800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003183"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":175,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.484180+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:06:45.484180+0000","last_peered":"2023-09-23T21:06:45.484180+0000","last_clean":"2023-09-23T21:06:45.484180+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:06:45.484180+0000","last_undegraded":"2023-09-23T21:06:45.484180+0000","last_fullsized":"2023-09-23T21:06:45.484180+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":44,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482168+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:06:45.482168+0000","last_peered":"2023-09-23T21:06:45.482168+0000","last_clean":"2023-09-23T21:06:45.482168+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:06:45.482168+0000","last_undegraded":"2023-09-23T21:06:45.482168+0000","last_fullsized":"2023-09-23T21:06:45.482168+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":317,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.483788+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:06:45.483788+0000","last_peered":"2023-09-23T21:06:45.483788+0000","last_clean":"2023-09-23T21:06:45.483788+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:06:45.483788+0000","last_undegraded":"2023-09-23T21:06:45.483788+0000","last_fullsized":"2023-09-23T21:06:45.483788+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":41,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481988+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:06:45.481988+0000","last_peered":"2023-09-23T21:06:45.481988+0000","last_clean":"2023-09-23T21:06:45.481988+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:06:45.481988+0000","last_undegraded":"2023-09-23T21:06:45.481988+0000","last_fullsized":"2023-09-23T21:06:45.481988+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":62,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481487+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:06:45.481487+0000","last_peered":"2023-09-23T21:06:45.481487+0000","last_clean":"2023-09-23T21:06:45.481487+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:06:45.481487+0000","last_undegraded":"2023-09-23T21:06:45.481487+0000","last_fullsized":"2023-09-23T21:06:45.481487+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":271,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.489655+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:06:45.489655+0000","last_peered":"2023-09-23T21:06:45.489655+0000","last_clean":"2023-09-23T21:06:45.489655+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:06:45.489655+0000","last_undegraded":"2023-09-23T21:06:45.489655+0000","last_fullsized":"2023-09-23T21:06:45.489655+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":41,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.498646+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:06:45.498646+0000","last_peered":"2023-09-23T21:06:45.498646+0000","last_clean":"2023-09-23T21:06:45.498646+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:06:45.498646+0000","last_undegraded":"2023-09-23T21:06:45.498646+0000","last_fullsized":"2023-09-23T21:06:45.498646+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":44,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481405+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:06:45.481405+0000","last_peered":"2023-09-23T21:06:45.481405+0000","last_clean":"2023-09-23T21:06:45.481405+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:06:45.481405+0000","last_undegraded":"2023-09-23T21:06:45.481405+0000","last_fullsized":"2023-09-23T21:06:45.481405+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.535433+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:06:45.535433+0000","last_peered":"2023-09-23T21:06:45.535433+0000","last_clean":"2023-09-23T21:06:45.535433+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:06:45.535433+0000","last_undegraded":"2023-09-23T21:06:45.535433+0000","last_fullsized":"2023-09-23T21:06:45.535433+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":44,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482085+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:06:45.482085+0000","last_peered":"2023-09-23T21:06:45.482085+0000","last_clean":"2023-09-23T21:06:45.482085+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:06:45.482085+0000","last_undegraded":"2023-09-23T21:06:45.482085+0000","last_fullsized":"2023-09-23T21:06:45.482085+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":60,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482259+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:06:45.482259+0000","last_peered":"2023-09-23T21:06:45.482259+0000","last_clean":"2023-09-23T21:06:45.482259+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:06:45.482259+0000","last_undegraded":"2023-09-23T21:06:45.482259+0000","last_fullsized":"2023-09-23T21:06:45.482259+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":36,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.483836+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:06:45.483836+0000","last_peered":"2023-09-23T21:06:45.483836+0000","last_clean":"2023-09-23T21:06:45.483836+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:06:45.483836+0000","last_undegraded":"2023-09-23T21:06:45.483836+0000","last_fullsized":"2023-09-23T21:06:45.483836+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":351,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.489510+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:06:45.489510+0000","last_peered":"2023-09-23T21:06:45.489510+0000","last_clean":"2023-09-23T21:06:45.489510+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:06:45.489510+0000","last_undegraded":"2023-09-23T21:06:45.489510+0000","last_fullsized":"2023-09-23T21:06:45.489510+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":323,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.536888+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:06:45.536888+0000","last_peered":"2023-09-23T21:06:45.536888+0000","last_clean":"2023-09-23T21:06:45.536888+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:06:45.536888+0000","last_undegraded":"2023-09-23T21:06:45.536888+0000","last_fullsized":"2023-09-23T21:06:45.536888+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":95,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:49.156914+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:06:49.156914+0000","last_peered":"2023-09-23T21:06:49.156914+0000","last_clean":"2023-09-23T21:06:49.156914+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:06:49.156914+0000","last_undegraded":"2023-09-23T21:06:49.156914+0000","last_fullsized":"2023-09-23T21:06:49.156914+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":53,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482283+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:06:45.482283+0000","last_peered":"2023-09-23T21:06:45.482283+0000","last_clean":"2023-09-23T21:06:45.482283+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:06:45.482283+0000","last_undegraded":"2023-09-23T21:06:45.482283+0000","last_fullsized":"2023-09-23T21:06:45.482283+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.499581+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:06:45.499581+0000","last_peered":"2023-09-23T21:06:45.499581+0000","last_clean":"2023-09-23T21:06:45.499581+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:06:45.499581+0000","last_undegraded":"2023-09-23T21:06:45.499581+0000","last_fullsized":"2023-09-23T21:06:45.499581+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":62,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482355+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:06:45.482355+0000","last_peered":"2023-09-23T21:06:45.482355+0000","last_clean":"2023-09-23T21:06:45.482355+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:06:45.482355+0000","last_undegraded":"2023-09-23T21:06:45.482355+0000","last_fullsized":"2023-09-23T21:06:45.482355+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":205,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.536821+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:06:45.536821+0000","last_peered":"2023-09-23T21:06:45.536821+0000","last_clean":"2023-09-23T21:06:45.536821+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:06:45.536821+0000","last_undegraded":"2023-09-23T21:06:45.536821+0000","last_fullsized":"2023-09-23T21:06:45.536821+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":41,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482066+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:06:45.482066+0000","last_peered":"2023-09-23T21:06:45.482066+0000","last_clean":"2023-09-23T21:06:45.482066+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:06:45.482066+0000","last_undegraded":"2023-09-23T21:06:45.482066+0000","last_fullsized":"2023-09-23T21:06:45.482066+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":401,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.486821+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:06:45.486821+0000","last_peered":"2023-09-23T21:06:45.486821+0000","last_clean":"2023-09-23T21:06:45.486821+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:06:45.486821+0000","last_undegraded":"2023-09-23T21:06:45.486821+0000","last_fullsized":"2023-09-23T21:06:45.486821+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":44,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.499645+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:06:45.499645+0000","last_peered":"2023-09-23T21:06:45.499645+0000","last_clean":"2023-09-23T21:06:45.499645+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:06:45.499645+0000","last_undegraded":"2023-09-23T21:06:45.499645+0000","last_fullsized":"2023-09-23T21:06:45.499645+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"405'6398","reported_seq":7759,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:55.311023+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:06:55.311023+0000","last_peered":"2023-09-23T21:06:55.311023+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:06:55.311023+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"349'1628","ondisk_log_start":"349'1628","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316669952,"num_objects":4832,"num_object_clones":0,"num_object_copies":14496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6398,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-783},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":4831}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":109,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.537210+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:06:45.537210+0000","last_peered":"2023-09-23T21:06:45.537210+0000","last_clean":"2023-09-23T21:06:45.537210+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:06:45.537210+0000","last_undegraded":"2023-09-23T21:06:45.537210+0000","last_fullsized":"2023-09-23T21:06:45.537210+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.535040+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:06:45.535040+0000","last_peered":"2023-09-23T21:06:45.535040+0000","last_clean":"2023-09-23T21:06:45.535040+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:06:45.535040+0000","last_undegraded":"2023-09-23T21:06:45.535040+0000","last_fullsized":"2023-09-23T21:06:45.535040+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"405'6079","reported_seq":6152,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:55.307859+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:06:55.307859+0000","last_peered":"2023-09-23T21:06:55.307859+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:06:55.307859+0000","last_undegraded":"2023-09-23T21:06:55.307859+0000","last_fullsized":"2023-09-23T21:06:55.307859+0000","mapping_epoch":375,"log_start":"348'1312","ondisk_log_start":"348'1312","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4767,"log_dups_size":0,"ondisk_log_size":4767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309264384,"num_objects":4719,"num_object_clones":0,"num_object_copies":14157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4719,"num_objects_unfound":0,"num_objects_dirty":4719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6079,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":4719}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":102,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.505273+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:06:45.505273+0000","last_peered":"2023-09-23T21:06:45.505273+0000","last_clean":"2023-09-23T21:06:45.505273+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:06:45.505273+0000","last_undegraded":"2023-09-23T21:06:45.505273+0000","last_fullsized":"2023-09-23T21:06:45.505273+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":365,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.483778+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:06:45.483778+0000","last_peered":"2023-09-23T21:06:45.483778+0000","last_clean":"2023-09-23T21:06:45.483778+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:06:45.483778+0000","last_undegraded":"2023-09-23T21:06:45.483778+0000","last_fullsized":"2023-09-23T21:06:45.483778+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"405'6187","reported_seq":10577,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:55.566211+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:06:55.566211+0000","last_peered":"2023-09-23T21:06:55.566211+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:06:55.566211+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"349'2111","ondisk_log_start":"349'2111","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4076,"log_dups_size":0,"ondisk_log_size":4076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311361536,"num_objects":4751,"num_object_clones":0,"num_object_copies":14253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9492,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6187,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1791,"num_bytes_recovered":117374976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":354},{"shards":"1,6,7","objects":2011},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":32,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.535640+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:06:45.535640+0000","last_peered":"2023-09-23T21:06:45.535640+0000","last_clean":"2023-09-23T21:06:45.535640+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:06:45.535640+0000","last_undegraded":"2023-09-23T21:06:45.535640+0000","last_fullsized":"2023-09-23T21:06:45.535640+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":293,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.487170+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:06:45.487170+0000","last_peered":"2023-09-23T21:06:45.487170+0000","last_clean":"2023-09-23T21:06:45.487170+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:06:45.487170+0000","last_undegraded":"2023-09-23T21:06:45.487170+0000","last_fullsized":"2023-09-23T21:06:45.487170+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"405'6227","reported_seq":6849,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:55.635791+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:06:55.635791+0000","last_peered":"2023-09-23T21:06:55.635791+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:06:55.635791+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"349'2168","ondisk_log_start":"349'2168","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4059,"log_dups_size":0,"ondisk_log_size":4059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316080128,"num_objects":4823,"num_object_clones":0,"num_object_copies":14469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6227,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-430},{"shards":"2,6,7","objects":4820}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":44,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482370+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:06:45.482370+0000","last_peered":"2023-09-23T21:06:45.482370+0000","last_clean":"2023-09-23T21:06:45.482370+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:06:45.482370+0000","last_undegraded":"2023-09-23T21:06:45.482370+0000","last_fullsized":"2023-09-23T21:06:45.482370+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":41,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481634+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:06:45.481634+0000","last_peered":"2023-09-23T21:06:45.481634+0000","last_clean":"2023-09-23T21:06:45.481634+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:06:45.481634+0000","last_undegraded":"2023-09-23T21:06:45.481634+0000","last_fullsized":"2023-09-23T21:06:45.481634+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"405'6378","reported_seq":17872,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:55.673627+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:06:55.673627+0000","last_peered":"2023-09-23T21:06:55.673627+0000","last_clean":"2023-09-23T21:06:55.673627+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:06:55.673627+0000","last_undegraded":"2023-09-23T21:06:55.673627+0000","last_fullsized":"2023-09-23T21:06:55.673627+0000","mapping_epoch":400,"log_start":"349'2299","ondisk_log_start":"349'2299","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4079,"log_dups_size":0,"ondisk_log_size":4079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323092480,"num_objects":4930,"num_object_clones":0,"num_object_copies":14790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6378,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":61,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482126+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:06:45.482126+0000","last_peered":"2023-09-23T21:06:45.482126+0000","last_clean":"2023-09-23T21:06:45.482126+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:06:45.482126+0000","last_undegraded":"2023-09-23T21:06:45.482126+0000","last_fullsized":"2023-09-23T21:06:45.482126+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":44,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482185+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:06:45.482185+0000","last_peered":"2023-09-23T21:06:45.482185+0000","last_clean":"2023-09-23T21:06:45.482185+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:06:45.482185+0000","last_undegraded":"2023-09-23T21:06:45.482185+0000","last_fullsized":"2023-09-23T21:06:45.482185+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"405'6118","reported_seq":17224,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:54.126648+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:06:54.126648+0000","last_peered":"2023-09-23T21:06:54.126648+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:06:54.126648+0000","last_undegraded":"2023-09-23T21:06:54.126648+0000","last_fullsized":"2023-09-23T21:06:54.126648+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318767104,"num_objects":4864,"num_object_clones":0,"num_object_copies":14592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4864,"num_objects_unfound":0,"num_objects_dirty":4864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6118,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":4864}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":331,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.487248+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:06:45.487248+0000","last_peered":"2023-09-23T21:06:45.487248+0000","last_clean":"2023-09-23T21:06:45.487248+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:06:45.487248+0000","last_undegraded":"2023-09-23T21:06:45.487248+0000","last_fullsized":"2023-09-23T21:06:45.487248+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":415,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482976+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:06:45.482976+0000","last_peered":"2023-09-23T21:06:45.482976+0000","last_clean":"2023-09-23T21:06:45.482976+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:06:45.482976+0000","last_undegraded":"2023-09-23T21:06:45.482976+0000","last_fullsized":"2023-09-23T21:06:45.482976+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"405'6128","reported_seq":6219,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:55.658654+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:06:55.658654+0000","last_peered":"2023-09-23T21:06:55.658654+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:06:55.658654+0000","last_undegraded":"2023-09-23T21:06:55.658654+0000","last_fullsized":"2023-09-23T21:06:55.658654+0000","mapping_epoch":374,"log_start":"349'1994","ondisk_log_start":"349'1994","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4134,"log_dups_size":0,"ondisk_log_size":4134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312999936,"num_objects":4776,"num_object_clones":0,"num_object_copies":14328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9552,"num_objects_unfound":0,"num_objects_dirty":4776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6128,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4776}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":32,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.536695+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:06:45.536695+0000","last_peered":"2023-09-23T21:06:45.536695+0000","last_clean":"2023-09-23T21:06:45.536695+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:06:45.536695+0000","last_undegraded":"2023-09-23T21:06:45.536695+0000","last_fullsized":"2023-09-23T21:06:45.536695+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":444,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.489751+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:06:45.489751+0000","last_peered":"2023-09-23T21:06:45.489751+0000","last_clean":"2023-09-23T21:06:45.489751+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:06:45.489751+0000","last_undegraded":"2023-09-23T21:06:45.489751+0000","last_fullsized":"2023-09-23T21:06:45.489751+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"405'6171","reported_seq":6348,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:55.290520+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:06:55.290520+0000","last_peered":"2023-09-23T21:06:55.290520+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:06:55.290520+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"348'1394","ondisk_log_start":"348'1394","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4777,"log_dups_size":0,"ondisk_log_size":4777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313982976,"num_objects":4791,"num_object_clones":0,"num_object_copies":14373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6171,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-648},{"shards":"1,6,7","objects":4791}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.173503+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:06:53.173503+0000","last_peered":"2023-09-23T21:06:53.173503+0000","last_clean":"2023-09-23T21:06:53.173503+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:06:53.173503+0000","last_undegraded":"2023-09-23T21:06:53.173503+0000","last_fullsized":"2023-09-23T21:06:53.173503+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":192,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.503042+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:06:45.503042+0000","last_peered":"2023-09-23T21:06:45.503042+0000","last_clean":"2023-09-23T21:06:45.503042+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:06:45.503042+0000","last_undegraded":"2023-09-23T21:06:45.503042+0000","last_fullsized":"2023-09-23T21:06:45.503042+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"405'6189","reported_seq":11678,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:54.042173+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:06:54.042173+0000","last_peered":"2023-09-23T21:06:54.042173+0000","last_clean":"2023-09-23T21:06:54.042173+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:06:54.042173+0000","last_undegraded":"2023-09-23T21:06:54.042173+0000","last_fullsized":"2023-09-23T21:06:54.042173+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317194264,"num_objects":4841,"num_object_clones":0,"num_object_copies":14523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4841,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6189,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":299,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.501515+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:06:45.501515+0000","last_peered":"2023-09-23T21:06:45.501515+0000","last_clean":"2023-09-23T21:06:45.501515+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:06:45.501515+0000","last_undegraded":"2023-09-23T21:06:45.501515+0000","last_fullsized":"2023-09-23T21:06:45.501515+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":70,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.535719+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:06:45.535719+0000","last_peered":"2023-09-23T21:06:45.535719+0000","last_clean":"2023-09-23T21:06:45.535719+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:06:45.535719+0000","last_undegraded":"2023-09-23T21:06:45.535719+0000","last_fullsized":"2023-09-23T21:06:45.535719+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.483911+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:06:45.483911+0000","last_peered":"2023-09-23T21:06:45.483911+0000","last_clean":"2023-09-23T21:06:45.483911+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:06:45.483911+0000","last_undegraded":"2023-09-23T21:06:45.483911+0000","last_fullsized":"2023-09-23T21:06:45.483911+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"405'6196","reported_seq":7722,"reported_epoch":405,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:06:55.680317+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:06:55.680317+0000","last_peered":"2023-09-23T21:06:55.680317+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:06:55.680317+0000","last_undegraded":"2023-09-23T21:06:55.680317+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"349'2057","ondisk_log_start":"349'2057","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4139,"log_dups_size":0,"ondisk_log_size":4139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316538880,"num_objects":4830,"num_object_clones":0,"num_object_copies":14490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11959,"num_objects_unfound":0,"num_objects_dirty":4830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6196,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":700,"num_bytes_recovered":45875200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":2480},{"shards":"2,6","objects":17},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":380,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.501054+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:06:45.501054+0000","last_peered":"2023-09-23T21:06:45.501054+0000","last_clean":"2023-09-23T21:06:45.501054+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:06:45.501054+0000","last_undegraded":"2023-09-23T21:06:45.501054+0000","last_fullsized":"2023-09-23T21:06:45.501054+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":355,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.493664+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:06:45.493664+0000","last_peered":"2023-09-23T21:06:45.493664+0000","last_clean":"2023-09-23T21:06:45.493664+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:06:45.493664+0000","last_undegraded":"2023-09-23T21:06:45.493664+0000","last_fullsized":"2023-09-23T21:06:45.493664+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":36,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.501361+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:06:45.501361+0000","last_peered":"2023-09-23T21:06:45.501361+0000","last_clean":"2023-09-23T21:06:45.501361+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:06:45.501361+0000","last_undegraded":"2023-09-23T21:06:45.501361+0000","last_fullsized":"2023-09-23T21:06:45.501361+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"405'6255","reported_seq":6429,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:54.105497+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:06:54.105497+0000","last_peered":"2023-09-23T21:06:54.105497+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:06:54.105497+0000","last_undegraded":"2023-09-23T21:06:54.105497+0000","last_fullsized":"2023-09-23T21:06:54.105497+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315031552,"num_objects":4807,"num_object_clones":0,"num_object_copies":14421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9583,"num_objects_unfound":0,"num_objects_dirty":4807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6255,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":4807}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":381,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482899+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:06:45.482899+0000","last_peered":"2023-09-23T21:06:45.482899+0000","last_clean":"2023-09-23T21:06:45.482899+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:06:45.482899+0000","last_undegraded":"2023-09-23T21:06:45.482899+0000","last_fullsized":"2023-09-23T21:06:45.482899+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":299,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.505038+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:06:45.505038+0000","last_peered":"2023-09-23T21:06:45.505038+0000","last_clean":"2023-09-23T21:06:45.505038+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:06:45.505038+0000","last_undegraded":"2023-09-23T21:06:45.505038+0000","last_fullsized":"2023-09-23T21:06:45.505038+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":36,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.502962+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:06:45.502962+0000","last_peered":"2023-09-23T21:06:45.502962+0000","last_clean":"2023-09-23T21:06:45.502962+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:06:45.502962+0000","last_undegraded":"2023-09-23T21:06:45.502962+0000","last_fullsized":"2023-09-23T21:06:45.502962+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"405'6109","reported_seq":12360,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:55.532095+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:06:55.532095+0000","last_peered":"2023-09-23T21:06:55.532095+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:06:55.532095+0000","last_undegraded":"2023-09-23T21:06:55.532095+0000","last_fullsized":"2023-09-23T21:06:55.532095+0000","mapping_epoch":387,"log_start":"349'1980","ondisk_log_start":"349'1980","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312279040,"num_objects":4765,"num_object_clones":0,"num_object_copies":14295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4765,"num_objects_unfound":0,"num_objects_dirty":4765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6109,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4765}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":219,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.500401+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:06:45.500401+0000","last_peered":"2023-09-23T21:06:45.500401+0000","last_clean":"2023-09-23T21:06:45.500401+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:06:45.500401+0000","last_undegraded":"2023-09-23T21:06:45.500401+0000","last_fullsized":"2023-09-23T21:06:45.500401+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":701,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:06:45.483789+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:06:45.483789+0000","last_peered":"2023-09-23T21:06:45.483789+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:06:45.483789+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":54,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:49.872823+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:06:49.872823+0000","last_peered":"2023-09-23T21:06:49.872823+0000","last_clean":"2023-09-23T21:06:49.872823+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:06:49.872823+0000","last_undegraded":"2023-09-23T21:06:49.872823+0000","last_fullsized":"2023-09-23T21:06:49.872823+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":293,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.493972+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:06:45.493972+0000","last_peered":"2023-09-23T21:06:45.493972+0000","last_clean":"2023-09-23T21:06:45.493972+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:06:45.493972+0000","last_undegraded":"2023-09-23T21:06:45.493972+0000","last_fullsized":"2023-09-23T21:06:45.493972+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"405'6276","reported_seq":6382,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:06:55.662772+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:06:55.662772+0000","last_peered":"2023-09-23T21:06:55.662772+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:06:55.662772+0000","last_undegraded":"2023-09-23T21:06:55.662772+0000","last_fullsized":"2023-09-23T21:06:55.662772+0000","mapping_epoch":374,"log_start":"349'2205","ondisk_log_start":"349'2205","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4071,"log_dups_size":0,"ondisk_log_size":4071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317587456,"num_objects":4846,"num_object_clones":0,"num_object_copies":14538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9692,"num_objects_unfound":0,"num_objects_dirty":4846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6276,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":4846}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":36,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.483661+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:06:45.483661+0000","last_peered":"2023-09-23T21:06:45.483661+0000","last_clean":"2023-09-23T21:06:45.483661+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:06:45.483661+0000","last_undegraded":"2023-09-23T21:06:45.483661+0000","last_fullsized":"2023-09-23T21:06:45.483661+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":50,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481392+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:06:45.481392+0000","last_peered":"2023-09-23T21:06:45.481392+0000","last_clean":"2023-09-23T21:06:45.481392+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:06:45.481392+0000","last_undegraded":"2023-09-23T21:06:45.481392+0000","last_fullsized":"2023-09-23T21:06:45.481392+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":41,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481311+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:06:45.481311+0000","last_peered":"2023-09-23T21:06:45.481311+0000","last_clean":"2023-09-23T21:06:45.481311+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:06:45.481311+0000","last_undegraded":"2023-09-23T21:06:45.481311+0000","last_fullsized":"2023-09-23T21:06:45.481311+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"405'6184","reported_seq":9797,"reported_epoch":405,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:55.268386+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:06:55.268386+0000","last_peered":"2023-09-23T21:06:55.268386+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:06:55.268386+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:06:55.268386+0000","mapping_epoch":383,"log_start":"349'1426","ondisk_log_start":"349'1426","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4758,"log_dups_size":0,"ondisk_log_size":4758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317325312,"num_objects":4842,"num_object_clones":0,"num_object_copies":14526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":1091,"num_objects_unfound":0,"num_objects_dirty":4842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6184,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":1091},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":41,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481595+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:06:45.481595+0000","last_peered":"2023-09-23T21:06:45.481595+0000","last_clean":"2023-09-23T21:06:45.481595+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:06:45.481595+0000","last_undegraded":"2023-09-23T21:06:45.481595+0000","last_fullsized":"2023-09-23T21:06:45.481595+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":301,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.503345+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:06:45.503345+0000","last_peered":"2023-09-23T21:06:45.503345+0000","last_clean":"2023-09-23T21:06:45.503345+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:06:45.503345+0000","last_undegraded":"2023-09-23T21:06:45.503345+0000","last_fullsized":"2023-09-23T21:06:45.503345+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":80,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.535511+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:06:45.535511+0000","last_peered":"2023-09-23T21:06:45.535511+0000","last_clean":"2023-09-23T21:06:45.535511+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:06:45.535511+0000","last_undegraded":"2023-09-23T21:06:45.535511+0000","last_fullsized":"2023-09-23T21:06:45.535511+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"405'6151","reported_seq":19034,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:55.330671+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:06:55.330671+0000","last_peered":"2023-09-23T21:06:55.330671+0000","last_clean":"2023-09-23T21:06:55.330671+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:06:55.330671+0000","last_undegraded":"2023-09-23T21:06:55.330671+0000","last_fullsized":"2023-09-23T21:06:55.330671+0000","mapping_epoch":397,"log_start":"349'1445","ondisk_log_start":"349'1445","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4706,"log_dups_size":0,"ondisk_log_size":4706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308871168,"num_objects":4713,"num_object_clones":0,"num_object_copies":14139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6151,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":62,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.179005+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:06:53.179005+0000","last_peered":"2023-09-23T21:06:53.179005+0000","last_clean":"2023-09-23T21:06:53.179005+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:06:53.179005+0000","last_undegraded":"2023-09-23T21:06:53.179005+0000","last_fullsized":"2023-09-23T21:06:53.179005+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":36,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.504861+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:06:45.504861+0000","last_peered":"2023-09-23T21:06:45.504861+0000","last_clean":"2023-09-23T21:06:45.504861+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:06:45.504861+0000","last_undegraded":"2023-09-23T21:06:45.504861+0000","last_fullsized":"2023-09-23T21:06:45.504861+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":54,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481236+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:06:45.481236+0000","last_peered":"2023-09-23T21:06:45.481236+0000","last_clean":"2023-09-23T21:06:45.481236+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:06:45.481236+0000","last_undegraded":"2023-09-23T21:06:45.481236+0000","last_fullsized":"2023-09-23T21:06:45.481236+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"405'6300","reported_seq":7814,"reported_epoch":405,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:06:55.333249+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:06:55.333249+0000","last_peered":"2023-09-23T21:06:55.333249+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:06:55.333249+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:06:55.333249+0000","mapping_epoch":382,"log_start":"349'1612","ondisk_log_start":"349'1612","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320995328,"num_objects":4898,"num_object_clones":0,"num_object_copies":14694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4894,"num_objects_misplaced":4902,"num_objects_unfound":0,"num_objects_dirty":4898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6300,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":702,"num_bytes_recovered":46006272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":4},{"shards":"1,6,7","objects":4894}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":301,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.489667+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:06:45.489667+0000","last_peered":"2023-09-23T21:06:45.489667+0000","last_clean":"2023-09-23T21:06:45.489667+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:06:45.489667+0000","last_undegraded":"2023-09-23T21:06:45.489667+0000","last_fullsized":"2023-09-23T21:06:45.489667+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":45,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481737+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:06:45.481737+0000","last_peered":"2023-09-23T21:06:45.481737+0000","last_clean":"2023-09-23T21:06:45.481737+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:06:45.481737+0000","last_undegraded":"2023-09-23T21:06:45.481737+0000","last_fullsized":"2023-09-23T21:06:45.481737+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":90,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.535079+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:06:45.535079+0000","last_peered":"2023-09-23T21:06:45.535079+0000","last_clean":"2023-09-23T21:06:45.535079+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:06:45.535079+0000","last_undegraded":"2023-09-23T21:06:45.535079+0000","last_fullsized":"2023-09-23T21:06:45.535079+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":50,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481565+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:06:45.481565+0000","last_peered":"2023-09-23T21:06:45.481565+0000","last_clean":"2023-09-23T21:06:45.481565+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:06:45.481565+0000","last_undegraded":"2023-09-23T21:06:45.481565+0000","last_fullsized":"2023-09-23T21:06:45.481565+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":428,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.166075+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:06:53.166075+0000","last_peered":"2023-09-23T21:06:53.166075+0000","last_clean":"2023-09-23T21:06:53.166075+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:06:53.166075+0000","last_undegraded":"2023-09-23T21:06:53.166075+0000","last_fullsized":"2023-09-23T21:06:53.166075+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":67,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.537026+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:06:45.537026+0000","last_peered":"2023-09-23T21:06:45.537026+0000","last_clean":"2023-09-23T21:06:45.537026+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:06:45.537026+0000","last_undegraded":"2023-09-23T21:06:45.537026+0000","last_fullsized":"2023-09-23T21:06:45.537026+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":70,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482171+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:06:45.482171+0000","last_peered":"2023-09-23T21:06:45.482171+0000","last_clean":"2023-09-23T21:06:45.482171+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:06:45.482171+0000","last_undegraded":"2023-09-23T21:06:45.482171+0000","last_fullsized":"2023-09-23T21:06:45.482171+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":286,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.495451+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:06:45.495451+0000","last_peered":"2023-09-23T21:06:45.495451+0000","last_clean":"2023-09-23T21:06:45.495451+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:06:45.495451+0000","last_undegraded":"2023-09-23T21:06:45.495451+0000","last_fullsized":"2023-09-23T21:06:45.495451+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":62,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.499653+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:06:45.499653+0000","last_peered":"2023-09-23T21:06:45.499653+0000","last_clean":"2023-09-23T21:06:45.499653+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:06:45.499653+0000","last_undegraded":"2023-09-23T21:06:45.499653+0000","last_fullsized":"2023-09-23T21:06:45.499653+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":45,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.499558+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:06:45.499558+0000","last_peered":"2023-09-23T21:06:45.499558+0000","last_clean":"2023-09-23T21:06:45.499558+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:06:45.499558+0000","last_undegraded":"2023-09-23T21:06:45.499558+0000","last_fullsized":"2023-09-23T21:06:45.499558+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":53,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.481867+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:06:45.481867+0000","last_peered":"2023-09-23T21:06:45.481867+0000","last_clean":"2023-09-23T21:06:45.481867+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:06:45.481867+0000","last_undegraded":"2023-09-23T21:06:45.481867+0000","last_fullsized":"2023-09-23T21:06:45.481867+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.482070+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:06:45.482070+0000","last_peered":"2023-09-23T21:06:45.482070+0000","last_clean":"2023-09-23T21:06:45.482070+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:06:45.482070+0000","last_undegraded":"2023-09-23T21:06:45.482070+0000","last_fullsized":"2023-09-23T21:06:45.482070+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":32,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T21:06:45.536553+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:06:45.536553+0000","last_peered":"2023-09-23T21:06:45.536553+0000","last_clean":"2023-09-23T21:06:45.536553+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:06:45.536553+0000","last_undegraded":"2023-09-23T21:06:45.536553+0000","last_fullsized":"2023-09-23T21:06:45.536553+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5048041496,"num_objects":77028,"num_object_clones":0,"num_object_copies":231084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34444,"num_objects_misplaced":61127,"num_objects_unfound":0,"num_objects_dirty":77028,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99346,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23655,"num_bytes_recovered":1550254080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16917278720,"data_stored":16917266504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73245,"ondisk_log_size":73245,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201738,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463359,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3904412,"kb_used_data":3409684,"kb_used_omap":0,"kb_used_meta":494592,"kb_avail":89840740,"statfs":{"total":95995035648,"available":91996917760,"internally_reserved":0,"allocated":3491516416,"data_stored":3490331098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":506462208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.131,"15min":2.1230000000000002},"min":{"1min":0.254,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6689999999999996,"5min":7.2480000000000002,"15min":102.242},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.107,"15min":1.8440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":0.89300000000000002,"5min":9.7110000000000003,"15min":102.502},"last":0.30599999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":3.907,"15min":2.3660000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":0.55200000000000005,"5min":119.077,"15min":119.077},"last":0.255},{"interface":"front","average":{"1min":0.52900000000000003,"5min":3.8039999999999998,"15min":2.4590000000000001},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":2.8740000000000001,"5min":119.313,"15min":119.313},"last":1.593}]},{"osd":2,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":4.1219999999999999,"15min":2.9380000000000002},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.5910000000000002,"5min":123.283,"15min":176.45400000000001},"last":2.4889999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":4.1680000000000001,"15min":3.1080000000000001},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":1.3009999999999999,"5min":123.051,"15min":176.46799999999999},"last":0.68500000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.456,"15min":4.4130000000000003},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.26900000000000002},"max":{"1min":3.173,"5min":35.078000000000003,"15min":250.804},"last":2.5590000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.5409999999999999,"15min":4.3600000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":2.1850000000000001,"5min":35.133000000000003,"15min":250.446},"last":2.3029999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.1579999999999999,"15min":0.93600000000000005},"min":{"1min":0.32000000000000001,"5min":0.27800000000000002,"15min":0.24299999999999999},"max":{"1min":0.878,"5min":13.1,"15min":13.1},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.91100000000000003,"15min":0.96299999999999997},"min":{"1min":0.36699999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":2.9079999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.0980000000000001,"15min":1.5289999999999999},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.90700000000000003,"5min":11.281000000000001,"15min":105.121},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.208,"15min":1.585},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":8.6229999999999993,"15min":106.30500000000001},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.123,"15min":1.5089999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":0.70899999999999996,"5min":7.093,"15min":101.04600000000001},"last":1.623},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.1890000000000001,"15min":1.516},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":0.66800000000000004,"5min":7.1849999999999996,"15min":101.081},"last":0.42099999999999999}]}]},{"osd":1,"up_from":276,"seq":1185410973805,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3995804,"kb_used_data":3661908,"kb_used_omap":5,"kb_used_meta":333754,"kb_avail":89749348,"statfs":{"total":95995035648,"available":91903332352,"internally_reserved":0,"allocated":3749793792,"data_stored":3747788341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":341764835},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.30599999999999999,"5min":0.70399999999999996,"15min":0.48899999999999999},"min":{"1min":0.20100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.47899999999999998,"5min":11.959,"15min":11.959},"last":0.504},{"interface":"front","average":{"1min":0.29999999999999999,"5min":1.1319999999999999,"15min":0.76700000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.49099999999999999,"5min":18.047999999999998,"15min":18.047999999999998},"last":2.339}]},{"osd":2,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":2.2770000000000001,"15min":1.089},"min":{"1min":0.23400000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57299999999999995,"5min":78.409000000000006,"15min":78.409000000000006},"last":2.589},{"interface":"front","average":{"1min":0.35799999999999998,"5min":1.835,"15min":0.90400000000000003},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.22},"max":{"1min":0.56000000000000005,"5min":76.245000000000005,"15min":76.245000000000005},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":3.621,"15min":2.4300000000000002},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":0.71499999999999997,"5min":77.75,"15min":172.44399999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.35999999999999999,"5min":4.4020000000000001,"15min":2.5960000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":0.74399999999999999,"5min":77.849000000000004,"15min":172.58500000000001},"last":5.9299999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.6499999999999999,"15min":0.88600000000000001},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66000000000000003,"5min":17.006,"15min":17.006},"last":2.1160000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.86699999999999999,"15min":0.67500000000000004},"min":{"1min":0.379,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.68200000000000005,"5min":11.916,"15min":11.916},"last":0.58299999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":1.383,"15min":0.83299999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.72599999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":2.375},{"interface":"front","average":{"1min":0.45700000000000002,"5min":1.7869999999999999,"15min":0.95499999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.68799999999999994,"5min":17.076000000000001,"15min":17.076000000000001},"last":2.085}]},{"osd":6,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":2.8460000000000001,"15min":1.339},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.69499999999999995,"5min":127.995,"15min":127.995},"last":2.427},{"interface":"front","average":{"1min":0.47599999999999998,"5min":3.1909999999999998,"15min":1.4239999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.68100000000000005,"5min":126.836,"15min":126.836},"last":2.0539999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:05:56 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":3.069,"15min":1.4239999999999999},"min":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71299999999999997,"5min":151.791,"15min":151.791},"last":2.714},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.238,"15min":1.171},"min":{"1min":0.42699999999999999,"5min":0.40400000000000003,"15min":0.38},"max":{"1min":0.73799999999999999,"5min":151.809,"15min":151.809},"last":6.798}]}]},{"osd":0,"up_from":92,"seq":395136991545,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1388192,"kb_used_data":661656,"kb_used_omap":5,"kb_used_meta":726522,"kb_avail":92356960,"statfs":{"total":95995035648,"available":94573527040,"internally_reserved":0,"allocated":677535744,"data_stored":675640421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":743959162},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.6749999999999998,"15min":1.2669999999999999},"min":{"1min":0.23200000000000001,"5min":0.188,"15min":0.184},"max":{"1min":52.590000000000003,"5min":61.341999999999999,"15min":61.341999999999999},"last":1.175},{"interface":"front","average":{"1min":3.165,"5min":2.5579999999999998,"15min":1.266},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":52.276000000000003,"5min":61.299999999999997,"15min":61.299999999999997},"last":1.345}]},{"osd":2,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.7110000000000001,"15min":2.238},"min":{"1min":0.217,"5min":0.217,"15min":0.161},"max":{"1min":41.860999999999997,"5min":41.860999999999997,"15min":183.77099999999999},"last":0.432},{"interface":"front","average":{"1min":4.5780000000000003,"5min":2.1219999999999999,"15min":2.5070000000000001},"min":{"1min":0.27000000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":44.779000000000003,"5min":44.779000000000003,"15min":183.78700000000001},"last":0.93999999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.1319999999999999,"15min":3.3079999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":5.5810000000000004,"5min":11.555999999999999,"15min":186.143},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.3320000000000001,"15min":3.4380000000000002},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.23000000000000001},"max":{"1min":4.016,"5min":16.640000000000001,"15min":186.97399999999999},"last":1.022}]},{"osd":4,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.2090000000000001,"15min":1.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.5259999999999998,"5min":9.7080000000000002,"15min":10.512},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.80200000000000005,"15min":0.93799999999999994},"min":{"1min":0.31,"5min":0.31,"15min":0.26200000000000001},"max":{"1min":3.875,"5min":6.0490000000000004,"15min":11.493},"last":0.40000000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.3640000000000001,"15min":2.4830000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.492,"5min":9.7439999999999998,"15min":154.30600000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.093,"5min":0.90200000000000002,"15min":1.905},"min":{"1min":0.33800000000000002,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":11.478,"5min":11.478,"15min":154.42400000000001},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2149999999999999,"5min":5.7060000000000004,"15min":3.1709999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":136.404,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":8.1039999999999992,"5min":5.9210000000000003,"15min":3.3599999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":136.44999999999999,"5min":169.767,"15min":169.767},"last":0.97099999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.98599999999999999,"15min":2.4279999999999999},"min":{"1min":0.35199999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":164.26400000000001},"last":0.503},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.107,"15min":2.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":1.6719999999999999,"5min":6.5410000000000004,"15min":164.23099999999999},"last":0.996}]}]},{"osd":2,"up_from":129,"seq":554050781436,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3924452,"kb_used_data":3307484,"kb_used_omap":7,"kb_used_meta":616952,"kb_avail":89820700,"statfs":{"total":95995035648,"available":91976396800,"internally_reserved":0,"allocated":3386863616,"data_stored":3384906751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":631758929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.302,"15min":1.3919999999999999},"min":{"1min":0.19400000000000001,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":1.7470000000000001,"5min":15.667,"15min":26.356000000000002},"last":6.8440000000000003},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.4419999999999999,"15min":1.556},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.663,"5min":12.445,"15min":26.341000000000001},"last":5.6529999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:05:55 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":5.5549999999999997,"15min":2.343},"min":{"1min":0.22500000000000001,"5min":0.19,"15min":0.188},"max":{"1min":1.9419999999999999,"5min":212.12,"15min":212.12},"last":4.9089999999999998},{"interface":"front","average":{"1min":0.35399999999999998,"5min":5.5369999999999999,"15min":2.1349999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.79100000000000004,"5min":213.02799999999999,"15min":213.02799999999999},"last":2.101}]},{"osd":3,"last update":"Sat Sep 23 21:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":2.2480000000000002,"15min":3.3039999999999998},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":4.3559999999999999,"5min":66.085999999999999,"15min":151.43799999999999},"last":4.952},{"interface":"front","average":{"1min":0.59599999999999997,"5min":2.222,"15min":3.3260000000000001},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":1.4930000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":1.613}]},{"osd":4,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.101,"15min":1.1419999999999999},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.4570000000000001,"5min":12.586,"15min":12.586},"last":1.3340000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.1870000000000001,"15min":1.1910000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.575,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":5.5590000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.29,"15min":2.5760000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.6379999999999999,"5min":9.9909999999999997,"15min":173.34},"last":5.5960000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.079,"15min":2.3809999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":2.1949999999999998,"5min":13.701000000000001,"15min":173.15000000000001},"last":5.492}]},{"osd":6,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":3.2869999999999999,"15min":1.883},"min":{"1min":0.34200000000000003,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":2.1789999999999998,"5min":175.72499999999999,"15min":175.72499999999999},"last":5.5259999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":3.3250000000000002,"15min":1.984},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":1.5329999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":1.369}]},{"osd":7,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.6060000000000001,"15min":2.7330000000000001},"min":{"1min":0.41399999999999998,"5min":0.40799999999999997,"15min":0.32700000000000001},"max":{"1min":9.6910000000000007,"5min":12.609,"15min":177.30799999999999},"last":1.395},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.355,"15min":2.6520000000000001},"min":{"1min":0.38900000000000001,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.615,"5min":10.016,"15min":177.40899999999999},"last":5.6219999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160030,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":546520,"kb_used_data":7568,"kb_used_omap":2,"kb_used_meta":538941,"kb_avail":93198632,"statfs":{"total":95995035648,"available":95435399168,"internally_reserved":0,"allocated":7749632,"data_stored":5624105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":551875637},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.5580000000000001,"15min":1.7350000000000001},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.18099999999999999},"max":{"1min":3.8570000000000002,"5min":44.963000000000001,"15min":95.507999999999996},"last":2.597},{"interface":"front","average":{"1min":0.438,"5min":1.02,"15min":1.698},"min":{"1min":0.219,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":1.2909999999999999,"5min":9.7759999999999998,"15min":95.259},"last":0.71199999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:05:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.339,"15min":1.0129999999999999},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.7639999999999993,"5min":12.035,"15min":19.786999999999999},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.6160000000000001,"15min":1.2110000000000001},"min":{"1min":0.253,"5min":0.182,"15min":0.182},"max":{"1min":2.3159999999999998,"5min":15.763999999999999,"15min":15.763999999999999},"last":0.77300000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.464,"15min":1.2290000000000001},"min":{"1min":0.29699999999999999,"5min":0.223,"15min":0.182},"max":{"1min":13.944000000000001,"5min":13.944000000000001,"15min":13.944000000000001},"last":2.6200000000000001},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.2470000000000001,"15min":1.2909999999999999},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":4.6349999999999998,"5min":12.096,"15min":19.754999999999999},"last":3.5859999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":2.645,"15min":2.1930000000000001},"min":{"1min":0.34799999999999998,"5min":0.307,"15min":0.222},"max":{"1min":8.1750000000000007,"5min":70.728999999999999,"15min":111.36},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1389999999999998,"15min":1.9019999999999999},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":4.1100000000000003,"5min":70.712000000000003,"15min":112.473},"last":2.5600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.0740000000000001,"15min":1.5129999999999999},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":1.325,"5min":9.6359999999999992,"15min":64.528000000000006},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.79,"15min":1.6220000000000001},"min":{"1min":0.42099999999999999,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":8.2490000000000006,"5min":10.617000000000001,"15min":14.054},"last":3.633}]},{"osd":6,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.857,"15min":1.6919999999999999},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.312},"max":{"1min":8.2780000000000005,"5min":10.52,"15min":19.568000000000001},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.4650000000000001,"15min":1.4750000000000001},"min":{"1min":0.39100000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":4.3920000000000003,"5min":14.356,"15min":15.573},"last":7.6749999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":2.1930000000000001,"15min":2.6099999999999999},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":1.3480000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.341,"5min":2.8090000000000002,"15min":2.8490000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":8.218,"5min":73.364999999999995,"15min":222.11699999999999},"last":4.8129999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986648,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1807056,"kb_used_data":1260488,"kb_used_omap":0,"kb_used_meta":546560,"kb_avail":91938096,"statfs":{"total":95995035648,"available":94144610304,"internally_reserved":0,"allocated":1290739712,"data_stored":1289557613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":559677440},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.2869999999999999,"15min":2.782},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.53200000000000003,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.119,"15min":2.8029999999999999},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.223},"max":{"1min":3.8959999999999999,"5min":8.5220000000000002,"15min":144.77000000000001},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:06:07 2023","interfaces":[{"interface":"back","average":{"1min":6.4459999999999997,"5min":6.3929999999999998,"15min":2.6269999999999998},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":118.411,"5min":118.411,"15min":118.411},"last":2.2010000000000001},{"interface":"front","average":{"1min":6.274,"5min":6.1340000000000003,"15min":2.4740000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":118.075,"5min":118.075,"15min":118.075},"last":0.26700000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":5.8410000000000002,"15min":3.2130000000000001},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":3.452,"5min":164.517,"15min":164.517},"last":3.1829999999999998},{"interface":"front","average":{"1min":0.46999999999999997,"5min":5.665,"15min":3.274},"min":{"1min":0.35599999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":0.56999999999999995,"5min":164.46799999999999,"15min":164.46799999999999},"last":3.3300000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:06 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.2709999999999999,"15min":2.2610000000000001},"min":{"1min":0.32500000000000001,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.8420000000000001,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.2829999999999999,"15min":2.3889999999999998},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.82799999999999996,"5min":7.5579999999999998,"15min":142.34700000000001},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.82499999999999996,"15min":1.0609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.24099999999999999},"max":{"1min":2.9049999999999998,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.875,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29899999999999999,"15min":0.21099999999999999},"max":{"1min":0.58299999999999996,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.004,"15min":0.96699999999999997},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":0.70599999999999996,"5min":11.800000000000001,"15min":11.887},"last":0.309},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.111,"15min":0.92100000000000004},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":2.3620000000000001,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.33000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":3.0110000000000001,"15min":1.8280000000000001},"min":{"1min":0.42799999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":0.72799999999999998,"5min":145.721,"15min":145.721},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":3.282,"15min":1.992},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":0.78100000000000003,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.39000000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725012,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1297660,"kb_used_data":711476,"kb_used_omap":0,"kb_used_meta":586176,"kb_avail":92447492,"statfs":{"total":95995035648,"available":94666231808,"internally_reserved":0,"allocated":728551424,"data_stored":727385525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":600244224},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.68500000000000005,"15min":1.3540000000000001},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":10.016999999999999,"5min":10.016999999999999,"15min":80.105999999999995},"last":0.248},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.89300000000000002,"15min":1.462},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":2.919,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.51300000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":4.1200000000000001,"15min":1.6699999999999999},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":0.67200000000000004,"5min":137.143,"15min":137.143},"last":1.9119999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":3.8199999999999998,"15min":1.5800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.247},"max":{"1min":0.66200000000000003,"5min":139.73699999999999,"15min":139.73699999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.3980000000000001,"15min":1.569},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":8.3770000000000007,"5min":103.63800000000001,"15min":103.63800000000001},"last":9.1240000000000006},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.6309999999999998,"15min":1.696},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":6.7640000000000002,"5min":103.664,"15min":103.664},"last":1.8460000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.3149999999999999,"15min":2.8359999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":8.0860000000000003,"5min":97.349999999999994,"15min":153.60300000000001},"last":1.972},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.2570000000000001,"15min":2.7559999999999998},"min":{"1min":0.38300000000000001,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":4.6609999999999996,"5min":97.316999999999993,"15min":153.774},"last":0.437}]},{"osd":4,"last update":"Sat Sep 23 21:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.69799999999999995,"15min":0.81699999999999995},"min":{"1min":0.33100000000000002,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":0.69899999999999995,"5min":12.739000000000001,"15min":12.739000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.88,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":0.59599999999999997,"5min":10.792,"15min":12.539},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.7150000000000001,"15min":1.1839999999999999},"min":{"1min":0.376,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":1.276,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.625,"5min":1.8420000000000001,"15min":1.3220000000000001},"min":{"1min":0.32400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6699999999999999,"5min":70.266000000000005,"15min":70.266000000000005},"last":1.8600000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.014,"15min":1.2230000000000001},"min":{"1min":0.40799999999999997,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.65200000000000002,"15min":1.0780000000000001},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.46100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":672071680,"data_stored":672071680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3726376960,"data_stored":3726376960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3371241472,"data_stored":3371237400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2490368,"data_stored":2490368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1275990016,"data_stored":1275985944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":720306176,"data_stored":720306176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3468890112,"data_stored":3468886040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3679911936,"data_stored":3679911936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:06:59.567 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:06:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:59.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:06:59.624 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:06:59 smithi121 ceph-mon[211526]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34446/231156 objects degraded (14.902%); 61127/231156 objects misplaced (26.444%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:59.624 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:06:59 smithi121 ceph-mon[219709]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34446/231156 objects degraded (14.902%); 61127/231156 objects misplaced (26.444%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:59.676 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:06:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:06:59 smithi123 ceph-mon[193903]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 34446/231156 objects degraded (14.902%); 61127/231156 objects misplaced (26.444%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:06:59.774 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:06:59.817 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:06:59.875 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:06:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:06:59.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:06:59.985 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:00.027 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:00.079 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:00.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:00.183 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:00.265 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:00.379 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:00.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:00.486 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:00.574 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:00.813 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:00.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:00.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:00.879 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:07:00.928 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:00.960 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/784799843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:00.963 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:01.039 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:01.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:01.147 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:01.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/784799843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:01.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/784799843' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:01.195 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:01.238 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:01.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:01.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:01.341 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:01.405 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:01.471 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:07:01.471 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:07:01.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:07:01.471 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:07:01.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97176 10.d 10:bf52c426:::benchmark_data_smithi121_358375_object12431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 505.96649199199999, 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0090121330000001, 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97176 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.472 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499973+0000", 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499972+0000", 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499975+0000", 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2090000000000001e-06 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499984+0000", 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.422e-06 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.499988+0000", 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8739999999999997e-06 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500030+0000", 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2366000000000001e-05 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.500039+0000", 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5569999999999996e-06 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242363+0000", 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.74232412699999994 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242370+0000", 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.041e-06 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482610+0000", 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2402398800000001 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482781+0000", 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000170884 2023-09-23T21:07:01.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.482955+0000", 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017400099999999999 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.490821+0000", 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078662190000000007 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.493967+0000", 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031467309999999998 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506247+0000", 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012279684000000001 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.506289+0000", 2023-09-23T21:07:01.479 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2329000000000001e-05 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.508985+0000", 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026955099999999999 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97184 10.a 10:54b30599:::benchmark_data_smithi121_358375_object12439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:07:01.480 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 505.87914777100002, 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.974572057, 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97184 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587317+0000", 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587316+0000", 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587321+0000", 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3270000000000002e-06 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587333+0000", 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2383e-05 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587338+0000", 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9189999999999998e-06 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587383+0000", 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5014e-05 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.587389+0000", 2023-09-23T21:07:01.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9660000000000001e-06 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242357+0000", 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.65496830699999997 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242367+0000", 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0266e-05 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538285+0000", 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2959178060000001 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538406+0000", 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120308 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.538531+0000", 2023-09-23T21:07:01.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000125761 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.549226+0000", 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010694789999999999 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.551472+0000", 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002246315 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559794+0000", 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083212719999999993 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.559841+0000", 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7197999999999998e-05 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.561889+0000", 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020482159999999998 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:97187 10.0 10:057fb271:::benchmark_data_smithi121_358375_object12442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 505.85561811500003, 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8667573179999999, 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.489 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 97187 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610847+0000", 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610846+0000", 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610850+0000", 2023-09-23T21:07:01.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2509999999999996e-06 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610863+0000", 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2995e-05 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610869+0000", 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1480000000000003e-06 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610945+0000", 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5284000000000006e-05 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:35.610953+0000", 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3750000000000003e-06 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242784+0000", 2023-09-23T21:07:01.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63183069599999997 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:36.242792+0000", 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6510000000000007e-06 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.434568+0000", 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1917759189999999 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.435371+0000", 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000802339 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451218+0000", 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015847757000000001 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451292+0000", 2023-09-23T21:07:01.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3386000000000005e-05 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.451393+0000", 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101216 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.462865+0000", 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011472379 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.466387+0000", 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035211050000000001 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.474961+0000", 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085745549999999993 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.475028+0000", 2023-09-23T21:07:01.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7105000000000002e-05 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T20:58:37.477604+0000", 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025758389999999999 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.498 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50073 11.b 11:da39f2df:::benchmark_data_smithi121_477989_object50072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 204.06142559099999, 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2188515999999998, 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50073 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405103+0000", 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3901999999999999e-05 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405111+0000", 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4460000000000001e-06 2023-09-23T21:07:01.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405116+0000", 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0629999999999996e-06 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416824+0000", 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011707555 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416833+0000", 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1540000000000008e-06 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400621+0000", 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98378816000000002 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400630+0000", 2023-09-23T21:07:01.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6950000000000006e-06 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.622813+0000", 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.222183198 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.623891+0000", 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001077929 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.504 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50086 11.b 11:d8a7cc4c:::benchmark_data_smithi121_477989_object50085:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 204.06133865199999, 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2482345829999999, 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50086 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405188+0000", 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2147e-05 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405196+0000", 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1799999999999996e-06 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405201+0000", 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9339999999999997e-06 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416857+0000", 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011656025 2023-09-23T21:07:01.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416865+0000", 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8329999999999994e-06 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400651+0000", 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98378581600000004 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400658+0000", 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.08e-06 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.623919+0000", 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2232614909999999 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.624169+0000", 2023-09-23T21:07:01.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024978700000000001 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.624334+0000", 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165315 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631814+0000", 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.007479482 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634635+0000", 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028213819999999999 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.650585+0000", 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015949393999999999 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.650633+0000", 2023-09-23T21:07:01.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8591999999999998e-05 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653361+0000", 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002727602 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50100 11.3 11:c9d587e5:::benchmark_data_smithi121_477989_object50099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:01.513 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 204.037865334, 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2031901700000001, 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50100 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.514 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428599+0000", 2023-09-23T21:07:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428631+0000", 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1956999999999999e-05 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428639+0000", 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8269999999999998e-06 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428644+0000", 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3550000000000004e-06 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428697+0000", 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2799000000000001e-05 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428706+0000", 2023-09-23T21:07:01.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0659999999999993e-06 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.399543+0000", 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97083674099999995 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.399551+0000", 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5429999999999994e-06 2023-09-23T21:07:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607385+0000", 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2078334150000001 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607502+0000", 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117125 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:07:01.519 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607664+0000", 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016177800000000001 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.614069+0000", 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064056510000000001 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629191+0000", 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015121252 2023-09-23T21:07:01.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629222+0000", 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1405000000000002e-05 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629265+0000", 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2824000000000002e-05 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631790+0000", 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025248559999999998 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50110 11.b 11:d197f0fb:::benchmark_data_smithi121_477989_object50109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 204.03192808599999, 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2210893860000001, 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50110 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434536+0000", 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434604+0000", 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8382999999999999e-05 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434616+0000", 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2139e-05 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434623+0000", 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6510000000000004e-06 2023-09-23T21:07:01.525 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434802+0000", 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017905300000000001 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434811+0000", 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3820000000000004e-06 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400678+0000", 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96586726099999998 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400684+0000", 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2689999999999996e-06 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.625756+0000", 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2250717149999999 2023-09-23T21:07:01.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.625892+0000", 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000135933 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.626051+0000", 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015894499999999999 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631821+0000", 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0057699200000000004 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634669+0000", 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028480839999999999 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653436+0000", 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018767817999999999 2023-09-23T21:07:01.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653479+0000", 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2070000000000002e-05 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655626+0000", 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002147599 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.530 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50121 11.b 11:d6d11928:::benchmark_data_smithi121_477989_object50120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 203.84147553299999, 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.033318296, 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.531 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50121 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624988+0000", 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625021+0000", 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3223999999999999e-05 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.533 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625034+0000", 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3191e-05 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625041+0000", 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1130000000000003e-06 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625217+0000", 2023-09-23T21:07:01.534 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000175921 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625227+0000", 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8390000000000002e-06 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400706+0000", 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77547898800000004 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400713+0000", 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9789999999999996e-06 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627606+0000", 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.226892662 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627745+0000", 2023-09-23T21:07:01.536 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013900300000000001 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627884+0000", 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000138932 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634608+0000", 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0067245739999999997 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634714+0000", 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010603300000000001 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655900+0000", 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021185137999999999 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655945+0000", 2023-09-23T21:07:01.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5346000000000003e-05 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.658308+0000", 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002362762 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50132 11.a 11:593fc135:::benchmark_data_smithi121_477989_object50131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:01.539 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 203.82372835000001, 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9071047490000002, 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50132 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.540 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642735+0000", 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642763+0000", 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7504e-05 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642777+0000", 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3684e-05 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642781+0000", 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9110000000000001e-06 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642843+0000", 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1890999999999994e-05 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642855+0000", 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1999e-05 2023-09-23T21:07:01.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.396214+0000", 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75335825499999998 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.396226+0000", 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2177e-05 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540693+0000", 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.144467481 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540826+0000", 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013311 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540929+0000", 2023-09-23T21:07:01.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010215799999999999 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547150+0000", 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062215100000000004 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547194+0000", 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3538999999999998e-05 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547204+0000", 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0873000000000001e-05 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547271+0000", 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6261999999999997e-05 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.549841+0000", 2023-09-23T21:07:01.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025705569999999998 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88131 11.9 11:9f73b961:::benchmark_data_smithi121_477989_object88130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.745423+0000", 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 152.72104207999999, 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1797363559999998, 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:01.548 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88131 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745423+0000", 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745423+0000", 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745422+0000", 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745500+0000", 2023-09-23T21:07:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8769000000000004e-05 2023-09-23T21:07:01.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745526+0000", 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5327e-05 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745544+0000", 2023-09-23T21:07:01.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8533999999999999e-05 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746670+0000", 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011258279999999999 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746810+0000", 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013983100000000001 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746968+0000", 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015768100000000001 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768727+0000", 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021759737000000001 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768736+0000", 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3599999999999996e-06 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772896+0000", 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0041604820000001 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772908+0000", 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1946e-05 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.906549+0000", 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.133640728 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.909440+0000", 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028911649999999998 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.924853+0000", 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015413377000000001 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.925159+0000", 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00030577099999999998 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88138 11.9 11:97b0517a:::benchmark_data_smithi121_477989_object88137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 152.719808817, 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2300521670000002, 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88138 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.559 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746706+0000", 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0890000000000002e-05 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746713+0000", 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2639999999999997e-06 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746717+0000", 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0899999999999998e-06 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.753050+0000", 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0063330749999999996 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.753173+0000", 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012362899999999999 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.753316+0000", 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000142149 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768761+0000", 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015445296000000001 2023-09-23T21:07:01.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768768+0000", 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.905e-06 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772942+0000", 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.004173762 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772951+0000", 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4150000000000005e-06 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.909629+0000", 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1366782120000001 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.910655+0000", 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001025518 2023-09-23T21:07:01.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.976619+0000", 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.065964751000000002 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.976708+0000", 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8727000000000006e-05 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.566 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88123 11.4 11:2a8ebd38:::benchmark_data_smithi121_477989_object88122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.759676+0000", 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 152.706788386, 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6256372400000001, 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.567 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88123 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759676+0000", 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759676+0000", 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759675+0000", 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759724+0000", 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8857000000000003e-05 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759737+0000", 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3185e-05 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759743+0000", 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4309999999999999e-06 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768032+0000", 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0082888340000000001 2023-09-23T21:07:01.570 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768043+0000", 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.039e-05 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767443+0000", 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99940036899999996 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767455+0000", 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2167000000000001e-05 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.383270+0000", 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6158152189999999 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385314+0000", 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00204339 2023-09-23T21:07:01.572 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88130 11.4 11:2d9f9be9:::benchmark_data_smithi121_477989_object88129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.767371+0000", 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 152.69909366499999, 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.618074526, 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.573 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88130 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767371+0000", 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767371+0000", 2023-09-23T21:07:01.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767370+0000", 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767417+0000", 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6795000000000002e-05 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767430+0000", 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3681e-05 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767437+0000", 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8730000000000001e-06 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768067+0000", 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00062932299999999999 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768074+0000", 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7039999999999994e-06 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767486+0000", 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99941159099999999 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767495+0000", 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7420000000000003e-06 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385352+0000", 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6178574299999999 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385446+0000", 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3614000000000002e-05 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88143 11.4 11:20aacae1:::benchmark_data_smithi121_477989_object88142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 152.699020472, 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6459621049999997, 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.579 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88143 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.580 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767481+0000", 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7512000000000003e-05 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767489+0000", 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5429999999999996e-06 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767493+0000", 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.279e-06 2023-09-23T21:07:01.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768835+0000", 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001341255 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768843+0000", 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9899999999999997e-06 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767519+0000", 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99867673400000001 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767528+0000", 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3119999999999994e-06 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385470+0000", 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.617942703 2023-09-23T21:07:01.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.386142+0000", 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00067205699999999999 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.386575+0000", 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043233799999999999 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.585 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.402719+0000", 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016144479999999999 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.406453+0000", 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037342669999999999 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.409931+0000", 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034772359999999999 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.409993+0000", 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2419e-05 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.413406+0000", 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034134259999999998 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.587 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88145 11.4 11:238c6080:::benchmark_data_smithi121_477989_object88144:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.769668+0000", 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 152.69679701300001, 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6476349180000001, 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:01.588 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88145 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769668+0000", 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769668+0000", 2023-09-23T21:07:01.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769667+0000", 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769698+0000", 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0843999999999998e-05 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769706+0000", 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6850000000000001e-06 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769711+0000", 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9289999999999997e-06 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769775+0000", 2023-09-23T21:07:01.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4603000000000005e-05 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769785+0000", 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1949999999999992e-06 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.770501+0000", 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0007162979999999 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.770513+0000", 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2418999999999999e-05 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.399922+0000", 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6294089870000001 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.400078+0000", 2023-09-23T21:07:01.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015531400000000001 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.400281+0000", 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000202995 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.406464+0000", 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061830230000000002 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.409889+0000", 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034254329999999999 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.413472+0000", 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035832640000000001 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.413534+0000", 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2169999999999996e-05 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.417303+0000", 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003768291 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88155 11.d e370/364 11:b7e40311:::benchmark_data_smithi121_477989_object88154:head v 370'5510, mlcod=367'5508)", 2023-09-23T21:07:01.596 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.067298+0000", 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 150.39916731299999, 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.751661201000001, 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067298+0000", 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.597 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067298+0000", 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067297+0000", 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067345+0000", 2023-09-23T21:07:01.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7917000000000002e-05 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067358+0000", 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2945000000000001e-05 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067365+0000", 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7700000000000001e-06 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.811897+0000", 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.744531416000001 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.811910+0000", 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2858e-05 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.818932+0000", 2023-09-23T21:07:01.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070229439999999997 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.818959+0000", 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6258000000000002e-05 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88162 11.d e370/364 11:b0e3ae3a:::benchmark_data_smithi121_477989_object88161:head v 370'5511, mlcod=367'5508)", 2023-09-23T21:07:01.601 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.408324+0000", 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 150.05814072999999, 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.412904649000001, 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408324+0000", 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408324+0000", 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408323+0000", 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408368+0000", 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5136000000000003e-05 2023-09-23T21:07:01.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408384+0000", 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6011999999999998e-05 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408391+0000", 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5039999999999999e-06 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.816146+0000", 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.407755393999999 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.816161+0000", 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4467999999999999e-05 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821197+0000", 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050365870000000004 2023-09-23T21:07:01.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821229+0000", 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1568e-05 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88168 11.d e370/364 11:b40d873b:::benchmark_data_smithi121_477989_object88167:head v 370'5512, mlcod=367'5508)", 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.427202+0000", 2023-09-23T21:07:01.606 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 150.03926274599999, 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.394081789000001, 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427202+0000", 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.607 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427202+0000", 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427201+0000", 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427246+0000", 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5252e-05 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427266+0000", 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9473000000000001e-05 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427273+0000", 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.515e-06 2023-09-23T21:07:01.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.817746+0000", 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.390472430999999 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.817758+0000", 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2257e-05 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821251+0000", 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034928160000000001 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821284+0000", 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3104e-05 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88182 11.d e370/364 11:b50fd016:::benchmark_data_smithi121_477989_object88181:head v 370'5513, mlcod=367'5508)", 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 150.00144143599999, 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.356282992000001, 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.612 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465088+0000", 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5238000000000001e-05 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465102+0000", 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.396e-05 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465108+0000", 2023-09-23T21:07:01.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.733e-06 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.818992+0000", 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.35388459 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.819016+0000", 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3479e-05 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821286+0000", 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022702270000000001 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821306+0000", 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0452999999999999e-05 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.616 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88186 11.d e370/364 11:b93f9f29:::benchmark_data_smithi121_477989_object88185:head v 370'5514, mlcod=367'5508)", 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.490693+0000", 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 149.97577171699999, 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.330656862000001, 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490693+0000", 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490693+0000", 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490692+0000", 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:01.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490727+0000", 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5203999999999997e-05 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490743+0000", 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5498e-05 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490749+0000", 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5200000000000003e-06 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.820123+0000", 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.3293739 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.820138+0000", 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4338999999999999e-05 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821337+0000", 2023-09-23T21:07:01.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011991079999999999 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821350+0000", 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3208e-05 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:01.621 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:07:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/784799843' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T21:07:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: pgmap v1270: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 33984/223470 objects degraded (15.207%); 59077/223470 objects misplaced (26.436%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 34446/231156 objects degraded (14.902%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/784799843' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T21:07:02.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: pgmap v1270: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 33984/223470 objects degraded (15.207%); 59077/223470 objects misplaced (26.436%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:07:02.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 34446/231156 objects degraded (14.902%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/784799843' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: failed to encode map e406 with expected crc 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: pgmap v1270: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 33984/223470 objects degraded (15.207%); 59077/223470 objects misplaced (26.436%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T21:07:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 34446/231156 objects degraded (14.902%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:02.396 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:07:02.441 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:02.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:02.481 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:07:02.549 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:02.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:02.814 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:02.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:02.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:07:02.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:07:02.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:02.944 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:02.989 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:03.064 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:03.066 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:03.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:03.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:03.174 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:03.180 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:07:03.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:02 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:07:03.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:03.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:03.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:03.218 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:03.221 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:03.226 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:07:03.236 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:03.298 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:07:03.350 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:07:03.387 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:07:03.417 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:03.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:03.428 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:07:03.476 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:07:03.513 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:07:03.522 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:03.529 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.529 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.529 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.529 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.529 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.530 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.530 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.530 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:07:03.568 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:03.803 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:03.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:03.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:03.909 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:03.955 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:04.129 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:04.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:04.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:04.133 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:03 smithi121 ceph-mon[211526]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 33984/223470 objects degraded (15.207%); 59077/223470 objects misplaced (26.436%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T21:07:04.134 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3216892244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:04.134 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:03 smithi121 ceph-mon[219709]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 33984/223470 objects degraded (15.207%); 59077/223470 objects misplaced (26.436%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T21:07:04.134 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3216892244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:03 smithi123 ceph-mon[193903]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 33984/223470 objects degraded (15.207%); 59077/223470 objects misplaced (26.436%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T21:07:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3216892244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:04.234 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:04.272 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:04.442 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:07:04.779 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:07:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:07:04] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:07:05.115 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:05.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:05.221 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:05.278 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:05.385 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:05.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:05.388 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:05 smithi123 ceph-mon[193903]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 33984/222114 objects degraded (15.300%); 58636/222114 objects misplaced (26.399%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T21:07:05.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:05 smithi121 ceph-mon[211526]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 33984/222114 objects degraded (15.300%); 58636/222114 objects misplaced (26.399%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T21:07:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:05 smithi121 ceph-mon[219709]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 33984/222114 objects degraded (15.300%); 58636/222114 objects misplaced (26.399%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T21:07:05.489 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:05.611 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:05.701 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:05.787 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:05.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:05.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:05.893 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:05.973 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:05.973 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:05.990 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:06.381 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:06.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:06.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:06.487 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:06.496 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 33984/222114 objects degraded (15.300%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:06.586 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:06.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 33984/222114 objects degraded (15.300%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:06.943 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:06.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:06.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 33984/222114 objects degraded (15.300%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:07.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:07.099 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:07.104 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:07:07.142 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:07:07.144 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:07:07.198 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:07.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:07.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:07.238 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:07:07.303 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:07.432 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:07.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:07 smithi121 ceph-mon[219709]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 33533/215931 objects degraded (15.529%); 57049/215931 objects misplaced (26.420%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:07:07.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:07 smithi121 ceph-mon[211526]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 33533/215931 objects degraded (15.529%); 57049/215931 objects misplaced (26.420%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:07:07.822 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:07 smithi123 ceph-mon[193903]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 33533/215931 objects degraded (15.529%); 57049/215931 objects misplaced (26.420%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:07:08.218 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:08.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:08.322 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:08.376 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1273,"stamp":"2023-09-23T21:07:07.032888+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4716042168,"num_objects":71977,"num_object_clones":0,"num_object_copies":215931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33533,"num_objects_misplaced":57049,"num_objects_unfound":0,"num_objects_dirty":71977,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":104721,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24707,"num_bytes_recovered":1614519564,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74336,"ondisk_log_size":74336,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16039964,"kb_used_data":12178916,"kb_used_omap":21,"kb_used_meta":3860842,"kb_avail":640176100,"statfs":{"total":671965249536,"available":655540326400,"internally_reserved":0,"allocated":12471209984,"data_stored":12459643579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3953502691},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,5,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-332595200,"num_objects":-5075,"num_object_clones":0,"num_object_copies":-15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-913,"num_objects_misplaced":-4078,"num_objects_unfound":0,"num_objects_dirty":-5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5075,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":935,"num_bytes_recovered":61276160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005025"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":176,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.175273+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:06:53.175273+0000","last_peered":"2023-09-23T21:06:53.175273+0000","last_clean":"2023-09-23T21:06:53.175273+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:06:53.175273+0000","last_undegraded":"2023-09-23T21:06:53.175273+0000","last_fullsized":"2023-09-23T21:06:53.175273+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":45,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.180173+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:06:53.180173+0000","last_peered":"2023-09-23T21:06:53.180173+0000","last_clean":"2023-09-23T21:06:53.180173+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:06:53.180173+0000","last_undegraded":"2023-09-23T21:06:53.180173+0000","last_fullsized":"2023-09-23T21:06:53.180173+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":318,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.174702+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:06:53.174702+0000","last_peered":"2023-09-23T21:06:53.174702+0000","last_clean":"2023-09-23T21:06:53.174702+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:06:53.174702+0000","last_undegraded":"2023-09-23T21:06:53.174702+0000","last_fullsized":"2023-09-23T21:06:53.174702+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":43,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:59.175044+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:06:59.175044+0000","last_peered":"2023-09-23T21:06:59.175044+0000","last_clean":"2023-09-23T21:06:59.175044+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:06:59.175044+0000","last_undegraded":"2023-09-23T21:06:59.175044+0000","last_fullsized":"2023-09-23T21:06:59.175044+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":63,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.173422+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:06:53.173422+0000","last_peered":"2023-09-23T21:06:53.173422+0000","last_clean":"2023-09-23T21:06:53.173422+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:06:53.173422+0000","last_undegraded":"2023-09-23T21:06:53.173422+0000","last_fullsized":"2023-09-23T21:06:53.173422+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.175692+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:06:53.175692+0000","last_peered":"2023-09-23T21:06:53.175692+0000","last_clean":"2023-09-23T21:06:53.175692+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:06:53.175692+0000","last_undegraded":"2023-09-23T21:06:53.175692+0000","last_fullsized":"2023-09-23T21:06:53.175692+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":42,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.179982+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:06:53.179982+0000","last_peered":"2023-09-23T21:06:53.179982+0000","last_clean":"2023-09-23T21:06:53.179982+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:06:53.179982+0000","last_undegraded":"2023-09-23T21:06:53.179982+0000","last_fullsized":"2023-09-23T21:06:53.179982+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":46,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.857292+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:07:00.857292+0000","last_peered":"2023-09-23T21:07:00.857292+0000","last_clean":"2023-09-23T21:07:00.857292+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:07:00.857292+0000","last_undegraded":"2023-09-23T21:07:00.857292+0000","last_fullsized":"2023-09-23T21:07:00.857292+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":29,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.171372+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:06:53.171372+0000","last_peered":"2023-09-23T21:06:53.171372+0000","last_clean":"2023-09-23T21:06:53.171372+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:06:53.171372+0000","last_undegraded":"2023-09-23T21:06:53.171372+0000","last_fullsized":"2023-09-23T21:06:53.171372+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":46,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.853751+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:07:00.853751+0000","last_peered":"2023-09-23T21:07:00.853751+0000","last_clean":"2023-09-23T21:07:00.853751+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:07:00.853751+0000","last_undegraded":"2023-09-23T21:07:00.853751+0000","last_fullsized":"2023-09-23T21:07:00.853751+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":61,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.180248+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:06:53.180248+0000","last_peered":"2023-09-23T21:06:53.180248+0000","last_clean":"2023-09-23T21:06:53.180248+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:06:53.180248+0000","last_undegraded":"2023-09-23T21:06:53.180248+0000","last_fullsized":"2023-09-23T21:06:53.180248+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":37,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.174963+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:06:53.174963+0000","last_peered":"2023-09-23T21:06:53.174963+0000","last_clean":"2023-09-23T21:06:53.174963+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:06:53.174963+0000","last_undegraded":"2023-09-23T21:06:53.174963+0000","last_fullsized":"2023-09-23T21:06:53.174963+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":352,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.178662+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:06:53.178662+0000","last_peered":"2023-09-23T21:06:53.178662+0000","last_clean":"2023-09-23T21:06:53.178662+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:06:53.178662+0000","last_undegraded":"2023-09-23T21:06:53.178662+0000","last_fullsized":"2023-09-23T21:06:53.178662+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":324,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.169134+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:06:53.169134+0000","last_peered":"2023-09-23T21:06:53.169134+0000","last_clean":"2023-09-23T21:06:53.169134+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:06:53.169134+0000","last_undegraded":"2023-09-23T21:06:53.169134+0000","last_fullsized":"2023-09-23T21:06:53.169134+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":96,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.165791+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:06:53.165791+0000","last_peered":"2023-09-23T21:06:53.165791+0000","last_clean":"2023-09-23T21:06:53.165791+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:06:53.165791+0000","last_undegraded":"2023-09-23T21:06:53.165791+0000","last_fullsized":"2023-09-23T21:06:53.165791+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":55,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.855084+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:07:00.855084+0000","last_peered":"2023-09-23T21:07:00.855084+0000","last_clean":"2023-09-23T21:07:00.855084+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:07:00.855084+0000","last_undegraded":"2023-09-23T21:07:00.855084+0000","last_fullsized":"2023-09-23T21:07:00.855084+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":64,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.855030+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:07:00.855030+0000","last_peered":"2023-09-23T21:07:00.855030+0000","last_clean":"2023-09-23T21:07:00.855030+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:07:00.855030+0000","last_undegraded":"2023-09-23T21:07:00.855030+0000","last_fullsized":"2023-09-23T21:07:00.855030+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":63,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.180319+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:06:53.180319+0000","last_peered":"2023-09-23T21:06:53.180319+0000","last_clean":"2023-09-23T21:06:53.180319+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:06:53.180319+0000","last_undegraded":"2023-09-23T21:06:53.180319+0000","last_fullsized":"2023-09-23T21:06:53.180319+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.168950+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:06:53.168950+0000","last_peered":"2023-09-23T21:06:53.168950+0000","last_clean":"2023-09-23T21:06:53.168950+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:06:53.168950+0000","last_undegraded":"2023-09-23T21:06:53.168950+0000","last_fullsized":"2023-09-23T21:06:53.168950+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":42,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.179761+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:06:53.179761+0000","last_peered":"2023-09-23T21:06:53.179761+0000","last_clean":"2023-09-23T21:06:53.179761+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:06:53.179761+0000","last_undegraded":"2023-09-23T21:06:53.179761+0000","last_fullsized":"2023-09-23T21:06:53.179761+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":402,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.180133+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:06:53.180133+0000","last_peered":"2023-09-23T21:06:53.180133+0000","last_clean":"2023-09-23T21:06:53.180133+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:06:53.180133+0000","last_undegraded":"2023-09-23T21:06:53.180133+0000","last_fullsized":"2023-09-23T21:06:53.180133+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.178931+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:06:53.178931+0000","last_peered":"2023-09-23T21:06:53.178931+0000","last_clean":"2023-09-23T21:06:53.178931+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:06:53.178931+0000","last_undegraded":"2023-09-23T21:06:53.178931+0000","last_fullsized":"2023-09-23T21:06:53.178931+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"406'6691","reported_seq":8053,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:05.342511+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:07:05.342511+0000","last_peered":"2023-09-23T21:07:05.342511+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:07:05.342511+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"349'1928","ondisk_log_start":"349'1928","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4763,"log_dups_size":0,"ondisk_log_size":4763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6691,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-1076},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":4538}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":110,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.169585+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:06:53.169585+0000","last_peered":"2023-09-23T21:06:53.169585+0000","last_clean":"2023-09-23T21:06:53.169585+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:06:53.169585+0000","last_undegraded":"2023-09-23T21:06:53.169585+0000","last_fullsized":"2023-09-23T21:06:53.169585+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":29,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.171585+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:06:53.171585+0000","last_peered":"2023-09-23T21:06:53.171585+0000","last_clean":"2023-09-23T21:06:53.171585+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:06:53.171585+0000","last_undegraded":"2023-09-23T21:06:53.171585+0000","last_fullsized":"2023-09-23T21:06:53.171585+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"406'6405","reported_seq":6480,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:05.276438+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:07:05.276438+0000","last_peered":"2023-09-23T21:07:05.276438+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:07:05.276438+0000","last_undegraded":"2023-09-23T21:07:05.276438+0000","last_fullsized":"2023-09-23T21:07:05.276438+0000","mapping_epoch":375,"log_start":"349'1712","ondisk_log_start":"349'1712","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4393,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6405,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":4393}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":104,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.343723+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:07:00.343723+0000","last_peered":"2023-09-23T21:07:00.343723+0000","last_clean":"2023-09-23T21:07:00.343723+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:07:00.343723+0000","last_undegraded":"2023-09-23T21:07:00.343723+0000","last_fullsized":"2023-09-23T21:07:00.343723+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":366,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.174628+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:06:53.174628+0000","last_peered":"2023-09-23T21:06:53.174628+0000","last_clean":"2023-09-23T21:06:53.174628+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:06:53.174628+0000","last_undegraded":"2023-09-23T21:06:53.174628+0000","last_fullsized":"2023-09-23T21:06:53.174628+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"406'6489","reported_seq":11783,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:05.695802+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:07:05.695802+0000","last_peered":"2023-09-23T21:07:05.695802+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:07:05.695802+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"350'2411","ondisk_log_start":"350'2411","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4078,"log_dups_size":0,"ondisk_log_size":4078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291569664,"num_objects":4449,"num_object_clones":0,"num_object_copies":13347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6489,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2093,"num_bytes_recovered":137166848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":52},{"shards":"1,6,7","objects":2011},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":33,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.171583+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:06:53.171583+0000","last_peered":"2023-09-23T21:06:53.171583+0000","last_clean":"2023-09-23T21:06:53.171583+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:06:53.171583+0000","last_undegraded":"2023-09-23T21:06:53.171583+0000","last_fullsized":"2023-09-23T21:06:53.171583+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":294,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.178785+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:06:53.178785+0000","last_peered":"2023-09-23T21:06:53.178785+0000","last_clean":"2023-09-23T21:06:53.178785+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:06:53.178785+0000","last_undegraded":"2023-09-23T21:06:53.178785+0000","last_fullsized":"2023-09-23T21:06:53.178785+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"406'6566","reported_seq":7189,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:05.573452+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:07:05.573452+0000","last_peered":"2023-09-23T21:07:05.573452+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:07:05.573452+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"350'2468","ondisk_log_start":"350'2468","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4098,"log_dups_size":0,"ondisk_log_size":4098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293863424,"num_objects":4484,"num_object_clones":0,"num_object_copies":13452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-769},{"shards":"2,6,7","objects":4481}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":46,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:59.874629+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:06:59.874629+0000","last_peered":"2023-09-23T21:06:59.874629+0000","last_clean":"2023-09-23T21:06:59.874629+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:06:59.874629+0000","last_undegraded":"2023-09-23T21:06:59.874629+0000","last_fullsized":"2023-09-23T21:06:59.874629+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":42,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.180525+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:06:53.180525+0000","last_peered":"2023-09-23T21:06:53.180525+0000","last_clean":"2023-09-23T21:06:53.180525+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:06:53.180525+0000","last_undegraded":"2023-09-23T21:06:53.180525+0000","last_fullsized":"2023-09-23T21:06:53.180525+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"406'6717","reported_seq":18212,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:05.623727+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:07:05.623727+0000","last_peered":"2023-09-23T21:07:05.623727+0000","last_clean":"2023-09-23T21:07:05.623727+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:07:05.623727+0000","last_undegraded":"2023-09-23T21:07:05.623727+0000","last_fullsized":"2023-09-23T21:07:05.623727+0000","mapping_epoch":400,"log_start":"350'2599","ondisk_log_start":"350'2599","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4118,"log_dups_size":0,"ondisk_log_size":4118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300875776,"num_objects":4591,"num_object_clones":0,"num_object_copies":13773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6717,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":63,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:59.874577+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:06:59.874577+0000","last_peered":"2023-09-23T21:06:59.874577+0000","last_clean":"2023-09-23T21:06:59.874577+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:06:59.874577+0000","last_undegraded":"2023-09-23T21:06:59.874577+0000","last_fullsized":"2023-09-23T21:06:59.874577+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":46,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:59.874606+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:06:59.874606+0000","last_peered":"2023-09-23T21:06:59.874606+0000","last_clean":"2023-09-23T21:06:59.874606+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:06:59.874606+0000","last_undegraded":"2023-09-23T21:06:59.874606+0000","last_fullsized":"2023-09-23T21:06:59.874606+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"406'6460","reported_seq":17567,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:04.189765+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:07:04.189765+0000","last_peered":"2023-09-23T21:07:04.189765+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:07:04.189765+0000","last_undegraded":"2023-09-23T21:07:04.189765+0000","last_fullsized":"2023-09-23T21:07:04.189765+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5972,"log_dups_size":0,"ondisk_log_size":5972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296353792,"num_objects":4522,"num_object_clones":0,"num_object_copies":13566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4522,"num_objects_unfound":0,"num_objects_dirty":4522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6460,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":4522}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":332,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.178851+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:06:53.178851+0000","last_peered":"2023-09-23T21:06:53.178851+0000","last_clean":"2023-09-23T21:06:53.178851+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:06:53.178851+0000","last_undegraded":"2023-09-23T21:06:53.178851+0000","last_fullsized":"2023-09-23T21:06:53.178851+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":416,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.171598+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:06:53.171598+0000","last_peered":"2023-09-23T21:06:53.171598+0000","last_clean":"2023-09-23T21:06:53.171598+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:06:53.171598+0000","last_undegraded":"2023-09-23T21:06:53.171598+0000","last_fullsized":"2023-09-23T21:06:53.171598+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"406'6432","reported_seq":6524,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:05.573419+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:07:05.573419+0000","last_peered":"2023-09-23T21:07:05.573419+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:07:05.573419+0000","last_undegraded":"2023-09-23T21:07:05.573419+0000","last_fullsized":"2023-09-23T21:07:05.573419+0000","mapping_epoch":374,"log_start":"349'2294","ondisk_log_start":"349'2294","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4138,"log_dups_size":0,"ondisk_log_size":4138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293076992,"num_objects":4472,"num_object_clones":0,"num_object_copies":13416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8944,"num_objects_unfound":0,"num_objects_dirty":4472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6432,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4472}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":33,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.168814+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:06:53.168814+0000","last_peered":"2023-09-23T21:06:53.168814+0000","last_clean":"2023-09-23T21:06:53.168814+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:06:53.168814+0000","last_undegraded":"2023-09-23T21:06:53.168814+0000","last_fullsized":"2023-09-23T21:06:53.168814+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":445,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.176336+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:06:53.176336+0000","last_peered":"2023-09-23T21:06:53.176336+0000","last_clean":"2023-09-23T21:06:53.176336+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:06:53.176336+0000","last_undegraded":"2023-09-23T21:06:53.176336+0000","last_fullsized":"2023-09-23T21:06:53.176336+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"406'6513","reported_seq":6690,"reported_epoch":406,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:05.268402+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:07:05.268402+0000","last_peered":"2023-09-23T21:07:05.268402+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:07:05.268402+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"349'1794","ondisk_log_start":"349'1794","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291569664,"num_objects":4449,"num_object_clones":0,"num_object_copies":13347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6513,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-990},{"shards":"1,6,7","objects":4449}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":55,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:54.873801+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:06:54.873801+0000","last_peered":"2023-09-23T21:06:54.873801+0000","last_clean":"2023-09-23T21:06:54.873801+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:06:54.873801+0000","last_undegraded":"2023-09-23T21:06:54.873801+0000","last_fullsized":"2023-09-23T21:06:54.873801+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":194,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.344279+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:07:00.344279+0000","last_peered":"2023-09-23T21:07:00.344279+0000","last_clean":"2023-09-23T21:07:00.344279+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:07:00.344279+0000","last_undegraded":"2023-09-23T21:07:00.344279+0000","last_fullsized":"2023-09-23T21:07:00.344279+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"406'6491","reported_seq":11981,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:04.151659+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:07:04.151659+0000","last_peered":"2023-09-23T21:07:04.151659+0000","last_clean":"2023-09-23T21:07:04.151659+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:07:04.151659+0000","last_undegraded":"2023-09-23T21:07:04.151659+0000","last_fullsized":"2023-09-23T21:07:04.151659+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5982,"log_dups_size":0,"ondisk_log_size":5982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297402392,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6491,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":300,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.186253+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:06:53.186253+0000","last_peered":"2023-09-23T21:06:53.186253+0000","last_clean":"2023-09-23T21:06:53.186253+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:06:53.186253+0000","last_undegraded":"2023-09-23T21:06:53.186253+0000","last_fullsized":"2023-09-23T21:06:53.186253+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":71,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.171680+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:06:53.171680+0000","last_peered":"2023-09-23T21:06:53.171680+0000","last_clean":"2023-09-23T21:06:53.171680+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:06:53.171680+0000","last_undegraded":"2023-09-23T21:06:53.171680+0000","last_fullsized":"2023-09-23T21:06:53.171680+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":55,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.175038+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:06:53.175038+0000","last_peered":"2023-09-23T21:06:53.175038+0000","last_clean":"2023-09-23T21:06:53.175038+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:06:53.175038+0000","last_undegraded":"2023-09-23T21:06:53.175038+0000","last_fullsized":"2023-09-23T21:06:53.175038+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"406'6515","reported_seq":8690,"reported_epoch":406,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:07:05.667792+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:07:05.667792+0000","last_peered":"2023-09-23T21:07:05.667792+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:07:05.667792+0000","last_undegraded":"2023-09-23T21:07:05.667792+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"350'2457","ondisk_log_start":"350'2457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4058,"log_dups_size":0,"ondisk_log_size":4058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295632896,"num_objects":4511,"num_object_clones":0,"num_object_copies":13533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11311,"num_objects_unfound":0,"num_objects_dirty":4511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6515,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1024,"num_bytes_recovered":67108864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":2156},{"shards":"2,6","objects":22},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":381,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.175505+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:06:53.175505+0000","last_peered":"2023-09-23T21:06:53.175505+0000","last_clean":"2023-09-23T21:06:53.175505+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:06:53.175505+0000","last_undegraded":"2023-09-23T21:06:53.175505+0000","last_fullsized":"2023-09-23T21:06:53.175505+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":356,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.182270+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:06:53.182270+0000","last_peered":"2023-09-23T21:06:53.182270+0000","last_clean":"2023-09-23T21:06:53.182270+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:06:53.182270+0000","last_undegraded":"2023-09-23T21:06:53.182270+0000","last_fullsized":"2023-09-23T21:06:53.182270+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":37,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.186122+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:06:53.186122+0000","last_peered":"2023-09-23T21:06:53.186122+0000","last_clean":"2023-09-23T21:06:53.186122+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:06:53.186122+0000","last_undegraded":"2023-09-23T21:06:53.186122+0000","last_fullsized":"2023-09-23T21:06:53.186122+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"406'6544","reported_seq":6719,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:04.092207+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:07:04.092207+0000","last_peered":"2023-09-23T21:07:04.092207+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:07:04.092207+0000","last_undegraded":"2023-09-23T21:07:04.092207+0000","last_fullsized":"2023-09-23T21:07:04.092207+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296091648,"num_objects":4518,"num_object_clones":0,"num_object_copies":13554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9007,"num_objects_unfound":0,"num_objects_dirty":4518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6544,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":4518}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":382,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.164524+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:06:53.164524+0000","last_peered":"2023-09-23T21:06:53.164524+0000","last_clean":"2023-09-23T21:06:53.164524+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:06:53.164524+0000","last_undegraded":"2023-09-23T21:06:53.164524+0000","last_fullsized":"2023-09-23T21:06:53.164524+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":300,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.177273+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:06:53.177273+0000","last_peered":"2023-09-23T21:06:53.177273+0000","last_clean":"2023-09-23T21:06:53.177273+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:06:53.177273+0000","last_undegraded":"2023-09-23T21:06:53.177273+0000","last_fullsized":"2023-09-23T21:06:53.177273+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":37,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.193126+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:06:53.193126+0000","last_peered":"2023-09-23T21:06:53.193126+0000","last_clean":"2023-09-23T21:06:53.193126+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:06:53.193126+0000","last_undegraded":"2023-09-23T21:06:53.193126+0000","last_fullsized":"2023-09-23T21:06:53.193126+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"406'6420","reported_seq":12672,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:05.653745+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:07:05.653745+0000","last_peered":"2023-09-23T21:07:05.653745+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:07:05.653745+0000","last_undegraded":"2023-09-23T21:07:05.653745+0000","last_fullsized":"2023-09-23T21:07:05.653745+0000","mapping_epoch":387,"log_start":"350'2280","ondisk_log_start":"350'2280","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4140,"log_dups_size":0,"ondisk_log_size":4140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291897344,"num_objects":4454,"num_object_clones":0,"num_object_copies":13362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4454,"num_objects_unfound":0,"num_objects_dirty":4454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6420,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":4454}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":220,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.165925+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:06:53.165925+0000","last_peered":"2023-09-23T21:06:53.165925+0000","last_clean":"2023-09-23T21:06:53.165925+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:06:53.165925+0000","last_undegraded":"2023-09-23T21:06:53.165925+0000","last_fullsized":"2023-09-23T21:06:53.165925+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":703,"reported_epoch":405,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:00.343476+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:07:00.343476+0000","last_peered":"2023-09-23T21:07:00.343476+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:07:00.343476+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":55,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.173741+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:06:53.173741+0000","last_peered":"2023-09-23T21:06:53.173741+0000","last_clean":"2023-09-23T21:06:53.173741+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:06:53.173741+0000","last_undegraded":"2023-09-23T21:06:53.173741+0000","last_fullsized":"2023-09-23T21:06:53.173741+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":294,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.183613+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:06:53.183613+0000","last_peered":"2023-09-23T21:06:53.183613+0000","last_clean":"2023-09-23T21:06:53.183613+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:06:53.183613+0000","last_undegraded":"2023-09-23T21:06:53.183613+0000","last_fullsized":"2023-09-23T21:06:53.183613+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"406'6595","reported_seq":6703,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:05.649532+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:07:05.649532+0000","last_peered":"2023-09-23T21:07:05.649532+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:07:05.649532+0000","last_undegraded":"2023-09-23T21:07:05.649532+0000","last_fullsized":"2023-09-23T21:07:05.649532+0000","mapping_epoch":374,"log_start":"350'2505","ondisk_log_start":"350'2505","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4090,"log_dups_size":0,"ondisk_log_size":4090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296681472,"num_objects":4527,"num_object_clones":0,"num_object_copies":13581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9054,"num_objects_unfound":0,"num_objects_dirty":4527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6595,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":4527}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":37,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.174689+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:06:53.174689+0000","last_peered":"2023-09-23T21:06:53.174689+0000","last_clean":"2023-09-23T21:06:53.174689+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:06:53.174689+0000","last_undegraded":"2023-09-23T21:06:53.174689+0000","last_fullsized":"2023-09-23T21:06:53.174689+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.866163+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:07:00.866163+0000","last_peered":"2023-09-23T21:07:00.866163+0000","last_clean":"2023-09-23T21:07:00.866163+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:07:00.866163+0000","last_undegraded":"2023-09-23T21:07:00.866163+0000","last_fullsized":"2023-09-23T21:07:00.866163+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":42,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.179330+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:06:53.179330+0000","last_peered":"2023-09-23T21:06:53.179330+0000","last_clean":"2023-09-23T21:06:53.179330+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:06:53.179330+0000","last_undegraded":"2023-09-23T21:06:53.179330+0000","last_fullsized":"2023-09-23T21:06:53.179330+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"406'6502","reported_seq":10115,"reported_epoch":406,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:05.353691+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:07:05.353691+0000","last_peered":"2023-09-23T21:07:05.353691+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:07:05.353691+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:07:05.353691+0000","mapping_epoch":383,"log_start":"349'1726","ondisk_log_start":"349'1726","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4776,"log_dups_size":0,"ondisk_log_size":4776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296484864,"num_objects":4524,"num_object_clones":0,"num_object_copies":13572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":773,"num_objects_unfound":0,"num_objects_dirty":4524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":773},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":43,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.869477+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:07:00.869477+0000","last_peered":"2023-09-23T21:07:00.869477+0000","last_clean":"2023-09-23T21:07:00.869477+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:07:00.869477+0000","last_undegraded":"2023-09-23T21:07:00.869477+0000","last_fullsized":"2023-09-23T21:07:00.869477+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":302,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.188208+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:06:53.188208+0000","last_peered":"2023-09-23T21:06:53.188208+0000","last_clean":"2023-09-23T21:06:53.188208+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:06:53.188208+0000","last_undegraded":"2023-09-23T21:06:53.188208+0000","last_fullsized":"2023-09-23T21:06:53.188208+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":81,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.172000+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:06:53.172000+0000","last_peered":"2023-09-23T21:06:53.172000+0000","last_clean":"2023-09-23T21:06:53.172000+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:06:53.172000+0000","last_undegraded":"2023-09-23T21:06:53.172000+0000","last_fullsized":"2023-09-23T21:06:53.172000+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"406'6471","reported_seq":19356,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:05.323569+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:07:05.323569+0000","last_peered":"2023-09-23T21:07:05.323569+0000","last_clean":"2023-09-23T21:07:05.323569+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:07:05.323569+0000","last_undegraded":"2023-09-23T21:07:05.323569+0000","last_fullsized":"2023-09-23T21:07:05.323569+0000","mapping_epoch":397,"log_start":"349'1745","ondisk_log_start":"349'1745","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4726,"log_dups_size":0,"ondisk_log_size":4726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6471,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":63,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:54.874020+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:06:54.874020+0000","last_peered":"2023-09-23T21:06:54.874020+0000","last_clean":"2023-09-23T21:06:54.874020+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:06:54.874020+0000","last_undegraded":"2023-09-23T21:06:54.874020+0000","last_fullsized":"2023-09-23T21:06:54.874020+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":37,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.188437+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:06:53.188437+0000","last_peered":"2023-09-23T21:06:53.188437+0000","last_clean":"2023-09-23T21:06:53.188437+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:06:53.188437+0000","last_undegraded":"2023-09-23T21:06:53.188437+0000","last_fullsized":"2023-09-23T21:06:53.188437+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":55,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.173223+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:06:53.173223+0000","last_peered":"2023-09-23T21:06:53.173223+0000","last_clean":"2023-09-23T21:06:53.173223+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:06:53.173223+0000","last_undegraded":"2023-09-23T21:06:53.173223+0000","last_fullsized":"2023-09-23T21:06:53.173223+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"406'6610","reported_seq":8742,"reported_epoch":406,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:05.351582+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:07:05.351582+0000","last_peered":"2023-09-23T21:07:05.351582+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:07:05.351582+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:07:05.351582+0000","mapping_epoch":382,"log_start":"349'1912","ondisk_log_start":"349'1912","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4698,"log_dups_size":0,"ondisk_log_size":4698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300679168,"num_objects":4588,"num_object_clones":0,"num_object_copies":13764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4585,"num_objects_misplaced":4591,"num_objects_unfound":0,"num_objects_dirty":4588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6610,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1011,"num_bytes_recovered":66256896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":4585}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":302,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.175765+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:06:53.175765+0000","last_peered":"2023-09-23T21:06:53.175765+0000","last_clean":"2023-09-23T21:06:53.175765+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:06:53.175765+0000","last_undegraded":"2023-09-23T21:06:53.175765+0000","last_fullsized":"2023-09-23T21:06:53.175765+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":46,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.179484+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:06:53.179484+0000","last_peered":"2023-09-23T21:06:53.179484+0000","last_clean":"2023-09-23T21:06:53.179484+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:06:53.179484+0000","last_undegraded":"2023-09-23T21:06:53.179484+0000","last_fullsized":"2023-09-23T21:06:53.179484+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":91,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.171069+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:06:53.171069+0000","last_peered":"2023-09-23T21:06:53.171069+0000","last_clean":"2023-09-23T21:06:53.171069+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:06:53.171069+0000","last_undegraded":"2023-09-23T21:06:53.171069+0000","last_fullsized":"2023-09-23T21:06:53.171069+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":51,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.180446+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:06:53.180446+0000","last_peered":"2023-09-23T21:06:53.180446+0000","last_clean":"2023-09-23T21:06:53.180446+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:06:53.180446+0000","last_undegraded":"2023-09-23T21:06:53.180446+0000","last_fullsized":"2023-09-23T21:06:53.180446+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":429,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:54.172990+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:06:54.172990+0000","last_peered":"2023-09-23T21:06:54.172990+0000","last_clean":"2023-09-23T21:06:54.172990+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:06:54.172990+0000","last_undegraded":"2023-09-23T21:06:54.172990+0000","last_fullsized":"2023-09-23T21:06:54.172990+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":68,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.169431+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:06:53.169431+0000","last_peered":"2023-09-23T21:06:53.169431+0000","last_clean":"2023-09-23T21:06:53.169431+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:06:53.169431+0000","last_undegraded":"2023-09-23T21:06:53.169431+0000","last_fullsized":"2023-09-23T21:06:53.169431+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":71,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.178219+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:06:53.178219+0000","last_peered":"2023-09-23T21:06:53.178219+0000","last_clean":"2023-09-23T21:06:53.178219+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:06:53.178219+0000","last_undegraded":"2023-09-23T21:06:53.178219+0000","last_fullsized":"2023-09-23T21:06:53.178219+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":287,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.185150+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:06:53.185150+0000","last_peered":"2023-09-23T21:06:53.185150+0000","last_clean":"2023-09-23T21:06:53.185150+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:06:53.185150+0000","last_undegraded":"2023-09-23T21:06:53.185150+0000","last_fullsized":"2023-09-23T21:06:53.185150+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":63,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.173887+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:06:53.173887+0000","last_peered":"2023-09-23T21:06:53.173887+0000","last_clean":"2023-09-23T21:06:53.173887+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:06:53.173887+0000","last_undegraded":"2023-09-23T21:06:53.173887+0000","last_fullsized":"2023-09-23T21:06:53.173887+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":46,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.178752+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:06:53.178752+0000","last_peered":"2023-09-23T21:06:53.178752+0000","last_clean":"2023-09-23T21:06:53.178752+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:06:53.178752+0000","last_undegraded":"2023-09-23T21:06:53.178752+0000","last_fullsized":"2023-09-23T21:06:53.178752+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.173315+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:06:53.173315+0000","last_peered":"2023-09-23T21:06:53.173315+0000","last_clean":"2023-09-23T21:06:53.173315+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:06:53.173315+0000","last_undegraded":"2023-09-23T21:06:53.173315+0000","last_fullsized":"2023-09-23T21:06:53.173315+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":49,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.865766+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:07:00.865766+0000","last_peered":"2023-09-23T21:07:00.865766+0000","last_clean":"2023-09-23T21:07:00.865766+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:07:00.865766+0000","last_undegraded":"2023-09-23T21:07:00.865766+0000","last_fullsized":"2023-09-23T21:07:00.865766+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":33,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T21:06:53.168537+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:06:53.168537+0000","last_peered":"2023-09-23T21:06:53.168537+0000","last_clean":"2023-09-23T21:06:53.168537+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:06:53.168537+0000","last_undegraded":"2023-09-23T21:06:53.168537+0000","last_fullsized":"2023-09-23T21:06:53.168537+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4715446296,"num_objects":71953,"num_object_clones":0,"num_object_copies":215859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33531,"num_objects_misplaced":57049,"num_objects_unfound":0,"num_objects_dirty":71953,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":104421,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24590,"num_bytes_recovered":1611530240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16002789376,"data_stored":16002777160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74120,"ondisk_log_size":74120,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201740,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463361,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3679080,"kb_used_data":3184480,"kb_used_omap":0,"kb_used_meta":494592,"kb_avail":90066072,"statfs":{"total":95995035648,"available":92227657728,"internally_reserved":0,"allocated":3260907520,"data_stored":3259718367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":506462208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.131,"15min":2.1230000000000002},"min":{"1min":0.254,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6689999999999996,"5min":7.2480000000000002,"15min":102.242},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.107,"15min":1.8440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":0.89300000000000002,"5min":9.7110000000000003,"15min":102.502},"last":2.2120000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":3.907,"15min":2.3660000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.24299999999999999},"max":{"1min":0.55200000000000005,"5min":119.077,"15min":119.077},"last":6.8200000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":3.8039999999999998,"15min":2.4590000000000001},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.24199999999999999},"max":{"1min":2.8740000000000001,"5min":119.313,"15min":119.313},"last":6.0119999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":4.1219999999999999,"15min":2.9380000000000002},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.5910000000000002,"5min":123.283,"15min":176.45400000000001},"last":6.4180000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":4.1680000000000001,"15min":3.1080000000000001},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":1.3009999999999999,"5min":123.051,"15min":176.46799999999999},"last":3.9550000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.456,"15min":4.4130000000000003},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.26900000000000002},"max":{"1min":3.173,"5min":35.078000000000003,"15min":250.804},"last":2.319},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.5409999999999999,"15min":4.3600000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":2.1850000000000001,"5min":35.133000000000003,"15min":250.446},"last":4.1120000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.1579999999999999,"15min":0.93600000000000005},"min":{"1min":0.32000000000000001,"5min":0.27800000000000002,"15min":0.24299999999999999},"max":{"1min":0.878,"5min":13.1,"15min":13.1},"last":2.2469999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.91100000000000003,"15min":0.96299999999999997},"min":{"1min":0.36699999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":2.9079999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.0980000000000001,"15min":1.5289999999999999},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.90700000000000003,"5min":11.281000000000001,"15min":105.121},"last":2.2709999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.208,"15min":1.585},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":8.6229999999999993,"15min":106.30500000000001},"last":2.0339999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.123,"15min":1.5089999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":0.70899999999999996,"5min":7.093,"15min":101.04600000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.1890000000000001,"15min":1.516},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":0.66800000000000004,"5min":7.1849999999999996,"15min":101.081},"last":2.1909999999999998}]}]},{"osd":1,"up_from":276,"seq":1185410973807,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3761260,"kb_used_data":3431524,"kb_used_omap":5,"kb_used_meta":329658,"kb_avail":89983892,"statfs":{"total":95995035648,"available":92143505408,"internally_reserved":0,"allocated":3513880576,"data_stored":3511867438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":337570531},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":0.85599999999999998,"15min":0.55200000000000005},"min":{"1min":0.20599999999999999,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":9.9469999999999992,"5min":11.959,"15min":11.959},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.256,"15min":0.81499999999999995},"min":{"1min":0.216,"5min":0.193,"15min":0.193},"max":{"1min":5.093,"5min":18.047999999999998,"15min":18.047999999999998},"last":2.202}]},{"osd":2,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.407,"15min":1.1359999999999999},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.0670000000000002,"5min":17.254000000000001,"15min":78.409000000000006},"last":2.3100000000000001},{"interface":"front","average":{"1min":1.55,"5min":1.03,"15min":0.97999999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.22},"max":{"1min":12.478999999999999,"5min":12.478999999999999,"15min":76.245000000000005},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.849,"15min":2.4670000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":7.3099999999999996,"5min":74.527000000000001,"15min":172.44399999999999},"last":4.2389999999999999},{"interface":"front","average":{"1min":1.907,"5min":2.5369999999999999,"15min":2.6880000000000002},"min":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":7.9050000000000002,"5min":72.216999999999999,"15min":172.58500000000001},"last":2.238}]},{"osd":4,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.472,"15min":0.93400000000000005},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.7069999999999999,"5min":17.006,"15min":17.006},"last":0.432},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.92600000000000005,"15min":0.70299999999999996},"min":{"1min":0.35699999999999998,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":4.7690000000000001,"5min":11.916,"15min":11.916},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.446,"15min":0.88300000000000001},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.6269999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":7.5090000000000003},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.589,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.3860000000000001,"5min":17.076000000000001,"15min":17.076000000000001},"last":4.1539999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.577,"15min":1.3879999999999999},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.6020000000000003,"5min":17.102,"15min":127.995},"last":3.044},{"interface":"front","average":{"1min":1.127,"5min":1.647,"15min":1.466},"min":{"1min":0.41699999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.375,"5min":17.302,"15min":126.836},"last":4.117}]},{"osd":7,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.9180000000000001,"15min":1.484},"min":{"1min":0.442,"5min":0.38200000000000001,"15min":0.36499999999999999},"max":{"1min":7.048,"5min":151.791,"15min":151.791},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.49,"5min":2.3889999999999998,"15min":1.2310000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.3739999999999997,"5min":151.809,"15min":151.809},"last":0.56999999999999995}]}]},{"osd":0,"up_from":92,"seq":395136991547,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353840,"kb_used_data":619048,"kb_used_omap":5,"kb_used_meta":734714,"kb_avail":92391312,"statfs":{"total":95995035648,"available":94608703488,"internally_reserved":0,"allocated":633905152,"data_stored":632002142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":752347770},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":2.9500000000000002,"15min":1.4830000000000001},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":40.284999999999997,"5min":61.341999999999999,"15min":61.341999999999999},"last":1.071},{"interface":"front","average":{"1min":3.4319999999999999,"5min":2.9670000000000001,"15min":1.4510000000000001},"min":{"1min":0.223,"5min":0.217,"15min":0.217},"max":{"1min":40.213000000000001,"5min":61.299999999999997,"15min":61.299999999999997},"last":2.7599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.7110000000000001,"15min":2.238},"min":{"1min":0.217,"5min":0.217,"15min":0.161},"max":{"1min":41.860999999999997,"5min":41.860999999999997,"15min":183.77099999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":4.5780000000000003,"5min":2.1219999999999999,"15min":2.5070000000000001},"min":{"1min":0.27000000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":44.779000000000003,"5min":44.779000000000003,"15min":183.78700000000001},"last":2.5739999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.1319999999999999,"15min":3.3079999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":5.5810000000000004,"5min":11.555999999999999,"15min":186.143},"last":1.1100000000000001},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.3320000000000001,"15min":3.4380000000000002},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.23000000000000001},"max":{"1min":4.016,"5min":16.640000000000001,"15min":186.97399999999999},"last":4.3200000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.2090000000000001,"15min":1.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.5259999999999998,"5min":9.7080000000000002,"15min":10.512},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.80200000000000005,"15min":0.93799999999999994},"min":{"1min":0.31,"5min":0.31,"15min":0.26200000000000001},"max":{"1min":3.875,"5min":6.0490000000000004,"15min":11.493},"last":0.74399999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.3640000000000001,"15min":2.4830000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.492,"5min":9.7439999999999998,"15min":154.30600000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.093,"5min":0.90200000000000002,"15min":1.905},"min":{"1min":0.33800000000000002,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":11.478,"5min":11.478,"15min":154.42400000000001},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2149999999999999,"5min":5.7060000000000004,"15min":3.1709999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":136.404,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":8.1039999999999992,"5min":5.9210000000000003,"15min":3.3599999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":136.44999999999999,"5min":169.767,"15min":169.767},"last":2.3650000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.98599999999999999,"15min":2.4279999999999999},"min":{"1min":0.35199999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":164.26400000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.107,"15min":2.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":1.6719999999999999,"5min":6.5410000000000004,"15min":164.23099999999999},"last":2.4049999999999998}]}]},{"osd":2,"up_from":129,"seq":554050781438,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3721716,"kb_used_data":3108844,"kb_used_omap":7,"kb_used_meta":612856,"kb_avail":90023436,"statfs":{"total":95995035648,"available":92183998464,"internally_reserved":0,"allocated":3183456256,"data_stored":3181491704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":627564625},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.302,"15min":1.3919999999999999},"min":{"1min":0.19400000000000001,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":1.7470000000000001,"5min":15.667,"15min":26.356000000000002},"last":4.3470000000000004},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.4419999999999999,"15min":1.556},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.663,"5min":12.445,"15min":26.341000000000001},"last":6.3579999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":4.9199999999999999,"15min":2.379},"min":{"1min":0.23599999999999999,"5min":0.19,"15min":0.188},"max":{"1min":4.9089999999999998,"5min":212.12,"15min":212.12},"last":0.377},{"interface":"front","average":{"1min":1.232,"5min":4.9390000000000001,"15min":2.1890000000000001},"min":{"1min":0.24299999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.4219999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":1.26}]},{"osd":3,"last update":"Sat Sep 23 21:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":2.2480000000000002,"15min":3.3039999999999998},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":4.3559999999999999,"5min":66.085999999999999,"15min":151.43799999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":2.222,"15min":3.3260000000000001},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":1.4930000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":0.85099999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.101,"15min":1.1419999999999999},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.4570000000000001,"5min":12.586,"15min":12.586},"last":1.294},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.1870000000000001,"15min":1.1910000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.575,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":6.2619999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.29,"15min":2.5760000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.6379999999999999,"5min":9.9909999999999997,"15min":173.34},"last":6.2169999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.079,"15min":2.3809999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":2.1949999999999998,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.40000000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":3.2869999999999999,"15min":1.883},"min":{"1min":0.34200000000000003,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":2.1789999999999998,"5min":175.72499999999999,"15min":175.72499999999999},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":3.3250000000000002,"15min":1.984},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":1.5329999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":1.722}]},{"osd":7,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.6060000000000001,"15min":2.7330000000000001},"min":{"1min":0.41399999999999998,"5min":0.40799999999999997,"15min":0.32700000000000001},"max":{"1min":9.6910000000000007,"5min":12.609,"15min":177.30799999999999},"last":1.4359999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.355,"15min":2.6520000000000001},"min":{"1min":0.38900000000000001,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.615,"5min":10.016,"15min":177.40899999999999},"last":6.2930000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160032,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542312,"kb_used_data":7456,"kb_used_omap":2,"kb_used_meta":534845,"kb_avail":93202840,"statfs":{"total":95995035648,"available":95439708160,"internally_reserved":0,"allocated":7634944,"data_stored":5501730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":547681333},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.5580000000000001,"15min":1.7350000000000001},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.18099999999999999},"max":{"1min":3.8570000000000002,"5min":44.963000000000001,"15min":95.507999999999996},"last":3.5579999999999998},{"interface":"front","average":{"1min":0.438,"5min":1.02,"15min":1.698},"min":{"1min":0.219,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":1.2909999999999999,"5min":9.7759999999999998,"15min":95.259},"last":5.9470000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:06:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.347,"15min":1.0580000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7619999999999996,"5min":12.035,"15min":19.786999999999999},"last":11.629},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.661,"15min":1.292},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":8.3059999999999992,"5min":15.763999999999999,"15min":15.763999999999999},"last":1.863}]},{"osd":2,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.464,"15min":1.2290000000000001},"min":{"1min":0.29699999999999999,"5min":0.223,"15min":0.182},"max":{"1min":13.944000000000001,"5min":13.944000000000001,"15min":13.944000000000001},"last":2.0680000000000001},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.2470000000000001,"15min":1.2909999999999999},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":4.6349999999999998,"5min":12.096,"15min":19.754999999999999},"last":7.2199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":2.645,"15min":2.1930000000000001},"min":{"1min":0.34799999999999998,"5min":0.307,"15min":0.222},"max":{"1min":8.1750000000000007,"5min":70.728999999999999,"15min":111.36},"last":1.79},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1389999999999998,"15min":1.9019999999999999},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":4.1100000000000003,"5min":70.712000000000003,"15min":112.473},"last":2.105}]},{"osd":5,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.0740000000000001,"15min":1.5129999999999999},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":1.325,"5min":9.6359999999999992,"15min":64.528000000000006},"last":6.0650000000000004},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.79,"15min":1.6220000000000001},"min":{"1min":0.42099999999999999,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":8.2490000000000006,"5min":10.617000000000001,"15min":14.054},"last":1.7549999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.857,"15min":1.6919999999999999},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.312},"max":{"1min":8.2780000000000005,"5min":10.52,"15min":19.568000000000001},"last":1.8400000000000001},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.4650000000000001,"15min":1.4750000000000001},"min":{"1min":0.39100000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":4.3920000000000003,"5min":14.356,"15min":15.573},"last":2.145}]},{"osd":7,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":2.1930000000000001,"15min":2.6099999999999999},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":1.3480000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":6.0940000000000003},{"interface":"front","average":{"1min":1.341,"5min":2.8090000000000002,"15min":2.8490000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":8.218,"5min":73.364999999999995,"15min":222.11699999999999},"last":1.8169999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986650,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1731944,"kb_used_data":1174880,"kb_used_omap":0,"kb_used_meta":557056,"kb_avail":92013208,"statfs":{"total":95995035648,"available":94221524992,"internally_reserved":0,"allocated":1203077120,"data_stored":1201887351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":570425344},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.2869999999999999,"15min":2.782},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.53200000000000003,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.119,"15min":2.8029999999999999},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.223},"max":{"1min":3.8959999999999999,"5min":8.5220000000000002,"15min":144.77000000000001},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:06:07 2023","interfaces":[{"interface":"back","average":{"1min":6.4459999999999997,"5min":6.3929999999999998,"15min":2.6269999999999998},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":118.411,"5min":118.411,"15min":118.411},"last":3.3740000000000001},{"interface":"front","average":{"1min":6.274,"5min":6.1340000000000003,"15min":2.4740000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":118.075,"5min":118.075,"15min":118.075},"last":0.97099999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":5.8410000000000002,"15min":3.2130000000000001},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":3.452,"5min":164.517,"15min":164.517},"last":10.247999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":5.665,"15min":3.274},"min":{"1min":0.35599999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":0.56999999999999995,"5min":164.46799999999999,"15min":164.46799999999999},"last":2.5790000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:06:06 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.2709999999999999,"15min":2.2610000000000001},"min":{"1min":0.32500000000000001,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.8420000000000001,"5min":8.7420000000000009,"15min":142.28399999999999},"last":1.1950000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.2829999999999999,"15min":2.3889999999999998},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.82799999999999996,"5min":7.5579999999999998,"15min":142.34700000000001},"last":2.2389999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.82499999999999996,"15min":1.0609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.24099999999999999},"max":{"1min":2.9049999999999998,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.875,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29899999999999999,"15min":0.21099999999999999},"max":{"1min":0.58299999999999996,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.004,"15min":0.96699999999999997},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":0.70599999999999996,"5min":11.800000000000001,"15min":11.887},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.111,"15min":0.92100000000000004},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":2.3620000000000001,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":2.6059999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":3.0110000000000001,"15min":1.8280000000000001},"min":{"1min":0.42799999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":0.72799999999999998,"5min":145.721,"15min":145.721},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":3.282,"15min":1.992},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":0.78100000000000003,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.51100000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725014,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1249812,"kb_used_data":652684,"kb_used_omap":0,"kb_used_meta":597120,"kb_avail":92495340,"statfs":{"total":95995035648,"available":94715228160,"internally_reserved":0,"allocated":668348416,"data_stored":667174847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":611450880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.68500000000000005,"15min":1.3540000000000001},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":10.016999999999999,"5min":10.016999999999999,"15min":80.105999999999995},"last":0.499},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.89300000000000002,"15min":1.462},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":2.919,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.58199999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":4.1200000000000001,"15min":1.6699999999999999},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":0.67200000000000004,"5min":137.143,"15min":137.143},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.47199999999999998,"5min":3.8199999999999998,"15min":1.5800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.247},"max":{"1min":0.66200000000000003,"5min":139.73699999999999,"15min":139.73699999999999},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.3980000000000001,"15min":1.569},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":8.3770000000000007,"5min":103.63800000000001,"15min":103.63800000000001},"last":73.787999999999997},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.6309999999999998,"15min":1.696},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":6.7640000000000002,"5min":103.664,"15min":103.664},"last":73.546999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.3149999999999999,"15min":2.8359999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":8.0860000000000003,"5min":97.349999999999994,"15min":153.60300000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.2570000000000001,"15min":2.7559999999999998},"min":{"1min":0.38300000000000001,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":4.6609999999999996,"5min":97.316999999999993,"15min":153.774},"last":0.59799999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.69799999999999995,"15min":0.81699999999999995},"min":{"1min":0.33100000000000002,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":0.69899999999999995,"5min":12.739000000000001,"15min":12.739000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.88,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.247},"max":{"1min":0.59599999999999997,"5min":10.792,"15min":12.539},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.7150000000000001,"15min":1.1839999999999999},"min":{"1min":0.376,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":1.276,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.625,"5min":1.8420000000000001,"15min":1.3220000000000001},"min":{"1min":0.32400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6699999999999999,"5min":70.266000000000005,"15min":70.266000000000005},"last":0.376}]},{"osd":7,"last update":"Sat Sep 23 21:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.014,"15min":1.2230000000000001},"min":{"1min":0.40799999999999997,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":0.68700000000000006,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.65200000000000002,"15min":1.0780000000000001},"min":{"1min":0.38300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.40899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":627507200,"data_stored":627507200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3496148992,"data_stored":3496148992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3164016640,"data_stored":3164012568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2424832,"data_stored":2424832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1191907328,"data_stored":1191903256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":659161088,"data_stored":659161088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3241283584,"data_stored":3241279512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3620339712,"data_stored":3620339712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:07:08.383 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:07:08.409 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:08.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:08 smithi121 ceph-mon[211526]: from='client.56530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:08.613 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:08 smithi121 ceph-mon[219709]: from='client.56530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:08.617 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:08.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:08.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:08 smithi123 ceph-mon[193903]: from='client.56530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:08.734 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:08.786 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:08.844 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:09.193 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:09.601 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:09 smithi121 ceph-mon[211526]: pgmap v1274: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 33533/215931 objects degraded (15.529%); 57049/215931 objects misplaced (26.420%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:07:09.601 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1233932787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:09.601 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:09 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:09.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:09 smithi121 ceph-mon[219709]: pgmap v1274: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 33533/215931 objects degraded (15.529%); 57049/215931 objects misplaced (26.420%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:07:09.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1233932787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:09.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:09 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:09.603 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:07:09.720 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:09.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:09.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:09.823 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:09.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:09 smithi123 ceph-mon[193903]: pgmap v1274: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 33533/215931 objects degraded (15.529%); 57049/215931 objects misplaced (26.420%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T21:07:09.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1233932787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:09.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:09 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:09.885 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:10.032 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:10.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:10.141 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:10.198 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:10.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:10 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:10.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:10 smithi123 ceph-mon[193903]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:10 smithi123 ceph-mon[193903]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:10 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:10.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:10 smithi123 ceph-mon[193903]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T21:07:10.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:10 smithi123 ceph-mon[193903]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.831 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:10.831 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:10.861 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:10.861 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[211526]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.861 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[211526]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.861 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:10.861 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[211526]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T21:07:10.861 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[211526]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:10.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[219709]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[219709]: failed to encode map e407 with expected crc 2023-09-23T21:07:10.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:10.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[219709]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T21:07:10.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:10 smithi121 ceph-mon[219709]: failed to encode map e407 with expected crc 2023-09-23T21:07:11.716 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:11.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:11.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:11.823 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:11.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:11.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:11.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[211526]: pgmap v1276: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 33052/208500 objects degraded (15.852%); 55073/208500 objects misplaced (26.414%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:11.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 33533/215931 objects degraded (15.529%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:11.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:11.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:11.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[219709]: pgmap v1276: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 33052/208500 objects degraded (15.852%); 55073/208500 objects misplaced (26.414%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:11.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 33533/215931 objects degraded (15.529%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:11.929 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:11.969 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:11 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:11.970 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:11 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:11.970 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:11 smithi123 ceph-mon[193903]: pgmap v1276: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 33052/208500 objects degraded (15.852%); 55073/208500 objects misplaced (26.414%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:11.970 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 33533/215931 objects degraded (15.529%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:11.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:07:12.481 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:12.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:12.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:12.515 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:07:12.586 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:12.725 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:12.764 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:12.765 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:12.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:13.050 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:13.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:07:13.161 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:13.190 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:07:13.211 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.240 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:07:13.278 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:13.287 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:07:13.323 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:07:13.383 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:13.387 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:07:13.416 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.425 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:07:13.467 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:07:13.475 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:13.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:13.499 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.499 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.499 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:07:13.581 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:13.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:07:13.622 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.676 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:13.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:13.781 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:13.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:13 smithi121 ceph-mon[211526]: pgmap v1277: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 33052/208500 objects degraded (15.852%); 55073/208500 objects misplaced (26.414%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:13.895 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2463526419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:13.895 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:13 smithi121 ceph-mon[219709]: pgmap v1277: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 33052/208500 objects degraded (15.852%); 55073/208500 objects misplaced (26.414%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:13.895 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2463526419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:13.896 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:13 smithi123 ceph-mon[193903]: pgmap v1277: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 33052/208500 objects degraded (15.852%); 55073/208500 objects misplaced (26.414%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2463526419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:14.033 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:07:14.033 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 33052/208500 objects degraded (15.852%), 7 pgs degraded, 6 pgs undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1690,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":407,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+remapped+backfill_wait","count":6},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":69500,"data_bytes":4553709496,"bytes_used":16000425984,"bytes_avail":655964823552,"bytes_total":671965249536,"degraded_objects":33052,"degraded_total":208500,"degraded_ratio":0.15852278177458035,"misplaced_objects":55073,"misplaced_total":208500,"misplaced_ratio":0.26413908872901681,"recovering_objects_per_sec":93,"recovering_bytes_per_sec":6151511,"recovering_keys_per_sec":0,"num_objects_recovered":939,"num_bytes_recovered":61538304,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":498},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T21:06:55.027429+0000","services":{}},"progress_events":{"2639f328-dcb0-4a01-b6c7-21332e7147ac":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 40s)","progress":0.84269660711288452,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:07:14.669 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:14.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:14.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:14.780 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:14.837 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:14.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:07:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:07:14] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2023-09-23T21:07:14.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3468202601' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:07:14.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3468202601' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:07:14.921 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:14.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:14.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:14.936 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3468202601' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:07:15.033 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:15.069 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:15.077 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:07:15.077 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:07:15.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:07:15.077 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:07:15.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.077 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140894 10.c 10:33443f8a:::benchmark_data_smithi121_358375_object56149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:07:15.077 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.11165523300002, 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5509230220000001, 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140894 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.078 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960828+0000", 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960827+0000", 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960833+0000", 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1330000000000004e-06 2023-09-23T21:07:15.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960845+0000", 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2065999999999999e-05 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960850+0000", 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1599999999999997e-06 2023-09-23T21:07:15.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960895+0000", 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5522999999999998e-05 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.960988+0000", 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2343999999999995e-05 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.961068+0000", 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9864999999999995e-05 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.964285+0000", 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0032171370000000001 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.964723+0000", 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043790000000000002 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973518+0000", 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0087954379999999992 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973531+0000", 2023-09-23T21:07:15.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2510999999999999e-05 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322237+0000", 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34870620200000002 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322246+0000", 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3449999999999995e-06 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511643+0000", 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1893964910000001 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511751+0000", 2023-09-23T21:07:15.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107996 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140895 10.c 10:32d93067:::benchmark_data_smithi121_358375_object56150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.10703132200001, 2023-09-23T21:07:15.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.546411454, 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140895 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965452+0000", 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965450+0000", 2023-09-23T21:07:15.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965457+0000", 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.297e-06 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965471+0000", 2023-09-23T21:07:15.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4352000000000001e-05 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965476+0000", 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.198e-06 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965514+0000", 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7704999999999999e-05 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965608+0000", 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4121000000000002e-05 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,4", 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.965685+0000", 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7127999999999997e-05 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973585+0000", 2023-09-23T21:07:15.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078994029999999993 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.973593+0000", 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2810000000000008e-06 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322280+0000", 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34868769799999999 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322287+0000", 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1879999999999999e-06 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511780+0000", 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.189493559 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511863+0000", 2023-09-23T21:07:15.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2774000000000004e-05 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140896 10.1 10:86418234:::benchmark_data_smithi121_358375_object56151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.09744729900001, 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6998174590000001, 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.095 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140896 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975036+0000", 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975034+0000", 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975041+0000", 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4139999999999999e-06 2023-09-23T21:07:15.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975054+0000", 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3604999999999999e-05 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.975061+0000", 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3419999999999996e-06 2023-09-23T21:07:15.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.976385+0000", 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0013248260000000001 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.976396+0000", 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.076e-05 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320693+0000", 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.34429716599999999 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320701+0000", 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6899999999999992e-06 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.654437+0000", 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3337360949999999 2023-09-23T21:07:15.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.655243+0000", 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00080610999999999996 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668434+0000", 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013190676 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668502+0000", 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8547999999999995e-05 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.668880+0000", 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00037727300000000002 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674052+0000", 2023-09-23T21:07:15.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051727869999999999 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674083+0000", 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0043999999999999e-05 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674132+0000", 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9484000000000003e-05 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674191+0000", 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8567999999999999e-05 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.674853+0000", 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000662445 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.104 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140900 10.c 10:3c2d050d:::benchmark_data_smithi121_358375_object56155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.07482528999998, 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5351996919999999, 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140900 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997658+0000", 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997656+0000", 2023-09-23T21:07:15.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997665+0000", 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8799999999999997e-06 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997687+0000", 2023-09-23T21:07:15.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1967000000000002e-05 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997699+0000", 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2074e-05 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997758+0000", 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9375e-05 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:35.997768+0000", 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6749999999999997e-06 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322303+0000", 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.324535299 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322309+0000", 2023-09-23T21:07:15.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5690000000000004e-06 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.511887+0000", 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1895777999999999 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.512028+0000", 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014090500000000001 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.512592+0000", 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00056408399999999996 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525899+0000", 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013307461 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532566+0000", 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0066666620000000003 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532600+0000", 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4233000000000002e-05 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532699+0000", 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8743e-05 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532754+0000", 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4941000000000001e-05 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532857+0000", 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103405 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140903 10.c 10:38af2a49:::benchmark_data_smithi121_358375_object56158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 339.05300906000002, 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.513614249, 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140903 2023-09-23T21:07:15.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019474+0000", 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019472+0000", 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019481+0000", 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1039999999999996e-06 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019501+0000", 2023-09-23T21:07:15.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9706000000000001e-05 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019513+0000", 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2092999999999999e-05 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019575+0000", 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2136000000000003e-05 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.019587+0000", 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1616e-05 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322325+0000", 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30273779699999998 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.322334+0000", 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3780000000000007e-06 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520201+0000", 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1978669260000001 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520304+0000", 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102685 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.520390+0000", 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6036000000000001e-05 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.528925+0000", 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0085353589999999993 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532633+0000", 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037075699999999999 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532894+0000", 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00026169499999999999 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532952+0000", 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7717999999999999e-05 2023-09-23T21:07:15.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532991+0000", 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9443999999999997e-05 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533088+0000", 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6669000000000006e-05 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.122 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140920 10.11 10:8e38667f:::benchmark_data_smithi121_358375_object56175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 338.87241396299999, 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4951127319999999, 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.123 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140920 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200069+0000", 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200068+0000", 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200075+0000", 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4109999999999999e-06 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200092+0000", 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6860000000000001e-05 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200099+0000", 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2559999999999996e-06 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200156+0000", 2023-09-23T21:07:15.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6854000000000001e-05 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.200167+0000", 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0437e-05 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320655+0000", 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.120488398 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.320668+0000", 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3132999999999999e-05 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.652475+0000", 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3318069219999999 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.653395+0000", 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00092004800000000002 2023-09-23T21:07:15.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677570+0000", 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024174998999999999 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677680+0000", 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011025499999999999 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.677936+0000", 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00025574300000000001 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.679730+0000", 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0017943849999999999 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.692535+0000", 2023-09-23T21:07:15.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012804519 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.694272+0000", 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001737326 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.694330+0000", 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7287999999999998e-05 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.695182+0000", 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00085209999999999995 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140940 10.c 10:3f6dfdef:::benchmark_data_smithi121_358375_object56195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 338.61419781900003, 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0750470560000001, 2023-09-23T21:07:15.132 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140940 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458285+0000", 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458283+0000", 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458293+0000", 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9380000000000004e-06 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458312+0000", 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9083999999999999e-05 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458320+0000", 2023-09-23T21:07:15.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2300000000000008e-06 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458374+0000", 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4116000000000002e-05 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.458383+0000", 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4409999999999993e-06 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.522747+0000", 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.064364592 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.522997+0000", 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024926899999999999 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.523081+0000", 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4122999999999994e-05 2023-09-23T21:07:15.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532666+0000", 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.009585484 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.532923+0000", 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00025692899999999998 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533121+0000", 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000198209 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533191+0000", 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9379000000000001e-05 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533229+0000", 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8133999999999998e-05 2023-09-23T21:07:15.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533332+0000", 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103198 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140941 10.c 10:3b70ed7f:::benchmark_data_smithi121_358375_object56196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 338.61346074199997, 2023-09-23T21:07:15.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.075658805, 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140941 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459022+0000", 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459026+0000", 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9400000000000001e-06 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459035+0000", 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5290000000000008e-06 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459039+0000", 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0709999999999996e-06 2023-09-23T21:07:15.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459081+0000", 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2103e-05 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.459087+0000", 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3230000000000003e-06 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.525940+0000", 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.066852771 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.526063+0000", 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012287299999999999 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.526177+0000", 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113816 2023-09-23T21:07:15.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533150+0000", 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0069729099999999997 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533368+0000", 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000218383 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533381+0000", 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2839e-05 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534586+0000", 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001204552 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534618+0000", 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2261999999999997e-05 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534681+0000", 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3054999999999999e-05 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.45470.0:140942 10.c 10:3a89f6d8:::benchmark_data_smithi121_358375_object56197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 338.59084537299998, 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0531614490000001, 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.45470", 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 140942 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.149 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481638+0000", 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481636+0000", 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481643+0000", 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6429999999999995e-06 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481659+0000", 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5274e-05 2023-09-23T21:07:15.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481665+0000", 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3500000000000002e-06 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481716+0000", 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1357999999999999e-05 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:36.481726+0000", 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8099999999999992e-06 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.528966+0000", 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0472400340000001 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.529117+0000", 2023-09-23T21:07:15.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015073199999999999 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.529274+0000", 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000157431 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533184+0000", 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00390931 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533387+0000", 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000203301 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.533489+0000", 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000102224 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534716+0000", 2023-09-23T21:07:15.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012264190000000001 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534744+0000", 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8334999999999999e-05 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:01:37.534799+0000", 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4917999999999998e-05 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.156 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50076 11.5 11:a3fd1c51:::benchmark_data_smithi121_477989_object50075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 217.68039835799999, 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1126377949999999, 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.157 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50076 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392083+0000", 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392127+0000", 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3806e-05 2023-09-23T21:07:15.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392141+0000", 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4394e-05 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392150+0000", 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1650000000000006e-06 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392234+0000", 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4027e-05 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392388+0000", 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015483000000000001 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,6", 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392554+0000", 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016592399999999999 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.398697+0000", 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061424319999999998 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.403213+0000", 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045163499999999997 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.403226+0000", 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2938999999999999e-05 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394392+0000", 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99116629099999998 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394406+0000", 2023-09-23T21:07:15.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3896999999999999e-05 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504613+0000", 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.110206864 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504722+0000", 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010918500000000001 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.164 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50104 11.5 11:ac5fad04:::benchmark_data_smithi121_477989_object50103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 217.642832532, 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0882670750000001, 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.165 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50104 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:07:15.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429649+0000", 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429803+0000", 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015389900000000001 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429823+0000", 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9340000000000001e-05 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429833+0000", 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0005e-05 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429893+0000", 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0324e-05 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429904+0000", 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0781000000000001e-05 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394433+0000", 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96452942900000005 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394443+0000", 2023-09-23T21:07:15.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4879999999999996e-06 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504746+0000", 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.110303627 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504903+0000", 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015637600000000001 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.505086+0000", 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000183516 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.512945+0000", 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078586579999999993 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515525+0000", 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025799350000000002 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515589+0000", 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3974999999999996e-05 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515654+0000", 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4867000000000001e-05 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.517917+0000", 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022637120000000002 2023-09-23T21:07:15.172 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50113 11.5 11:ab5e54a5:::benchmark_data_smithi121_477989_object50112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 217.632793897, 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0819687199999999, 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.173 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50113 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439688+0000", 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439777+0000", 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9221000000000005e-05 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439790+0000", 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3013999999999999e-05 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439795+0000", 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2310000000000004e-06 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439871+0000", 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6036000000000002e-05 2023-09-23T21:07:15.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439883+0000", 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.186e-05 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394464+0000", 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95458152699999999 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394472+0000", 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8790000000000003e-06 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.506903+0000", 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1124306429999999 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.507065+0000", 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016177499999999999 2023-09-23T21:07:15.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.507239+0000", 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000174191 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515535+0000", 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0082961040000000003 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.517971+0000", 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024364360000000002 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.519321+0000", 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001349814 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.519388+0000", 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6958999999999994e-05 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.521658+0000", 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022694899999999999 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50122 11.5 11:a0d5e578:::benchmark_data_smithi121_477989_object50121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 217.44654945299999, 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.905001269, 2023-09-23T21:07:15.181 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50122 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625982+0000", 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9774999999999997e-05 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625994+0000", 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1986e-05 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625999+0000", 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9980000000000003e-06 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.626045+0000", 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5528999999999997e-05 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.626052+0000", 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5159999999999997e-06 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394493+0000", 2023-09-23T21:07:15.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76844040899999999 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394501+0000", 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5059999999999995e-06 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509037+0000", 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1145361 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.186 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509179+0000", 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014180599999999999 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509350+0000", 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017093699999999999 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515543+0000", 2023-09-23T21:07:15.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061929389999999997 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.522051+0000", 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065082919999999997 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.528556+0000", 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065045950000000002 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.528631+0000", 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4646000000000003e-05 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.530935+0000", 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023040959999999998 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.189 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50126 11.5 11:a51a9ec8:::benchmark_data_smithi121_477989_object50125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 217.44365980000001, 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9041968090000001, 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50126 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628822+0000", 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628877+0000", 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5653999999999999e-05 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628891+0000", 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3808000000000001e-05 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628897+0000", 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1460000000000001e-06 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628958+0000", 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0720000000000001e-05 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628968+0000", 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6229999999999992e-06 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394522+0000", 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76555460600000003 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394530+0000", 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9019999999999999e-06 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511058+0000", 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1165273579999999 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511192+0000", 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013458099999999999 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511361+0000", 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016914299999999999 2023-09-23T21:07:15.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515550+0000", 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041891139999999999 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.522105+0000", 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065545459999999996 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.531002+0000", 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088968529999999997 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.531064+0000", 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2463000000000001e-05 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.533020+0000", 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019555419999999998 2023-09-23T21:07:15.197 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78881 11.4 11:2f950127:::benchmark_data_smithi121_477989_object78880:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 179.18716681500001, 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30758963700000003, 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.198 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78881 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885361+0000", 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5769000000000002e-05 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885384+0000", 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3087000000000001e-05 2023-09-23T21:07:15.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885390+0000", 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7540000000000003e-06 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.887935+0000", 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025446190000000001 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.888057+0000", 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122161 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.888200+0000", 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014262500000000001 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893701+0000", 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005501695 2023-09-23T21:07:15.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.905399+0000", 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011697991 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192819+0000", 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.28741952999999998 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192867+0000", 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8605e-05 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192906+0000", 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8302e-05 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78885 11.4 11:2a17f6c4:::benchmark_data_smithi121_477989_object78884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:07:15.204 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 179.187077829, 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30759584699999998, 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78885 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.205 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885437+0000", 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2227999999999997e-05 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885444+0000", 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6939999999999995e-06 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885449+0000", 2023-09-23T21:07:15.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3950000000000003e-06 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890915+0000", 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0054663869999999996 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.891027+0000", 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011214600000000001 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.891156+0000", 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012827499999999999 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893843+0000", 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026873719999999999 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.905447+0000", 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011604353 2023-09-23T21:07:15.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192935+0000", 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.28748794300000002 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192973+0000", 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7407999999999998e-05 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.193001+0000", 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8166000000000001e-05 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78889 11.5 11:a04ebbe0:::benchmark_data_smithi121_477989_object78888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 179.186043552, 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30786717499999999, 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:15.211 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78889 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886438+0000", 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886479+0000", 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0985000000000001e-05 2023-09-23T21:07:15.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886493+0000", 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4032999999999999e-05 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886499+0000", 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1859999999999997e-06 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890610+0000", 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041114760000000002 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890765+0000", 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000154184 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890939+0000", 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017454800000000001 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893385+0000", 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002445984 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.899338+0000", 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059533499999999996 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192227+0000", 2023-09-23T21:07:15.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.29288863399999998 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192279+0000", 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2392999999999998e-05 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.194306+0000", 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020270679999999999 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.217 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88133 11.5 11:a2e8b6ce:::benchmark_data_smithi121_477989_object88132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 166.324566949, 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.015216702, 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.218 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88133 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747914+0000", 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747958+0000", 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.333e-05 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747970+0000", 2023-09-23T21:07:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2914999999999999e-05 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747977+0000", 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4620000000000001e-06 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752486+0000", 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045088680000000001 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752629+0000", 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014352899999999999 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752804+0000", 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017513500000000001 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.776368+0000", 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023563562999999999 2023-09-23T21:07:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.776380+0000", 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2476999999999999e-05 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763117+0000", 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98673686100000002 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763133+0000", 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5291e-05 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88146 11.5 11.f29ad9f5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 166.29964914000001, 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99033201000000004, 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88146 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772833+0000", 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772873+0000", 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0082e-05 2023-09-23T21:07:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772885+0000", 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2529e-05 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772893+0000", 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3520000000000001e-06 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.777718+0000", 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048257910000000003 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.777727+0000", 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4780000000000008e-06 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763156+0000", 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98542964799999999 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763166+0000", 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2620000000000004e-06 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88149 11.5 11.fd969135 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 166.28334599199999, 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97405727799999997, 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:07:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88149 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:07:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789135+0000", 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789175+0000", 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9626000000000003e-05 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:07:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789193+0000", 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7737999999999999e-05 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789199+0000", 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4500000000000001e-06 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789265+0000", 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5572000000000002e-05 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789281+0000", 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6612e-05 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763186+0000", 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.973904412 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763194+0000", 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5080000000000005e-06 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:07:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:07:15.234 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:07:15.294 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:15.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:15.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:15.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:15.467 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:15.623 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:15.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:15.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:15.657 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:07:15.657 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:07:15.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:15.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:15 smithi121 ceph-mon[211526]: pgmap v1278: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 33052/207129 objects degraded (15.957%); 54616/207129 objects misplaced (26.368%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:15.768 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:15 smithi121 ceph-mon[219709]: pgmap v1278: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 33052/207129 objects degraded (15.957%); 54616/207129 objects misplaced (26.368%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:15.780 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:15.821 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:15.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:15 smithi123 ceph-mon[193903]: pgmap v1278: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 499 op/s; 33052/207129 objects degraded (15.957%); 54616/207129 objects misplaced (26.368%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T21:07:16.047 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:16.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:16.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:16.125 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:07:16.154 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:16.190 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:16.245 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:16.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:16.355 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:16.511 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:16.716 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:07:16.798 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:16.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:16 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:16.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:16 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:16.804 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 33052/207129 objects degraded (15.957%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:16.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:16.909 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:17.004 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:17.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:16 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:17.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:16 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:17.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 33052/207129 objects degraded (15.957%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:16 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:16 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:17.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 33052/207129 objects degraded (15.957%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:17.576 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:17.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:17.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:17.655 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:17.681 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:17.817 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:17.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:07:17.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:07:17.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:17.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[211526]: pgmap v1279: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 32575/200982 objects degraded (16.208%); 53064/200982 objects misplaced (26.402%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:07:17.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:07:17.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:07:17.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:17.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:17 smithi121 ceph-mon[219709]: pgmap v1279: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 32575/200982 objects degraded (16.208%); 53064/200982 objects misplaced (26.402%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:07:18.221 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:07:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:07:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:17 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:18.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:17 smithi123 ceph-mon[193903]: pgmap v1279: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 498 op/s; 32575/200982 objects degraded (16.208%); 53064/200982 objects misplaced (26.402%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:07:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:18.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:18.331 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:18.369 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:18.441 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:18.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:18.447 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:18.548 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:18.586 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:18.655 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:18.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:18.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:18.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:18.826 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:18.983 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:07:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/180972772' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/180972772' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/180972772' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:19.898 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:19.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:19.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:20.001 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:20.046 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:20.076 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:20.124 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:20.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:20.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:20.235 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:20.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:20.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[219709]: failed to encode map e408 with expected crc 2023-09-23T21:07:20.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:20.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[219709]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T21:07:20.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[219709]: failed to encode map e408 with expected crc 2023-09-23T21:07:20.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[219709]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 32575/200982 objects degraded (16.208%); 53064/200982 objects misplaced (26.402%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T21:07:20.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:20.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[211526]: failed to encode map e408 with expected crc 2023-09-23T21:07:20.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:20.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[211526]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T21:07:20.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[211526]: failed to encode map e408 with expected crc 2023-09-23T21:07:20.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:20 smithi121 ceph-mon[211526]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 32575/200982 objects degraded (16.208%); 53064/200982 objects misplaced (26.402%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T21:07:20.344 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:20.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:20 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:07:20.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:20 smithi123 ceph-mon[193903]: failed to encode map e408 with expected crc 2023-09-23T21:07:20.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:20 smithi123 ceph-mon[193903]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T21:07:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:20 smithi123 ceph-mon[193903]: failed to encode map e408 with expected crc 2023-09-23T21:07:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:20 smithi123 ceph-mon[193903]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 4.1 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 32575/200982 objects degraded (16.208%); 53064/200982 objects misplaced (26.402%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T21:07:20.690 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:20.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:20.804 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:20.843 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:20.909 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:20.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:21.018 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:21.052 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:21.114 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:21.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:21.229 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:21.300 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:21 smithi121 ceph-mon[211526]: pgmap v1282: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 32059/193143 objects degraded (16.599%); 50920/193143 objects misplaced (26.364%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:21.300 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:21 smithi121 ceph-mon[219709]: pgmap v1282: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 32059/193143 objects degraded (16.599%); 50920/193143 objects misplaced (26.364%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:21.305 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:21 smithi123 ceph-mon[193903]: pgmap v1282: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 32059/193143 objects degraded (16.599%); 50920/193143 objects misplaced (26.364%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:22.134 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:07:22.159 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:07:22.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:22.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:22.290 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:22.320 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:22.377 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:22.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 32059/193143 objects degraded (16.599%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:22.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:22 smithi121 ceph-mon[219709]: from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:22.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 32059/193143 objects degraded (16.599%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:22.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:22 smithi121 ceph-mon[211526]: from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:22.453 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:07:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 32059/193143 objects degraded (16.599%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:22 smithi123 ceph-mon[193903]: from='client.46487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:22.483 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:22.521 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:22.578 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:22.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:22.691 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:22.793 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:23.407 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1282,"stamp":"2023-09-23T21:07:21.037984+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4218230712,"num_objects":64381,"num_object_clones":0,"num_object_copies":193143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32059,"num_objects_misplaced":50920,"num_objects_unfound":0,"num_objects_dirty":64381,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":112317,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26186,"num_bytes_recovered":1711447308,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75832,"ondisk_log_size":75832,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14847476,"kb_used_data":10949116,"kb_used_omap":21,"kb_used_meta":3898154,"kb_avail":641368588,"statfs":{"total":671965249536,"available":656761434112,"internally_reserved":0,"allocated":11211894784,"data_stored":11200266934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":3991710179},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,4,7,6,6,1,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-335478784,"num_objects":-5119,"num_object_clones":0,"num_object_copies":-15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-993,"num_objects_misplaced":-4153,"num_objects_unfound":0,"num_objects_dirty":-5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5119,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":998,"num_bytes_recovered":65404928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003458"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":178,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.578428+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:07:09.578428+0000","last_peered":"2023-09-23T21:07:09.578428+0000","last_clean":"2023-09-23T21:07:09.578428+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:07:09.578428+0000","last_undegraded":"2023-09-23T21:07:09.578428+0000","last_fullsized":"2023-09-23T21:07:09.578428+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":48,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:14.891979+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:07:14.891979+0000","last_peered":"2023-09-23T21:07:14.891979+0000","last_clean":"2023-09-23T21:07:14.891979+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:07:14.891979+0000","last_undegraded":"2023-09-23T21:07:14.891979+0000","last_fullsized":"2023-09-23T21:07:14.891979+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":320,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.577541+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:07:09.577541+0000","last_peered":"2023-09-23T21:07:09.577541+0000","last_clean":"2023-09-23T21:07:09.577541+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:07:09.577541+0000","last_undegraded":"2023-09-23T21:07:09.577541+0000","last_fullsized":"2023-09-23T21:07:09.577541+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":45,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.585659+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:07:09.585659+0000","last_peered":"2023-09-23T21:07:09.585659+0000","last_clean":"2023-09-23T21:07:09.585659+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:07:09.585659+0000","last_undegraded":"2023-09-23T21:07:09.585659+0000","last_fullsized":"2023-09-23T21:07:09.585659+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":66,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.887676+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:07:09.887676+0000","last_peered":"2023-09-23T21:07:09.887676+0000","last_clean":"2023-09-23T21:07:09.887676+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:07:09.887676+0000","last_undegraded":"2023-09-23T21:07:09.887676+0000","last_fullsized":"2023-09-23T21:07:09.887676+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":274,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.563743+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:07:09.563743+0000","last_peered":"2023-09-23T21:07:09.563743+0000","last_clean":"2023-09-23T21:07:09.563743+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:07:09.563743+0000","last_undegraded":"2023-09-23T21:07:09.563743+0000","last_fullsized":"2023-09-23T21:07:09.563743+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":44,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.584247+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:07:09.584247+0000","last_peered":"2023-09-23T21:07:09.584247+0000","last_clean":"2023-09-23T21:07:09.584247+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:07:09.584247+0000","last_undegraded":"2023-09-23T21:07:09.584247+0000","last_fullsized":"2023-09-23T21:07:09.584247+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":48,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.560994+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:07:09.560994+0000","last_peered":"2023-09-23T21:07:09.560994+0000","last_clean":"2023-09-23T21:07:09.560994+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:07:09.560994+0000","last_undegraded":"2023-09-23T21:07:09.560994+0000","last_fullsized":"2023-09-23T21:07:09.560994+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":30,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.865415+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:07:00.865415+0000","last_peered":"2023-09-23T21:07:00.865415+0000","last_clean":"2023-09-23T21:07:00.865415+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:07:00.865415+0000","last_undegraded":"2023-09-23T21:07:00.865415+0000","last_fullsized":"2023-09-23T21:07:00.865415+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":48,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.552950+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:07:09.552950+0000","last_peered":"2023-09-23T21:07:09.552950+0000","last_clean":"2023-09-23T21:07:09.552950+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:07:09.552950+0000","last_undegraded":"2023-09-23T21:07:09.552950+0000","last_fullsized":"2023-09-23T21:07:09.552950+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":64,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.887782+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:07:09.887782+0000","last_peered":"2023-09-23T21:07:09.887782+0000","last_clean":"2023-09-23T21:07:09.887782+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:07:09.887782+0000","last_undegraded":"2023-09-23T21:07:09.887782+0000","last_fullsized":"2023-09-23T21:07:09.887782+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":39,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.576856+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:07:09.576856+0000","last_peered":"2023-09-23T21:07:09.576856+0000","last_clean":"2023-09-23T21:07:09.576856+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:07:09.576856+0000","last_undegraded":"2023-09-23T21:07:09.576856+0000","last_fullsized":"2023-09-23T21:07:09.576856+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":354,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.566467+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:07:09.566467+0000","last_peered":"2023-09-23T21:07:09.566467+0000","last_clean":"2023-09-23T21:07:09.566467+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:07:09.566467+0000","last_undegraded":"2023-09-23T21:07:09.566467+0000","last_fullsized":"2023-09-23T21:07:09.566467+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":325,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.873657+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:07:00.873657+0000","last_peered":"2023-09-23T21:07:00.873657+0000","last_clean":"2023-09-23T21:07:00.873657+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:07:00.873657+0000","last_undegraded":"2023-09-23T21:07:00.873657+0000","last_fullsized":"2023-09-23T21:07:00.873657+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":98,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.588062+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:07:09.588062+0000","last_peered":"2023-09-23T21:07:09.588062+0000","last_clean":"2023-09-23T21:07:09.588062+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:07:09.588062+0000","last_undegraded":"2023-09-23T21:07:09.588062+0000","last_fullsized":"2023-09-23T21:07:09.588062+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":57,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.552783+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:07:09.552783+0000","last_peered":"2023-09-23T21:07:09.552783+0000","last_clean":"2023-09-23T21:07:09.552783+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:07:09.552783+0000","last_undegraded":"2023-09-23T21:07:09.552783+0000","last_fullsized":"2023-09-23T21:07:09.552783+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.553635+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:07:09.553635+0000","last_peered":"2023-09-23T21:07:09.553635+0000","last_clean":"2023-09-23T21:07:09.553635+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:07:09.553635+0000","last_undegraded":"2023-09-23T21:07:09.553635+0000","last_fullsized":"2023-09-23T21:07:09.553635+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":66,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.887792+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:07:09.887792+0000","last_peered":"2023-09-23T21:07:09.887792+0000","last_clean":"2023-09-23T21:07:09.887792+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:07:09.887792+0000","last_undegraded":"2023-09-23T21:07:09.887792+0000","last_fullsized":"2023-09-23T21:07:09.887792+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":207,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.873157+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:07:00.873157+0000","last_peered":"2023-09-23T21:07:00.873157+0000","last_clean":"2023-09-23T21:07:00.873157+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:07:00.873157+0000","last_undegraded":"2023-09-23T21:07:00.873157+0000","last_fullsized":"2023-09-23T21:07:00.873157+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":44,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.585728+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:07:09.585728+0000","last_peered":"2023-09-23T21:07:09.585728+0000","last_clean":"2023-09-23T21:07:09.585728+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:07:09.585728+0000","last_undegraded":"2023-09-23T21:07:09.585728+0000","last_fullsized":"2023-09-23T21:07:09.585728+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":404,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.564124+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:07:09.564124+0000","last_peered":"2023-09-23T21:07:09.564124+0000","last_clean":"2023-09-23T21:07:09.564124+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:07:09.564124+0000","last_undegraded":"2023-09-23T21:07:09.564124+0000","last_fullsized":"2023-09-23T21:07:09.564124+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.887811+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:07:09.887811+0000","last_peered":"2023-09-23T21:07:09.887811+0000","last_clean":"2023-09-23T21:07:09.887811+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:07:09.887811+0000","last_undegraded":"2023-09-23T21:07:09.887811+0000","last_fullsized":"2023-09-23T21:07:09.887811+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"408'7174","reported_seq":8537,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:20.411463+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:07:20.411463+0000","last_peered":"2023-09-23T21:07:20.411463+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:07:20.411463+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"349'2428","ondisk_log_start":"349'2428","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4746,"log_dups_size":0,"ondisk_log_size":4746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265814016,"num_objects":4056,"num_object_clones":0,"num_object_copies":12168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7174,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-1559},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":4055}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":111,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.873348+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:07:00.873348+0000","last_peered":"2023-09-23T21:07:00.873348+0000","last_clean":"2023-09-23T21:07:00.873348+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:07:00.873348+0000","last_undegraded":"2023-09-23T21:07:00.873348+0000","last_fullsized":"2023-09-23T21:07:00.873348+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":30,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.864507+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:07:00.864507+0000","last_peered":"2023-09-23T21:07:00.864507+0000","last_clean":"2023-09-23T21:07:00.864507+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:07:00.864507+0000","last_undegraded":"2023-09-23T21:07:00.864507+0000","last_fullsized":"2023-09-23T21:07:00.864507+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"408'6856","reported_seq":6934,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:20.240121+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:07:20.240121+0000","last_peered":"2023-09-23T21:07:20.240121+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:07:20.240121+0000","last_undegraded":"2023-09-23T21:07:20.240121+0000","last_fullsized":"2023-09-23T21:07:20.240121+0000","mapping_epoch":375,"log_start":"349'2112","ondisk_log_start":"349'2112","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4744,"log_dups_size":0,"ondisk_log_size":4744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258342912,"num_objects":3942,"num_object_clones":0,"num_object_copies":11826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3942,"num_objects_unfound":0,"num_objects_dirty":3942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6856,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":3942}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":106,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.584137+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:07:09.584137+0000","last_peered":"2023-09-23T21:07:09.584137+0000","last_clean":"2023-09-23T21:07:09.584137+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:07:09.584137+0000","last_undegraded":"2023-09-23T21:07:09.584137+0000","last_fullsized":"2023-09-23T21:07:09.584137+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":368,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.577464+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:07:09.577464+0000","last_peered":"2023-09-23T21:07:09.577464+0000","last_clean":"2023-09-23T21:07:09.577464+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:07:09.577464+0000","last_undegraded":"2023-09-23T21:07:09.577464+0000","last_fullsized":"2023-09-23T21:07:09.577464+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"408'6982","reported_seq":13759,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:20.718038+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:07:20.718038+0000","last_peered":"2023-09-23T21:07:20.718038+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:07:20.718038+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"353'2911","ondisk_log_start":"353'2911","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4071,"log_dups_size":0,"ondisk_log_size":4071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259260416,"num_objects":3956,"num_object_clones":0,"num_object_copies":11868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7902,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6982,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2586,"num_bytes_recovered":169476096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":1570},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":34,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.865388+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:07:00.865388+0000","last_peered":"2023-09-23T21:07:00.865388+0000","last_clean":"2023-09-23T21:07:00.865388+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:07:00.865388+0000","last_undegraded":"2023-09-23T21:07:00.865388+0000","last_fullsized":"2023-09-23T21:07:00.865388+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":296,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.564291+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:07:09.564291+0000","last_peered":"2023-09-23T21:07:09.564291+0000","last_clean":"2023-09-23T21:07:09.564291+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:07:09.564291+0000","last_undegraded":"2023-09-23T21:07:09.564291+0000","last_fullsized":"2023-09-23T21:07:09.564291+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"408'7058","reported_seq":7680,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:20.707264+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:07:20.707264+0000","last_peered":"2023-09-23T21:07:20.707264+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:07:20.707264+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"353'2968","ondisk_log_start":"353'2968","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4090,"log_dups_size":0,"ondisk_log_size":4090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261619712,"num_objects":3992,"num_object_clones":0,"num_object_copies":11976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7058,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-1261},{"shards":"2,6,7","objects":3989}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":48,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.552861+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:07:09.552861+0000","last_peered":"2023-09-23T21:07:09.552861+0000","last_clean":"2023-09-23T21:07:09.552861+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:07:09.552861+0000","last_undegraded":"2023-09-23T21:07:09.552861+0000","last_fullsized":"2023-09-23T21:07:09.552861+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":45,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:14.182140+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:07:14.182140+0000","last_peered":"2023-09-23T21:07:14.182140+0000","last_clean":"2023-09-23T21:07:14.182140+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:07:14.182140+0000","last_undegraded":"2023-09-23T21:07:14.182140+0000","last_fullsized":"2023-09-23T21:07:14.182140+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"408'7200","reported_seq":18697,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:20.604638+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:07:20.604638+0000","last_peered":"2023-09-23T21:07:20.604638+0000","last_clean":"2023-09-23T21:07:20.604638+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:07:20.604638+0000","last_undegraded":"2023-09-23T21:07:20.604638+0000","last_fullsized":"2023-09-23T21:07:20.604638+0000","mapping_epoch":400,"log_start":"353'3099","ondisk_log_start":"353'3099","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4101,"log_dups_size":0,"ondisk_log_size":4101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269221888,"num_objects":4108,"num_object_clones":0,"num_object_copies":12324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7200,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":65,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.553895+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:07:09.553895+0000","last_peered":"2023-09-23T21:07:09.553895+0000","last_clean":"2023-09-23T21:07:09.553895+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:07:09.553895+0000","last_undegraded":"2023-09-23T21:07:09.553895+0000","last_fullsized":"2023-09-23T21:07:09.553895+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":48,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.552711+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:07:09.552711+0000","last_peered":"2023-09-23T21:07:09.552711+0000","last_clean":"2023-09-23T21:07:09.552711+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:07:09.552711+0000","last_undegraded":"2023-09-23T21:07:09.552711+0000","last_fullsized":"2023-09-23T21:07:09.552711+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"408'6957","reported_seq":18067,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:19.189070+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:07:19.189070+0000","last_peered":"2023-09-23T21:07:19.189070+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:07:19.189070+0000","last_undegraded":"2023-09-23T21:07:19.189070+0000","last_fullsized":"2023-09-23T21:07:19.189070+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263782400,"num_objects":4025,"num_object_clones":0,"num_object_copies":12075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4025,"num_objects_unfound":0,"num_objects_dirty":4025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6957,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":4025}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":334,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.565563+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:07:09.565563+0000","last_peered":"2023-09-23T21:07:09.565563+0000","last_clean":"2023-09-23T21:07:09.565563+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:07:09.565563+0000","last_undegraded":"2023-09-23T21:07:09.565563+0000","last_fullsized":"2023-09-23T21:07:09.565563+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":418,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.590436+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:07:09.590436+0000","last_peered":"2023-09-23T21:07:09.590436+0000","last_clean":"2023-09-23T21:07:09.590436+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:07:09.590436+0000","last_undegraded":"2023-09-23T21:07:09.590436+0000","last_fullsized":"2023-09-23T21:07:09.590436+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"408'6899","reported_seq":6993,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:20.685863+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:07:20.685863+0000","last_peered":"2023-09-23T21:07:20.685863+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:07:20.685863+0000","last_undegraded":"2023-09-23T21:07:20.685863+0000","last_fullsized":"2023-09-23T21:07:20.685863+0000","mapping_epoch":374,"log_start":"353'2794","ondisk_log_start":"353'2794","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4105,"log_dups_size":0,"ondisk_log_size":4105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262471680,"num_objects":4005,"num_object_clones":0,"num_object_copies":12015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8010,"num_objects_unfound":0,"num_objects_dirty":4005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6899,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4005}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":34,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.873037+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:07:00.873037+0000","last_peered":"2023-09-23T21:07:00.873037+0000","last_clean":"2023-09-23T21:07:00.873037+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:07:00.873037+0000","last_undegraded":"2023-09-23T21:07:00.873037+0000","last_fullsized":"2023-09-23T21:07:00.873037+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":447,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.565780+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:07:09.565780+0000","last_peered":"2023-09-23T21:07:09.565780+0000","last_clean":"2023-09-23T21:07:09.565780+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:07:09.565780+0000","last_undegraded":"2023-09-23T21:07:09.565780+0000","last_fullsized":"2023-09-23T21:07:09.565780+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"408'6981","reported_seq":7162,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:20.386631+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:07:20.386631+0000","last_peered":"2023-09-23T21:07:20.386631+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:07:20.386631+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"349'2194","ondisk_log_start":"349'2194","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4787,"log_dups_size":0,"ondisk_log_size":4787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260898816,"num_objects":3981,"num_object_clones":0,"num_object_copies":11943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6981,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-1458},{"shards":"1,6,7","objects":3981}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":57,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.561159+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:07:09.561159+0000","last_peered":"2023-09-23T21:07:09.561159+0000","last_clean":"2023-09-23T21:07:09.561159+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:07:09.561159+0000","last_undegraded":"2023-09-23T21:07:09.561159+0000","last_fullsized":"2023-09-23T21:07:09.561159+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":196,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.589814+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:07:09.589814+0000","last_peered":"2023-09-23T21:07:09.589814+0000","last_clean":"2023-09-23T21:07:09.589814+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:07:09.589814+0000","last_undegraded":"2023-09-23T21:07:09.589814+0000","last_fullsized":"2023-09-23T21:07:09.589814+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"408'6954","reported_seq":12446,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.190403+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:07:19.190403+0000","last_peered":"2023-09-23T21:07:19.190403+0000","last_clean":"2023-09-23T21:07:19.190403+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:07:19.190403+0000","last_undegraded":"2023-09-23T21:07:19.190403+0000","last_fullsized":"2023-09-23T21:07:19.190403+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267059224,"num_objects":4076,"num_object_clones":0,"num_object_copies":12228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4076,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6954,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":302,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.581968+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:07:09.581968+0000","last_peered":"2023-09-23T21:07:09.581968+0000","last_clean":"2023-09-23T21:07:09.581968+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:07:09.581968+0000","last_undegraded":"2023-09-23T21:07:09.581968+0000","last_fullsized":"2023-09-23T21:07:09.581968+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":72,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.865439+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:07:00.865439+0000","last_peered":"2023-09-23T21:07:00.865439+0000","last_clean":"2023-09-23T21:07:00.865439+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:07:00.865439+0000","last_undegraded":"2023-09-23T21:07:00.865439+0000","last_fullsized":"2023-09-23T21:07:00.865439+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.002776+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:07:19.002776+0000","last_peered":"2023-09-23T21:07:19.002776+0000","last_clean":"2023-09-23T21:07:19.002776+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:07:19.002776+0000","last_undegraded":"2023-09-23T21:07:19.002776+0000","last_fullsized":"2023-09-23T21:07:19.002776+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"408'7004","reported_seq":10177,"reported_epoch":408,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:07:20.716436+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:07:20.716436+0000","last_peered":"2023-09-23T21:07:20.716436+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:07:20.716436+0000","last_undegraded":"2023-09-23T21:07:20.716436+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"353'2857","ondisk_log_start":"353'2857","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4147,"log_dups_size":0,"ondisk_log_size":4147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263585792,"num_objects":4022,"num_object_clones":0,"num_object_copies":12066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10315,"num_objects_unfound":0,"num_objects_dirty":4022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7004,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1522,"num_bytes_recovered":99745792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":1658},{"shards":"2,6","objects":31},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":383,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.578663+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:07:09.578663+0000","last_peered":"2023-09-23T21:07:09.578663+0000","last_clean":"2023-09-23T21:07:09.578663+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:07:09.578663+0000","last_undegraded":"2023-09-23T21:07:09.578663+0000","last_fullsized":"2023-09-23T21:07:09.578663+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":358,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.566086+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:07:09.566086+0000","last_peered":"2023-09-23T21:07:09.566086+0000","last_clean":"2023-09-23T21:07:09.566086+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:07:09.566086+0000","last_undegraded":"2023-09-23T21:07:09.566086+0000","last_fullsized":"2023-09-23T21:07:09.566086+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":40,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.011777+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:07:19.011777+0000","last_peered":"2023-09-23T21:07:19.011777+0000","last_clean":"2023-09-23T21:07:19.011777+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:07:19.011777+0000","last_undegraded":"2023-09-23T21:07:19.011777+0000","last_fullsized":"2023-09-23T21:07:19.011777+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"408'6990","reported_seq":7167,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:19.093058+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:07:19.093058+0000","last_peered":"2023-09-23T21:07:19.093058+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:07:19.093058+0000","last_undegraded":"2023-09-23T21:07:19.093058+0000","last_fullsized":"2023-09-23T21:07:19.093058+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5517,"log_dups_size":0,"ondisk_log_size":5517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266862592,"num_objects":4072,"num_object_clones":0,"num_object_copies":12216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8120,"num_objects_unfound":0,"num_objects_dirty":4072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6990,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":4072}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":385,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.013987+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:07:19.013987+0000","last_peered":"2023-09-23T21:07:19.013987+0000","last_clean":"2023-09-23T21:07:19.013987+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:07:19.013987+0000","last_undegraded":"2023-09-23T21:07:19.013987+0000","last_fullsized":"2023-09-23T21:07:19.013987+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":302,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.586245+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:07:09.586245+0000","last_peered":"2023-09-23T21:07:09.586245+0000","last_clean":"2023-09-23T21:07:09.586245+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:07:09.586245+0000","last_undegraded":"2023-09-23T21:07:09.586245+0000","last_fullsized":"2023-09-23T21:07:09.586245+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":39,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.589682+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:07:09.589682+0000","last_peered":"2023-09-23T21:07:09.589682+0000","last_clean":"2023-09-23T21:07:09.589682+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:07:09.589682+0000","last_undegraded":"2023-09-23T21:07:09.589682+0000","last_fullsized":"2023-09-23T21:07:09.589682+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"408'6876","reported_seq":13130,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:20.653492+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:07:20.653492+0000","last_peered":"2023-09-23T21:07:20.653492+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:07:20.653492+0000","last_undegraded":"2023-09-23T21:07:20.653492+0000","last_fullsized":"2023-09-23T21:07:20.653492+0000","mapping_epoch":387,"log_start":"353'2780","ondisk_log_start":"353'2780","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4096,"log_dups_size":0,"ondisk_log_size":4096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262012928,"num_objects":3998,"num_object_clones":0,"num_object_copies":11994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3998,"num_objects_unfound":0,"num_objects_dirty":3998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6876,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3998}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":223,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.014745+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:07:19.014745+0000","last_peered":"2023-09-23T21:07:19.014745+0000","last_clean":"2023-09-23T21:07:19.014745+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:07:19.014745+0000","last_undegraded":"2023-09-23T21:07:19.014745+0000","last_fullsized":"2023-09-23T21:07:19.014745+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":705,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:09.577178+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:07:09.577178+0000","last_peered":"2023-09-23T21:07:09.577178+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:07:09.577178+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":57,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.553547+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:07:09.553547+0000","last_peered":"2023-09-23T21:07:09.553547+0000","last_clean":"2023-09-23T21:07:09.553547+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:07:09.553547+0000","last_undegraded":"2023-09-23T21:07:09.553547+0000","last_fullsized":"2023-09-23T21:07:09.553547+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":296,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.565755+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:07:09.565755+0000","last_peered":"2023-09-23T21:07:09.565755+0000","last_clean":"2023-09-23T21:07:09.565755+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:07:09.565755+0000","last_undegraded":"2023-09-23T21:07:09.565755+0000","last_fullsized":"2023-09-23T21:07:09.565755+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"408'7065","reported_seq":7175,"reported_epoch":408,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:20.701397+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:07:20.701397+0000","last_peered":"2023-09-23T21:07:20.701397+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:07:20.701397+0000","last_undegraded":"2023-09-23T21:07:20.701397+0000","last_fullsized":"2023-09-23T21:07:20.701397+0000","mapping_epoch":374,"log_start":"353'3005","ondisk_log_start":"353'3005","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4060,"log_dups_size":0,"ondisk_log_size":4060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265879552,"num_objects":4057,"num_object_clones":0,"num_object_copies":12171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8114,"num_objects_unfound":0,"num_objects_dirty":4057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7065,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":4057}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":40,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:15.384814+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:07:15.384814+0000","last_peered":"2023-09-23T21:07:15.384814+0000","last_clean":"2023-09-23T21:07:15.384814+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:07:15.384814+0000","last_undegraded":"2023-09-23T21:07:15.384814+0000","last_fullsized":"2023-09-23T21:07:15.384814+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":54,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.583237+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:07:09.583237+0000","last_peered":"2023-09-23T21:07:09.583237+0000","last_clean":"2023-09-23T21:07:09.583237+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:07:09.583237+0000","last_undegraded":"2023-09-23T21:07:09.583237+0000","last_fullsized":"2023-09-23T21:07:09.583237+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":45,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.585307+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:07:09.585307+0000","last_peered":"2023-09-23T21:07:09.585307+0000","last_clean":"2023-09-23T21:07:09.585307+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:07:09.585307+0000","last_undegraded":"2023-09-23T21:07:09.585307+0000","last_fullsized":"2023-09-23T21:07:09.585307+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"408'6984","reported_seq":10599,"reported_epoch":408,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:20.394841+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:07:20.394841+0000","last_peered":"2023-09-23T21:07:20.394841+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:07:20.394841+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:07:20.394841+0000","mapping_epoch":383,"log_start":"349'2226","ondisk_log_start":"349'2226","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4758,"log_dups_size":0,"ondisk_log_size":4758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896512,"num_objects":4042,"num_object_clones":0,"num_object_copies":12126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":291,"num_objects_unfound":0,"num_objects_dirty":4042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6984,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":291},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":45,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.583719+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:07:09.583719+0000","last_peered":"2023-09-23T21:07:09.583719+0000","last_clean":"2023-09-23T21:07:09.583719+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:07:09.583719+0000","last_undegraded":"2023-09-23T21:07:09.583719+0000","last_fullsized":"2023-09-23T21:07:09.583719+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":304,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.586978+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:07:09.586978+0000","last_peered":"2023-09-23T21:07:09.586978+0000","last_clean":"2023-09-23T21:07:09.586978+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:07:09.586978+0000","last_undegraded":"2023-09-23T21:07:09.586978+0000","last_fullsized":"2023-09-23T21:07:09.586978+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":82,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.865165+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:07:00.865165+0000","last_peered":"2023-09-23T21:07:00.865165+0000","last_clean":"2023-09-23T21:07:00.865165+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:07:00.865165+0000","last_undegraded":"2023-09-23T21:07:00.865165+0000","last_fullsized":"2023-09-23T21:07:00.865165+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"408'6940","reported_seq":19827,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:20.298967+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:07:20.298967+0000","last_peered":"2023-09-23T21:07:20.298967+0000","last_clean":"2023-09-23T21:07:20.298967+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:07:20.298967+0000","last_undegraded":"2023-09-23T21:07:20.298967+0000","last_fullsized":"2023-09-23T21:07:20.298967+0000","mapping_epoch":397,"log_start":"349'2245","ondisk_log_start":"349'2245","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257163264,"num_objects":3924,"num_object_clones":0,"num_object_copies":11772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6940,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":65,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.553537+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:07:09.553537+0000","last_peered":"2023-09-23T21:07:09.553537+0000","last_clean":"2023-09-23T21:07:09.553537+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:07:09.553537+0000","last_undegraded":"2023-09-23T21:07:09.553537+0000","last_fullsized":"2023-09-23T21:07:09.553537+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":39,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.583677+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:07:09.583677+0000","last_peered":"2023-09-23T21:07:09.583677+0000","last_clean":"2023-09-23T21:07:09.583677+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:07:09.583677+0000","last_undegraded":"2023-09-23T21:07:09.583677+0000","last_fullsized":"2023-09-23T21:07:09.583677+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.559831+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:07:09.559831+0000","last_peered":"2023-09-23T21:07:09.559831+0000","last_clean":"2023-09-23T21:07:09.559831+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:07:09.559831+0000","last_undegraded":"2023-09-23T21:07:09.559831+0000","last_fullsized":"2023-09-23T21:07:09.559831+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"408'7097","reported_seq":10208,"reported_epoch":408,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:20.412371+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:07:20.412371+0000","last_peered":"2023-09-23T21:07:20.412371+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:07:20.412371+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:07:20.412371+0000","mapping_epoch":382,"log_start":"349'2312","ondisk_log_start":"349'2312","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4785,"log_dups_size":0,"ondisk_log_size":4785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268763136,"num_objects":4101,"num_object_clones":0,"num_object_copies":12303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4097,"num_objects_misplaced":4105,"num_objects_unfound":0,"num_objects_dirty":4101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7097,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1499,"num_bytes_recovered":98238464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":4},{"shards":"1,6,7","objects":4097}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":304,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.563826+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:07:09.563826+0000","last_peered":"2023-09-23T21:07:09.563826+0000","last_clean":"2023-09-23T21:07:09.563826+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:07:09.563826+0000","last_undegraded":"2023-09-23T21:07:09.563826+0000","last_fullsized":"2023-09-23T21:07:09.563826+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":49,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.585455+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:07:09.585455+0000","last_peered":"2023-09-23T21:07:09.585455+0000","last_clean":"2023-09-23T21:07:09.585455+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:07:09.585455+0000","last_undegraded":"2023-09-23T21:07:09.585455+0000","last_fullsized":"2023-09-23T21:07:09.585455+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":92,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.864583+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:07:00.864583+0000","last_peered":"2023-09-23T21:07:00.864583+0000","last_clean":"2023-09-23T21:07:00.864583+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:07:00.864583+0000","last_undegraded":"2023-09-23T21:07:00.864583+0000","last_fullsized":"2023-09-23T21:07:00.864583+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":53,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.584949+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:07:09.584949+0000","last_peered":"2023-09-23T21:07:09.584949+0000","last_clean":"2023-09-23T21:07:09.584949+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:07:09.584949+0000","last_undegraded":"2023-09-23T21:07:09.584949+0000","last_fullsized":"2023-09-23T21:07:09.584949+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":431,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.586545+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:07:09.586545+0000","last_peered":"2023-09-23T21:07:09.586545+0000","last_clean":"2023-09-23T21:07:09.586545+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:07:09.586545+0000","last_undegraded":"2023-09-23T21:07:09.586545+0000","last_fullsized":"2023-09-23T21:07:09.586545+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":69,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.873970+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:07:00.873970+0000","last_peered":"2023-09-23T21:07:00.873970+0000","last_clean":"2023-09-23T21:07:00.873970+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:07:00.873970+0000","last_undegraded":"2023-09-23T21:07:00.873970+0000","last_fullsized":"2023-09-23T21:07:00.873970+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":74,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:14.891937+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:07:14.891937+0000","last_peered":"2023-09-23T21:07:14.891937+0000","last_clean":"2023-09-23T21:07:14.891937+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:07:14.891937+0000","last_undegraded":"2023-09-23T21:07:14.891937+0000","last_fullsized":"2023-09-23T21:07:14.891937+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":289,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.567057+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:07:09.567057+0000","last_peered":"2023-09-23T21:07:09.567057+0000","last_clean":"2023-09-23T21:07:09.567057+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:07:09.567057+0000","last_undegraded":"2023-09-23T21:07:09.567057+0000","last_fullsized":"2023-09-23T21:07:09.567057+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":66,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:14.891910+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:07:14.891910+0000","last_peered":"2023-09-23T21:07:14.891910+0000","last_clean":"2023-09-23T21:07:14.891910+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:07:14.891910+0000","last_undegraded":"2023-09-23T21:07:14.891910+0000","last_fullsized":"2023-09-23T21:07:14.891910+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":48,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.553270+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:07:09.553270+0000","last_peered":"2023-09-23T21:07:09.553270+0000","last_clean":"2023-09-23T21:07:09.553270+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:07:09.553270+0000","last_undegraded":"2023-09-23T21:07:09.553270+0000","last_fullsized":"2023-09-23T21:07:09.553270+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":57,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.887704+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:07:09.887704+0000","last_peered":"2023-09-23T21:07:09.887704+0000","last_clean":"2023-09-23T21:07:09.887704+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:07:09.887704+0000","last_undegraded":"2023-09-23T21:07:09.887704+0000","last_fullsized":"2023-09-23T21:07:09.887704+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":51,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.584129+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:07:09.584129+0000","last_peered":"2023-09-23T21:07:09.584129+0000","last_clean":"2023-09-23T21:07:09.584129+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:07:09.584129+0000","last_undegraded":"2023-09-23T21:07:09.584129+0000","last_fullsized":"2023-09-23T21:07:09.584129+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":34,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T21:07:00.872928+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:07:00.872928+0000","last_peered":"2023-09-23T21:07:00.872928+0000","last_clean":"2023-09-23T21:07:00.872928+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:07:00.872928+0000","last_undegraded":"2023-09-23T21:07:00.872928+0000","last_fullsized":"2023-09-23T21:07:00.872928+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4217634840,"num_objects":64357,"num_object_clones":0,"num_object_copies":193071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32057,"num_objects_misplaced":50920,"num_objects_unfound":0,"num_objects_dirty":64357,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112017,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26069,"num_bytes_recovered":1708457984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14642589696,"data_stored":14642577480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75616,"ondisk_log_size":75616,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201743,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463364,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3358976,"kb_used_data":2854136,"kb_used_omap":0,"kb_used_meta":504832,"kb_avail":90386176,"statfs":{"total":95995035648,"available":92555444224,"internally_reserved":0,"allocated":2922635264,"data_stored":2921438441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":516947968},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.131,"15min":2.1230000000000002},"min":{"1min":0.254,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6689999999999996,"5min":7.2480000000000002,"15min":102.242},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.107,"15min":1.8440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":0.89300000000000002,"5min":9.7110000000000003,"15min":102.502},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":3.9460000000000002,"15min":2.379},"min":{"1min":0.251,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":11.743,"5min":119.077,"15min":119.077},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.901,"5min":3.7730000000000001,"15min":2.452},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":11.936,"5min":119.313,"15min":119.313},"last":0.70899999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":4.1219999999999999,"15min":2.9380000000000002},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.5910000000000002,"5min":123.283,"15min":176.45400000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":4.1680000000000001,"15min":3.1080000000000001},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":1.3009999999999999,"5min":123.051,"15min":176.46799999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.456,"15min":4.4130000000000003},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.26900000000000002},"max":{"1min":3.173,"5min":35.078000000000003,"15min":250.804},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.5409999999999999,"15min":4.3600000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":2.1850000000000001,"5min":35.133000000000003,"15min":250.446},"last":0.97199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.1579999999999999,"15min":0.93600000000000005},"min":{"1min":0.32000000000000001,"5min":0.27800000000000002,"15min":0.24299999999999999},"max":{"1min":0.878,"5min":13.1,"15min":13.1},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.91100000000000003,"15min":0.96299999999999997},"min":{"1min":0.36699999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":2.9079999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.0980000000000001,"15min":1.5289999999999999},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.90700000000000003,"5min":11.281000000000001,"15min":105.121},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.208,"15min":1.585},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":8.6229999999999993,"15min":106.30500000000001},"last":0.39500000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.2569999999999999,"15min":1.4710000000000001},"min":{"1min":0.34899999999999998,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.8079999999999998,"5min":7.8079999999999998,"15min":101.04600000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.2509999999999999,"15min":1.5029999999999999},"min":{"1min":0.39200000000000002,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":4.407,"5min":7.1849999999999996,"15min":101.081},"last":0.46400000000000002}]}]},{"osd":1,"up_from":276,"seq":1185410973810,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3450828,"kb_used_data":3098884,"kb_used_omap":5,"kb_used_meta":351802,"kb_avail":90294324,"statfs":{"total":95995035648,"available":92461387776,"internally_reserved":0,"allocated":3173257216,"data_stored":3171228704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":360245987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":0.85599999999999998,"15min":0.55200000000000005},"min":{"1min":0.20599999999999999,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":9.9469999999999992,"5min":11.959,"15min":11.959},"last":17.652999999999999},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.256,"15min":0.81499999999999995},"min":{"1min":0.216,"5min":0.193,"15min":0.193},"max":{"1min":5.093,"5min":18.047999999999998,"15min":18.047999999999998},"last":1.4970000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.407,"15min":1.1359999999999999},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.0670000000000002,"5min":17.254000000000001,"15min":78.409000000000006},"last":3.0019999999999998},{"interface":"front","average":{"1min":1.55,"5min":1.03,"15min":0.97999999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.22},"max":{"1min":12.478999999999999,"5min":12.478999999999999,"15min":76.245000000000005},"last":18.010000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.849,"15min":2.4670000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":7.3099999999999996,"5min":74.527000000000001,"15min":172.44399999999999},"last":17.812999999999999},{"interface":"front","average":{"1min":1.907,"5min":2.5369999999999999,"15min":2.6880000000000002},"min":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":7.9050000000000002,"5min":72.216999999999999,"15min":172.58500000000001},"last":1.405}]},{"osd":4,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.472,"15min":0.93400000000000005},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.7069999999999999,"5min":17.006,"15min":17.006},"last":2.3900000000000001},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.92600000000000005,"15min":0.70299999999999996},"min":{"1min":0.35699999999999998,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":4.7690000000000001,"5min":11.916,"15min":11.916},"last":17.423999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.446,"15min":0.88300000000000001},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.6269999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":1.4419999999999999},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.589,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.3860000000000001,"5min":17.076000000000001,"15min":17.076000000000001},"last":2.4260000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.577,"15min":1.3879999999999999},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.6020000000000003,"5min":17.102,"15min":127.995},"last":1.472},{"interface":"front","average":{"1min":1.127,"5min":1.647,"15min":1.466},"min":{"1min":0.41699999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.375,"5min":17.302,"15min":126.836},"last":2.4500000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.9180000000000001,"15min":1.484},"min":{"1min":0.442,"5min":0.38200000000000001,"15min":0.36499999999999999},"max":{"1min":7.048,"5min":151.791,"15min":151.791},"last":2.464},{"interface":"front","average":{"1min":1.49,"5min":2.3889999999999998,"15min":1.2310000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.3739999999999997,"5min":151.809,"15min":151.809},"last":17.475000000000001}]}]},{"osd":0,"up_from":92,"seq":395136991550,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1288012,"kb_used_data":557380,"kb_used_omap":5,"kb_used_meta":730618,"kb_avail":92457140,"statfs":{"total":95995035648,"available":94676111360,"internally_reserved":0,"allocated":570757120,"data_stored":568842588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":748153466},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":2.9500000000000002,"15min":1.4830000000000001},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":40.284999999999997,"5min":61.341999999999999,"15min":61.341999999999999},"last":8.9260000000000002},{"interface":"front","average":{"1min":3.4319999999999999,"5min":2.9670000000000001,"15min":1.4510000000000001},"min":{"1min":0.223,"5min":0.217,"15min":0.217},"max":{"1min":40.213000000000001,"5min":61.299999999999997,"15min":61.299999999999997},"last":6.9050000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.7110000000000001,"15min":2.238},"min":{"1min":0.217,"5min":0.217,"15min":0.161},"max":{"1min":41.860999999999997,"5min":41.860999999999997,"15min":183.77099999999999},"last":9.4789999999999992},{"interface":"front","average":{"1min":4.5780000000000003,"5min":2.1219999999999999,"15min":2.5070000000000001},"min":{"1min":0.27000000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":44.779000000000003,"5min":44.779000000000003,"15min":183.78700000000001},"last":2.879}]},{"osd":3,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.1319999999999999,"15min":3.3079999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":5.5810000000000004,"5min":11.555999999999999,"15min":186.143},"last":9.8119999999999994},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.3320000000000001,"15min":3.4380000000000002},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.23000000000000001},"max":{"1min":4.016,"5min":16.640000000000001,"15min":186.97399999999999},"last":0.45100000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.2090000000000001,"15min":1.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.5259999999999998,"5min":9.7080000000000002,"15min":10.512},"last":7.5449999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.80200000000000005,"15min":0.93799999999999994},"min":{"1min":0.31,"5min":0.31,"15min":0.26200000000000001},"max":{"1min":3.875,"5min":6.0490000000000004,"15min":11.493},"last":5.5579999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.3640000000000001,"15min":2.4830000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.492,"5min":9.7439999999999998,"15min":154.30600000000001},"last":7.5839999999999996},{"interface":"front","average":{"1min":1.093,"5min":0.90200000000000002,"15min":1.905},"min":{"1min":0.33800000000000002,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":11.478,"5min":11.478,"15min":154.42400000000001},"last":5.5979999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2149999999999999,"5min":5.7060000000000004,"15min":3.1709999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":136.404,"5min":170.12700000000001,"15min":170.12700000000001},"last":5.6289999999999996},{"interface":"front","average":{"1min":8.1039999999999992,"5min":5.9210000000000003,"15min":3.3599999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":136.44999999999999,"5min":169.767,"15min":169.767},"last":0.48399999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.98599999999999999,"15min":2.4279999999999999},"min":{"1min":0.35199999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":164.26400000000001},"last":10.74},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.107,"15min":2.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":1.6719999999999999,"5min":6.5410000000000004,"15min":164.23099999999999},"last":0.50700000000000001}]}]},{"osd":2,"up_from":129,"seq":554050781441,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3423360,"kb_used_data":2810488,"kb_used_omap":7,"kb_used_meta":612856,"kb_avail":90321792,"statfs":{"total":95995035648,"available":92489515008,"internally_reserved":0,"allocated":2877939712,"data_stored":2875971325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":627564625},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.302,"15min":1.3919999999999999},"min":{"1min":0.19400000000000001,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":1.7470000000000001,"5min":15.667,"15min":26.356000000000002},"last":3.8530000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.4419999999999999,"15min":1.556},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.663,"5min":12.445,"15min":26.341000000000001},"last":3.46}]},{"osd":1,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":4.9199999999999999,"15min":2.379},"min":{"1min":0.23599999999999999,"5min":0.19,"15min":0.188},"max":{"1min":4.9089999999999998,"5min":212.12,"15min":212.12},"last":0.93200000000000005},{"interface":"front","average":{"1min":1.232,"5min":4.9390000000000001,"15min":2.1890000000000001},"min":{"1min":0.24299999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.4219999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":1.0049999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":2.2480000000000002,"15min":3.3039999999999998},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":4.3559999999999999,"5min":66.085999999999999,"15min":151.43799999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":2.222,"15min":3.3260000000000001},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":1.4930000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":0.745}]},{"osd":4,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.101,"15min":1.1419999999999999},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.4570000000000001,"5min":12.586,"15min":12.586},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.1870000000000001,"15min":1.1910000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.575,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":1.026}]},{"osd":5,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.29,"15min":2.5760000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.6379999999999999,"5min":9.9909999999999997,"15min":173.34},"last":3.4129999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.079,"15min":2.3809999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":2.1949999999999998,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":3.2869999999999999,"15min":1.883},"min":{"1min":0.34200000000000003,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":2.1789999999999998,"5min":175.72499999999999,"15min":175.72499999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":3.3250000000000002,"15min":1.984},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":1.5329999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.69499999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.6060000000000001,"15min":2.7330000000000001},"min":{"1min":0.41399999999999998,"5min":0.40799999999999997,"15min":0.32700000000000001},"max":{"1min":9.6910000000000007,"5min":12.609,"15min":177.30799999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.355,"15min":2.6520000000000001},"min":{"1min":0.38900000000000001,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.615,"5min":10.016,"15min":177.40899999999999},"last":1.0580000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160035,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":555912,"kb_used_data":7104,"kb_used_omap":2,"kb_used_meta":548797,"kb_avail":93189240,"statfs":{"total":95995035648,"available":95425781760,"internally_reserved":0,"allocated":7274496,"data_stored":5125908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":561968181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.5580000000000001,"15min":1.7350000000000001},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.18099999999999999},"max":{"1min":3.8570000000000002,"5min":44.963000000000001,"15min":95.507999999999996},"last":6.6139999999999999},{"interface":"front","average":{"1min":0.438,"5min":1.02,"15min":1.698},"min":{"1min":0.219,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":1.2909999999999999,"5min":9.7759999999999998,"15min":95.259},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:06:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.347,"15min":1.0580000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7619999999999996,"5min":12.035,"15min":19.786999999999999},"last":8.3599999999999994},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.661,"15min":1.292},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":8.3059999999999992,"5min":15.763999999999999,"15min":15.763999999999999},"last":4.8810000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.464,"15min":1.2290000000000001},"min":{"1min":0.29699999999999999,"5min":0.223,"15min":0.182},"max":{"1min":13.944000000000001,"5min":13.944000000000001,"15min":13.944000000000001},"last":8.0289999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.2470000000000001,"15min":1.2909999999999999},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":4.6349999999999998,"5min":12.096,"15min":19.754999999999999},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":2.645,"15min":2.1930000000000001},"min":{"1min":0.34799999999999998,"5min":0.307,"15min":0.222},"max":{"1min":8.1750000000000007,"5min":70.728999999999999,"15min":111.36},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1389999999999998,"15min":1.9019999999999999},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":4.1100000000000003,"5min":70.712000000000003,"15min":112.473},"last":6.6859999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.0740000000000001,"15min":1.5129999999999999},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":1.325,"5min":9.6359999999999992,"15min":64.528000000000006},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.79,"15min":1.6220000000000001},"min":{"1min":0.42099999999999999,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":8.2490000000000006,"5min":10.617000000000001,"15min":14.054},"last":0.93799999999999994}]},{"osd":6,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.857,"15min":1.6919999999999999},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.312},"max":{"1min":8.2780000000000005,"5min":10.52,"15min":19.568000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.4650000000000001,"15min":1.4750000000000001},"min":{"1min":0.39100000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":4.3920000000000003,"5min":14.356,"15min":15.573},"last":6.6609999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":2.1930000000000001,"15min":2.6099999999999999},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":1.3480000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.625},{"interface":"front","average":{"1min":1.341,"5min":2.8090000000000002,"15min":2.8490000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":8.218,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.48699999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986653,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1609012,"kb_used_data":1057068,"kb_used_omap":0,"kb_used_meta":551936,"kb_avail":92136140,"statfs":{"total":95995035648,"available":94347407360,"internally_reserved":0,"allocated":1082437632,"data_stored":1081244028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":565182464},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.2869999999999999,"15min":2.782},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.53200000000000003,"5min":47.476999999999997,"15min":146.25299999999999},"last":2.246},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.119,"15min":2.8029999999999999},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.223},"max":{"1min":3.8959999999999999,"5min":8.5220000000000002,"15min":144.77000000000001},"last":4.9139999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":12.298,"5min":8.5619999999999994,"15min":3.3940000000000001},"min":{"1min":0.34899999999999998,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":127.411,"5min":127.411,"15min":127.411},"last":4.8819999999999997},{"interface":"front","average":{"1min":4.8810000000000002,"5min":6.7430000000000003,"15min":2.758},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":65.688999999999993,"5min":118.075,"15min":118.075},"last":1.2390000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":5.8410000000000002,"15min":3.2130000000000001},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":3.452,"5min":164.517,"15min":164.517},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":5.665,"15min":3.274},"min":{"1min":0.35599999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":0.56999999999999995,"5min":164.46799999999999,"15min":164.46799999999999},"last":1.1679999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2170000000000001,"15min":2.1280000000000001},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.2879999999999998,"5min":8.7420000000000009,"15min":142.28399999999999},"last":2.9950000000000001},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.177,"15min":2.3029999999999999},"min":{"1min":0.35299999999999998,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":8.2899999999999991,"5min":8.2899999999999991,"15min":142.34700000000001},"last":1.216}]},{"osd":5,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.82499999999999996,"15min":1.0609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.24099999999999999},"max":{"1min":2.9049999999999998,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.875,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29899999999999999,"15min":0.21099999999999999},"max":{"1min":0.58299999999999996,"5min":7.0179999999999998,"15min":72.290000000000006},"last":1.26}]},{"osd":6,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.004,"15min":0.96699999999999997},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":0.70599999999999996,"5min":11.800000000000001,"15min":11.887},"last":4.8399999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.111,"15min":0.92100000000000004},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":2.3620000000000001,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":2.948}]},{"osd":7,"last update":"Sat Sep 23 21:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":3.0110000000000001,"15min":1.8280000000000001},"min":{"1min":0.42799999999999999,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":0.72799999999999998,"5min":145.721,"15min":145.721},"last":1.4370000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":3.282,"15min":1.992},"min":{"1min":0.38,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":0.78100000000000003,"5min":148.45699999999999,"15min":148.45699999999999},"last":4.7969999999999997}]}]},{"osd":5,"up_from":39,"seq":167503725017,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1161376,"kb_used_data":564056,"kb_used_omap":0,"kb_used_meta":597312,"kb_avail":92583776,"statfs":{"total":95995035648,"available":94805786624,"internally_reserved":0,"allocated":577593344,"data_stored":576415940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":611647488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.68500000000000005,"15min":1.3540000000000001},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":10.016999999999999,"5min":10.016999999999999,"15min":80.105999999999995},"last":3.0209999999999999},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.89300000000000002,"15min":1.462},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":2.919,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.251}]},{"osd":1,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":13.016999999999999,"5min":6.4710000000000001,"15min":2.512},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":233.77099999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":13.031000000000001,"5min":6.1769999999999996,"15min":2.4220000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.247},"max":{"1min":233.75200000000001,"5min":233.75200000000001,"15min":233.75200000000001},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.3980000000000001,"15min":1.569},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":8.3770000000000007,"5min":103.63800000000001,"15min":103.63800000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.6309999999999998,"15min":1.696},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":6.7640000000000002,"5min":103.664,"15min":103.664},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.3149999999999999,"15min":2.8359999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":8.0860000000000003,"5min":97.349999999999994,"15min":153.60300000000001},"last":4.2460000000000004},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.2570000000000001,"15min":2.7559999999999998},"min":{"1min":0.38300000000000001,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":4.6609999999999996,"5min":97.316999999999993,"15min":153.774},"last":0.442}]},{"osd":4,"last update":"Sat Sep 23 21:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59999999999999998,"15min":0.77600000000000002},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":2.2320000000000002,"5min":3.3029999999999999,"15min":12.739000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.88900000000000001,"15min":0.98299999999999998},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.247},"max":{"1min":3.645,"5min":8.3740000000000006,"15min":12.539},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.7150000000000001,"15min":1.1839999999999999},"min":{"1min":0.376,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":1.276,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.625,"5min":1.8420000000000001,"15min":1.3220000000000001},"min":{"1min":0.32400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6699999999999999,"5min":70.266000000000005,"15min":70.266000000000005},"last":0.377}]},{"osd":7,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.143,"15min":1.2010000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":5.0119999999999996,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65800000000000003,"15min":1.054},"min":{"1min":0.38500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.5339999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.46700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":565248000,"data_stored":565248000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3151429632,"data_stored":3151429632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2858029056,"data_stored":2858024984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1835008,"data_stored":1835008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1070206976,"data_stored":1070202904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":566099968,"data_stored":566099968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2899578880,"data_stored":2899574808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3530162176,"data_stored":3530162176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:07:23.416 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:23.421 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:23 smithi121 ceph-mon[211526]: pgmap v1283: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 32059/193143 objects degraded (16.599%); 50920/193143 objects misplaced (26.364%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:23.421 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:23 smithi121 ceph-mon[219709]: pgmap v1283: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 32059/193143 objects degraded (16.599%); 50920/193143 objects misplaced (26.364%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:23.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:23 smithi123 ceph-mon[193903]: pgmap v1283: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 32059/193143 objects degraded (16.599%); 50920/193143 objects misplaced (26.364%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:23.525 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:23.564 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:23.634 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:23.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:23.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:23.752 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:23.799 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:23.892 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:23.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:24.006 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:24.109 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:24.110 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:07:24.110 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:07:24.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.110 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:113941 11.6 11:6fcce8e3:::benchmark_data_smithi121_477989_object25752:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:07:24.110 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.068030+0000", 2023-09-23T21:07:24.110 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.029626649000000001, 2023-09-23T21:07:24.110 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.029661427000000001, 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 113941 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.068030+0000", 2023-09-23T21:07:24.111 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.068030+0000", 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.068029+0000", 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.068032+0000", 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9330000000000001e-06 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.068038+0000", 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8919999999999997e-06 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.068043+0000", 2023-09-23T21:07:24.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6639999999999997e-06 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.089957+0000", 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.021913683999999999 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.090077+0000", 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000120461 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2,4", 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.090180+0000", 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000102938 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:113932 11.6 11:60c72dbe:::benchmark_data_smithi121_477989_object25743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.038183+0000", 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.059473946999999999, 2023-09-23T21:07:24.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.059592646999999999, 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 113932 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.038183+0000", 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.038183+0000", 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.038181+0000", 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.038186+0000", 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3920000000000002e-06 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.038195+0000", 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4400000000000005e-06 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.038200+0000", 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1660000000000002e-06 2023-09-23T21:07:24.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.086888+0000", 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.048688002000000001 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.087048+0000", 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016010899999999999 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2,4", 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.087209+0000", 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016051600000000001 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:113947 11.e 11:7779cf0a:::benchmark_data_smithi121_477989_object25758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.091377+0000", 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0062799809999999996, 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064739849999999998, 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.120 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 113947 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091377+0000", 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091377+0000", 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091375+0000", 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091383+0000", 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1599999999999998e-06 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091397+0000", 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3424999999999999e-05 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091405+0000", 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7630000000000006e-06 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091597+0000", 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019207600000000001 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.092647+0000", 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001049108 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:113950 11.c 11:3cffe708:::benchmark_data_smithi121_477989_object25761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.091413+0000", 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0062430740000000004, 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065046260000000003, 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 113950 2023-09-23T21:07:24.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091413+0000", 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091413+0000", 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.126 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091413+0000", 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091416+0000", 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2679999999999999e-06 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.127 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091422+0000", 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7869999999999998e-06 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091427+0000", 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2170000000000002e-06 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094156+0000", 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027282589999999998 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094256+0000", 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000100519 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094403+0000", 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000146608 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:113951 11.e 11:78b19ab2:::benchmark_data_smithi121_477989_object25762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.091440+0000", 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0062168190000000002, 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065534419999999996, 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "delayed", 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 113951 2023-09-23T21:07:24.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091440+0000", 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091440+0000", 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091439+0000", 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091442+0000", 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.638e-06 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091448+0000", 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6350000000000003e-06 2023-09-23T21:07:24.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091452+0000", 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8890000000000001e-06 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.092677+0000", 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012240630000000001 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095328+0000", 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026518900000000001 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:113919 11.e 11:7e200171:::benchmark_data_smithi121_477989_object25730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.008996+0000", 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.088660487999999996, 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089063449000000003, 2023-09-23T21:07:24.134 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 113919 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.008996+0000", 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.008996+0000", 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.008991+0000", 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999943 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.009006+0000", 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5534000000000001e-05 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.009026+0000", 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9877999999999998e-05 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.009039+0000", 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3138e-05 2023-09-23T21:07:24.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.017732+0000", 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086926850000000003 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for degraded object", 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.018616+0000", 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00088461799999999999 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.070396+0000", 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.051779804999999998 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.070510+0000", 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114123 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2,4,6", 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.070675+0000", 2023-09-23T21:07:24.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016519200000000001 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.083004+0000", 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012328644 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.086810+0000", 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0038055490000000001 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.086856+0000", 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5976999999999997e-05 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.087431+0000", 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057507999999999995 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095354+0000", 2023-09-23T21:07:24.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079229360000000002 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095404+0000", 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9965e-05 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:113943 11.2 e408/403 11:4b0fbcec:::benchmark_data_smithi121_477989_object25754:head v 408'7100, mlcod=408'7098)", 2023-09-23T21:07:24.142 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.091478+0000", 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0061785950000000003, 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0066984879999999998, 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091478+0000", 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091478+0000", 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091477+0000", 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091484+0000", 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8079999999999999e-06 2023-09-23T21:07:24.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091491+0000", 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.34e-06 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.091497+0000", 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2900000000000002e-06 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.092052+0000", 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00055566600000000002 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.092075+0000", 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2371999999999999e-05 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:113957 11.4 e408/369 11:23109491:::benchmark_data_smithi121_477989_object25768:head v 408'7146, mlcod=0'0)", 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.094929+0000", 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0027274410000000002, 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033074749999999998, 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094929+0000", 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094929+0000", 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094928+0000", 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094937+0000", 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8680000000000005e-06 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094946+0000", 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6339999999999996e-06 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.094958+0000", 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1698e-05 2023-09-23T21:07:24.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095012+0000", 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4543999999999998e-05 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095037+0000", 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4451999999999999e-05 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.150 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:113949 11.2 e408/403)", 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:24.095250+0000", 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0024069019999999998, 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030495409999999998, 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:24.151 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095250+0000", 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095250+0000", 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095249+0000", 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:07:24.152 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095255+0000", 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2890000000000003e-06 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095263+0000", 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2130000000000008e-06 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:24.153 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:24.095275+0000", 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1426e-05 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 9 2023-09-23T21:07:24.154 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:07:24.155 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:07:24.196 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:07:24.247 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:07:24.280 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:07:24.319 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:07:24.362 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:07:24.401 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:07:24.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/868774230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:24.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/868774230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:24.430 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:07:24.448 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.455 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:07:24.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:24.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/868774230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:24.559 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:24.648 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:24.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:24.820 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:24.867 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:24.942 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:24.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:24.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:25.052 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:25.098 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:25.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:07:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:07:24] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2023-09-23T21:07:25.168 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:25.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:25.282 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:25.307 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:25.349 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:25.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:25.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:25.452 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:25 smithi123 ceph-mon[193903]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 32059/191742 objects degraded (16.720%); 50435/191742 objects misplaced (26.304%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:25.500 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:25.576 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:25.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:25.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:25.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:25 smithi121 ceph-mon[211526]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 32059/191742 objects degraded (16.720%); 50435/191742 objects misplaced (26.304%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:25.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:25 smithi121 ceph-mon[219709]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 32059/191742 objects degraded (16.720%); 50435/191742 objects misplaced (26.304%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T21:07:25.685 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:25.766 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:25.790 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:25.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:07:25.921 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:25.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:25.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:26.041 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:26.126 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:26.237 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:26.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:26.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:26.341 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:26.372 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:26.416 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:07:26.438 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:26.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:26.545 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:26.651 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:26.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 32059/191742 objects degraded (16.720%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:26.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 32059/191742 objects degraded (16.720%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:26.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 32059/191742 objects degraded (16.720%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:26.800 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:26.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:26.907 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:26.933 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:27.391 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:27.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:27.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:27.503 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:27.540 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:27.590 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:27.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:27 smithi121 ceph-mon[211526]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 31564/185334 objects degraded (17.031%); 48797/185334 objects misplaced (26.329%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T21:07:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:27 smithi121 ceph-mon[219709]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 31564/185334 objects degraded (17.031%); 48797/185334 objects misplaced (26.329%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T21:07:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:27.665 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:27.703 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:27.721 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:27.792 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:27.844 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:07:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:27 smithi123 ceph-mon[193903]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 31564/185334 objects degraded (17.031%); 48797/185334 objects misplaced (26.329%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T21:07:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:28.326 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:28.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:28.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:28.432 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:28.460 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:28.533 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:28.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:28.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:28.745 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:28.930 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:28.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:28.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:29.039 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:29.083 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:07:29.091 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:29.157 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:29.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2260145161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:07:29.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2260145161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:07:29.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:29.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:29.269 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:29.342 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:29.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2260145161' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:07:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[219709]: pgmap v1286: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 31564/185334 objects degraded (17.031%); 48797/185334 objects misplaced (26.329%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T21:07:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2260145161' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:07:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[219709]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T21:07:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[219709]: failed to encode map e409 with expected crc 2023-09-23T21:07:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[219709]: failed to encode map e409 with expected crc 2023-09-23T21:07:30.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:30.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[211526]: pgmap v1286: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 31564/185334 objects degraded (17.031%); 48797/185334 objects misplaced (26.329%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T21:07:30.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2260145161' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:07:30.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[211526]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T21:07:30.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[211526]: failed to encode map e409 with expected crc 2023-09-23T21:07:30.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:30 smithi121 ceph-mon[211526]: failed to encode map e409 with expected crc 2023-09-23T21:07:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:30 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:30 smithi123 ceph-mon[193903]: pgmap v1286: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 31564/185334 objects degraded (17.031%); 48797/185334 objects misplaced (26.329%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T21:07:30.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2260145161' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:07:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:30 smithi123 ceph-mon[193903]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T21:07:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:30 smithi123 ceph-mon[193903]: failed to encode map e409 with expected crc 2023-09-23T21:07:30.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:30 smithi123 ceph-mon[193903]: failed to encode map e409 with expected crc 2023-09-23T21:07:30.799 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:30.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:30.903 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:30.983 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:31.280 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:31.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:31.399 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:31.499 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:31.711 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:31.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:31.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:31.836 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:31.857 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:31.934 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:31.977 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:31.977 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:32.288 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:32 smithi121 ceph-mon[219709]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 31139/177720 objects degraded (17.521%); 46638/177720 objects misplaced (26.242%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:07:32.288 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31139/177720 objects degraded (17.521%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:32.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:32 smithi121 ceph-mon[211526]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 31139/177720 objects degraded (17.521%); 46638/177720 objects misplaced (26.242%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:07:32.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31139/177720 objects degraded (17.521%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:32.366 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:32.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:32.403 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:07:32.423 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:07:32.471 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:32.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:32 smithi123 ceph-mon[193903]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 31139/177720 objects degraded (17.521%); 46638/177720 objects misplaced (26.242%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:07:32.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31139/177720 objects degraded (17.521%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:32.511 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:32.592 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:32.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:32.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:32.706 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:32.767 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:33.411 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:07:33.549 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:33.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:33.657 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:33.683 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:33.746 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:33.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:33.853 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:33.937 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:33.961 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:07:34.178 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:34 smithi121 ceph-mon[211526]: from='client.46505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:34.178 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:34 smithi121 ceph-mon[211526]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 31139/177720 objects degraded (17.521%); 46638/177720 objects misplaced (26.242%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:07:34.178 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:34 smithi121 ceph-mon[219709]: from='client.46505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:34.178 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:34 smithi121 ceph-mon[219709]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 31139/177720 objects degraded (17.521%); 46638/177720 objects misplaced (26.242%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:07:34.189 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1288,"stamp":"2023-09-23T21:07:31.044664+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3881310136,"num_objects":59240,"num_object_clones":0,"num_object_copies":177720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31139,"num_objects_misplaced":46638,"num_objects_unfound":0,"num_objects_dirty":59240,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":117458,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27171,"num_bytes_recovered":1776000268,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76673,"ondisk_log_size":76673,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14033104,"kb_used_data":10101528,"kb_used_omap":21,"kb_used_meta":3931242,"kb_avail":642182960,"statfs":{"total":671965249536,"available":657595351040,"internally_reserved":0,"allocated":10343964672,"data_stored":10332302273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4025592291},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-336920576,"num_objects":-5141,"num_object_clones":0,"num_object_copies":-15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-920,"num_objects_misplaced":-4282,"num_objects_unfound":0,"num_objects_dirty":-5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5141,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":985,"num_bytes_recovered":64552960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006680"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":179,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.002951+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:07:19.002951+0000","last_peered":"2023-09-23T21:07:19.002951+0000","last_clean":"2023-09-23T21:07:19.002951+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:07:19.002951+0000","last_undegraded":"2023-09-23T21:07:19.002951+0000","last_fullsized":"2023-09-23T21:07:19.002951+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":49,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000482+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:07:19.000482+0000","last_peered":"2023-09-23T21:07:19.000482+0000","last_clean":"2023-09-23T21:07:19.000482+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:07:19.000482+0000","last_undegraded":"2023-09-23T21:07:19.000482+0000","last_fullsized":"2023-09-23T21:07:19.000482+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":322,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:25.437341+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:07:25.437341+0000","last_peered":"2023-09-23T21:07:25.437341+0000","last_clean":"2023-09-23T21:07:25.437341+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:07:25.437341+0000","last_undegraded":"2023-09-23T21:07:25.437341+0000","last_fullsized":"2023-09-23T21:07:25.437341+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":46,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.027162+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:07:19.027162+0000","last_peered":"2023-09-23T21:07:19.027162+0000","last_clean":"2023-09-23T21:07:19.027162+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:07:19.027162+0000","last_undegraded":"2023-09-23T21:07:19.027162+0000","last_fullsized":"2023-09-23T21:07:19.027162+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":67,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000057+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:07:19.000057+0000","last_peered":"2023-09-23T21:07:19.000057+0000","last_clean":"2023-09-23T21:07:19.000057+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:07:19.000057+0000","last_undegraded":"2023-09-23T21:07:19.000057+0000","last_fullsized":"2023-09-23T21:07:19.000057+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":275,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.982962+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:07:18.982962+0000","last_peered":"2023-09-23T21:07:18.982962+0000","last_clean":"2023-09-23T21:07:18.982962+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:07:18.982962+0000","last_undegraded":"2023-09-23T21:07:18.982962+0000","last_fullsized":"2023-09-23T21:07:18.982962+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":46,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:24.186682+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:07:24.186682+0000","last_peered":"2023-09-23T21:07:24.186682+0000","last_clean":"2023-09-23T21:07:24.186682+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:07:24.186682+0000","last_undegraded":"2023-09-23T21:07:24.186682+0000","last_fullsized":"2023-09-23T21:07:24.186682+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":49,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.999986+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:07:18.999986+0000","last_peered":"2023-09-23T21:07:18.999986+0000","last_clean":"2023-09-23T21:07:18.999986+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:07:18.999986+0000","last_undegraded":"2023-09-23T21:07:18.999986+0000","last_fullsized":"2023-09-23T21:07:18.999986+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":31,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.575602+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:07:09.575602+0000","last_peered":"2023-09-23T21:07:09.575602+0000","last_clean":"2023-09-23T21:07:09.575602+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:07:09.575602+0000","last_undegraded":"2023-09-23T21:07:09.575602+0000","last_fullsized":"2023-09-23T21:07:09.575602+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":49,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000559+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:07:19.000559+0000","last_peered":"2023-09-23T21:07:19.000559+0000","last_clean":"2023-09-23T21:07:19.000559+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:07:19.000559+0000","last_undegraded":"2023-09-23T21:07:19.000559+0000","last_fullsized":"2023-09-23T21:07:19.000559+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":65,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000565+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:07:19.000565+0000","last_peered":"2023-09-23T21:07:19.000565+0000","last_clean":"2023-09-23T21:07:19.000565+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:07:19.000565+0000","last_undegraded":"2023-09-23T21:07:19.000565+0000","last_fullsized":"2023-09-23T21:07:19.000565+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.002734+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:07:19.002734+0000","last_peered":"2023-09-23T21:07:19.002734+0000","last_clean":"2023-09-23T21:07:19.002734+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:07:19.002734+0000","last_undegraded":"2023-09-23T21:07:19.002734+0000","last_fullsized":"2023-09-23T21:07:19.002734+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":355,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.982575+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:07:18.982575+0000","last_peered":"2023-09-23T21:07:18.982575+0000","last_clean":"2023-09-23T21:07:18.982575+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:07:18.982575+0000","last_undegraded":"2023-09-23T21:07:18.982575+0000","last_fullsized":"2023-09-23T21:07:18.982575+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":326,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.562630+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:07:09.562630+0000","last_peered":"2023-09-23T21:07:09.562630+0000","last_clean":"2023-09-23T21:07:09.562630+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:07:09.562630+0000","last_undegraded":"2023-09-23T21:07:09.562630+0000","last_fullsized":"2023-09-23T21:07:09.562630+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":99,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.014667+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:07:19.014667+0000","last_peered":"2023-09-23T21:07:19.014667+0000","last_clean":"2023-09-23T21:07:19.014667+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:07:19.014667+0000","last_undegraded":"2023-09-23T21:07:19.014667+0000","last_fullsized":"2023-09-23T21:07:19.014667+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":58,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000375+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:07:19.000375+0000","last_peered":"2023-09-23T21:07:19.000375+0000","last_clean":"2023-09-23T21:07:19.000375+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:07:19.000375+0000","last_undegraded":"2023-09-23T21:07:19.000375+0000","last_fullsized":"2023-09-23T21:07:19.000375+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":67,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000312+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:07:19.000312+0000","last_peered":"2023-09-23T21:07:19.000312+0000","last_clean":"2023-09-23T21:07:19.000312+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:07:19.000312+0000","last_undegraded":"2023-09-23T21:07:19.000312+0000","last_fullsized":"2023-09-23T21:07:19.000312+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":67,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000646+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:07:19.000646+0000","last_peered":"2023-09-23T21:07:19.000646+0000","last_clean":"2023-09-23T21:07:19.000646+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:07:19.000646+0000","last_undegraded":"2023-09-23T21:07:19.000646+0000","last_fullsized":"2023-09-23T21:07:19.000646+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":208,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.562573+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:07:09.562573+0000","last_peered":"2023-09-23T21:07:09.562573+0000","last_clean":"2023-09-23T21:07:09.562573+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:07:09.562573+0000","last_undegraded":"2023-09-23T21:07:09.562573+0000","last_fullsized":"2023-09-23T21:07:09.562573+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":46,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:24.186648+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:07:24.186648+0000","last_peered":"2023-09-23T21:07:24.186648+0000","last_clean":"2023-09-23T21:07:24.186648+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:07:24.186648+0000","last_undegraded":"2023-09-23T21:07:24.186648+0000","last_fullsized":"2023-09-23T21:07:24.186648+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":405,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.981982+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:07:18.981982+0000","last_peered":"2023-09-23T21:07:18.981982+0000","last_clean":"2023-09-23T21:07:18.981982+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:07:18.981982+0000","last_undegraded":"2023-09-23T21:07:18.981982+0000","last_fullsized":"2023-09-23T21:07:18.981982+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":49,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000490+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:07:19.000490+0000","last_peered":"2023-09-23T21:07:19.000490+0000","last_clean":"2023-09-23T21:07:19.000490+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:07:19.000490+0000","last_undegraded":"2023-09-23T21:07:19.000490+0000","last_fullsized":"2023-09-23T21:07:19.000490+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"409'7518","reported_seq":8884,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:30.475645+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:07:30.475645+0000","last_peered":"2023-09-23T21:07:30.475645+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:07:30.475645+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"353'2828","ondisk_log_start":"353'2828","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4690,"log_dups_size":0,"ondisk_log_size":4690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243269632,"num_objects":3712,"num_object_clones":0,"num_object_copies":11136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7518,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-1903},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":3711}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":112,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.563755+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:07:09.563755+0000","last_peered":"2023-09-23T21:07:09.563755+0000","last_clean":"2023-09-23T21:07:09.563755+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:07:09.563755+0000","last_undegraded":"2023-09-23T21:07:09.563755+0000","last_fullsized":"2023-09-23T21:07:09.563755+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":31,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.575523+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:07:09.575523+0000","last_peered":"2023-09-23T21:07:09.575523+0000","last_clean":"2023-09-23T21:07:09.575523+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:07:09.575523+0000","last_undegraded":"2023-09-23T21:07:09.575523+0000","last_fullsized":"2023-09-23T21:07:09.575523+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"409'7181","reported_seq":7260,"reported_epoch":409,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:30.458913+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:07:30.458913+0000","last_peered":"2023-09-23T21:07:30.458913+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:07:30.458913+0000","last_undegraded":"2023-09-23T21:07:30.458913+0000","last_fullsized":"2023-09-23T21:07:30.458913+0000","mapping_epoch":375,"log_start":"350'2412","ondisk_log_start":"350'2412","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4769,"log_dups_size":0,"ondisk_log_size":4769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237043712,"num_objects":3617,"num_object_clones":0,"num_object_copies":10851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3617,"num_objects_unfound":0,"num_objects_dirty":3617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7181,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":3617}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":107,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.003200+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:07:19.003200+0000","last_peered":"2023-09-23T21:07:19.003200+0000","last_clean":"2023-09-23T21:07:19.003200+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:07:19.003200+0000","last_undegraded":"2023-09-23T21:07:19.003200+0000","last_fullsized":"2023-09-23T21:07:19.003200+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":369,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.002368+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:07:19.002368+0000","last_peered":"2023-09-23T21:07:19.002368+0000","last_clean":"2023-09-23T21:07:19.002368+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:07:19.002368+0000","last_undegraded":"2023-09-23T21:07:19.002368+0000","last_fullsized":"2023-09-23T21:07:19.002368+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"409'7270","reported_seq":14913,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:30.725824+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:07:30.725824+0000","last_peered":"2023-09-23T21:07:30.725824+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:07:30.725824+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"358'3211","ondisk_log_start":"358'3211","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4059,"log_dups_size":0,"ondisk_log_size":4059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240386048,"num_objects":3668,"num_object_clones":0,"num_object_copies":11004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7270,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2874,"num_bytes_recovered":188350464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":1282},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":35,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.576445+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:07:09.576445+0000","last_peered":"2023-09-23T21:07:09.576445+0000","last_clean":"2023-09-23T21:07:09.576445+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:07:09.576445+0000","last_undegraded":"2023-09-23T21:07:09.576445+0000","last_fullsized":"2023-09-23T21:07:09.576445+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":297,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.982292+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:07:18.982292+0000","last_peered":"2023-09-23T21:07:18.982292+0000","last_clean":"2023-09-23T21:07:18.982292+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:07:18.982292+0000","last_undegraded":"2023-09-23T21:07:18.982292+0000","last_fullsized":"2023-09-23T21:07:18.982292+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"409'7370","reported_seq":7996,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:30.692341+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:07:30.692341+0000","last_peered":"2023-09-23T21:07:30.692341+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:07:30.692341+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"358'3268","ondisk_log_start":"358'3268","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4102,"log_dups_size":0,"ondisk_log_size":4102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241172480,"num_objects":3680,"num_object_clones":0,"num_object_copies":11040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7370,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-1573},{"shards":"2,6,7","objects":3677}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":49,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000646+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:07:19.000646+0000","last_peered":"2023-09-23T21:07:19.000646+0000","last_clean":"2023-09-23T21:07:19.000646+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:07:19.000646+0000","last_undegraded":"2023-09-23T21:07:19.000646+0000","last_fullsized":"2023-09-23T21:07:19.000646+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":46,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.026779+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:07:19.026779+0000","last_peered":"2023-09-23T21:07:19.026779+0000","last_clean":"2023-09-23T21:07:19.026779+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:07:19.026779+0000","last_undegraded":"2023-09-23T21:07:19.026779+0000","last_fullsized":"2023-09-23T21:07:19.026779+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"409'7522","reported_seq":19020,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:30.715816+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:07:30.715816+0000","last_peered":"2023-09-23T21:07:30.715816+0000","last_clean":"2023-09-23T21:07:30.715816+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:07:30.715816+0000","last_undegraded":"2023-09-23T21:07:30.715816+0000","last_fullsized":"2023-09-23T21:07:30.715816+0000","mapping_epoch":400,"log_start":"358'3399","ondisk_log_start":"358'3399","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4123,"log_dups_size":0,"ondisk_log_size":4123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248119296,"num_objects":3786,"num_object_clones":0,"num_object_copies":11358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7522,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":66,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000327+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:07:19.000327+0000","last_peered":"2023-09-23T21:07:19.000327+0000","last_clean":"2023-09-23T21:07:19.000327+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:07:19.000327+0000","last_undegraded":"2023-09-23T21:07:19.000327+0000","last_fullsized":"2023-09-23T21:07:19.000327+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":49,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000257+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:07:19.000257+0000","last_peered":"2023-09-23T21:07:19.000257+0000","last_clean":"2023-09-23T21:07:19.000257+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:07:19.000257+0000","last_undegraded":"2023-09-23T21:07:19.000257+0000","last_fullsized":"2023-09-23T21:07:19.000257+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"409'7262","reported_seq":18373,"reported_epoch":409,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:29.210671+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:07:29.210671+0000","last_peered":"2023-09-23T21:07:29.210671+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:07:29.210671+0000","last_undegraded":"2023-09-23T21:07:29.210671+0000","last_fullsized":"2023-09-23T21:07:29.210671+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":6774,"log_dups_size":0,"ondisk_log_size":6774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243793920,"num_objects":3720,"num_object_clones":0,"num_object_copies":11160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3720,"num_objects_unfound":0,"num_objects_dirty":3720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7262,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":3720}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":335,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.982348+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:07:18.982348+0000","last_peered":"2023-09-23T21:07:18.982348+0000","last_clean":"2023-09-23T21:07:18.982348+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:07:18.982348+0000","last_undegraded":"2023-09-23T21:07:18.982348+0000","last_fullsized":"2023-09-23T21:07:18.982348+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":419,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.014000+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:07:19.014000+0000","last_peered":"2023-09-23T21:07:19.014000+0000","last_clean":"2023-09-23T21:07:19.014000+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:07:19.014000+0000","last_undegraded":"2023-09-23T21:07:19.014000+0000","last_fullsized":"2023-09-23T21:07:19.014000+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"409'7243","reported_seq":7338,"reported_epoch":409,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:30.663787+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:07:30.663787+0000","last_peered":"2023-09-23T21:07:30.663787+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:07:30.663787+0000","last_undegraded":"2023-09-23T21:07:30.663787+0000","last_fullsized":"2023-09-23T21:07:30.663787+0000","mapping_epoch":374,"log_start":"356'3094","ondisk_log_start":"356'3094","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4149,"log_dups_size":0,"ondisk_log_size":4149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239927296,"num_objects":3661,"num_object_clones":0,"num_object_copies":10983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7322,"num_objects_unfound":0,"num_objects_dirty":3661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7243,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3661}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":35,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.562439+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:07:09.562439+0000","last_peered":"2023-09-23T21:07:09.562439+0000","last_clean":"2023-09-23T21:07:09.562439+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:07:09.562439+0000","last_undegraded":"2023-09-23T21:07:09.562439+0000","last_fullsized":"2023-09-23T21:07:09.562439+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":448,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.982853+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:07:18.982853+0000","last_peered":"2023-09-23T21:07:18.982853+0000","last_clean":"2023-09-23T21:07:18.982853+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:07:18.982853+0000","last_undegraded":"2023-09-23T21:07:18.982853+0000","last_fullsized":"2023-09-23T21:07:18.982853+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"409'7289","reported_seq":7471,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:30.410873+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:07:30.410873+0000","last_peered":"2023-09-23T21:07:30.410873+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:07:30.410873+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"350'2594","ondisk_log_start":"350'2594","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240713728,"num_objects":3673,"num_object_clones":0,"num_object_copies":11019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7289,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-1766},{"shards":"1,6,7","objects":3673}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":58,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000673+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:07:19.000673+0000","last_peered":"2023-09-23T21:07:19.000673+0000","last_clean":"2023-09-23T21:07:19.000673+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:07:19.000673+0000","last_undegraded":"2023-09-23T21:07:19.000673+0000","last_fullsized":"2023-09-23T21:07:19.000673+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":197,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.010040+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:07:19.010040+0000","last_peered":"2023-09-23T21:07:19.010040+0000","last_clean":"2023-09-23T21:07:19.010040+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:07:19.010040+0000","last_undegraded":"2023-09-23T21:07:19.010040+0000","last_fullsized":"2023-09-23T21:07:19.010040+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"409'7252","reported_seq":12745,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.165668+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:07:29.165668+0000","last_peered":"2023-09-23T21:07:29.165668+0000","last_clean":"2023-09-23T21:07:29.165668+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:07:29.165668+0000","last_undegraded":"2023-09-23T21:07:29.165668+0000","last_fullsized":"2023-09-23T21:07:29.165668+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247529496,"num_objects":3778,"num_object_clones":0,"num_object_copies":11334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3778,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7252,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":303,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.011923+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:07:19.011923+0000","last_peered":"2023-09-23T21:07:19.011923+0000","last_clean":"2023-09-23T21:07:19.011923+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:07:19.011923+0000","last_undegraded":"2023-09-23T21:07:19.011923+0000","last_fullsized":"2023-09-23T21:07:19.011923+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":73,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.576511+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:07:09.576511+0000","last_peered":"2023-09-23T21:07:09.576511+0000","last_clean":"2023-09-23T21:07:09.576511+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:07:09.576511+0000","last_undegraded":"2023-09-23T21:07:09.576511+0000","last_fullsized":"2023-09-23T21:07:09.576511+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":59,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:20.413077+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:07:20.413077+0000","last_peered":"2023-09-23T21:07:20.413077+0000","last_clean":"2023-09-23T21:07:20.413077+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:07:20.413077+0000","last_undegraded":"2023-09-23T21:07:20.413077+0000","last_fullsized":"2023-09-23T21:07:20.413077+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"409'7348","reported_seq":11227,"reported_epoch":409,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:07:30.728895+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:07:30.728895+0000","last_peered":"2023-09-23T21:07:30.728895+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:07:30.728895+0000","last_undegraded":"2023-09-23T21:07:30.728895+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"357'3257","ondisk_log_start":"357'3257","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4091,"log_dups_size":0,"ondisk_log_size":4091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241041408,"num_objects":3678,"num_object_clones":0,"num_object_copies":11034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9609,"num_objects_unfound":0,"num_objects_dirty":3678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7348,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1875,"num_bytes_recovered":122880000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":1305},{"shards":"2,6","objects":40},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":384,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.003069+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:07:19.003069+0000","last_peered":"2023-09-23T21:07:19.003069+0000","last_clean":"2023-09-23T21:07:19.003069+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:07:19.003069+0000","last_undegraded":"2023-09-23T21:07:19.003069+0000","last_fullsized":"2023-09-23T21:07:19.003069+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":359,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.982376+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:07:18.982376+0000","last_peered":"2023-09-23T21:07:18.982376+0000","last_clean":"2023-09-23T21:07:18.982376+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:07:18.982376+0000","last_undegraded":"2023-09-23T21:07:18.982376+0000","last_fullsized":"2023-09-23T21:07:18.982376+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":41,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:20.413127+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:07:20.413127+0000","last_peered":"2023-09-23T21:07:20.413127+0000","last_clean":"2023-09-23T21:07:20.413127+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:07:20.413127+0000","last_undegraded":"2023-09-23T21:07:20.413127+0000","last_fullsized":"2023-09-23T21:07:20.413127+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"409'7311","reported_seq":7489,"reported_epoch":409,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:29.234641+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:07:29.234641+0000","last_peered":"2023-09-23T21:07:29.234641+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:07:29.234641+0000","last_undegraded":"2023-09-23T21:07:29.234641+0000","last_fullsized":"2023-09-23T21:07:29.234641+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245825536,"num_objects":3751,"num_object_clones":0,"num_object_copies":11253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7480,"num_objects_unfound":0,"num_objects_dirty":3751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7311,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":3751}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":386,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.239592+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:07:19.239592+0000","last_peered":"2023-09-23T21:07:19.239592+0000","last_clean":"2023-09-23T21:07:19.239592+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:07:19.239592+0000","last_undegraded":"2023-09-23T21:07:19.239592+0000","last_fullsized":"2023-09-23T21:07:19.239592+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":303,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.003254+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:07:19.003254+0000","last_peered":"2023-09-23T21:07:19.003254+0000","last_clean":"2023-09-23T21:07:19.003254+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:07:19.003254+0000","last_undegraded":"2023-09-23T21:07:19.003254+0000","last_fullsized":"2023-09-23T21:07:19.003254+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":41,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:25.447976+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:07:25.447976+0000","last_peered":"2023-09-23T21:07:25.447976+0000","last_clean":"2023-09-23T21:07:25.447976+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:07:25.447976+0000","last_undegraded":"2023-09-23T21:07:25.447976+0000","last_fullsized":"2023-09-23T21:07:25.447976+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"409'7202","reported_seq":13457,"reported_epoch":409,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:30.704844+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:07:30.704844+0000","last_peered":"2023-09-23T21:07:30.704844+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:07:30.704844+0000","last_undegraded":"2023-09-23T21:07:30.704844+0000","last_fullsized":"2023-09-23T21:07:30.704844+0000","mapping_epoch":387,"log_start":"356'3080","ondisk_log_start":"356'3080","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4122,"log_dups_size":0,"ondisk_log_size":4122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648192,"num_objects":3672,"num_object_clones":0,"num_object_copies":11016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3672,"num_objects_unfound":0,"num_objects_dirty":3672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7202,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3672}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":224,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.239621+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:07:19.239621+0000","last_peered":"2023-09-23T21:07:19.239621+0000","last_clean":"2023-09-23T21:07:19.239621+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:07:19.239621+0000","last_undegraded":"2023-09-23T21:07:19.239621+0000","last_fullsized":"2023-09-23T21:07:19.239621+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":706,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:19.002329+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:07:19.002329+0000","last_peered":"2023-09-23T21:07:19.002329+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:07:19.002329+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":58,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000197+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:07:19.000197+0000","last_peered":"2023-09-23T21:07:19.000197+0000","last_clean":"2023-09-23T21:07:19.000197+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:07:19.000197+0000","last_undegraded":"2023-09-23T21:07:19.000197+0000","last_fullsized":"2023-09-23T21:07:19.000197+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":297,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.982834+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:07:18.982834+0000","last_peered":"2023-09-23T21:07:18.982834+0000","last_clean":"2023-09-23T21:07:18.982834+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:07:18.982834+0000","last_undegraded":"2023-09-23T21:07:18.982834+0000","last_fullsized":"2023-09-23T21:07:18.982834+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"409'7395","reported_seq":7506,"reported_epoch":409,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:30.659527+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:07:30.659527+0000","last_peered":"2023-09-23T21:07:30.659527+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:07:30.659527+0000","last_undegraded":"2023-09-23T21:07:30.659527+0000","last_fullsized":"2023-09-23T21:07:30.659527+0000","mapping_epoch":374,"log_start":"358'3305","ondisk_log_start":"358'3305","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4090,"log_dups_size":0,"ondisk_log_size":4090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244252672,"num_objects":3727,"num_object_clones":0,"num_object_copies":11181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7454,"num_objects_unfound":0,"num_objects_dirty":3727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7395,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":3727}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":41,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.002195+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:07:19.002195+0000","last_peered":"2023-09-23T21:07:19.002195+0000","last_clean":"2023-09-23T21:07:19.002195+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:07:19.002195+0000","last_undegraded":"2023-09-23T21:07:19.002195+0000","last_fullsized":"2023-09-23T21:07:19.002195+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":55,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.023415+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:07:19.023415+0000","last_peered":"2023-09-23T21:07:19.023415+0000","last_clean":"2023-09-23T21:07:19.023415+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:07:19.023415+0000","last_undegraded":"2023-09-23T21:07:19.023415+0000","last_fullsized":"2023-09-23T21:07:19.023415+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":46,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.026806+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:07:19.026806+0000","last_peered":"2023-09-23T21:07:19.026806+0000","last_clean":"2023-09-23T21:07:19.026806+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:07:19.026806+0000","last_undegraded":"2023-09-23T21:07:19.026806+0000","last_fullsized":"2023-09-23T21:07:19.026806+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"409'7272","reported_seq":10889,"reported_epoch":409,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:30.456053+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:07:30.456053+0000","last_peered":"2023-09-23T21:07:30.456053+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:07:30.456053+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:07:30.456053+0000","mapping_epoch":383,"log_start":"350'2526","ondisk_log_start":"350'2526","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4746,"log_dups_size":0,"ondisk_log_size":4746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246022144,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3751,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7272,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":3},{"shards":"1,2,7","objects":3751}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":46,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.026035+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:07:19.026035+0000","last_peered":"2023-09-23T21:07:19.026035+0000","last_clean":"2023-09-23T21:07:19.026035+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:07:19.026035+0000","last_undegraded":"2023-09-23T21:07:19.026035+0000","last_fullsized":"2023-09-23T21:07:19.026035+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":305,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.014155+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:07:19.014155+0000","last_peered":"2023-09-23T21:07:19.014155+0000","last_clean":"2023-09-23T21:07:19.014155+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:07:19.014155+0000","last_undegraded":"2023-09-23T21:07:19.014155+0000","last_fullsized":"2023-09-23T21:07:19.014155+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":83,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.575853+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:07:09.575853+0000","last_peered":"2023-09-23T21:07:09.575853+0000","last_clean":"2023-09-23T21:07:09.575853+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:07:09.575853+0000","last_undegraded":"2023-09-23T21:07:09.575853+0000","last_fullsized":"2023-09-23T21:07:09.575853+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"409'7282","reported_seq":20170,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:30.376771+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:07:30.376771+0000","last_peered":"2023-09-23T21:07:30.376771+0000","last_clean":"2023-09-23T21:07:30.376771+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:07:30.376771+0000","last_undegraded":"2023-09-23T21:07:30.376771+0000","last_fullsized":"2023-09-23T21:07:30.376771+0000","mapping_epoch":397,"log_start":"350'2545","ondisk_log_start":"350'2545","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4737,"log_dups_size":0,"ondisk_log_size":4737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234749952,"num_objects":3582,"num_object_clones":0,"num_object_copies":10746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7282,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":66,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000580+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:07:19.000580+0000","last_peered":"2023-09-23T21:07:19.000580+0000","last_clean":"2023-09-23T21:07:19.000580+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:07:19.000580+0000","last_undegraded":"2023-09-23T21:07:19.000580+0000","last_fullsized":"2023-09-23T21:07:19.000580+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":41,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:25.439868+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:07:25.439868+0000","last_peered":"2023-09-23T21:07:25.439868+0000","last_clean":"2023-09-23T21:07:25.439868+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:07:25.439868+0000","last_undegraded":"2023-09-23T21:07:25.439868+0000","last_fullsized":"2023-09-23T21:07:25.439868+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.978348+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:07:18.978348+0000","last_peered":"2023-09-23T21:07:18.978348+0000","last_clean":"2023-09-23T21:07:18.978348+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:07:18.978348+0000","last_undegraded":"2023-09-23T21:07:18.978348+0000","last_fullsized":"2023-09-23T21:07:18.978348+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"409'7441","reported_seq":11239,"reported_epoch":409,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:30.467584+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:07:30.467584+0000","last_peered":"2023-09-23T21:07:30.467584+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:07:30.467584+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:07:30.467584+0000","mapping_epoch":382,"log_start":"350'2712","ondisk_log_start":"350'2712","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4729,"log_dups_size":0,"ondisk_log_size":4729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246218752,"num_objects":3757,"num_object_clones":0,"num_object_copies":11271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3753,"num_objects_misplaced":3761,"num_objects_unfound":0,"num_objects_dirty":3757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7441,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1843,"num_bytes_recovered":120782848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":4},{"shards":"1,6,7","objects":3753}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":305,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.983028+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:07:18.983028+0000","last_peered":"2023-09-23T21:07:18.983028+0000","last_clean":"2023-09-23T21:07:18.983028+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:07:18.983028+0000","last_undegraded":"2023-09-23T21:07:18.983028+0000","last_fullsized":"2023-09-23T21:07:18.983028+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":50,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.026978+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:07:19.026978+0000","last_peered":"2023-09-23T21:07:19.026978+0000","last_clean":"2023-09-23T21:07:19.026978+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:07:19.026978+0000","last_undegraded":"2023-09-23T21:07:19.026978+0000","last_fullsized":"2023-09-23T21:07:19.026978+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.575484+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:07:09.575484+0000","last_peered":"2023-09-23T21:07:09.575484+0000","last_clean":"2023-09-23T21:07:09.575484+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:07:09.575484+0000","last_undegraded":"2023-09-23T21:07:09.575484+0000","last_fullsized":"2023-09-23T21:07:09.575484+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":55,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068290+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:07:29.068290+0000","last_peered":"2023-09-23T21:07:29.068290+0000","last_clean":"2023-09-23T21:07:29.068290+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:07:29.068290+0000","last_undegraded":"2023-09-23T21:07:29.068290+0000","last_fullsized":"2023-09-23T21:07:29.068290+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":432,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.014088+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:07:19.014088+0000","last_peered":"2023-09-23T21:07:19.014088+0000","last_clean":"2023-09-23T21:07:19.014088+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:07:19.014088+0000","last_undegraded":"2023-09-23T21:07:19.014088+0000","last_fullsized":"2023-09-23T21:07:19.014088+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":70,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.562857+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:07:09.562857+0000","last_peered":"2023-09-23T21:07:09.562857+0000","last_clean":"2023-09-23T21:07:09.562857+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:07:09.562857+0000","last_undegraded":"2023-09-23T21:07:09.562857+0000","last_fullsized":"2023-09-23T21:07:09.562857+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000415+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:07:19.000415+0000","last_peered":"2023-09-23T21:07:19.000415+0000","last_clean":"2023-09-23T21:07:19.000415+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:07:19.000415+0000","last_undegraded":"2023-09-23T21:07:19.000415+0000","last_fullsized":"2023-09-23T21:07:19.000415+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":290,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:18.983514+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:07:18.983514+0000","last_peered":"2023-09-23T21:07:18.983514+0000","last_clean":"2023-09-23T21:07:18.983514+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:07:18.983514+0000","last_undegraded":"2023-09-23T21:07:18.983514+0000","last_fullsized":"2023-09-23T21:07:18.983514+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":67,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000391+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:07:19.000391+0000","last_peered":"2023-09-23T21:07:19.000391+0000","last_clean":"2023-09-23T21:07:19.000391+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:07:19.000391+0000","last_undegraded":"2023-09-23T21:07:19.000391+0000","last_fullsized":"2023-09-23T21:07:19.000391+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":49,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000540+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:07:19.000540+0000","last_peered":"2023-09-23T21:07:19.000540+0000","last_clean":"2023-09-23T21:07:19.000540+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:07:19.000540+0000","last_undegraded":"2023-09-23T21:07:19.000540+0000","last_fullsized":"2023-09-23T21:07:19.000540+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":58,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.000480+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:07:19.000480+0000","last_peered":"2023-09-23T21:07:19.000480+0000","last_clean":"2023-09-23T21:07:19.000480+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:07:19.000480+0000","last_undegraded":"2023-09-23T21:07:19.000480+0000","last_fullsized":"2023-09-23T21:07:19.000480+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":52,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T21:07:19.030139+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:07:19.030139+0000","last_peered":"2023-09-23T21:07:19.030139+0000","last_clean":"2023-09-23T21:07:19.030139+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:07:19.030139+0000","last_undegraded":"2023-09-23T21:07:19.030139+0000","last_fullsized":"2023-09-23T21:07:19.030139+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":35,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T21:07:09.562117+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:07:09.562117+0000","last_peered":"2023-09-23T21:07:09.562117+0000","last_clean":"2023-09-23T21:07:09.562117+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:07:09.562117+0000","last_undegraded":"2023-09-23T21:07:09.562117+0000","last_fullsized":"2023-09-23T21:07:09.562117+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3880714264,"num_objects":59216,"num_object_clones":0,"num_object_copies":177648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31137,"num_objects_misplaced":46638,"num_objects_unfound":0,"num_objects_dirty":59216,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117158,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27054,"num_bytes_recovered":1773010944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13719842816,"data_stored":13719830600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76457,"ondisk_log_size":76457,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201745,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463366,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3138636,"kb_used_data":2633604,"kb_used_omap":0,"kb_used_meta":504768,"kb_avail":90606516,"statfs":{"total":95995035648,"available":92781072384,"internally_reserved":0,"allocated":2696810496,"data_stored":2695609838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":516882432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.131,"15min":2.1230000000000002},"min":{"1min":0.254,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6689999999999996,"5min":7.2480000000000002,"15min":102.242},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.107,"15min":1.8440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":0.89300000000000002,"5min":9.7110000000000003,"15min":102.502},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":3.9460000000000002,"15min":2.379},"min":{"1min":0.251,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":11.743,"5min":119.077,"15min":119.077},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.901,"5min":3.7730000000000001,"15min":2.452},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":11.936,"5min":119.313,"15min":119.313},"last":2.726}]},{"osd":2,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":4.1219999999999999,"15min":2.9380000000000002},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.224},"max":{"1min":2.5910000000000002,"5min":123.283,"15min":176.45400000000001},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":4.1680000000000001,"15min":3.1080000000000001},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":1.3009999999999999,"5min":123.051,"15min":176.46799999999999},"last":2.589}]},{"osd":3,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":1.456,"15min":4.4130000000000003},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.26900000000000002},"max":{"1min":3.173,"5min":35.078000000000003,"15min":250.804},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.5409999999999999,"15min":4.3600000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":2.1850000000000001,"5min":35.133000000000003,"15min":250.446},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.1579999999999999,"15min":0.93600000000000005},"min":{"1min":0.32000000000000001,"5min":0.27800000000000002,"15min":0.24299999999999999},"max":{"1min":0.878,"5min":13.1,"15min":13.1},"last":0.376},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.91100000000000003,"15min":0.96299999999999997},"min":{"1min":0.36699999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":2.9079999999999999,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 21:06:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.0980000000000001,"15min":1.5289999999999999},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.90700000000000003,"5min":11.281000000000001,"15min":105.121},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.208,"15min":1.585},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":8.6229999999999993,"15min":106.30500000000001},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.2569999999999999,"15min":1.4710000000000001},"min":{"1min":0.34899999999999998,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.8079999999999998,"5min":7.8079999999999998,"15min":101.04600000000001},"last":2.8340000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.2509999999999999,"15min":1.5029999999999999},"min":{"1min":0.39200000000000002,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":4.407,"5min":7.1849999999999996,"15min":101.081},"last":0.56499999999999995}]}]},{"osd":1,"up_from":276,"seq":1185410973812,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3244888,"kb_used_data":2866832,"kb_used_omap":5,"kb_used_meta":378042,"kb_avail":90500264,"statfs":{"total":95995035648,"available":92672270336,"internally_reserved":0,"allocated":2935635968,"data_stored":2933603621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":387115747},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":0.85599999999999998,"15min":0.55200000000000005},"min":{"1min":0.20599999999999999,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":9.9469999999999992,"5min":11.959,"15min":11.959},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.256,"15min":0.81499999999999995},"min":{"1min":0.216,"5min":0.193,"15min":0.193},"max":{"1min":5.093,"5min":18.047999999999998,"15min":18.047999999999998},"last":3.3900000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.407,"15min":1.1359999999999999},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.0670000000000002,"5min":17.254000000000001,"15min":78.409000000000006},"last":1.232},{"interface":"front","average":{"1min":1.55,"5min":1.03,"15min":0.97999999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.22},"max":{"1min":12.478999999999999,"5min":12.478999999999999,"15min":76.245000000000005},"last":0.57299999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.849,"15min":2.4670000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":7.3099999999999996,"5min":74.527000000000001,"15min":172.44399999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.907,"5min":2.5369999999999999,"15min":2.6880000000000002},"min":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":7.9050000000000002,"5min":72.216999999999999,"15min":172.58500000000001},"last":3.4449999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.472,"15min":0.93400000000000005},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.7069999999999999,"5min":17.006,"15min":17.006},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.92600000000000005,"15min":0.70299999999999996},"min":{"1min":0.35699999999999998,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":4.7690000000000001,"5min":11.916,"15min":11.916},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.446,"15min":0.88300000000000001},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.6269999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":1.29},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.589,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.3860000000000001,"5min":17.076000000000001,"15min":17.076000000000001},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.577,"15min":1.3879999999999999},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.6020000000000003,"5min":17.102,"15min":127.995},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.127,"5min":1.647,"15min":1.466},"min":{"1min":0.41699999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.375,"5min":17.302,"15min":126.836},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.9180000000000001,"15min":1.484},"min":{"1min":0.442,"5min":0.38200000000000001,"15min":0.36499999999999999},"max":{"1min":7.048,"5min":151.791,"15min":151.791},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.49,"5min":2.3889999999999998,"15min":1.2310000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.3739999999999997,"5min":151.809,"15min":151.809},"last":0.51600000000000001}]}]},{"osd":0,"up_from":92,"seq":395136991552,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1238424,"kb_used_data":515984,"kb_used_omap":5,"kb_used_meta":722426,"kb_avail":92506728,"statfs":{"total":95995035648,"available":94726889472,"internally_reserved":0,"allocated":528367616,"data_stored":526449249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":739764858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":2.9500000000000002,"15min":1.4830000000000001},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":40.284999999999997,"5min":61.341999999999999,"15min":61.341999999999999},"last":15.958},{"interface":"front","average":{"1min":3.4319999999999999,"5min":2.9670000000000001,"15min":1.4510000000000001},"min":{"1min":0.223,"5min":0.217,"15min":0.217},"max":{"1min":40.213000000000001,"5min":61.299999999999997,"15min":61.299999999999997},"last":2.927}]},{"osd":2,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.7110000000000001,"15min":2.238},"min":{"1min":0.217,"5min":0.217,"15min":0.161},"max":{"1min":41.860999999999997,"5min":41.860999999999997,"15min":183.77099999999999},"last":10.539},{"interface":"front","average":{"1min":4.5780000000000003,"5min":2.1219999999999999,"15min":2.5070000000000001},"min":{"1min":0.27000000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":44.779000000000003,"5min":44.779000000000003,"15min":183.78700000000001},"last":2.5430000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.1319999999999999,"15min":3.3079999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":5.5810000000000004,"5min":11.555999999999999,"15min":186.143},"last":10.702},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.3320000000000001,"15min":3.4380000000000002},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.23000000000000001},"max":{"1min":4.016,"5min":16.640000000000001,"15min":186.97399999999999},"last":2.6539999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.2090000000000001,"15min":1.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.5259999999999998,"5min":9.7080000000000002,"15min":10.512},"last":2.4790000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.80200000000000005,"15min":0.93799999999999994},"min":{"1min":0.31,"5min":0.31,"15min":0.26200000000000001},"max":{"1min":3.875,"5min":6.0490000000000004,"15min":11.493},"last":10.586}]},{"osd":5,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.3640000000000001,"15min":2.4830000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.492,"5min":9.7439999999999998,"15min":154.30600000000001},"last":2.512},{"interface":"front","average":{"1min":1.093,"5min":0.90200000000000002,"15min":1.905},"min":{"1min":0.33800000000000002,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":11.478,"5min":11.478,"15min":154.42400000000001},"last":10.618}]},{"osd":6,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2149999999999999,"5min":5.7060000000000004,"15min":3.1709999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":136.404,"5min":170.12700000000001,"15min":170.12700000000001},"last":10.654},{"interface":"front","average":{"1min":8.1039999999999992,"5min":5.9210000000000003,"15min":3.3599999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":136.44999999999999,"5min":169.767,"15min":169.767},"last":2.6120000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.98599999999999999,"15min":2.4279999999999999},"min":{"1min":0.35199999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":164.26400000000001},"last":10.679},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.107,"15min":2.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":1.6719999999999999,"5min":6.5410000000000004,"15min":164.23099999999999},"last":2.5800000000000001}]}]},{"osd":2,"up_from":129,"seq":554050781443,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3234384,"kb_used_data":2607560,"kb_used_omap":7,"kb_used_meta":626808,"kb_avail":90510768,"statfs":{"total":95995035648,"available":92683026432,"internally_reserved":0,"allocated":2670141440,"data_stored":2668165366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":641851473},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.302,"15min":1.3919999999999999},"min":{"1min":0.19400000000000001,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":1.7470000000000001,"5min":15.667,"15min":26.356000000000002},"last":6.4169999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.4419999999999999,"15min":1.556},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.663,"5min":12.445,"15min":26.341000000000001},"last":11.928000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":4.9199999999999999,"15min":2.379},"min":{"1min":0.23599999999999999,"5min":0.19,"15min":0.188},"max":{"1min":4.9089999999999998,"5min":212.12,"15min":212.12},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.232,"5min":4.9390000000000001,"15min":2.1890000000000001},"min":{"1min":0.24299999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.4219999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":6.5300000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":2.2480000000000002,"15min":3.3039999999999998},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":4.3559999999999999,"5min":66.085999999999999,"15min":151.43799999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":2.222,"15min":3.3260000000000001},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":1.4930000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":6.4470000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.101,"15min":1.1419999999999999},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":1.4570000000000001,"5min":12.586,"15min":12.586},"last":6.383},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.1870000000000001,"15min":1.1910000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.575,"5min":9.9529999999999994,"15min":9.9529999999999994},"last":8.0850000000000009}]},{"osd":5,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.29,"15min":2.5760000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":1.6379999999999999,"5min":9.9909999999999997,"15min":173.34},"last":8.1470000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.079,"15min":2.3809999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29399999999999998},"max":{"1min":2.1949999999999998,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":3.2869999999999999,"15min":1.883},"min":{"1min":0.34200000000000003,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":2.1789999999999998,"5min":175.72499999999999,"15min":175.72499999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":3.3250000000000002,"15min":1.984},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.33000000000000002},"max":{"1min":1.5329999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":6.5049999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:06:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.6060000000000001,"15min":2.7330000000000001},"min":{"1min":0.41399999999999998,"5min":0.40799999999999997,"15min":0.32700000000000001},"max":{"1min":9.6910000000000007,"5min":12.609,"15min":177.30799999999999},"last":6.4809999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.355,"15min":2.6520000000000001},"min":{"1min":0.38900000000000001,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":1.615,"5min":10.016,"15min":177.40899999999999},"last":8.1219999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160037,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":568152,"kb_used_data":7056,"kb_used_omap":2,"kb_used_meta":561085,"kb_avail":93177000,"statfs":{"total":95995035648,"available":95413248000,"internally_reserved":0,"allocated":7225344,"data_stored":5069069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":574551093},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.5580000000000001,"15min":1.7350000000000001},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.18099999999999999},"max":{"1min":3.8570000000000002,"5min":44.963000000000001,"15min":95.507999999999996},"last":2.1829999999999998},{"interface":"front","average":{"1min":0.438,"5min":1.02,"15min":1.698},"min":{"1min":0.219,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":1.2909999999999999,"5min":9.7759999999999998,"15min":95.259},"last":0.67400000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:06:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.347,"15min":1.0580000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7619999999999996,"5min":12.035,"15min":19.786999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.661,"15min":1.292},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":8.3059999999999992,"5min":15.763999999999999,"15min":15.763999999999999},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.464,"15min":1.2290000000000001},"min":{"1min":0.29699999999999999,"5min":0.223,"15min":0.182},"max":{"1min":13.944000000000001,"5min":13.944000000000001,"15min":13.944000000000001},"last":4.3490000000000002},{"interface":"front","average":{"1min":0.70199999999999996,"5min":1.2470000000000001,"15min":1.2909999999999999},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":4.6349999999999998,"5min":12.096,"15min":19.754999999999999},"last":2.2440000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":2.645,"15min":2.1930000000000001},"min":{"1min":0.34799999999999998,"5min":0.307,"15min":0.222},"max":{"1min":8.1750000000000007,"5min":70.728999999999999,"15min":111.36},"last":1.6799999999999999},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1389999999999998,"15min":1.9019999999999999},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":4.1100000000000003,"5min":70.712000000000003,"15min":112.473},"last":2.2549999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.0740000000000001,"15min":1.5129999999999999},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":1.325,"5min":9.6359999999999992,"15min":64.528000000000006},"last":0.626},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.79,"15min":1.6220000000000001},"min":{"1min":0.42099999999999999,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":8.2490000000000006,"5min":10.617000000000001,"15min":14.054},"last":1.74}]},{"osd":6,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.857,"15min":1.6919999999999999},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.312},"max":{"1min":8.2780000000000005,"5min":10.52,"15min":19.568000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.4650000000000001,"15min":1.4750000000000001},"min":{"1min":0.39100000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":4.3920000000000003,"5min":14.356,"15min":15.573},"last":2.2280000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:06:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":2.1930000000000001,"15min":2.6099999999999999},"min":{"1min":0.40600000000000003,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":1.3480000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.341,"5min":2.8090000000000002,"15min":2.8490000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":8.218,"5min":73.364999999999995,"15min":222.11699999999999},"last":1.71}]}]},{"osd":4,"up_from":31,"seq":133143986655,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1519168,"kb_used_data":973240,"kb_used_omap":0,"kb_used_meta":545920,"kb_avail":92225984,"statfs":{"total":95995035648,"available":94439407616,"internally_reserved":0,"allocated":996597760,"data_stored":995400321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":559022080},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.2869999999999999,"15min":2.782},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.20399999999999999},"max":{"1min":0.53200000000000003,"5min":47.476999999999997,"15min":146.25299999999999},"last":1.667},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.119,"15min":2.8029999999999999},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.223},"max":{"1min":3.8959999999999999,"5min":8.5220000000000002,"15min":144.77000000000001},"last":0.748}]},{"osd":1,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":12.298,"5min":8.5619999999999994,"15min":3.3940000000000001},"min":{"1min":0.34899999999999998,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":127.411,"5min":127.411,"15min":127.411},"last":4.3920000000000003},{"interface":"front","average":{"1min":4.8810000000000002,"5min":6.7430000000000003,"15min":2.758},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":65.688999999999993,"5min":118.075,"15min":118.075},"last":1.135}]},{"osd":2,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":5.8410000000000002,"15min":3.2130000000000001},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":3.452,"5min":164.517,"15min":164.517},"last":3.669},{"interface":"front","average":{"1min":0.46999999999999997,"5min":5.665,"15min":3.274},"min":{"1min":0.35599999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":0.56999999999999995,"5min":164.46799999999999,"15min":164.46799999999999},"last":1.532}]},{"osd":3,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2170000000000001,"15min":2.1280000000000001},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.2879999999999998,"5min":8.7420000000000009,"15min":142.28399999999999},"last":3.7690000000000001},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.177,"15min":2.3029999999999999},"min":{"1min":0.35299999999999998,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":8.2899999999999991,"5min":8.2899999999999991,"15min":142.34700000000001},"last":2.9630000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.81999999999999995,"15min":1.052},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":2.2629999999999999,"5min":8.7089999999999996,"15min":72.304000000000002},"last":3.7290000000000001},{"interface":"front","average":{"1min":1.042,"5min":0.96799999999999997,"15min":1.1220000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.21099999999999999},"max":{"1min":4.282,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.34300000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.004,"15min":0.96699999999999997},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":0.70599999999999996,"5min":11.800000000000001,"15min":11.887},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.111,"15min":0.92100000000000004},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":2.3620000000000001,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":3.7000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.601,"15min":1.6040000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":4.3129999999999997,"5min":145.721,"15min":145.721},"last":3.6219999999999999},{"interface":"front","average":{"1min":1.347,"5min":2.9489999999999998,"15min":1.776},"min":{"1min":0.38500000000000001,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":4.7969999999999997,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.52400000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1089452,"kb_used_data":497252,"kb_used_omap":0,"kb_used_meta":592192,"kb_avail":92655700,"statfs":{"total":95995035648,"available":94879436800,"internally_reserved":0,"allocated":509186048,"data_stored":508004809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606404608},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.68500000000000005,"15min":1.3540000000000001},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":10.016999999999999,"5min":10.016999999999999,"15min":80.105999999999995},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.89300000000000002,"15min":1.462},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":2.919,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.222}]},{"osd":1,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":13.016999999999999,"5min":6.4710000000000001,"15min":2.512},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":233.77099999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":3.9399999999999999},{"interface":"front","average":{"1min":13.031000000000001,"5min":6.1769999999999996,"15min":2.4220000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.247},"max":{"1min":233.75200000000001,"5min":233.75200000000001,"15min":233.75200000000001},"last":4.1029999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.3980000000000001,"15min":1.569},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":8.3770000000000007,"5min":103.63800000000001,"15min":103.63800000000001},"last":1.919},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.6309999999999998,"15min":1.696},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":6.7640000000000002,"5min":103.664,"15min":103.664},"last":1.4410000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.3149999999999999,"15min":2.8359999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":8.0860000000000003,"5min":97.349999999999994,"15min":153.60300000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.2570000000000001,"15min":2.7559999999999998},"min":{"1min":0.38300000000000001,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":4.6609999999999996,"5min":97.316999999999993,"15min":153.774},"last":1.6970000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59999999999999998,"15min":0.77600000000000002},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":2.2320000000000002,"5min":3.3029999999999999,"15min":12.739000000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.88900000000000001,"15min":0.98299999999999998},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.247},"max":{"1min":3.645,"5min":8.3740000000000006,"15min":12.539},"last":0.28000000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.7150000000000001,"15min":1.1839999999999999},"min":{"1min":0.376,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":1.276,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.318},{"interface":"front","average":{"1min":0.625,"5min":1.8420000000000001,"15min":1.3220000000000001},"min":{"1min":0.32400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6699999999999999,"5min":70.266000000000005,"15min":70.266000000000005},"last":4.0880000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.143,"15min":1.2010000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":5.0119999999999996,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65800000000000003,"15min":1.054},"min":{"1min":0.38500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.5339999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.39300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":522715136,"data_stored":522715136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2915827712,"data_stored":2915827712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2651066368,"data_stored":2651062296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1835008,"data_stored":1835008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":986451968,"data_stored":986447896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":504627200,"data_stored":504627200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2667450368,"data_stored":2667446296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3469869056,"data_stored":3469869056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:07:34.194 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:07:34.279 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:34.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:34.386 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:34.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:34 smithi123 ceph-mon[193903]: from='client.46505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:34.414 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:34 smithi123 ceph-mon[193903]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 31139/177720 objects degraded (17.521%); 46638/177720 objects misplaced (26.242%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:07:34.415 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:34.483 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:34.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:34.591 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:34.626 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:34.693 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:34.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:34.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:34.804 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:34.880 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.055 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:35.069 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:07:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:07:34] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.43.0" 2023-09-23T21:07:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:35.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:35.171 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:35.200 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.227 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:35.257 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:35.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:35.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:35.288 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:35.368 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:35.482 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.689 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:35.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:35.761 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:35.798 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:35.907 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:35.993 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:07:36.050 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:07:36.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:07:36.143 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:07:36.144 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:36.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:36.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:36.253 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:36.274 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:36 smithi121 ceph-mon[211526]: pgmap v1290: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 31139/176238 objects degraded (17.669%); 46149/176238 objects misplaced (26.186%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:07:36.275 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3174064384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:36.275 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:36 smithi121 ceph-mon[219709]: pgmap v1290: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 31139/176238 objects degraded (17.669%); 46149/176238 objects misplaced (26.186%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:07:36.275 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3174064384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:36.278 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:36.320 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:36.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:36.326 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:07:36.326 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.327 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:07:36.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:36 smithi123 ceph-mon[193903]: pgmap v1290: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 31139/176238 objects degraded (17.669%); 46149/176238 objects misplaced (26.186%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:07:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3174064384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:36.498 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:37.107 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31139/176238 objects degraded (17.669%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:37.107 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:37 smithi121 ceph-mon[219709]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 30473/170055 objects degraded (17.919%); 44628/170055 objects misplaced (26.243%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:07:37.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31139/176238 objects degraded (17.669%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:37 smithi121 ceph-mon[211526]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 30473/170055 objects degraded (17.919%); 44628/170055 objects misplaced (26.243%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:07:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31139/176238 objects degraded (17.669%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:37 smithi123 ceph-mon[193903]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 30473/170055 objects degraded (17.919%); 44628/170055 objects misplaced (26.243%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:07:39.062 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:39.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:39.173 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:39.202 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:39.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:39 smithi121 ceph-mon[211526]: pgmap v1292: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 30473/170055 objects degraded (17.919%); 44628/170055 objects misplaced (26.243%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:07:39.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:39 smithi121 ceph-mon[219709]: pgmap v1292: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 30473/170055 objects degraded (17.919%); 44628/170055 objects misplaced (26.243%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:07:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:39 smithi123 ceph-mon[193903]: pgmap v1292: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 30473/170055 objects degraded (17.919%); 44628/170055 objects misplaced (26.243%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T21:07:39.790 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:39.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:39.902 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:39.936 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:40.011 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:40.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:40.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:40.086 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:07:40.121 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:40.151 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:40.211 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:40.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:40.317 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:40.503 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:40.518 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:07:40.518 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30473/170055 objects degraded (17.919%), 7 pgs degraded, 6 pgs undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1717,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":409,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+remapped+backfill_wait","count":6},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":56685,"data_bytes":3713865656,"bytes_used":13909872640,"bytes_avail":658055376896,"bytes_total":671965249536,"degraded_objects":30473,"degraded_total":170055,"degraded_ratio":0.17919496633442122,"misplaced_objects":44628,"misplaced_total":170055,"misplaced_ratio":0.26243274234806385,"recovering_objects_per_sec":116,"recovering_bytes_per_sec":7606375,"recovering_keys_per_sec":0,"num_objects_recovered":1162,"num_bytes_recovered":76152832,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":508},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T21:06:55.027429+0000","services":{}},"progress_events":{"2639f328-dcb0-4a01-b6c7-21332e7147ac":{"message":"Global Recovery Event (4m)\n [=======================.....] (remaining: 44s)","progress":0.84269660711288452,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:07:40.805 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:07:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:07:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:91726 10.2 10.67783df2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T20:57:43.511418+0000", 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 597.28572507800004, 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73011041099999996, 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 91726 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511418+0000", 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511418+0000", 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511416+0000", 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511422+0000", 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9599999999999997e-06 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511432+0000", 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8719999999999997e-06 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511441+0000", 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2909999999999996e-06 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511493+0000", 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1400000000000003e-05 2023-09-23T21:07:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:43.511502+0000", 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.38e-06 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241515+0000", 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73001329000000004 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T20:57:44.241528+0000", 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2694000000000001e-05 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140910 10.12 10:4837830a:::benchmark_data_smithi121_358375_object56165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.102455+0000", 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.69468767900003, 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1947788660000001, 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.811 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140910 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102455+0000", 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.812 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102455+0000", 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102453+0000", 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102463+0000", 2023-09-23T21:07:40.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8449999999999998e-06 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102481+0000", 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8437999999999999e-05 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102492+0000", 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.047e-05 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102549+0000", 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7170999999999999e-05 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.102559+0000", 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0237999999999999e-05 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.815 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424192+0000", 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.321633053 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424199+0000", 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0870000000000001e-06 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.036174+0000", 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6119743660000001 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.036312+0000", 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013814200000000001 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.036596+0000", 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00028440600000000001 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.293913+0000", 2023-09-23T21:07:40.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25731645600000003 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.293966+0000", 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3164999999999997e-05 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.296611+0000", 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026455139999999999 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.296678+0000", 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6105999999999994e-05 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297234+0000", 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00055633400000000002 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.819 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140922 10.2 10:4783e976:::benchmark_data_smithi121_358375_object56177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.206018+0000", 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.59112427299999, 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0967715469999999, 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140922 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.820 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206018+0000", 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206018+0000", 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206017+0000", 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206024+0000", 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5529999999999998e-06 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206037+0000", 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3502e-05 2023-09-23T21:07:40.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206046+0000", 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8319999999999995e-06 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206112+0000", 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6548e-05 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.206126+0000", 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3871e-05 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424107+0000", 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.21798082399999999 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424119+0000", 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2058e-05 2023-09-23T21:07:40.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.285655+0000", 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8615357120000002 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.287673+0000", 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020179690000000001 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.290434+0000", 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027612829999999998 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.296758+0000", 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063240839999999998 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297811+0000", 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0010529599999999999 2023-09-23T21:07:40.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.298391+0000", 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057977500000000004 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.299305+0000", 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00091387200000000003 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.302790+0000", 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00348504 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140925 10.2 10:46517284:::benchmark_data_smithi121_358375_object56180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.213680+0000", 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.58346252600001, 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1225043499999998, 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.828 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140925 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213680+0000", 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.829 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213680+0000", 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213678+0000", 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213688+0000", 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6809999999999993e-06 2023-09-23T21:07:40.830 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213703+0000", 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5082000000000001e-05 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213712+0000", 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1060000000000006e-06 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.833 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213775+0000", 2023-09-23T21:07:40.833 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2755000000000005e-05 2023-09-23T21:07:40.833 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.833 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.833 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.833 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.213786+0000", 2023-09-23T21:07:40.833 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1973999999999999e-05 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424140+0000", 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.210353283 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424147+0000", 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1380000000000001e-06 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.303011+0000", 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8788636099999998 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.305262+0000", 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022513820000000001 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.308610+0000", 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033481280000000001 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.332116+0000", 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.023506422999999999 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.332590+0000", 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00047317300000000002 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.836 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.332729+0000", 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139398 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.333445+0000", 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00071641100000000002 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336184+0000", 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027390510000000002 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140934 10.2 10:4673e96c:::benchmark_data_smithi121_358375_object56189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e321)", 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:36.283977+0000", 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.51316607699999, 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0573845450000001, 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.838 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140934 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283977+0000", 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.839 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283977+0000", 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283975+0000", 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283981+0000", 2023-09-23T21:07:40.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.913e-06 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283992+0000", 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0363999999999999e-05 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.283997+0000", 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7910000000000001e-06 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.284040+0000", 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3022000000000002e-05 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.284048+0000", 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2520000000000002e-06 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424166+0000", 2023-09-23T21:07:40.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14011805399999999 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:36.424173+0000", 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8689999999999998e-06 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.312215+0000", 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.888042714 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.314573+0000", 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023574109999999998 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.318102+0000", 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003529435 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336333+0000", 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018230745 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336933+0000", 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00060003200000000002 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.337441+0000", 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050855500000000005 2023-09-23T21:07:40.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.338135+0000", 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00069391699999999995 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.341361+0000", 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003225819 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.846 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140943 10.12 10:492afa11:::benchmark_data_smithi121_358375_object56198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.514244+0000", 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 363.28289913200001, 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.783543908, 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.847 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140943 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514244+0000", 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514244+0000", 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514242+0000", 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.514252+0000", 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0256e-05 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.548140+0000", 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.033888505999999999 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.548154+0000", 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3625000000000001e-05 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.037993+0000", 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.48983858 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.038159+0000", 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016624700000000001 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.038384+0000", 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00022518899999999999 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.293920+0000", 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25553648400000001 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297302+0000", 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003381472 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297361+0000", 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8845000000000003e-05 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297420+0000", 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9021999999999998e-05 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.297787+0000", 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00036763300000000002 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140955 10.2 10:435f7b81:::benchmark_data_smithi121_358375_object56210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.557460+0000", 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 363.239682227, 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.788443177, 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140955 2023-09-23T21:07:40.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557460+0000", 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557460+0000", 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557458+0000", 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557467+0000", 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4740000000000005e-06 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557479+0000", 2023-09-23T21:07:40.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1987999999999999e-05 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.557489+0000", 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5889999999999992e-06 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.322214+0000", 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.764725766 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.324762+0000", 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025475789999999999 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.328397+0000", 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036347319999999999 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.336482+0000", 2023-09-23T21:07:40.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0080858570000000001 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.341956+0000", 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054735859999999999 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.342423+0000", 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00046714599999999999 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.343086+0000", 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00066331699999999996 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.345904+0000", 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028171120000000001 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.860 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140956 10.12 10:4f74787e:::benchmark_data_smithi121_358375_object56211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.562575+0000", 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 363.23456774099998, 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.793809465, 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140956 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.861 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562575+0000", 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562575+0000", 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.862 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562573+0000", 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562582+0000", 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6419999999999996e-06 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562596+0000", 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3827e-05 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.562606+0000", 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0251000000000001e-05 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.345960+0000", 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7833538820000001 2023-09-23T21:07:40.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.346160+0000", 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020042000000000001 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.346599+0000", 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00043851799999999999 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.865 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.350847+0000", 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042478890000000004 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.350857+0000", 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0258000000000001e-05 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.355774+0000", 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049169360000000002 2023-09-23T21:07:40.866 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.355811+0000", 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7187000000000002e-05 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356384+0000", 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057330499999999997 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:140957 10.2 10:42e2eb9f:::benchmark_data_smithi121_358375_object56212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.563491+0000", 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 363.23365209500002, 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8074938659999999, 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 140957 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.868 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563491+0000", 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563491+0000", 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563489+0000", 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563496+0000", 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2979999999999997e-06 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563508+0000", 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1928e-05 2023-09-23T21:07:40.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.563516+0000", 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2220000000000002e-06 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.346650+0000", 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.783134309 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.349101+0000", 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002450972 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.352487+0000", 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033854990000000001 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366032+0000", 2023-09-23T21:07:40.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013545171 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366538+0000", 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050632300000000004 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.367103+0000", 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00056445900000000001 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.367988+0000", 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00088564700000000004 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.370984+0000", 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029961340000000001 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141012 10.12 10:4b4d14f7:::benchmark_data_smithi121_358375_object56267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.723211+0000", 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 363.07393215799999, 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6431093799999998, 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141012 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723211+0000", 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723211+0000", 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723208+0000", 2023-09-23T21:07:40.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723219+0000", 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0429e-05 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723233+0000", 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3854e-05 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.723242+0000", 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3659999999999998e-06 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.348512+0000", 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.625270223 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.348662+0000", 2023-09-23T21:07:40.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014937300000000001 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.349009+0000", 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00034730699999999999 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356434+0000", 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074248750000000001 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.365465+0000", 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0090311019999999992 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.365476+0000", 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1039999999999999e-05 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.365544+0000", 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7500000000000001e-05 2023-09-23T21:07:40.880 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366320+0000", 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00077635300000000003 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141018 10.2 10:4724d2b5:::benchmark_data_smithi121_358375_object56273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.732723+0000", 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 363.06441942700002, 2023-09-23T21:07:40.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6436562540000001, 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141018 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732723+0000", 2023-09-23T21:07:40.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732723+0000", 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732721+0000", 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732729+0000", 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2729999999999997e-06 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732737+0000", 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8610000000000004e-06 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.732744+0000", 2023-09-23T21:07:40.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2650000000000002e-06 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.355973+0000", 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.623229349 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.358295+0000", 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002321427 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.361636+0000", 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003341157 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371580+0000", 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.009944652 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371760+0000", 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000179514 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.372291+0000", 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00053071699999999999 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.373187+0000", 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089667000000000002 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376379+0000", 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031922399999999998 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141049 10.12 10:486305eb:::benchmark_data_smithi121_358375_object56304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.799440+0000", 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 362.99770314099999, 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5723892450000001, 2023-09-23T21:07:40.888 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141049 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799440+0000", 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799440+0000", 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799438+0000", 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799448+0000", 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.077e-05 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799466+0000", 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7416999999999999e-05 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.799477+0000", 2023-09-23T21:07:40.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.131e-05 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356459+0000", 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.556982026 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.356634+0000", 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017506999999999999 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.357018+0000", 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00038430700000000002 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366425+0000", 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094066500000000008 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.366550+0000", 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012518800000000001 2023-09-23T21:07:40.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371053+0000", 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004503155 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371122+0000", 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8137000000000002e-05 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371829+0000", 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00070721499999999997 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141098 10.12 10:4b556bc3:::benchmark_data_smithi121_358375_object56353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:37.986992+0000", 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 362.810150163, 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.42552641, 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.895 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141098 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986992+0000", 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986992+0000", 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986991+0000", 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.986997+0000", 2023-09-23T21:07:40.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3400000000000003e-06 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.987007+0000", 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6770000000000007e-06 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:37.987013+0000", 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.874e-06 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.371859+0000", 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3848462750000001 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.372023+0000", 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016354499999999999 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.372386+0000", 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00036363600000000003 2023-09-23T21:07:40.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378916+0000", 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065301129999999997 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386328+0000", 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.007411829 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.406124+0000", 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.019796022 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.406173+0000", 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8587000000000001e-05 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.412519+0000", 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063460089999999997 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.901 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141102 10.12 10:4b3897e2:::benchmark_data_smithi121_358375_object56357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.001005+0000", 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 362.79613733899998, 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4121252740000001, 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.902 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141102 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001005+0000", 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001005+0000", 2023-09-23T21:07:40.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001004+0000", 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001012+0000", 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3110000000000006e-06 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001026+0000", 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4273e-05 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.001036+0000", 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8900000000000002e-06 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.373843+0000", 2023-09-23T21:07:40.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3728068849999999 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.373969+0000", 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012648600000000001 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.374219+0000", 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024986799999999999 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378935+0000", 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047160769999999999 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386385+0000", 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074491829999999998 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.412569+0000", 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026184926000000001 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.412628+0000", 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9061000000000003e-05 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413131+0000", 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050210599999999999 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.908 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141120 10.12 10:4b149f10:::benchmark_data_smithi121_358375_object56375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.063325+0000", 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 362.733817612, 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.350347706, 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.909 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141120 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063325+0000", 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063325+0000", 2023-09-23T21:07:40.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063323+0000", 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063332+0000", 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7700000000000007e-06 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063351+0000", 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9137999999999999e-05 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.063361+0000", 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9329999999999996e-06 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376082+0000", 2023-09-23T21:07:40.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3127212770000001 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376247+0000", 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016494200000000001 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376632+0000", 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00038421499999999998 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378953+0000", 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023215639999999999 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386392+0000", 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074385110000000001 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413179+0000", 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.026787261 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413234+0000", 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4830999999999997e-05 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.413673+0000", 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00043899400000000001 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.915 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141122 10.2 10:400b253d:::benchmark_data_smithi121_358375_object56377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.070895+0000", 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 362.72624806900001, 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3548394610000001, 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.916 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141122 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070895+0000", 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070895+0000", 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070893+0000", 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070901+0000", 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9789999999999993e-06 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070916+0000", 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5024999999999999e-05 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.070926+0000", 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.91e-06 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.376577+0000", 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3056514889999999 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378571+0000", 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019931689999999999 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.382394+0000", 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0038230619999999999 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386569+0000", 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041757859999999999 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.396732+0000", 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010162642 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.421647+0000", 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.024914828999999999 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.422506+0000", 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00085867199999999999 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.425734+0000", 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032284150000000001 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:141127 10.12 10:49d57e63:::benchmark_data_smithi121_358375_object56382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:38.103373+0000", 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 362.69377013600001, 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3186484979999999, 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 141127 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.923 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103373+0000", 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103373+0000", 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103371+0000", 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103379+0000", 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7620000000000001e-06 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103396+0000", 2023-09-23T21:07:40.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7099999999999999e-05 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:38.103406+0000", 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5240000000000006e-06 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.378966+0000", 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.27556041 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.379096+0000", 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012971799999999999 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.379492+0000", 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039616199999999998 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386370+0000", 2023-09-23T21:07:40.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068777669999999999 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.386398+0000", 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8461000000000001e-05 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.421131+0000", 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.034733030999999998 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.421194+0000", 2023-09-23T21:07:40.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2656000000000005e-05 2023-09-23T21:07:40.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:39.422021+0000", 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00082691700000000004 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88124 11.c 11:386a4abc:::benchmark_data_smithi121_477989_object88123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:07:40.930 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 192.06207977700001, 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6441045829999998, 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88124 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735062+0000", 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735084+0000", 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.234e-05 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735099+0000", 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.508e-05 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735108+0000", 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3689999999999996e-06 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735157+0000", 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8273999999999999e-05 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735257+0000", 2023-09-23T21:07:40.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000100432 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735419+0000", 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000162158 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.741925+0000", 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00650594 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.744875+0000", 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002949981 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757406+0000", 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012530777999999999 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757417+0000", 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0699e-05 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770169+0000", 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0127527590000001 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770180+0000", 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0428000000000001e-05 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.379055+0000", 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6088747780000001 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.379167+0000", 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112796 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88126 11.c 11:3f012f98:::benchmark_data_smithi121_477989_object88125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 192.06041272799999, 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6457891419999999, 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88126 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.939 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736728+0000", 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736758+0000", 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0073000000000002e-05 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736775+0000", 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6716000000000001e-05 2023-09-23T21:07:40.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736784+0000", 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1719999999999996e-06 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738664+0000", 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00187952 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738803+0000", 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013961799999999999 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738967+0000", 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016343 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.744883+0000", 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0059158769999999999 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757481+0000", 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012598689 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757491+0000", 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.098e-06 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770209+0000", 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.012718877 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770217+0000", 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1679999999999998e-06 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382403+0000", 2023-09-23T21:07:40.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6121860369999998 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382519+0000", 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116488 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.946 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88142 11.c 11:3d9b819d:::benchmark_data_smithi121_477989_object88141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 192.040811572, 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6474134190000003, 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88142 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:07:40.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756330+0000", 2023-09-23T21:07:40.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756356+0000", 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6440000000000001e-05 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756382+0000", 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5948999999999998e-05 2023-09-23T21:07:40.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756392+0000", 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5999999999999996e-06 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757514+0000", 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001122088 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757521+0000", 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8099999999999998e-06 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770236+0000", 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0127141230000001 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770241+0000", 2023-09-23T21:07:40.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9309999999999996e-06 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382568+0000", 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.612326293 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382717+0000", 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014953000000000001 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.383252+0000", 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00053481999999999998 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.388314+0000", 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050622560000000002 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.396573+0000", 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0082585120000000008 2023-09-23T21:07:40.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.399933+0000", 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033602250000000001 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.399996+0000", 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2965000000000004e-05 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:07:40.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.403745+0000", 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037484340000000001 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:07:40.955 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:07:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:40.956 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:40.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:40.957 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/470693528' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:07:40.957 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/470693528' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:07:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/470693528' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:07:41.058 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:41.158 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:41.751 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:07:41.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[211526]: pgmap v1293: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 29865/162354 objects degraded (18.395%); 42602/162354 objects misplaced (26.240%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:07:41.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/905495655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 30473/170055 objects degraded (17.919%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:41.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[219709]: pgmap v1293: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 29865/162354 objects degraded (18.395%); 42602/162354 objects misplaced (26.240%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:07:41.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/905495655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 30473/170055 objects degraded (17.919%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:41 smithi123 ceph-mon[193903]: pgmap v1293: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 29865/162354 objects degraded (18.395%); 42602/162354 objects misplaced (26.240%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:07:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/905495655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:07:41.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 30473/170055 objects degraded (17.919%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:42.501 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:42.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:42.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:42.522 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:07:42.522 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:07:42.610 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:42.707 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:42.732 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:07:42.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:42.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[219709]: failed to encode map e410 with expected crc 2023-09-23T21:07:42.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[219709]: failed to encode map e410 with expected crc 2023-09-23T21:07:42.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:42.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[219709]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T21:07:42.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:42.817 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:42.818 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:42.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:42.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[211526]: failed to encode map e410 with expected crc 2023-09-23T21:07:42.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[211526]: failed to encode map e410 with expected crc 2023-09-23T21:07:42.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:42.818 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[211526]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T21:07:42.819 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:42.819 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:42.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:42.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:42.921 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:42.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:42 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:42.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:42 smithi123 ceph-mon[193903]: failed to encode map e410 with expected crc 2023-09-23T21:07:42.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:42 smithi123 ceph-mon[193903]: failed to encode map e410 with expected crc 2023-09-23T21:07:42.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:42 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:07:42.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:42 smithi123 ceph-mon[193903]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T21:07:42.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:07:42.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:42.947 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:42.993 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:42.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:43.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:43.100 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:43.131 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:43.182 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:43.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:43.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:43.296 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:43.340 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:43.395 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:43.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:43.507 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:43.542 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:43.599 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:43.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:43.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:43.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:43 smithi121 ceph-mon[219709]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 29865/162354 objects degraded (18.395%); 42602/162354 objects misplaced (26.240%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T21:07:43.646 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:43.673 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:43.704 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:43.732 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:43.737 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:43.795 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:43.799 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:43 smithi123 ceph-mon[193903]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 29865/162354 objects degraded (18.395%); 42602/162354 objects misplaced (26.240%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T21:07:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:43.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:43.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:43 smithi121 ceph-mon[211526]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 511 op/s; 29865/162354 objects degraded (18.395%); 42602/162354 objects misplaced (26.240%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T21:07:43.969 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:44.278 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:44.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:44.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:44.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:44.480 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:45.134 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:45.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:45.147 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:07:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:07:44] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2023-09-23T21:07:45.248 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:45.333 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:45.391 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:07:45.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:45 smithi121 ceph-mon[219709]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 29865/160941 objects degraded (18.556%); 42127/160941 objects misplaced (26.175%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T21:07:45.467 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:45 smithi121 ceph-mon[211526]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 29865/160941 objects degraded (18.556%); 42127/160941 objects misplaced (26.175%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T21:07:45.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:45 smithi123 ceph-mon[193903]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 29865/160941 objects degraded (18.556%); 42127/160941 objects misplaced (26.175%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T21:07:45.509 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:45.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:45.618 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:45.664 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:45.731 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:45.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:45.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:45.844 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:45.870 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:45.911 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:45.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:45.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:46.018 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:46.043 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:46.193 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:46.219 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.260 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:46.368 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:46.408 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.471 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:46.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:46.577 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:46.639 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:46.719 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:46.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 29865/160941 objects degraded (18.556%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:46.793 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 29865/160941 objects degraded (18.556%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:46.795 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 29865/160941 objects degraded (18.556%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:46.827 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:46.939 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:47.095 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:47.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:47.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:47.279 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:47.548 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:47.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:47.663 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:47.711 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:47.774 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:47.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:47.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:47.782 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:47.882 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:07:47.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:47 smithi121 ceph-mon[211526]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 29221/154815 objects degraded (18.875%); 40490/154815 objects misplaced (26.154%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:07:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:47 smithi121 ceph-mon[219709]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 29221/154815 objects degraded (18.875%); 40490/154815 objects misplaced (26.154%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:07:47.917 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:47 smithi123 ceph-mon[193903]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 29221/154815 objects degraded (18.875%); 40490/154815 objects misplaced (26.154%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:07:47.920 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:47.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:07:47.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:47.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:07:47.985 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:07:48.046 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:07:48.084 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:48.093 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:48.093 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:48.119 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:48.130 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:07:48.169 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:48.181 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:07:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:48.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:48.245 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:07:48.274 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:07:48.277 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:07:48.278 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:07:48.282 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:48.320 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:07:48.365 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:07:48.399 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:48.400 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.400 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.400 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.400 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.401 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.401 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.401 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.401 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:07:48.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/168119708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:48.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/168119708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:48.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/168119708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:07:49.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:49 smithi121 ceph-mon[211526]: from='client.46526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:49.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:49 smithi121 ceph-mon[211526]: pgmap v1298: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 29221/154815 objects degraded (18.875%); 40490/154815 objects misplaced (26.154%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:07:49.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:49 smithi121 ceph-mon[219709]: from='client.46526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:49.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:49 smithi121 ceph-mon[219709]: pgmap v1298: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 29221/154815 objects degraded (18.875%); 40490/154815 objects misplaced (26.154%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:07:49.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:49 smithi123 ceph-mon[193903]: from='client.46526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:07:49.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:49 smithi123 ceph-mon[193903]: pgmap v1298: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 29221/154815 objects degraded (18.875%); 40490/154815 objects misplaced (26.154%); 8.6 MiB/s, 138 objects/s recovering 2023-09-23T21:07:50.426 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:50.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:50.535 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:50.619 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:51.336 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:07:51.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:51 smithi121 ceph-mon[211526]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 28639/147690 objects degraded (19.391%); 38566/147690 objects misplaced (26.113%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T21:07:51.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:51 smithi121 ceph-mon[219709]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 28639/147690 objects degraded (19.391%); 38566/147690 objects misplaced (26.113%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T21:07:51.421 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:51.422 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1297,"stamp":"2023-09-23T21:07:47.055814+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3380942776,"num_objects":51605,"num_object_clones":0,"num_object_copies":154815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29221,"num_objects_misplaced":40490,"num_objects_unfound":0,"num_objects_dirty":51605,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":125093,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29235,"num_bytes_recovered":1911266572,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78208,"ondisk_log_size":78208,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12909776,"kb_used_data":8900632,"kb_used_omap":21,"kb_used_meta":4008746,"kb_avail":643306288,"statfs":{"total":671965249536,"available":658745638912,"internally_reserved":0,"allocated":9114247168,"data_stored":9102538715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4104956387},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,8,4,3,5,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-332922880,"num_objects":-5080,"num_object_clones":0,"num_object_copies":-15240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1252,"num_objects_misplaced":-4138,"num_objects_unfound":0,"num_objects_dirty":-5080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5080,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1381,"num_bytes_recovered":90505216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004588"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":181,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:40.536910+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:07:40.536910+0000","last_peered":"2023-09-23T21:07:40.536910+0000","last_clean":"2023-09-23T21:07:40.536910+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:07:40.536910+0000","last_undegraded":"2023-09-23T21:07:40.536910+0000","last_fullsized":"2023-09-23T21:07:40.536910+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":50,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062051+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:07:29.062051+0000","last_peered":"2023-09-23T21:07:29.062051+0000","last_clean":"2023-09-23T21:07:29.062051+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:07:29.062051+0000","last_undegraded":"2023-09-23T21:07:29.062051+0000","last_fullsized":"2023-09-23T21:07:29.062051+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":323,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.072020+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:07:29.072020+0000","last_peered":"2023-09-23T21:07:29.072020+0000","last_clean":"2023-09-23T21:07:29.072020+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:07:29.072020+0000","last_undegraded":"2023-09-23T21:07:29.072020+0000","last_fullsized":"2023-09-23T21:07:29.072020+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":47,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068556+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:07:29.068556+0000","last_peered":"2023-09-23T21:07:29.068556+0000","last_clean":"2023-09-23T21:07:29.068556+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:07:29.068556+0000","last_undegraded":"2023-09-23T21:07:29.068556+0000","last_fullsized":"2023-09-23T21:07:29.068556+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":68,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.061816+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:07:29.061816+0000","last_peered":"2023-09-23T21:07:29.061816+0000","last_clean":"2023-09-23T21:07:29.061816+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:07:29.061816+0000","last_undegraded":"2023-09-23T21:07:29.061816+0000","last_fullsized":"2023-09-23T21:07:29.061816+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":276,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.056569+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:07:29.056569+0000","last_peered":"2023-09-23T21:07:29.056569+0000","last_clean":"2023-09-23T21:07:29.056569+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:07:29.056569+0000","last_undegraded":"2023-09-23T21:07:29.056569+0000","last_fullsized":"2023-09-23T21:07:29.056569+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":47,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068388+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:07:29.068388+0000","last_peered":"2023-09-23T21:07:29.068388+0000","last_clean":"2023-09-23T21:07:29.068388+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:07:29.068388+0000","last_undegraded":"2023-09-23T21:07:29.068388+0000","last_fullsized":"2023-09-23T21:07:29.068388+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":50,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.061770+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:07:29.061770+0000","last_peered":"2023-09-23T21:07:29.061770+0000","last_clean":"2023-09-23T21:07:29.061770+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:07:29.061770+0000","last_undegraded":"2023-09-23T21:07:29.061770+0000","last_fullsized":"2023-09-23T21:07:29.061770+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":33,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.053178+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:07:29.053178+0000","last_peered":"2023-09-23T21:07:29.053178+0000","last_clean":"2023-09-23T21:07:29.053178+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:07:29.053178+0000","last_undegraded":"2023-09-23T21:07:29.053178+0000","last_fullsized":"2023-09-23T21:07:29.053178+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062990+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:07:29.062990+0000","last_peered":"2023-09-23T21:07:29.062990+0000","last_clean":"2023-09-23T21:07:29.062990+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:07:29.062990+0000","last_undegraded":"2023-09-23T21:07:29.062990+0000","last_fullsized":"2023-09-23T21:07:29.062990+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":66,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062537+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:07:29.062537+0000","last_peered":"2023-09-23T21:07:29.062537+0000","last_clean":"2023-09-23T21:07:29.062537+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:07:29.062537+0000","last_undegraded":"2023-09-23T21:07:29.062537+0000","last_fullsized":"2023-09-23T21:07:29.062537+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:35.513328+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:07:35.513328+0000","last_peered":"2023-09-23T21:07:35.513328+0000","last_clean":"2023-09-23T21:07:35.513328+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:07:35.513328+0000","last_undegraded":"2023-09-23T21:07:35.513328+0000","last_fullsized":"2023-09-23T21:07:35.513328+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":356,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.063044+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:07:29.063044+0000","last_peered":"2023-09-23T21:07:29.063044+0000","last_clean":"2023-09-23T21:07:29.063044+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:07:29.063044+0000","last_undegraded":"2023-09-23T21:07:29.063044+0000","last_fullsized":"2023-09-23T21:07:29.063044+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":328,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.072090+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:07:29.072090+0000","last_peered":"2023-09-23T21:07:29.072090+0000","last_clean":"2023-09-23T21:07:29.072090+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:07:29.072090+0000","last_undegraded":"2023-09-23T21:07:29.072090+0000","last_fullsized":"2023-09-23T21:07:29.072090+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":100,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.066075+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:07:29.066075+0000","last_peered":"2023-09-23T21:07:29.066075+0000","last_clean":"2023-09-23T21:07:29.066075+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:07:29.066075+0000","last_undegraded":"2023-09-23T21:07:29.066075+0000","last_fullsized":"2023-09-23T21:07:29.066075+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":59,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062385+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:07:29.062385+0000","last_peered":"2023-09-23T21:07:29.062385+0000","last_clean":"2023-09-23T21:07:29.062385+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:07:29.062385+0000","last_undegraded":"2023-09-23T21:07:29.062385+0000","last_fullsized":"2023-09-23T21:07:29.062385+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062387+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:07:29.062387+0000","last_peered":"2023-09-23T21:07:29.062387+0000","last_clean":"2023-09-23T21:07:29.062387+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:07:29.062387+0000","last_undegraded":"2023-09-23T21:07:29.062387+0000","last_fullsized":"2023-09-23T21:07:29.062387+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062151+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:07:29.062151+0000","last_peered":"2023-09-23T21:07:29.062151+0000","last_clean":"2023-09-23T21:07:29.062151+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:07:29.062151+0000","last_undegraded":"2023-09-23T21:07:29.062151+0000","last_fullsized":"2023-09-23T21:07:29.062151+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":210,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.072055+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:07:29.072055+0000","last_peered":"2023-09-23T21:07:29.072055+0000","last_clean":"2023-09-23T21:07:29.072055+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:07:29.072055+0000","last_undegraded":"2023-09-23T21:07:29.072055+0000","last_fullsized":"2023-09-23T21:07:29.072055+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":47,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068592+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:07:29.068592+0000","last_peered":"2023-09-23T21:07:29.068592+0000","last_clean":"2023-09-23T21:07:29.068592+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:07:29.068592+0000","last_undegraded":"2023-09-23T21:07:29.068592+0000","last_fullsized":"2023-09-23T21:07:29.068592+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":406,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.055314+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:07:29.055314+0000","last_peered":"2023-09-23T21:07:29.055314+0000","last_clean":"2023-09-23T21:07:29.055314+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:07:29.055314+0000","last_undegraded":"2023-09-23T21:07:29.055314+0000","last_fullsized":"2023-09-23T21:07:29.055314+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":50,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.065622+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:07:29.065622+0000","last_peered":"2023-09-23T21:07:29.065622+0000","last_clean":"2023-09-23T21:07:29.065622+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:07:29.065622+0000","last_undegraded":"2023-09-23T21:07:29.065622+0000","last_fullsized":"2023-09-23T21:07:29.065622+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"410'7999","reported_seq":9366,"reported_epoch":410,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:45.504755+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:07:45.504755+0000","last_peered":"2023-09-23T21:07:45.504755+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:07:45.504755+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"356'3228","ondisk_log_start":"356'3228","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4771,"log_dups_size":0,"ondisk_log_size":4771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211746816,"num_objects":3231,"num_object_clones":0,"num_object_copies":9693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7999,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-2384},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":3230}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":114,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.073986+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:07:29.073986+0000","last_peered":"2023-09-23T21:07:29.073986+0000","last_clean":"2023-09-23T21:07:29.073986+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:07:29.073986+0000","last_undegraded":"2023-09-23T21:07:29.073986+0000","last_fullsized":"2023-09-23T21:07:29.073986+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":33,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.052700+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:07:29.052700+0000","last_peered":"2023-09-23T21:07:29.052700+0000","last_clean":"2023-09-23T21:07:29.052700+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:07:29.052700+0000","last_undegraded":"2023-09-23T21:07:29.052700+0000","last_fullsized":"2023-09-23T21:07:29.052700+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"410'7631","reported_seq":7711,"reported_epoch":410,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:45.482317+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:07:45.482317+0000","last_peered":"2023-09-23T21:07:45.482317+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:07:45.482317+0000","last_undegraded":"2023-09-23T21:07:45.482317+0000","last_fullsized":"2023-09-23T21:07:45.482317+0000","mapping_epoch":375,"log_start":"353'2912","ondisk_log_start":"353'2912","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207552512,"num_objects":3167,"num_object_clones":0,"num_object_copies":9501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3167,"num_objects_unfound":0,"num_objects_dirty":3167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7631,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":3167}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":108,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.076561+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:07:29.076561+0000","last_peered":"2023-09-23T21:07:29.076561+0000","last_clean":"2023-09-23T21:07:29.076561+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:07:29.076561+0000","last_undegraded":"2023-09-23T21:07:29.076561+0000","last_fullsized":"2023-09-23T21:07:29.076561+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":370,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.071952+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:07:29.071952+0000","last_peered":"2023-09-23T21:07:29.071952+0000","last_clean":"2023-09-23T21:07:29.071952+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:07:29.071952+0000","last_undegraded":"2023-09-23T21:07:29.071952+0000","last_fullsized":"2023-09-23T21:07:29.071952+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"410'7750","reported_seq":16834,"reported_epoch":410,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:45.728379+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:07:45.728379+0000","last_peered":"2023-09-23T21:07:45.728379+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:07:45.728379+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"358'3611","ondisk_log_start":"358'3611","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4139,"log_dups_size":0,"ondisk_log_size":4139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208928768,"num_objects":3188,"num_object_clones":0,"num_object_copies":9564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7750,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3354,"num_bytes_recovered":219807744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":802},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":37,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.053605+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:07:29.053605+0000","last_peered":"2023-09-23T21:07:29.053605+0000","last_clean":"2023-09-23T21:07:29.053605+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:07:29.053605+0000","last_undegraded":"2023-09-23T21:07:29.053605+0000","last_fullsized":"2023-09-23T21:07:29.053605+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":298,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.060355+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:07:29.060355+0000","last_peered":"2023-09-23T21:07:29.060355+0000","last_clean":"2023-09-23T21:07:29.060355+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:07:29.060355+0000","last_undegraded":"2023-09-23T21:07:29.060355+0000","last_fullsized":"2023-09-23T21:07:29.060355+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"410'7845","reported_seq":8472,"reported_epoch":410,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:45.725028+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:07:45.725028+0000","last_peered":"2023-09-23T21:07:45.725028+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:07:45.725028+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"359'3768","ondisk_log_start":"359'3768","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4077,"log_dups_size":0,"ondisk_log_size":4077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210042880,"num_objects":3205,"num_object_clones":0,"num_object_copies":9615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7845,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-2048},{"shards":"2,6,7","objects":3202}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":50,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062463+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:07:29.062463+0000","last_peered":"2023-09-23T21:07:29.062463+0000","last_clean":"2023-09-23T21:07:29.062463+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:07:29.062463+0000","last_undegraded":"2023-09-23T21:07:29.062463+0000","last_fullsized":"2023-09-23T21:07:29.062463+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":47,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068373+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:07:29.068373+0000","last_peered":"2023-09-23T21:07:29.068373+0000","last_clean":"2023-09-23T21:07:29.068373+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:07:29.068373+0000","last_undegraded":"2023-09-23T21:07:29.068373+0000","last_fullsized":"2023-09-23T21:07:29.068373+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"410'7993","reported_seq":19492,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:45.665353+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:07:45.665353+0000","last_peered":"2023-09-23T21:07:45.665353+0000","last_clean":"2023-09-23T21:07:45.665353+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:07:45.665353+0000","last_undegraded":"2023-09-23T21:07:45.665353+0000","last_fullsized":"2023-09-23T21:07:45.665353+0000","mapping_epoch":400,"log_start":"359'3899","ondisk_log_start":"359'3899","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217251840,"num_objects":3315,"num_object_clones":0,"num_object_copies":9945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7993,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":67,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062046+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:07:29.062046+0000","last_peered":"2023-09-23T21:07:29.062046+0000","last_clean":"2023-09-23T21:07:29.062046+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:07:29.062046+0000","last_undegraded":"2023-09-23T21:07:29.062046+0000","last_fullsized":"2023-09-23T21:07:29.062046+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":50,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062314+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:07:29.062314+0000","last_peered":"2023-09-23T21:07:29.062314+0000","last_clean":"2023-09-23T21:07:29.062314+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:07:29.062314+0000","last_undegraded":"2023-09-23T21:07:29.062314+0000","last_fullsized":"2023-09-23T21:07:29.062314+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"410'7734","reported_seq":18846,"reported_epoch":410,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:44.172295+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:07:44.172295+0000","last_peered":"2023-09-23T21:07:44.172295+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:07:44.172295+0000","last_undegraded":"2023-09-23T21:07:44.172295+0000","last_fullsized":"2023-09-23T21:07:44.172295+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":7246,"log_dups_size":0,"ondisk_log_size":7246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212860928,"num_objects":3248,"num_object_clones":0,"num_object_copies":9744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3248,"num_objects_unfound":0,"num_objects_dirty":3248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7734,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":3248}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":336,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.060421+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:07:29.060421+0000","last_peered":"2023-09-23T21:07:29.060421+0000","last_clean":"2023-09-23T21:07:29.060421+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:07:29.060421+0000","last_undegraded":"2023-09-23T21:07:29.060421+0000","last_fullsized":"2023-09-23T21:07:29.060421+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":420,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.064090+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:07:29.064090+0000","last_peered":"2023-09-23T21:07:29.064090+0000","last_clean":"2023-09-23T21:07:29.064090+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:07:29.064090+0000","last_undegraded":"2023-09-23T21:07:29.064090+0000","last_fullsized":"2023-09-23T21:07:29.064090+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"410'7740","reported_seq":7836,"reported_epoch":410,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:45.709901+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:07:45.709901+0000","last_peered":"2023-09-23T21:07:45.709901+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:07:45.709901+0000","last_undegraded":"2023-09-23T21:07:45.709901+0000","last_fullsized":"2023-09-23T21:07:45.709901+0000","mapping_epoch":374,"log_start":"358'3594","ondisk_log_start":"358'3594","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4146,"log_dups_size":0,"ondisk_log_size":4146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207355904,"num_objects":3164,"num_object_clones":0,"num_object_copies":9492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6328,"num_objects_unfound":0,"num_objects_dirty":3164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7740,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":3164}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":37,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.071940+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:07:29.071940+0000","last_peered":"2023-09-23T21:07:29.071940+0000","last_clean":"2023-09-23T21:07:29.071940+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:07:29.071940+0000","last_undegraded":"2023-09-23T21:07:29.071940+0000","last_fullsized":"2023-09-23T21:07:29.071940+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":449,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.055939+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:07:29.055939+0000","last_peered":"2023-09-23T21:07:29.055939+0000","last_clean":"2023-09-23T21:07:29.055939+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:07:29.055939+0000","last_undegraded":"2023-09-23T21:07:29.055939+0000","last_fullsized":"2023-09-23T21:07:29.055939+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"410'7747","reported_seq":7930,"reported_epoch":410,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:45.504258+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:07:45.504258+0000","last_peered":"2023-09-23T21:07:45.504258+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:07:45.504258+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"353'2994","ondisk_log_start":"353'2994","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4753,"log_dups_size":0,"ondisk_log_size":4753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210698240,"num_objects":3215,"num_object_clones":0,"num_object_copies":9645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7747,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-2224},{"shards":"1,6,7","objects":3215}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":59,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.061897+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:07:29.061897+0000","last_peered":"2023-09-23T21:07:29.061897+0000","last_clean":"2023-09-23T21:07:29.061897+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:07:29.061897+0000","last_undegraded":"2023-09-23T21:07:29.061897+0000","last_fullsized":"2023-09-23T21:07:29.061897+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":198,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.071280+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:07:29.071280+0000","last_peered":"2023-09-23T21:07:29.071280+0000","last_clean":"2023-09-23T21:07:29.071280+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:07:29.071280+0000","last_undegraded":"2023-09-23T21:07:29.071280+0000","last_fullsized":"2023-09-23T21:07:29.071280+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"410'7741","reported_seq":13236,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:44.172165+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:07:44.172165+0000","last_peered":"2023-09-23T21:07:44.172165+0000","last_clean":"2023-09-23T21:07:44.172165+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:07:44.172165+0000","last_undegraded":"2023-09-23T21:07:44.172165+0000","last_fullsized":"2023-09-23T21:07:44.172165+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":7232,"log_dups_size":0,"ondisk_log_size":7232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215482392,"num_objects":3289,"num_object_clones":0,"num_object_copies":9867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3289,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7741,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":304,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.074577+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:07:29.074577+0000","last_peered":"2023-09-23T21:07:29.074577+0000","last_clean":"2023-09-23T21:07:29.074577+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:07:29.074577+0000","last_undegraded":"2023-09-23T21:07:29.074577+0000","last_fullsized":"2023-09-23T21:07:29.074577+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":75,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.053673+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:07:29.053673+0000","last_peered":"2023-09-23T21:07:29.053673+0000","last_clean":"2023-09-23T21:07:29.053673+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:07:29.053673+0000","last_undegraded":"2023-09-23T21:07:29.053673+0000","last_fullsized":"2023-09-23T21:07:29.053673+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":60,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.071285+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:07:29.071285+0000","last_peered":"2023-09-23T21:07:29.071285+0000","last_clean":"2023-09-23T21:07:29.071285+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:07:29.071285+0000","last_undegraded":"2023-09-23T21:07:29.071285+0000","last_fullsized":"2023-09-23T21:07:29.071285+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"410'7811","reported_seq":12944,"reported_epoch":410,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:07:45.732816+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:07:45.732816+0000","last_peered":"2023-09-23T21:07:45.732816+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:07:45.732816+0000","last_undegraded":"2023-09-23T21:07:45.732816+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"358'3657","ondisk_log_start":"358'3657","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4154,"log_dups_size":0,"ondisk_log_size":4154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210698240,"num_objects":3215,"num_object_clones":0,"num_object_copies":9645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8357,"num_objects_unfound":0,"num_objects_dirty":3215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7811,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2501,"num_bytes_recovered":163905536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":679},{"shards":"2,6","objects":203},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":385,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.071755+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:07:29.071755+0000","last_peered":"2023-09-23T21:07:29.071755+0000","last_clean":"2023-09-23T21:07:29.071755+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:07:29.071755+0000","last_undegraded":"2023-09-23T21:07:29.071755+0000","last_fullsized":"2023-09-23T21:07:29.071755+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":360,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.061548+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:07:29.061548+0000","last_peered":"2023-09-23T21:07:29.061548+0000","last_clean":"2023-09-23T21:07:29.061548+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:07:29.061548+0000","last_undegraded":"2023-09-23T21:07:29.061548+0000","last_fullsized":"2023-09-23T21:07:29.061548+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":42,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.074493+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:07:29.074493+0000","last_peered":"2023-09-23T21:07:29.074493+0000","last_clean":"2023-09-23T21:07:29.074493+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:07:29.074493+0000","last_undegraded":"2023-09-23T21:07:29.074493+0000","last_fullsized":"2023-09-23T21:07:29.074493+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"410'7799","reported_seq":7978,"reported_epoch":410,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:44.188704+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:07:44.188704+0000","last_peered":"2023-09-23T21:07:44.188704+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:07:44.188704+0000","last_undegraded":"2023-09-23T21:07:44.188704+0000","last_fullsized":"2023-09-23T21:07:44.188704+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":6326,"log_dups_size":0,"ondisk_log_size":6326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213843968,"num_objects":3263,"num_object_clones":0,"num_object_copies":9789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6505,"num_objects_unfound":0,"num_objects_dirty":3263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7799,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":3263}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":387,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.064523+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:07:29.064523+0000","last_peered":"2023-09-23T21:07:29.064523+0000","last_clean":"2023-09-23T21:07:29.064523+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:07:29.064523+0000","last_undegraded":"2023-09-23T21:07:29.064523+0000","last_fullsized":"2023-09-23T21:07:29.064523+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":304,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.077329+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:07:29.077329+0000","last_peered":"2023-09-23T21:07:29.077329+0000","last_clean":"2023-09-23T21:07:29.077329+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:07:29.077329+0000","last_undegraded":"2023-09-23T21:07:29.077329+0000","last_fullsized":"2023-09-23T21:07:29.077329+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":42,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.071154+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:07:29.071154+0000","last_peered":"2023-09-23T21:07:29.071154+0000","last_clean":"2023-09-23T21:07:29.071154+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:07:29.071154+0000","last_undegraded":"2023-09-23T21:07:29.071154+0000","last_fullsized":"2023-09-23T21:07:29.071154+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"410'7662","reported_seq":13918,"reported_epoch":410,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:45.687277+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:07:45.687277+0000","last_peered":"2023-09-23T21:07:45.687277+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:07:45.687277+0000","last_undegraded":"2023-09-23T21:07:45.687277+0000","last_fullsized":"2023-09-23T21:07:45.687277+0000","mapping_epoch":387,"log_start":"358'3580","ondisk_log_start":"358'3580","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4082,"log_dups_size":0,"ondisk_log_size":4082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210501632,"num_objects":3212,"num_object_clones":0,"num_object_copies":9636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3212,"num_objects_unfound":0,"num_objects_dirty":3212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7662,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":3212}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":225,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.066146+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:07:29.066146+0000","last_peered":"2023-09-23T21:07:29.066146+0000","last_clean":"2023-09-23T21:07:29.066146+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:07:29.066146+0000","last_undegraded":"2023-09-23T21:07:29.066146+0000","last_fullsized":"2023-09-23T21:07:29.066146+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":707,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:29.070427+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:07:29.070427+0000","last_peered":"2023-09-23T21:07:29.070427+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:07:29.070427+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":59,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062305+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:07:29.062305+0000","last_peered":"2023-09-23T21:07:29.062305+0000","last_clean":"2023-09-23T21:07:29.062305+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:07:29.062305+0000","last_undegraded":"2023-09-23T21:07:29.062305+0000","last_fullsized":"2023-09-23T21:07:29.062305+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":298,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.058601+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:07:29.058601+0000","last_peered":"2023-09-23T21:07:29.058601+0000","last_clean":"2023-09-23T21:07:29.058601+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:07:29.058601+0000","last_undegraded":"2023-09-23T21:07:29.058601+0000","last_fullsized":"2023-09-23T21:07:29.058601+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"410'7928","reported_seq":8040,"reported_epoch":410,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:45.725646+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:07:45.725646+0000","last_peered":"2023-09-23T21:07:45.725646+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:07:45.725646+0000","last_undegraded":"2023-09-23T21:07:45.725646+0000","last_fullsized":"2023-09-23T21:07:45.725646+0000","mapping_epoch":374,"log_start":"359'3805","ondisk_log_start":"359'3805","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4123,"log_dups_size":0,"ondisk_log_size":4123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6388,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7928,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":3194}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":42,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.070322+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:07:29.070322+0000","last_peered":"2023-09-23T21:07:29.070322+0000","last_clean":"2023-09-23T21:07:29.070322+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:07:29.070322+0000","last_undegraded":"2023-09-23T21:07:29.070322+0000","last_fullsized":"2023-09-23T21:07:29.070322+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":56,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068864+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:07:29.068864+0000","last_peered":"2023-09-23T21:07:29.068864+0000","last_clean":"2023-09-23T21:07:29.068864+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:07:29.068864+0000","last_undegraded":"2023-09-23T21:07:29.068864+0000","last_fullsized":"2023-09-23T21:07:29.068864+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":47,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068262+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:07:29.068262+0000","last_peered":"2023-09-23T21:07:29.068262+0000","last_clean":"2023-09-23T21:07:29.068262+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:07:29.068262+0000","last_undegraded":"2023-09-23T21:07:29.068262+0000","last_fullsized":"2023-09-23T21:07:29.068262+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"410'7753","reported_seq":12333,"reported_epoch":410,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:45.525991+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:07:45.525991+0000","last_peered":"2023-09-23T21:07:45.525991+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:07:45.525991+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:07:45.525991+0000","mapping_epoch":383,"log_start":"353'3026","ondisk_log_start":"353'3026","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4727,"log_dups_size":0,"ondisk_log_size":4727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214499328,"num_objects":3273,"num_object_clones":0,"num_object_copies":9819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3270,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":3273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7753,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2240,"num_bytes_recovered":146800640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":3},{"shards":"1,2,7","objects":3270}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":47,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.073298+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:07:29.073298+0000","last_peered":"2023-09-23T21:07:29.073298+0000","last_clean":"2023-09-23T21:07:29.073298+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:07:29.073298+0000","last_undegraded":"2023-09-23T21:07:29.073298+0000","last_fullsized":"2023-09-23T21:07:29.073298+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":306,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.076651+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:07:29.076651+0000","last_peered":"2023-09-23T21:07:29.076651+0000","last_clean":"2023-09-23T21:07:29.076651+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:07:29.076651+0000","last_undegraded":"2023-09-23T21:07:29.076651+0000","last_fullsized":"2023-09-23T21:07:29.076651+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":85,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.053107+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:07:29.053107+0000","last_peered":"2023-09-23T21:07:29.053107+0000","last_clean":"2023-09-23T21:07:29.053107+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:07:29.053107+0000","last_undegraded":"2023-09-23T21:07:29.053107+0000","last_fullsized":"2023-09-23T21:07:29.053107+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"410'7741","reported_seq":20630,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:45.522435+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:07:45.522435+0000","last_peered":"2023-09-23T21:07:45.522435+0000","last_clean":"2023-09-23T21:07:45.522435+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:07:45.522435+0000","last_undegraded":"2023-09-23T21:07:45.522435+0000","last_fullsized":"2023-09-23T21:07:45.522435+0000","mapping_epoch":397,"log_start":"353'3045","ondisk_log_start":"353'3045","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4696,"log_dups_size":0,"ondisk_log_size":4696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204668928,"num_objects":3123,"num_object_clones":0,"num_object_copies":9369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7741,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":67,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.065716+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:07:29.065716+0000","last_peered":"2023-09-23T21:07:29.065716+0000","last_clean":"2023-09-23T21:07:29.065716+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:07:29.065716+0000","last_undegraded":"2023-09-23T21:07:29.065716+0000","last_fullsized":"2023-09-23T21:07:29.065716+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.076148+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:07:29.076148+0000","last_peered":"2023-09-23T21:07:29.076148+0000","last_clean":"2023-09-23T21:07:29.076148+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:07:29.076148+0000","last_undegraded":"2023-09-23T21:07:29.076148+0000","last_fullsized":"2023-09-23T21:07:29.076148+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.061684+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:07:29.061684+0000","last_peered":"2023-09-23T21:07:29.061684+0000","last_clean":"2023-09-23T21:07:29.061684+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:07:29.061684+0000","last_undegraded":"2023-09-23T21:07:29.061684+0000","last_fullsized":"2023-09-23T21:07:29.061684+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"410'7919","reported_seq":12675,"reported_epoch":410,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:45.497782+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:07:45.497782+0000","last_peered":"2023-09-23T21:07:45.497782+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:07:45.497782+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:07:45.497782+0000","mapping_epoch":382,"log_start":"356'3212","ondisk_log_start":"356'3212","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4707,"log_dups_size":0,"ondisk_log_size":4707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214892544,"num_objects":3279,"num_object_clones":0,"num_object_copies":9837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3276,"num_objects_misplaced":3282,"num_objects_unfound":0,"num_objects_dirty":3279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7919,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2320,"num_bytes_recovered":152043520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":3276}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":306,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.056631+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:07:29.056631+0000","last_peered":"2023-09-23T21:07:29.056631+0000","last_clean":"2023-09-23T21:07:29.056631+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:07:29.056631+0000","last_undegraded":"2023-09-23T21:07:29.056631+0000","last_fullsized":"2023-09-23T21:07:29.056631+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":51,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068442+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:07:29.068442+0000","last_peered":"2023-09-23T21:07:29.068442+0000","last_clean":"2023-09-23T21:07:29.068442+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:07:29.068442+0000","last_undegraded":"2023-09-23T21:07:29.068442+0000","last_fullsized":"2023-09-23T21:07:29.068442+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.052551+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:07:29.052551+0000","last_peered":"2023-09-23T21:07:29.052551+0000","last_clean":"2023-09-23T21:07:29.052551+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:07:29.052551+0000","last_undegraded":"2023-09-23T21:07:29.052551+0000","last_fullsized":"2023-09-23T21:07:29.052551+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":56,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.192149+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:07:29.192149+0000","last_peered":"2023-09-23T21:07:29.192149+0000","last_clean":"2023-09-23T21:07:29.192149+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:07:29.192149+0000","last_undegraded":"2023-09-23T21:07:29.192149+0000","last_fullsized":"2023-09-23T21:07:29.192149+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":433,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.063833+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:07:29.063833+0000","last_peered":"2023-09-23T21:07:29.063833+0000","last_clean":"2023-09-23T21:07:29.063833+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:07:29.063833+0000","last_undegraded":"2023-09-23T21:07:29.063833+0000","last_fullsized":"2023-09-23T21:07:29.063833+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":72,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.073585+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:07:29.073585+0000","last_peered":"2023-09-23T21:07:29.073585+0000","last_clean":"2023-09-23T21:07:29.073585+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:07:29.073585+0000","last_undegraded":"2023-09-23T21:07:29.073585+0000","last_fullsized":"2023-09-23T21:07:29.073585+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062127+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:07:29.062127+0000","last_peered":"2023-09-23T21:07:29.062127+0000","last_clean":"2023-09-23T21:07:29.062127+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:07:29.062127+0000","last_undegraded":"2023-09-23T21:07:29.062127+0000","last_fullsized":"2023-09-23T21:07:29.062127+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":291,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068372+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:07:29.068372+0000","last_peered":"2023-09-23T21:07:29.068372+0000","last_clean":"2023-09-23T21:07:29.068372+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:07:29.068372+0000","last_undegraded":"2023-09-23T21:07:29.068372+0000","last_fullsized":"2023-09-23T21:07:29.068372+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":68,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062461+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:07:29.062461+0000","last_peered":"2023-09-23T21:07:29.062461+0000","last_clean":"2023-09-23T21:07:29.062461+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:07:29.062461+0000","last_undegraded":"2023-09-23T21:07:29.062461+0000","last_fullsized":"2023-09-23T21:07:29.062461+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":50,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062649+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:07:29.062649+0000","last_peered":"2023-09-23T21:07:29.062649+0000","last_clean":"2023-09-23T21:07:29.062649+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:07:29.062649+0000","last_undegraded":"2023-09-23T21:07:29.062649+0000","last_fullsized":"2023-09-23T21:07:29.062649+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":59,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.062883+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:07:29.062883+0000","last_peered":"2023-09-23T21:07:29.062883+0000","last_clean":"2023-09-23T21:07:29.062883+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:07:29.062883+0000","last_undegraded":"2023-09-23T21:07:29.062883+0000","last_fullsized":"2023-09-23T21:07:29.062883+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":53,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.068242+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:07:29.068242+0000","last_peered":"2023-09-23T21:07:29.068242+0000","last_clean":"2023-09-23T21:07:29.068242+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:07:29.068242+0000","last_undegraded":"2023-09-23T21:07:29.068242+0000","last_fullsized":"2023-09-23T21:07:29.068242+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":37,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T21:07:29.072423+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:07:29.072423+0000","last_peered":"2023-09-23T21:07:29.072423+0000","last_clean":"2023-09-23T21:07:29.072423+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:07:29.072423+0000","last_undegraded":"2023-09-23T21:07:29.072423+0000","last_fullsized":"2023-09-23T21:07:29.072423+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3380346904,"num_objects":51581,"num_object_clones":0,"num_object_copies":154743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29219,"num_objects_misplaced":40490,"num_objects_unfound":0,"num_objects_dirty":51581,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124793,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29118,"num_bytes_recovered":1908277248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12390182912,"data_stored":12390170696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77992,"ondisk_log_size":77992,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201748,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463369,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2806936,"kb_used_data":2291664,"kb_used_omap":0,"kb_used_meta":515136,"kb_avail":90938216,"statfs":{"total":95995035648,"available":93120733184,"internally_reserved":0,"allocated":2346663936,"data_stored":2345459443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":527499264},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":6.6479999999999997,"5min":2.0830000000000002,"15min":2.2610000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":99.533000000000001,"5min":99.533000000000001,"15min":102.242},"last":3.145},{"interface":"front","average":{"1min":6.8070000000000004,"5min":2.1949999999999998,"15min":2.0470000000000002},"min":{"1min":0.248,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":99.325000000000003,"5min":99.325000000000003,"15min":102.502},"last":6.04}]},{"osd":1,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":3.9460000000000002,"15min":2.379},"min":{"1min":0.251,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":11.743,"5min":119.077,"15min":119.077},"last":1.5920000000000001},{"interface":"front","average":{"1min":1.901,"5min":3.7730000000000001,"15min":2.452},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":11.936,"5min":119.313,"15min":119.313},"last":2.54}]},{"osd":2,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":13.304,"5min":6.1219999999999999,"15min":3.726},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":199.23400000000001,"5min":199.23400000000001,"15min":199.23400000000001},"last":4.6139999999999999},{"interface":"front","average":{"1min":12.943,"5min":6.1870000000000003,"15min":3.8719999999999999},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.84599999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":0.98599999999999999,"15min":4.0119999999999996},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":3.194,"5min":5.9420000000000002,"15min":250.804},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.45,"15min":4.0099999999999998},"min":{"1min":0.371,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":13.319000000000001,"5min":13.319000000000001,"15min":250.446},"last":4.1520000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.353,"15min":0.97099999999999997},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.24299999999999999},"max":{"1min":6.3689999999999998,"5min":13.1,"15min":13.1},"last":2.032},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.98699999999999999,"15min":0.95199999999999996},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":4.3849999999999998,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.333,"15min":1.105},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":6.343,"5min":11.281000000000001,"15min":18.962},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.325,"5min":1.286,"15min":1.212},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":6.6890000000000001,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.2569999999999999,"15min":1.4710000000000001},"min":{"1min":0.34899999999999998,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.8079999999999998,"5min":7.8079999999999998,"15min":101.04600000000001},"last":3.1139999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.2509999999999999,"15min":1.5029999999999999},"min":{"1min":0.39200000000000002,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":4.407,"5min":7.1849999999999996,"15min":101.081},"last":0.52900000000000003}]}]},{"osd":1,"up_from":276,"seq":1185410973815,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2941224,"kb_used_data":2540832,"kb_used_omap":5,"kb_used_meta":400186,"kb_avail":90803928,"statfs":{"total":95995035648,"available":92983222272,"internally_reserved":0,"allocated":2601811968,"data_stored":2599771934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":409791203},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,3,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":0.85599999999999998,"15min":0.55200000000000005},"min":{"1min":0.20599999999999999,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":9.9469999999999992,"5min":11.959,"15min":11.959},"last":2.3599999999999999},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.256,"15min":0.81499999999999995},"min":{"1min":0.216,"5min":0.193,"15min":0.193},"max":{"1min":5.093,"5min":18.047999999999998,"15min":18.047999999999998},"last":2.371}]},{"osd":2,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.407,"15min":1.1359999999999999},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.0670000000000002,"5min":17.254000000000001,"15min":78.409000000000006},"last":2.48},{"interface":"front","average":{"1min":1.55,"5min":1.03,"15min":0.97999999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.22},"max":{"1min":12.478999999999999,"5min":12.478999999999999,"15min":76.245000000000005},"last":12.936}]},{"osd":3,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.849,"15min":2.4670000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":7.3099999999999996,"5min":74.527000000000001,"15min":172.44399999999999},"last":4.7999999999999998},{"interface":"front","average":{"1min":1.907,"5min":2.5369999999999999,"15min":2.6880000000000002},"min":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":7.9050000000000002,"5min":72.216999999999999,"15min":172.58500000000001},"last":2.4449999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.472,"15min":0.93400000000000005},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.7069999999999999,"5min":17.006,"15min":17.006},"last":3.4140000000000001},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.92600000000000005,"15min":0.70299999999999996},"min":{"1min":0.35699999999999998,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":4.7690000000000001,"5min":11.916,"15min":11.916},"last":4.8700000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.446,"15min":0.88300000000000001},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.6269999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":2.4199999999999999},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.589,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.3860000000000001,"5min":17.076000000000001,"15min":17.076000000000001},"last":3.3769999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.577,"15min":1.3879999999999999},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.6020000000000003,"5min":17.102,"15min":127.995},"last":2.4990000000000001},{"interface":"front","average":{"1min":1.127,"5min":1.647,"15min":1.466},"min":{"1min":0.41699999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.375,"5min":17.302,"15min":126.836},"last":3.4580000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.9180000000000001,"15min":1.484},"min":{"1min":0.442,"5min":0.38200000000000001,"15min":0.36499999999999999},"max":{"1min":7.048,"5min":151.791,"15min":151.791},"last":3.4369999999999998},{"interface":"front","average":{"1min":1.49,"5min":2.3889999999999998,"15min":1.2310000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.3739999999999997,"5min":151.809,"15min":151.809},"last":4.8419999999999996}]}]},{"osd":0,"up_from":92,"seq":395136991555,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1190564,"kb_used_data":459932,"kb_used_omap":5,"kb_used_meta":730618,"kb_avail":92554588,"statfs":{"total":95995035648,"available":94775898112,"internally_reserved":0,"allocated":470970368,"data_stored":469048166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":748153466},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":2.9500000000000002,"15min":1.4830000000000001},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":40.284999999999997,"5min":61.341999999999999,"15min":61.341999999999999},"last":2.9020000000000001},{"interface":"front","average":{"1min":3.4319999999999999,"5min":2.9670000000000001,"15min":1.4510000000000001},"min":{"1min":0.223,"5min":0.217,"15min":0.217},"max":{"1min":40.213000000000001,"5min":61.299999999999997,"15min":61.299999999999997},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":1.7110000000000001,"15min":2.238},"min":{"1min":0.217,"5min":0.217,"15min":0.161},"max":{"1min":41.860999999999997,"5min":41.860999999999997,"15min":183.77099999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":4.5780000000000003,"5min":2.1219999999999999,"15min":2.5070000000000001},"min":{"1min":0.27000000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":44.779000000000003,"5min":44.779000000000003,"15min":183.78700000000001},"last":0.56599999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.1319999999999999,"15min":3.3079999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":5.5810000000000004,"5min":11.555999999999999,"15min":186.143},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.3320000000000001,"15min":3.4380000000000002},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.23000000000000001},"max":{"1min":4.016,"5min":16.640000000000001,"15min":186.97399999999999},"last":0.94999999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.2090000000000001,"15min":1.286},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.5259999999999998,"5min":9.7080000000000002,"15min":10.512},"last":2.6680000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.80200000000000005,"15min":0.93799999999999994},"min":{"1min":0.31,"5min":0.31,"15min":0.26200000000000001},"max":{"1min":3.875,"5min":6.0490000000000004,"15min":11.493},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.3640000000000001,"15min":2.4830000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.492,"5min":9.7439999999999998,"15min":154.30600000000001},"last":2.7120000000000002},{"interface":"front","average":{"1min":1.093,"5min":0.90200000000000002,"15min":1.905},"min":{"1min":0.33800000000000002,"5min":0.33500000000000002,"15min":0.28699999999999998},"max":{"1min":11.478,"5min":11.478,"15min":154.42400000000001},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2149999999999999,"5min":5.7060000000000004,"15min":3.1709999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":136.404,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":8.1039999999999992,"5min":5.9210000000000003,"15min":3.3599999999999999},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.28299999999999997},"max":{"1min":136.44999999999999,"5min":169.767,"15min":169.767},"last":0.60599999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.98599999999999999,"15min":2.4279999999999999},"min":{"1min":0.35199999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.6840000000000002,"5min":6.6840000000000002,"15min":164.26400000000001},"last":0.437},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.107,"15min":2.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.379,"15min":0.34599999999999997},"max":{"1min":1.6719999999999999,"5min":6.5410000000000004,"15min":164.23099999999999},"last":0.53300000000000003}]}]},{"osd":2,"up_from":129,"seq":554050781446,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2948712,"kb_used_data":2317792,"kb_used_omap":7,"kb_used_meta":630904,"kb_avail":90796440,"statfs":{"total":95995035648,"available":92975554560,"internally_reserved":0,"allocated":2373419008,"data_stored":2371435264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":646045777},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.4450000000000001,"15min":1.3959999999999999},"min":{"1min":0.20499999999999999,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":6.8440000000000003,"5min":15.667,"15min":26.356000000000002},"last":3.5459999999999998},{"interface":"front","average":{"1min":6.4569999999999999,"5min":2.0830000000000002,"15min":1.8320000000000001},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":23.963999999999999,"5min":23.963999999999999,"15min":26.341000000000001},"last":5.2599999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":4.9199999999999999,"15min":2.379},"min":{"1min":0.23599999999999999,"5min":0.19,"15min":0.188},"max":{"1min":4.9089999999999998,"5min":212.12,"15min":212.12},"last":3.8490000000000002},{"interface":"front","average":{"1min":1.232,"5min":4.9390000000000001,"15min":2.1890000000000001},"min":{"1min":0.24299999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.4219999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":2.9140000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":2.2480000000000002,"15min":3.3039999999999998},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":4.3559999999999999,"5min":66.085999999999999,"15min":151.43799999999999},"last":8.3580000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":2.222,"15min":3.3260000000000001},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":1.4930000000000001,"5min":66.102999999999994,"15min":152.20699999999999},"last":3.7919999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":1.153,"15min":1.113},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":6.383,"5min":12.586,"15min":12.586},"last":0.39800000000000002},{"interface":"front","average":{"1min":4.1820000000000004,"5min":1.5700000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":14.672000000000001,"5min":14.672000000000001,"15min":14.672000000000001},"last":5.1639999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":4.367,"5min":1.7050000000000001,"15min":2.129},"min":{"1min":0.36799999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":14.710000000000001,"5min":14.710000000000001,"15min":173.34},"last":5.202},{"interface":"front","average":{"1min":2.097,"5min":1.173,"15min":1.796},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":8.9740000000000002,"5min":13.701000000000001,"15min":173.15000000000001},"last":3.835}]},{"osd":6,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":3.3940000000000001,"15min":1.946},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":9.0329999999999995,"5min":175.72499999999999,"15min":175.72499999999999},"last":3.8050000000000002},{"interface":"front","average":{"1min":1.901,"5min":3.375,"15min":1.9550000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.5049999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.35999999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.655,"15min":2.081},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":6.4809999999999999,"5min":12.609,"15min":177.30799999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":4.2240000000000002,"5min":1.645,"15min":2.1989999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33600000000000002},"max":{"1min":14.736000000000001,"5min":14.736000000000001,"15min":177.40899999999999},"last":5.226}]}]},{"osd":3,"up_from":148,"seq":635655160040,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":564008,"kb_used_data":7008,"kb_used_omap":2,"kb_used_meta":556989,"kb_avail":93181144,"statfs":{"total":95995035648,"available":95417491456,"internally_reserved":0,"allocated":7176192,"data_stored":5012230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":570356789},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":1.4910000000000001,"15min":1.774},"min":{"1min":0.249,"5min":0.20999999999999999,"15min":0.184},"max":{"1min":8.8629999999999995,"5min":44.963000000000001,"15min":95.507999999999996},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.8919999999999999,"5min":0.94799999999999995,"15min":1.6679999999999999},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.17199999999999999},"max":{"1min":7.7850000000000001,"5min":9.7370000000000001,"15min":95.259},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:06:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.347,"15min":1.0580000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7619999999999996,"5min":12.035,"15min":19.786999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.661,"15min":1.292},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":8.3059999999999992,"5min":15.763999999999999,"15min":15.763999999999999},"last":12.43}]},{"osd":2,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4129999999999998,"5min":1.4750000000000001,"15min":1.351},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.182},"max":{"1min":14.243,"5min":14.243,"15min":14.243},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.6059999999999999,"5min":1.325,"15min":1.3420000000000001},"min":{"1min":0.34399999999999997,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":7.8120000000000003,"5min":9.7140000000000004,"15min":19.754999999999999},"last":0.55600000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4980000000000002,"5min":2.5470000000000002,"15min":2.2559999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.222},"max":{"1min":38.600000000000001,"5min":70.728999999999999,"15min":111.36},"last":12.356999999999999},{"interface":"front","average":{"1min":3.7919999999999998,"5min":2.4529999999999998,"15min":2.0539999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.252},"max":{"1min":38.524000000000001,"5min":70.712000000000003,"15min":112.473},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":1.575,"15min":1.581},"min":{"1min":0.41199999999999998,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":36.488999999999997,"5min":36.488999999999997,"15min":64.528000000000006},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.5110000000000001,"5min":1.6879999999999999,"15min":1.647},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":36.295000000000002,"5min":36.295000000000002,"15min":36.295000000000002},"last":12.382999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.3740000000000001,"15min":1.623},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.312},"max":{"1min":10.827999999999999,"5min":10.827999999999999,"15min":19.568000000000001},"last":12.317},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.448,"15min":1.5029999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":7.6749999999999998,"5min":14.356,"15min":15.573},"last":1.087}]},{"osd":7,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.3599999999999999,"15min":2.5510000000000002},"min":{"1min":0.5,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":7.7370000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.2549999999999999,"5min":2.3530000000000002,"15min":2.8170000000000002},"min":{"1min":0.48099999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":7.4249999999999998,"5min":73.364999999999995,"15min":222.11699999999999},"last":12.407}]}]},{"osd":4,"up_from":31,"seq":133143986658,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1411480,"kb_used_data":850064,"kb_used_omap":0,"kb_used_meta":561408,"kb_avail":92333672,"statfs":{"total":95995035648,"available":94549680128,"internally_reserved":0,"allocated":870465536,"data_stored":869260427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":574881792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.4770000000000001,"15min":2.6949999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":12.869,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.2849999999999999,"15min":2.6920000000000002},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.223},"max":{"1min":12.848000000000001,"5min":12.848000000000001,"15min":144.77000000000001},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":12.298,"5min":8.5619999999999994,"15min":3.3940000000000001},"min":{"1min":0.34899999999999998,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":127.411,"5min":127.411,"15min":127.411},"last":0.40300000000000002},{"interface":"front","average":{"1min":4.8810000000000002,"5min":6.7430000000000003,"15min":2.758},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":65.688999999999993,"5min":118.075,"15min":118.075},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":6.7910000000000004,"15min":3.5299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":71.454999999999998,"5min":164.517,"15min":164.517},"last":3.0070000000000001},{"interface":"front","average":{"1min":5.343,"5min":6.282,"15min":3.5099999999999998},"min":{"1min":0.34899999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":71.191999999999993,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2170000000000001,"15min":2.1280000000000001},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.2879999999999998,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.177,"15min":2.3029999999999999},"min":{"1min":0.35299999999999998,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":8.2899999999999991,"5min":8.2899999999999991,"15min":142.34700000000001},"last":1.139}]},{"osd":5,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.81999999999999995,"15min":1.052},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":2.2629999999999999,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.376},{"interface":"front","average":{"1min":1.042,"5min":0.96799999999999997,"15min":1.1220000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.21099999999999999},"max":{"1min":4.282,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.38800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.1779999999999999,"15min":0.98199999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":4.8399999999999999,"5min":11.800000000000001,"15min":11.887},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.2410000000000001,"15min":0.93700000000000006},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0419999999999998,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.601,"15min":1.6040000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":4.3129999999999997,"5min":145.721,"15min":145.721},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.347,"5min":2.9489999999999998,"15min":1.776},"min":{"1min":0.38500000000000001,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":4.7969999999999997,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.56000000000000005}]}]},{"osd":5,"up_from":39,"seq":167503725022,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1046852,"kb_used_data":433340,"kb_used_omap":0,"kb_used_meta":613504,"kb_avail":92698300,"statfs":{"total":95995035648,"available":94923059200,"internally_reserved":0,"allocated":443740160,"data_stored":442551251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":628228096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.68500000000000005,"15min":1.3540000000000001},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":10.016999999999999,"5min":10.016999999999999,"15min":80.105999999999995},"last":3.1070000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.89300000000000002,"15min":1.462},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":2.919,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":13.016999999999999,"5min":6.4710000000000001,"15min":2.512},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":233.77099999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":7.3230000000000004},{"interface":"front","average":{"1min":13.031000000000001,"5min":6.1769999999999996,"15min":2.4220000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.247},"max":{"1min":233.75200000000001,"5min":233.75200000000001,"15min":233.75200000000001},"last":7.3419999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.3980000000000001,"15min":1.569},"min":{"1min":0.33300000000000002,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":8.3770000000000007,"5min":103.63800000000001,"15min":103.63800000000001},"last":0.504},{"interface":"front","average":{"1min":1.0920000000000001,"5min":2.6309999999999998,"15min":1.696},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.26200000000000001},"max":{"1min":6.7640000000000002,"5min":103.664,"15min":103.664},"last":1.5189999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.3149999999999999,"15min":2.8359999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":8.0860000000000003,"5min":97.349999999999994,"15min":153.60300000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.2570000000000001,"15min":2.7559999999999998},"min":{"1min":0.38300000000000001,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":4.6609999999999996,"5min":97.316999999999993,"15min":153.774},"last":0.57799999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59999999999999998,"15min":0.77600000000000002},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":2.2320000000000002,"5min":3.3029999999999999,"15min":12.739000000000001},"last":0.314},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.88900000000000001,"15min":0.98299999999999998},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.247},"max":{"1min":3.645,"5min":8.3740000000000006,"15min":12.539},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:06:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.7150000000000001,"15min":1.1839999999999999},"min":{"1min":0.376,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":1.276,"5min":71.230000000000004,"15min":71.230000000000004},"last":0.62},{"interface":"front","average":{"1min":0.625,"5min":1.8420000000000001,"15min":1.3220000000000001},"min":{"1min":0.32400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6699999999999999,"5min":70.266000000000005,"15min":70.266000000000005},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.143,"15min":1.2010000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":5.0119999999999996,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65800000000000003,"15min":1.054},"min":{"1min":0.38500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.5339999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.45500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":462028800,"data_stored":462028800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2572156928,"data_stored":2572156928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2355367936,"data_stored":2355363864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1769472,"data_stored":1769472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":861474816,"data_stored":861470744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":438697984,"data_stored":438697984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2321223680,"data_stored":2321219608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3377463296,"data_stored":3377463296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:07:51.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:51.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:51.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:51 smithi123 ceph-mon[193903]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 28639/147690 objects degraded (19.391%); 38566/147690 objects misplaced (26.113%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T21:07:51.555 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:51.649 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:51.764 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:51.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:51.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:51.884 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:51.932 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:51.994 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:51.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:51.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:52.099 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:52.182 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 28639/147690 objects degraded (19.391%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:52.328 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:52.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:52.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:52.333 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 28639/147690 objects degraded (19.391%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:52.333 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 28639/147690 objects degraded (19.391%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:52.385 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:52.433 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:52.517 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:52.708 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:52.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:52.817 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:52.855 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:52.930 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:52.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:52.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:53.035 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:07:53.084 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:53.167 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:07:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:53.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:07:53.213 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:07:53.276 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:07:53.311 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:53.397 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:07:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:53.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:07:53.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2814929769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:53.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:53 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:53.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:53 smithi121 ceph-mon[219709]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 28639/147690 objects degraded (19.391%); 38566/147690 objects misplaced (26.113%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:07:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2814929769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:53 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:53.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:53 smithi121 ceph-mon[211526]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 28639/147690 objects degraded (19.391%); 38566/147690 objects misplaced (26.113%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:07:53.470 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:07:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2814929769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:53.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:53 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:07:53.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:53 smithi123 ceph-mon[193903]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 28639/147690 objects degraded (19.391%); 38566/147690 objects misplaced (26.113%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:07:53.505 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:53.580 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:54.455 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:07:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:54 smithi123 ceph-mon[193903]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.479 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:54.480 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.480 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:54.480 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T21:07:54.480 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.480 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.480 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[211526]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.481 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:54 smithi121 ceph-mon[219709]: failed to encode map e411 with expected crc 2023-09-23T21:07:54.837 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:54.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:54.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:54.948 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:55.034 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:55.107 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:55.156 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:55.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:07:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:07:54] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2023-09-23T21:07:55.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:55.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:55.275 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:55.372 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:55.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:55 smithi123 ceph-mon[193903]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 28639/146193 objects degraded (19.590%); 38067/146193 objects misplaced (26.039%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T21:07:55.495 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:55 smithi121 ceph-mon[211526]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 28639/146193 objects degraded (19.590%); 38067/146193 objects misplaced (26.039%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T21:07:55.499 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:55 smithi121 ceph-mon[219709]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 28639/146193 objects degraded (19.590%); 38067/146193 objects misplaced (26.039%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T21:07:55.574 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:55.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:55.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:55.623 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:55.690 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:07:55.731 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:07:55.731 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:07:55.785 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:56.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 28639/146193 objects degraded (19.590%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:56.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 28639/146193 objects degraded (19.590%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:56.920 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:07:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 28639/146193 objects degraded (19.590%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:07:57.236 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:07:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:57.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:07:57.349 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:57.437 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:57.563 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:57 smithi121 ceph-mon[211526]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 27985/140124 objects degraded (19.972%); 36599/140124 objects misplaced (26.119%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:07:57.563 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:57.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:57 smithi121 ceph-mon[219709]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 27985/140124 objects degraded (19.972%); 36599/140124 objects misplaced (26.119%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:07:57.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:57 smithi123 ceph-mon[193903]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 27985/140124 objects degraded (19.972%); 36599/140124 objects misplaced (26.119%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:07:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:07:58.226 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:07:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:58.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:07:58.335 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:07:58.377 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:07:58.418 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:58.798 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:07:58.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:58.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:07:58.903 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:07:58.939 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:59.003 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:07:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:59.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:07:59.242 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:07:59 smithi123 ceph-mon[193903]: pgmap v1304: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 27985/140124 objects degraded (19.972%); 36599/140124 objects misplaced (26.119%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:07:59.242 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:07:59 smithi121 ceph-mon[211526]: pgmap v1304: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 27985/140124 objects degraded (19.972%); 36599/140124 objects misplaced (26.119%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:07:59.245 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:07:59.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:07:59.343 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:07:59.348 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:07:59.393 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:07:59.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:07:59 smithi121 ceph-mon[219709]: pgmap v1304: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 27985/140124 objects degraded (19.972%); 36599/140124 objects misplaced (26.119%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T21:07:59.437 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:07:59.437 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:07:59.477 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:07:59.519 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:07:59.535 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:59.562 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:07:59.576 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.576 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.576 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.576 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.577 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.577 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.577 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.577 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:07:59.601 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:07:59.628 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:07:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:07:59.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:07:59.734 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:07:59.821 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:00.050 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:08:00.075 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:08:00.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3704940898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:00 smithi121 ceph-mon[211526]: from='client.36543 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:00.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3704940898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:00.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:00 smithi121 ceph-mon[219709]: from='client.36543 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3704940898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:00 smithi123 ceph-mon[193903]: from='client.36543 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:00.634 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:00.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:00.743 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:00.841 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:01.484 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:01 smithi123 ceph-mon[193903]: pgmap v1305: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 27356/132375 objects degraded (20.666%); 34508/132375 objects misplaced (26.068%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:01.552 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:01 smithi121 ceph-mon[219709]: pgmap v1305: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 27356/132375 objects degraded (20.666%); 34508/132375 objects misplaced (26.068%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:01.554 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:01 smithi121 ceph-mon[211526]: pgmap v1305: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 27356/132375 objects degraded (20.666%); 34508/132375 objects misplaced (26.068%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:01.802 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1304,"stamp":"2023-09-23T21:07:59.061270+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3060012984,"num_objects":46708,"num_object_clones":0,"num_object_copies":140124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27985,"num_objects_misplaced":36599,"num_objects_unfound":0,"num_objects_dirty":46708,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":129990,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30613,"num_bytes_recovered":2001575180,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79105,"ondisk_log_size":79105,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12089268,"kb_used_data":8054652,"kb_used_omap":21,"kb_used_meta":4034474,"kb_avail":644126796,"statfs":{"total":671965249536,"available":659585839104,"internally_reserved":0,"allocated":8247963648,"data_stored":8236212942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4131301859},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-320929792,"num_objects":-4897,"num_object_clones":0,"num_object_copies":-14691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1236,"num_objects_misplaced":-3891,"num_objects_unfound":0,"num_objects_dirty":-4897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4897,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1378,"num_bytes_recovered":90308608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004799"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":182,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.616515+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:07:41.616515+0000","last_peered":"2023-09-23T21:07:41.616515+0000","last_clean":"2023-09-23T21:07:41.616515+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:07:41.616515+0000","last_undegraded":"2023-09-23T21:07:41.616515+0000","last_fullsized":"2023-09-23T21:07:41.616515+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":51,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.612574+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:07:41.612574+0000","last_peered":"2023-09-23T21:07:41.612574+0000","last_clean":"2023-09-23T21:07:41.612574+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:07:41.612574+0000","last_undegraded":"2023-09-23T21:07:41.612574+0000","last_fullsized":"2023-09-23T21:07:41.612574+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":324,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615247+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:07:41.615247+0000","last_peered":"2023-09-23T21:07:41.615247+0000","last_clean":"2023-09-23T21:07:41.615247+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:07:41.615247+0000","last_undegraded":"2023-09-23T21:07:41.615247+0000","last_fullsized":"2023-09-23T21:07:41.615247+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":48,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.623990+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:07:41.623990+0000","last_peered":"2023-09-23T21:07:41.623990+0000","last_clean":"2023-09-23T21:07:41.623990+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:07:41.623990+0000","last_undegraded":"2023-09-23T21:07:41.623990+0000","last_fullsized":"2023-09-23T21:07:41.623990+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":69,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.613317+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:07:41.613317+0000","last_peered":"2023-09-23T21:07:41.613317+0000","last_clean":"2023-09-23T21:07:41.613317+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:07:41.613317+0000","last_undegraded":"2023-09-23T21:07:41.613317+0000","last_fullsized":"2023-09-23T21:07:41.613317+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":277,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.610799+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:07:41.610799+0000","last_peered":"2023-09-23T21:07:41.610799+0000","last_clean":"2023-09-23T21:07:41.610799+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:07:41.610799+0000","last_undegraded":"2023-09-23T21:07:41.610799+0000","last_fullsized":"2023-09-23T21:07:41.610799+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":48,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.624260+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:07:41.624260+0000","last_peered":"2023-09-23T21:07:41.624260+0000","last_clean":"2023-09-23T21:07:41.624260+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:07:41.624260+0000","last_undegraded":"2023-09-23T21:07:41.624260+0000","last_fullsized":"2023-09-23T21:07:41.624260+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":51,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.612530+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:07:41.612530+0000","last_peered":"2023-09-23T21:07:41.612530+0000","last_clean":"2023-09-23T21:07:41.612530+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:07:41.612530+0000","last_undegraded":"2023-09-23T21:07:41.612530+0000","last_fullsized":"2023-09-23T21:07:41.612530+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":34,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615158+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:07:41.615158+0000","last_peered":"2023-09-23T21:07:41.615158+0000","last_clean":"2023-09-23T21:07:41.615158+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:07:41.615158+0000","last_undegraded":"2023-09-23T21:07:41.615158+0000","last_fullsized":"2023-09-23T21:07:41.615158+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":51,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.612408+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:07:41.612408+0000","last_peered":"2023-09-23T21:07:41.612408+0000","last_clean":"2023-09-23T21:07:41.612408+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:07:41.612408+0000","last_undegraded":"2023-09-23T21:07:41.612408+0000","last_fullsized":"2023-09-23T21:07:41.612408+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":67,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.614718+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:07:41.614718+0000","last_peered":"2023-09-23T21:07:41.614718+0000","last_clean":"2023-09-23T21:07:41.614718+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:07:41.614718+0000","last_undegraded":"2023-09-23T21:07:41.614718+0000","last_fullsized":"2023-09-23T21:07:41.614718+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":43,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.616168+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:07:41.616168+0000","last_peered":"2023-09-23T21:07:41.616168+0000","last_clean":"2023-09-23T21:07:41.616168+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:07:41.616168+0000","last_undegraded":"2023-09-23T21:07:41.616168+0000","last_fullsized":"2023-09-23T21:07:41.616168+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":357,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.613396+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:07:41.613396+0000","last_peered":"2023-09-23T21:07:41.613396+0000","last_clean":"2023-09-23T21:07:41.613396+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:07:41.613396+0000","last_undegraded":"2023-09-23T21:07:41.613396+0000","last_fullsized":"2023-09-23T21:07:41.613396+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":329,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.607772+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:07:41.607772+0000","last_peered":"2023-09-23T21:07:41.607772+0000","last_clean":"2023-09-23T21:07:41.607772+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:07:41.607772+0000","last_undegraded":"2023-09-23T21:07:41.607772+0000","last_fullsized":"2023-09-23T21:07:41.607772+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":101,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.617826+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:07:41.617826+0000","last_peered":"2023-09-23T21:07:41.617826+0000","last_clean":"2023-09-23T21:07:41.617826+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:07:41.617826+0000","last_undegraded":"2023-09-23T21:07:41.617826+0000","last_fullsized":"2023-09-23T21:07:41.617826+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":60,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611483+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:07:41.611483+0000","last_peered":"2023-09-23T21:07:41.611483+0000","last_clean":"2023-09-23T21:07:41.611483+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:07:41.611483+0000","last_undegraded":"2023-09-23T21:07:41.611483+0000","last_fullsized":"2023-09-23T21:07:41.611483+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.613001+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:07:41.613001+0000","last_peered":"2023-09-23T21:07:41.613001+0000","last_clean":"2023-09-23T21:07:41.613001+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:07:41.613001+0000","last_undegraded":"2023-09-23T21:07:41.613001+0000","last_fullsized":"2023-09-23T21:07:41.613001+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.614793+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:07:41.614793+0000","last_peered":"2023-09-23T21:07:41.614793+0000","last_clean":"2023-09-23T21:07:41.614793+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:07:41.614793+0000","last_undegraded":"2023-09-23T21:07:41.614793+0000","last_fullsized":"2023-09-23T21:07:41.614793+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.607804+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:07:41.607804+0000","last_peered":"2023-09-23T21:07:41.607804+0000","last_clean":"2023-09-23T21:07:41.607804+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:07:41.607804+0000","last_undegraded":"2023-09-23T21:07:41.607804+0000","last_fullsized":"2023-09-23T21:07:41.607804+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":48,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.624053+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:07:41.624053+0000","last_peered":"2023-09-23T21:07:41.624053+0000","last_clean":"2023-09-23T21:07:41.624053+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:07:41.624053+0000","last_undegraded":"2023-09-23T21:07:41.624053+0000","last_fullsized":"2023-09-23T21:07:41.624053+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":407,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.610529+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:07:41.610529+0000","last_peered":"2023-09-23T21:07:41.610529+0000","last_clean":"2023-09-23T21:07:41.610529+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:07:41.610529+0000","last_undegraded":"2023-09-23T21:07:41.610529+0000","last_fullsized":"2023-09-23T21:07:41.610529+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":51,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611956+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:07:41.611956+0000","last_peered":"2023-09-23T21:07:41.611956+0000","last_clean":"2023-09-23T21:07:41.611956+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:07:41.611956+0000","last_undegraded":"2023-09-23T21:07:41.611956+0000","last_fullsized":"2023-09-23T21:07:41.611956+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"411'8304","reported_seq":9672,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:55.542944+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:07:55.542944+0000","last_peered":"2023-09-23T21:07:55.542944+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:07:55.542944+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"358'3528","ondisk_log_start":"358'3528","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4776,"log_dups_size":0,"ondisk_log_size":4776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8304,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-2689},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":2925}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":115,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.609515+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:07:41.609515+0000","last_peered":"2023-09-23T21:07:41.609515+0000","last_clean":"2023-09-23T21:07:41.609515+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:07:41.609515+0000","last_undegraded":"2023-09-23T21:07:41.609515+0000","last_fullsized":"2023-09-23T21:07:41.609515+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":34,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.614867+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:07:41.614867+0000","last_peered":"2023-09-23T21:07:41.614867+0000","last_clean":"2023-09-23T21:07:41.614867+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:07:41.614867+0000","last_undegraded":"2023-09-23T21:07:41.614867+0000","last_fullsized":"2023-09-23T21:07:41.614867+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"411'7900","reported_seq":7981,"reported_epoch":411,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:55.437087+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:07:55.437087+0000","last_peered":"2023-09-23T21:07:55.437087+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:07:55.437087+0000","last_undegraded":"2023-09-23T21:07:55.437087+0000","last_fullsized":"2023-09-23T21:07:55.437087+0000","mapping_epoch":375,"log_start":"358'3212","ondisk_log_start":"358'3212","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189923328,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2898,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7900,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":2898}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":109,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.621206+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:07:41.621206+0000","last_peered":"2023-09-23T21:07:41.621206+0000","last_clean":"2023-09-23T21:07:41.621206+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:07:41.621206+0000","last_undegraded":"2023-09-23T21:07:41.621206+0000","last_fullsized":"2023-09-23T21:07:41.621206+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":371,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615178+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:07:41.615178+0000","last_peered":"2023-09-23T21:07:41.615178+0000","last_clean":"2023-09-23T21:07:41.615178+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:07:41.615178+0000","last_undegraded":"2023-09-23T21:07:41.615178+0000","last_fullsized":"2023-09-23T21:07:41.615178+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"411'8040","reported_seq":17998,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:55.744993+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:07:55.744993+0000","last_peered":"2023-09-23T21:07:55.744993+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:07:55.744993+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"359'3911","ondisk_log_start":"359'3911","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189923328,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5784,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8040,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3645,"num_bytes_recovered":238878720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":511},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":6}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":38,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615862+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:07:41.615862+0000","last_peered":"2023-09-23T21:07:41.615862+0000","last_clean":"2023-09-23T21:07:41.615862+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:07:41.615862+0000","last_undegraded":"2023-09-23T21:07:41.615862+0000","last_fullsized":"2023-09-23T21:07:41.615862+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":299,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.610913+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:07:41.610913+0000","last_peered":"2023-09-23T21:07:41.610913+0000","last_clean":"2023-09-23T21:07:41.610913+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:07:41.610913+0000","last_undegraded":"2023-09-23T21:07:41.610913+0000","last_fullsized":"2023-09-23T21:07:41.610913+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"411'8131","reported_seq":8759,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:55.702564+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:07:55.702564+0000","last_peered":"2023-09-23T21:07:55.702564+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:07:55.702564+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"359'4068","ondisk_log_start":"359'4068","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4063,"log_dups_size":0,"ondisk_log_size":4063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191299584,"num_objects":2919,"num_object_clones":0,"num_object_copies":8757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8131,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-2334},{"shards":"2,6,7","objects":2916}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":51,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611527+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:07:41.611527+0000","last_peered":"2023-09-23T21:07:41.611527+0000","last_clean":"2023-09-23T21:07:41.611527+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:07:41.611527+0000","last_undegraded":"2023-09-23T21:07:41.611527+0000","last_fullsized":"2023-09-23T21:07:41.611527+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":48,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.623987+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:07:41.623987+0000","last_peered":"2023-09-23T21:07:41.623987+0000","last_clean":"2023-09-23T21:07:41.623987+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:07:41.623987+0000","last_undegraded":"2023-09-23T21:07:41.623987+0000","last_fullsized":"2023-09-23T21:07:41.623987+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"411'8327","reported_seq":19827,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:55.747982+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:07:55.747982+0000","last_peered":"2023-09-23T21:07:55.747982+0000","last_clean":"2023-09-23T21:07:55.747982+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:07:55.747982+0000","last_undegraded":"2023-09-23T21:07:55.747982+0000","last_fullsized":"2023-09-23T21:07:55.747982+0000","mapping_epoch":400,"log_start":"360'4199","ondisk_log_start":"360'4199","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4128,"log_dups_size":0,"ondisk_log_size":4128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195362816,"num_objects":2981,"num_object_clones":0,"num_object_copies":8943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8327,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":68,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611220+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:07:41.611220+0000","last_peered":"2023-09-23T21:07:41.611220+0000","last_clean":"2023-09-23T21:07:41.611220+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:07:41.611220+0000","last_undegraded":"2023-09-23T21:07:41.611220+0000","last_fullsized":"2023-09-23T21:07:41.611220+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":51,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611443+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:07:41.611443+0000","last_peered":"2023-09-23T21:07:41.611443+0000","last_clean":"2023-09-23T21:07:41.611443+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:07:41.611443+0000","last_undegraded":"2023-09-23T21:07:41.611443+0000","last_fullsized":"2023-09-23T21:07:41.611443+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"411'8031","reported_seq":19144,"reported_epoch":411,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:54.243525+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:07:54.243525+0000","last_peered":"2023-09-23T21:07:54.243525+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:07:54.243525+0000","last_undegraded":"2023-09-23T21:07:54.243525+0000","last_fullsized":"2023-09-23T21:07:54.243525+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":7543,"log_dups_size":0,"ondisk_log_size":7543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193396736,"num_objects":2951,"num_object_clones":0,"num_object_copies":8853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2951,"num_objects_unfound":0,"num_objects_dirty":2951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8031,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":2951}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":337,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.610972+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:07:41.610972+0000","last_peered":"2023-09-23T21:07:41.610972+0000","last_clean":"2023-09-23T21:07:41.610972+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:07:41.610972+0000","last_undegraded":"2023-09-23T21:07:41.610972+0000","last_fullsized":"2023-09-23T21:07:41.610972+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":421,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.616552+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:07:41.616552+0000","last_peered":"2023-09-23T21:07:41.616552+0000","last_clean":"2023-09-23T21:07:41.616552+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:07:41.616552+0000","last_undegraded":"2023-09-23T21:07:41.616552+0000","last_fullsized":"2023-09-23T21:07:41.616552+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"411'8027","reported_seq":8124,"reported_epoch":411,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:55.657920+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:07:55.657920+0000","last_peered":"2023-09-23T21:07:55.657920+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:07:55.657920+0000","last_undegraded":"2023-09-23T21:07:55.657920+0000","last_fullsized":"2023-09-23T21:07:55.657920+0000","mapping_epoch":374,"log_start":"359'3894","ondisk_log_start":"359'3894","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4133,"log_dups_size":0,"ondisk_log_size":4133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188547072,"num_objects":2877,"num_object_clones":0,"num_object_copies":8631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5754,"num_objects_unfound":0,"num_objects_dirty":2877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8027,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":2877}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":38,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.607683+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:07:41.607683+0000","last_peered":"2023-09-23T21:07:41.607683+0000","last_clean":"2023-09-23T21:07:41.607683+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:07:41.607683+0000","last_undegraded":"2023-09-23T21:07:41.607683+0000","last_fullsized":"2023-09-23T21:07:41.607683+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":450,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611024+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:07:41.611024+0000","last_peered":"2023-09-23T21:07:41.611024+0000","last_clean":"2023-09-23T21:07:41.611024+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:07:41.611024+0000","last_undegraded":"2023-09-23T21:07:41.611024+0000","last_fullsized":"2023-09-23T21:07:41.611024+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"411'8045","reported_seq":8229,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:55.498803+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:07:55.498803+0000","last_peered":"2023-09-23T21:07:55.498803+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:07:55.498803+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"358'3294","ondisk_log_start":"358'3294","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4751,"log_dups_size":0,"ondisk_log_size":4751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191168512,"num_objects":2917,"num_object_clones":0,"num_object_copies":8751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8045,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-2522},{"shards":"1,6,7","objects":2917}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":60,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615043+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:07:41.615043+0000","last_peered":"2023-09-23T21:07:41.615043+0000","last_clean":"2023-09-23T21:07:41.615043+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:07:41.615043+0000","last_undegraded":"2023-09-23T21:07:41.615043+0000","last_fullsized":"2023-09-23T21:07:41.615043+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":199,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.623237+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:07:41.623237+0000","last_peered":"2023-09-23T21:07:41.623237+0000","last_clean":"2023-09-23T21:07:41.623237+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:07:41.623237+0000","last_undegraded":"2023-09-23T21:07:41.623237+0000","last_fullsized":"2023-09-23T21:07:41.623237+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"411'8048","reported_seq":13544,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:54.261205+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:07:54.261205+0000","last_peered":"2023-09-23T21:07:54.261205+0000","last_clean":"2023-09-23T21:07:54.261205+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:07:54.261205+0000","last_undegraded":"2023-09-23T21:07:54.261205+0000","last_fullsized":"2023-09-23T21:07:54.261205+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":7539,"log_dups_size":0,"ondisk_log_size":7539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195362840,"num_objects":2982,"num_object_clones":0,"num_object_copies":8946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2982,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8048,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":305,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.617701+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:07:41.617701+0000","last_peered":"2023-09-23T21:07:41.617701+0000","last_clean":"2023-09-23T21:07:41.617701+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:07:41.617701+0000","last_undegraded":"2023-09-23T21:07:41.617701+0000","last_fullsized":"2023-09-23T21:07:41.617701+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615931+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:07:41.615931+0000","last_peered":"2023-09-23T21:07:41.615931+0000","last_clean":"2023-09-23T21:07:41.615931+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:07:41.615931+0000","last_undegraded":"2023-09-23T21:07:41.615931+0000","last_fullsized":"2023-09-23T21:07:41.615931+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.616235+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:07:41.616235+0000","last_peered":"2023-09-23T21:07:41.616235+0000","last_clean":"2023-09-23T21:07:41.616235+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:07:41.616235+0000","last_undegraded":"2023-09-23T21:07:41.616235+0000","last_fullsized":"2023-09-23T21:07:41.616235+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"411'8128","reported_seq":14128,"reported_epoch":411,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:07:55.753435+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:07:55.753435+0000","last_peered":"2023-09-23T21:07:55.753435+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:07:55.753435+0000","last_undegraded":"2023-09-23T21:07:55.753435+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"359'4057","ondisk_log_start":"359'4057","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4071,"log_dups_size":0,"ondisk_log_size":4071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189923328,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7491,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8128,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2934,"num_bytes_recovered":192282624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":246},{"shards":"2,6","objects":319},{"shards":"2,6,7","objects":2333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":386,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.616757+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:07:41.616757+0000","last_peered":"2023-09-23T21:07:41.616757+0000","last_clean":"2023-09-23T21:07:41.616757+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:07:41.616757+0000","last_undegraded":"2023-09-23T21:07:41.616757+0000","last_fullsized":"2023-09-23T21:07:41.616757+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":361,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611585+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:07:41.611585+0000","last_peered":"2023-09-23T21:07:41.611585+0000","last_clean":"2023-09-23T21:07:41.611585+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:07:41.611585+0000","last_undegraded":"2023-09-23T21:07:41.611585+0000","last_fullsized":"2023-09-23T21:07:41.611585+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":43,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.617571+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:07:41.617571+0000","last_peered":"2023-09-23T21:07:41.617571+0000","last_clean":"2023-09-23T21:07:41.617571+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:07:41.617571+0000","last_undegraded":"2023-09-23T21:07:41.617571+0000","last_fullsized":"2023-09-23T21:07:41.617571+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"411'8125","reported_seq":8305,"reported_epoch":411,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:54.272462+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:07:54.272462+0000","last_peered":"2023-09-23T21:07:54.272462+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:07:54.272462+0000","last_undegraded":"2023-09-23T21:07:54.272462+0000","last_fullsized":"2023-09-23T21:07:54.272462+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":6652,"log_dups_size":0,"ondisk_log_size":6652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192479232,"num_objects":2937,"num_object_clones":0,"num_object_copies":8811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5856,"num_objects_unfound":0,"num_objects_dirty":2937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8125,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2937}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":388,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.616697+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:07:41.616697+0000","last_peered":"2023-09-23T21:07:41.616697+0000","last_clean":"2023-09-23T21:07:41.616697+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:07:41.616697+0000","last_undegraded":"2023-09-23T21:07:41.616697+0000","last_fullsized":"2023-09-23T21:07:41.616697+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":306,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:50.549825+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:07:50.549825+0000","last_peered":"2023-09-23T21:07:50.549825+0000","last_clean":"2023-09-23T21:07:50.549825+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:07:50.549825+0000","last_undegraded":"2023-09-23T21:07:50.549825+0000","last_fullsized":"2023-09-23T21:07:50.549825+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":43,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.623107+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:07:41.623107+0000","last_peered":"2023-09-23T21:07:41.623107+0000","last_clean":"2023-09-23T21:07:41.623107+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:07:41.623107+0000","last_undegraded":"2023-09-23T21:07:41.623107+0000","last_fullsized":"2023-09-23T21:07:41.623107+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"411'7974","reported_seq":14231,"reported_epoch":411,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:55.698482+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:07:55.698482+0000","last_peered":"2023-09-23T21:07:55.698482+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:07:55.698482+0000","last_undegraded":"2023-09-23T21:07:55.698482+0000","last_fullsized":"2023-09-23T21:07:55.698482+0000","mapping_epoch":387,"log_start":"359'3880","ondisk_log_start":"359'3880","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190054400,"num_objects":2900,"num_object_clones":0,"num_object_copies":8700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2900,"num_objects_unfound":0,"num_objects_dirty":2900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7974,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","1","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2900}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":226,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.617892+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:07:41.617892+0000","last_peered":"2023-09-23T21:07:41.617892+0000","last_clean":"2023-09-23T21:07:41.617892+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:07:41.617892+0000","last_undegraded":"2023-09-23T21:07:41.617892+0000","last_fullsized":"2023-09-23T21:07:41.617892+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":708,"reported_epoch":410,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:41.618250+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:07:41.618250+0000","last_peered":"2023-09-23T21:07:41.618250+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:07:41.618250+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":60,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.612924+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:07:41.612924+0000","last_peered":"2023-09-23T21:07:41.612924+0000","last_clean":"2023-09-23T21:07:41.612924+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:07:41.612924+0000","last_undegraded":"2023-09-23T21:07:41.612924+0000","last_fullsized":"2023-09-23T21:07:41.612924+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":299,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615954+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:07:41.615954+0000","last_peered":"2023-09-23T21:07:41.615954+0000","last_clean":"2023-09-23T21:07:41.615954+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:07:41.615954+0000","last_undegraded":"2023-09-23T21:07:41.615954+0000","last_fullsized":"2023-09-23T21:07:41.615954+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"411'8229","reported_seq":8342,"reported_epoch":411,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:07:55.745092+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:07:55.745092+0000","last_peered":"2023-09-23T21:07:55.745092+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:07:55.745092+0000","last_undegraded":"2023-09-23T21:07:55.745092+0000","last_fullsized":"2023-09-23T21:07:55.745092+0000","mapping_epoch":374,"log_start":"359'4105","ondisk_log_start":"359'4105","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4124,"log_dups_size":0,"ondisk_log_size":4124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189595648,"num_objects":2893,"num_object_clones":0,"num_object_copies":8679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5786,"num_objects_unfound":0,"num_objects_dirty":2893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8229,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":2893}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":43,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.618138+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:07:41.618138+0000","last_peered":"2023-09-23T21:07:41.618138+0000","last_clean":"2023-09-23T21:07:41.618138+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:07:41.618138+0000","last_undegraded":"2023-09-23T21:07:41.618138+0000","last_fullsized":"2023-09-23T21:07:41.618138+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":57,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.625911+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:07:41.625911+0000","last_peered":"2023-09-23T21:07:41.625911+0000","last_clean":"2023-09-23T21:07:41.625911+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:07:41.625911+0000","last_undegraded":"2023-09-23T21:07:41.625911+0000","last_fullsized":"2023-09-23T21:07:41.625911+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":48,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.623681+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:07:41.623681+0000","last_peered":"2023-09-23T21:07:41.623681+0000","last_clean":"2023-09-23T21:07:41.623681+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:07:41.623681+0000","last_undegraded":"2023-09-23T21:07:41.623681+0000","last_fullsized":"2023-09-23T21:07:41.623681+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"411'8083","reported_seq":13327,"reported_epoch":411,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:55.548119+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:07:55.548119+0000","last_peered":"2023-09-23T21:07:55.548119+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:07:55.548119+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:07:55.548119+0000","mapping_epoch":383,"log_start":"358'3326","ondisk_log_start":"358'3326","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4757,"log_dups_size":0,"ondisk_log_size":4757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192872448,"num_objects":2943,"num_object_clones":0,"num_object_copies":8829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2939,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":2943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8083,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2571,"num_bytes_recovered":168493056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":4},{"shards":"1,2,7","objects":2939}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":48,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.624629+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:07:41.624629+0000","last_peered":"2023-09-23T21:07:41.624629+0000","last_clean":"2023-09-23T21:07:41.624629+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:07:41.624629+0000","last_undegraded":"2023-09-23T21:07:41.624629+0000","last_fullsized":"2023-09-23T21:07:41.624629+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":307,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.618555+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:07:41.618555+0000","last_peered":"2023-09-23T21:07:41.618555+0000","last_clean":"2023-09-23T21:07:41.618555+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:07:41.618555+0000","last_undegraded":"2023-09-23T21:07:41.618555+0000","last_fullsized":"2023-09-23T21:07:41.618555+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":86,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.615350+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:07:41.615350+0000","last_peered":"2023-09-23T21:07:41.615350+0000","last_clean":"2023-09-23T21:07:41.615350+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:07:41.615350+0000","last_undegraded":"2023-09-23T21:07:41.615350+0000","last_fullsized":"2023-09-23T21:07:41.615350+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"411'8056","reported_seq":20946,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:55.532711+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:07:55.532711+0000","last_peered":"2023-09-23T21:07:55.532711+0000","last_clean":"2023-09-23T21:07:55.532711+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:07:55.532711+0000","last_undegraded":"2023-09-23T21:07:55.532711+0000","last_fullsized":"2023-09-23T21:07:55.532711+0000","mapping_epoch":397,"log_start":"358'3345","ondisk_log_start":"358'3345","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4711,"log_dups_size":0,"ondisk_log_size":4711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184025088,"num_objects":2808,"num_object_clones":0,"num_object_copies":8424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8056,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":68,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.612004+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:07:41.612004+0000","last_peered":"2023-09-23T21:07:41.612004+0000","last_clean":"2023-09-23T21:07:41.612004+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:07:41.612004+0000","last_undegraded":"2023-09-23T21:07:41.612004+0000","last_fullsized":"2023-09-23T21:07:41.612004+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":43,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.620804+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:07:41.620804+0000","last_peered":"2023-09-23T21:07:41.620804+0000","last_clean":"2023-09-23T21:07:41.620804+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:07:41.620804+0000","last_undegraded":"2023-09-23T21:07:41.620804+0000","last_fullsized":"2023-09-23T21:07:41.620804+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":60,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.612380+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:07:41.612380+0000","last_peered":"2023-09-23T21:07:41.612380+0000","last_clean":"2023-09-23T21:07:41.612380+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:07:41.612380+0000","last_undegraded":"2023-09-23T21:07:41.612380+0000","last_fullsized":"2023-09-23T21:07:41.612380+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"411'8242","reported_seq":13645,"reported_epoch":411,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:07:55.500974+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:07:55.500974+0000","last_peered":"2023-09-23T21:07:55.500974+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:07:55.500974+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:07:55.500974+0000","mapping_epoch":382,"log_start":"358'3512","ondisk_log_start":"358'3512","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4730,"log_dups_size":0,"ondisk_log_size":4730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193724416,"num_objects":2956,"num_object_clones":0,"num_object_copies":8868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2953,"num_objects_misplaced":2959,"num_objects_unfound":0,"num_objects_dirty":2956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8242,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2643,"num_bytes_recovered":173211648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":2953}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":307,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.610868+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:07:41.610868+0000","last_peered":"2023-09-23T21:07:41.610868+0000","last_clean":"2023-09-23T21:07:41.610868+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:07:41.610868+0000","last_undegraded":"2023-09-23T21:07:41.610868+0000","last_fullsized":"2023-09-23T21:07:41.610868+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":52,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.623811+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:07:41.623811+0000","last_peered":"2023-09-23T21:07:41.623811+0000","last_clean":"2023-09-23T21:07:41.623811+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:07:41.623811+0000","last_undegraded":"2023-09-23T21:07:41.623811+0000","last_fullsized":"2023-09-23T21:07:41.623811+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":96,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.614914+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:07:41.614914+0000","last_peered":"2023-09-23T21:07:41.614914+0000","last_clean":"2023-09-23T21:07:41.614914+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:07:41.614914+0000","last_undegraded":"2023-09-23T21:07:41.614914+0000","last_fullsized":"2023-09-23T21:07:41.614914+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":57,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.623939+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:07:41.623939+0000","last_peered":"2023-09-23T21:07:41.623939+0000","last_clean":"2023-09-23T21:07:41.623939+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:07:41.623939+0000","last_undegraded":"2023-09-23T21:07:41.623939+0000","last_fullsized":"2023-09-23T21:07:41.623939+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":434,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.617343+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:07:41.617343+0000","last_peered":"2023-09-23T21:07:41.617343+0000","last_clean":"2023-09-23T21:07:41.617343+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:07:41.617343+0000","last_undegraded":"2023-09-23T21:07:41.617343+0000","last_fullsized":"2023-09-23T21:07:41.617343+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":73,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.608289+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:07:41.608289+0000","last_peered":"2023-09-23T21:07:41.608289+0000","last_clean":"2023-09-23T21:07:41.608289+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:07:41.608289+0000","last_undegraded":"2023-09-23T21:07:41.608289+0000","last_fullsized":"2023-09-23T21:07:41.608289+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611306+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:07:41.611306+0000","last_peered":"2023-09-23T21:07:41.611306+0000","last_clean":"2023-09-23T21:07:41.611306+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:07:41.611306+0000","last_undegraded":"2023-09-23T21:07:41.611306+0000","last_fullsized":"2023-09-23T21:07:41.611306+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":292,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.617497+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:07:41.617497+0000","last_peered":"2023-09-23T21:07:41.617497+0000","last_clean":"2023-09-23T21:07:41.617497+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:07:41.617497+0000","last_undegraded":"2023-09-23T21:07:41.617497+0000","last_fullsized":"2023-09-23T21:07:41.617497+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":69,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.613068+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:07:41.613068+0000","last_peered":"2023-09-23T21:07:41.613068+0000","last_clean":"2023-09-23T21:07:41.613068+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:07:41.613068+0000","last_undegraded":"2023-09-23T21:07:41.613068+0000","last_fullsized":"2023-09-23T21:07:41.613068+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":51,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.611841+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:07:41.611841+0000","last_peered":"2023-09-23T21:07:41.611841+0000","last_clean":"2023-09-23T21:07:41.611841+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:07:41.611841+0000","last_undegraded":"2023-09-23T21:07:41.611841+0000","last_fullsized":"2023-09-23T21:07:41.611841+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":60,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.612222+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:07:41.612222+0000","last_peered":"2023-09-23T21:07:41.612222+0000","last_clean":"2023-09-23T21:07:41.612222+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:07:41.612222+0000","last_undegraded":"2023-09-23T21:07:41.612222+0000","last_fullsized":"2023-09-23T21:07:41.612222+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":54,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.624146+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:07:41.624146+0000","last_peered":"2023-09-23T21:07:41.624146+0000","last_clean":"2023-09-23T21:07:41.624146+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:07:41.624146+0000","last_undegraded":"2023-09-23T21:07:41.624146+0000","last_fullsized":"2023-09-23T21:07:41.624146+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T21:07:41.607246+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:07:41.607246+0000","last_peered":"2023-09-23T21:07:41.607246+0000","last_clean":"2023-09-23T21:07:41.607246+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:07:41.607246+0000","last_undegraded":"2023-09-23T21:07:41.607246+0000","last_fullsized":"2023-09-23T21:07:41.607246+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3059417112,"num_objects":46684,"num_object_clones":0,"num_object_copies":140052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27983,"num_objects_misplaced":36599,"num_objects_unfound":0,"num_objects_dirty":46684,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129690,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30496,"num_bytes_recovered":1998585856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11438665728,"data_stored":11438653512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78889,"ondisk_log_size":78889,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201751,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463371,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2588260,"kb_used_data":2073180,"kb_used_omap":0,"kb_used_meta":515072,"kb_avail":91156892,"statfs":{"total":95995035648,"available":93344657408,"internally_reserved":0,"allocated":2122936320,"data_stored":2121727992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":527433728},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":6.6479999999999997,"5min":2.0830000000000002,"15min":2.2610000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":99.533000000000001,"5min":99.533000000000001,"15min":102.242},"last":0.24099999999999999},{"interface":"front","average":{"1min":6.8070000000000004,"5min":2.1949999999999998,"15min":2.0470000000000002},"min":{"1min":0.248,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":99.325000000000003,"5min":99.325000000000003,"15min":102.502},"last":1.4239999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":3.9460000000000002,"15min":2.379},"min":{"1min":0.251,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":11.743,"5min":119.077,"15min":119.077},"last":2.9089999999999998},{"interface":"front","average":{"1min":1.901,"5min":3.7730000000000001,"15min":2.452},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":11.936,"5min":119.313,"15min":119.313},"last":0.627}]},{"osd":2,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":13.304,"5min":6.1219999999999999,"15min":3.726},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":199.23400000000001,"5min":199.23400000000001,"15min":199.23400000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":12.943,"5min":6.1870000000000003,"15min":3.8719999999999999},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.08000000000001},"last":2.556}]},{"osd":3,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":0.98599999999999999,"15min":4.0119999999999996},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":3.194,"5min":5.9420000000000002,"15min":250.804},"last":13.340999999999999},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.45,"15min":4.0099999999999998},"min":{"1min":0.371,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":13.319000000000001,"5min":13.319000000000001,"15min":250.446},"last":2.3599999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.353,"15min":0.97099999999999997},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.24299999999999999},"max":{"1min":6.3689999999999998,"5min":13.1,"15min":13.1},"last":1.3600000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.98699999999999999,"15min":0.95199999999999996},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":4.3849999999999998,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.333,"15min":1.105},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":6.343,"5min":11.281000000000001,"15min":18.962},"last":1.397},{"interface":"front","average":{"1min":1.325,"5min":1.286,"15min":1.212},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":6.6890000000000001,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.2569999999999999,"15min":1.4710000000000001},"min":{"1min":0.34899999999999998,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.8079999999999998,"5min":7.8079999999999998,"15min":101.04600000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.2509999999999999,"15min":1.5029999999999999},"min":{"1min":0.39200000000000002,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":4.407,"5min":7.1849999999999996,"15min":101.081},"last":0.52600000000000002}]}]},{"osd":1,"up_from":276,"seq":1185410973817,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2750008,"kb_used_data":2321840,"kb_used_omap":5,"kb_used_meta":428090,"kb_avail":90995144,"statfs":{"total":95995035648,"available":93179027456,"internally_reserved":0,"allocated":2377564160,"data_stored":2375516439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":438364899},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":0.85599999999999998,"15min":0.55200000000000005},"min":{"1min":0.20599999999999999,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":9.9469999999999992,"5min":11.959,"15min":11.959},"last":1.4159999999999999},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.256,"15min":0.81499999999999995},"min":{"1min":0.216,"5min":0.193,"15min":0.193},"max":{"1min":5.093,"5min":18.047999999999998,"15min":18.047999999999998},"last":2.3929999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.407,"15min":1.1359999999999999},"min":{"1min":0.23200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.0670000000000002,"5min":17.254000000000001,"15min":78.409000000000006},"last":2.3719999999999999},{"interface":"front","average":{"1min":1.55,"5min":1.03,"15min":0.97999999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.22},"max":{"1min":12.478999999999999,"5min":12.478999999999999,"15min":76.245000000000005},"last":1.4370000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.849,"15min":2.4670000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":7.3099999999999996,"5min":74.527000000000001,"15min":172.44399999999999},"last":1.397},{"interface":"front","average":{"1min":1.907,"5min":2.5369999999999999,"15min":2.6880000000000002},"min":{"1min":0.28599999999999998,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":7.9050000000000002,"5min":72.216999999999999,"15min":172.58500000000001},"last":2.3500000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.472,"15min":0.93400000000000005},"min":{"1min":0.32500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.7069999999999999,"5min":17.006,"15min":17.006},"last":2.173},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.92600000000000005,"15min":0.70299999999999996},"min":{"1min":0.35699999999999998,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":4.7690000000000001,"5min":11.916,"15min":11.916},"last":1.3460000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.446,"15min":0.88300000000000001},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.6269999999999998,"5min":17.155000000000001,"15min":17.155000000000001},"last":2.3260000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.589,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.3860000000000001,"5min":17.076000000000001,"15min":17.076000000000001},"last":2.21}]},{"osd":6,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.577,"15min":1.3879999999999999},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.6020000000000003,"5min":17.102,"15min":127.995},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.127,"5min":1.647,"15min":1.466},"min":{"1min":0.41699999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.375,"5min":17.302,"15min":126.836},"last":2.2570000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.9180000000000001,"15min":1.484},"min":{"1min":0.442,"5min":0.38200000000000001,"15min":0.36499999999999999},"max":{"1min":7.048,"5min":151.791,"15min":151.791},"last":2.2330000000000001},{"interface":"front","average":{"1min":1.49,"5min":2.3889999999999998,"15min":1.2310000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.3739999999999997,"5min":151.809,"15min":151.809},"last":1.375}]}]},{"osd":0,"up_from":92,"seq":395136991557,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1160756,"kb_used_data":420268,"kb_used_omap":5,"kb_used_meta":740474,"kb_avail":92584396,"statfs":{"total":95995035648,"available":94806421504,"internally_reserved":0,"allocated":430354432,"data_stored":428424543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":758246010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:06:58 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":2.9500000000000002,"15min":1.4830000000000001},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":40.284999999999997,"5min":61.341999999999999,"15min":61.341999999999999},"last":1.4019999999999999},{"interface":"front","average":{"1min":3.4319999999999999,"5min":2.9670000000000001,"15min":1.4510000000000001},"min":{"1min":0.223,"5min":0.217,"15min":0.217},"max":{"1min":40.213000000000001,"5min":61.299999999999997,"15min":61.299999999999997},"last":1.7569999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":5.468,"5min":2.4609999999999999,"15min":2.4790000000000001},"min":{"1min":0.24099999999999999,"5min":0.217,"15min":0.161},"max":{"1min":45.912999999999997,"5min":45.912999999999997,"15min":183.77099999999999},"last":1.9339999999999999},{"interface":"front","average":{"1min":5.2930000000000001,"5min":2.5070000000000001,"15min":2.6949999999999998},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":46.234000000000002,"5min":46.234000000000002,"15min":183.78700000000001},"last":2}]},{"osd":3,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.276,"15min":3.1070000000000002},"min":{"1min":0.28999999999999998,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":12.113,"5min":12.113,"15min":186.143},"last":4.7300000000000004},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.1759999999999999,"15min":3.1629999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":12.348000000000001,"5min":12.348000000000001,"15min":186.97399999999999},"last":4.8129999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":1.2789999999999999,"15min":1.3600000000000001},"min":{"1min":0.376,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":10.814,"5min":10.814,"15min":10.814},"last":4.9059999999999997},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.03,"15min":0.96799999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.586,"5min":10.586,"15min":11.493},"last":1.538}]},{"osd":5,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.454,"15min":2.0590000000000002},"min":{"1min":0.34399999999999997,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":10.763,"5min":10.763,"15min":154.30600000000001},"last":4.8639999999999999},{"interface":"front","average":{"1min":2.129,"5min":1.147,"15min":1.5189999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28699999999999998},"max":{"1min":10.618,"5min":11.478,"15min":154.42400000000001},"last":1.4790000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":5.9500000000000002,"15min":3.214},"min":{"1min":0.35199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.654,"5min":170.12700000000001,"15min":170.12700000000001},"last":1.5109999999999999},{"interface":"front","average":{"1min":1.494,"5min":5.9000000000000004,"15min":3.3380000000000001},"min":{"1min":0.39900000000000002,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":5.7750000000000004,"5min":169.767,"15min":169.767},"last":1.827}]},{"osd":7,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.347,"15min":2.2509999999999999},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.74,"5min":10.74,"15min":164.26400000000001},"last":2.4580000000000002},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.0720000000000001,"15min":2.2549999999999999},"min":{"1min":0.44,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":5.8159999999999998,"5min":5.8159999999999998,"15min":164.23099999999999},"last":2.4809999999999999}]}]},{"osd":2,"up_from":129,"seq":554050781448,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2754232,"kb_used_data":2127408,"kb_used_omap":7,"kb_used_meta":626808,"kb_avail":90990920,"statfs":{"total":95995035648,"available":93174702080,"internally_reserved":0,"allocated":2178465792,"data_stored":2176474361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":641851473},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.4450000000000001,"15min":1.3959999999999999},"min":{"1min":0.20499999999999999,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":6.8440000000000003,"5min":15.667,"15min":26.356000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":6.4569999999999999,"5min":2.0830000000000002,"15min":1.8320000000000001},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":23.963999999999999,"5min":23.963999999999999,"15min":26.341000000000001},"last":13.734999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:06:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":4.9199999999999999,"15min":2.379},"min":{"1min":0.23599999999999999,"5min":0.19,"15min":0.188},"max":{"1min":4.9089999999999998,"5min":212.12,"15min":212.12},"last":12.680999999999999},{"interface":"front","average":{"1min":1.232,"5min":4.9390000000000001,"15min":2.1890000000000001},"min":{"1min":0.24299999999999999,"5min":0.214,"15min":0.214},"max":{"1min":6.4219999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":0.63300000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:07:47 2023","interfaces":[{"interface":"back","average":{"1min":10.468999999999999,"5min":2.8500000000000001,"15min":3.2440000000000002},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":138.90299999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":12.452},{"interface":"front","average":{"1min":10.629,"5min":2.8900000000000001,"15min":3.2090000000000001},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":138.87700000000001,"5min":138.87700000000001,"15min":138.87700000000001},"last":0.27600000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":1.153,"15min":1.113},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":6.383,"5min":12.586,"15min":12.586},"last":0.33300000000000002},{"interface":"front","average":{"1min":4.1820000000000004,"5min":1.5700000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":14.672000000000001,"5min":14.672000000000001,"15min":14.672000000000001},"last":13.824999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":4.367,"5min":1.7050000000000001,"15min":2.129},"min":{"1min":0.36799999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":14.710000000000001,"5min":14.710000000000001,"15min":173.34},"last":13.779999999999999},{"interface":"front","average":{"1min":2.097,"5min":1.173,"15min":1.796},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":8.9740000000000002,"5min":13.701000000000001,"15min":173.15000000000001},"last":12.368}]},{"osd":6,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":3.3940000000000001,"15min":1.946},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":9.0329999999999995,"5min":175.72499999999999,"15min":175.72499999999999},"last":12.323},{"interface":"front","average":{"1min":1.901,"5min":3.375,"15min":1.9550000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.5049999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.433}]},{"osd":7,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.655,"15min":2.081},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":6.4809999999999999,"5min":12.609,"15min":177.30799999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":4.2240000000000002,"5min":1.645,"15min":2.1989999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33600000000000002},"max":{"1min":14.736000000000001,"5min":14.736000000000001,"15min":177.40899999999999},"last":13.804}]}]},{"osd":3,"up_from":148,"seq":635655160042,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":555640,"kb_used_data":6832,"kb_used_omap":2,"kb_used_meta":548797,"kb_avail":93189512,"statfs":{"total":95995035648,"available":95426060288,"internally_reserved":0,"allocated":6995968,"data_stored":4824319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":561968181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":1.4910000000000001,"15min":1.774},"min":{"1min":0.249,"5min":0.20999999999999999,"15min":0.184},"max":{"1min":8.8629999999999995,"5min":44.963000000000001,"15min":95.507999999999996},"last":0.20300000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":0.94799999999999995,"15min":1.6679999999999999},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.17199999999999999},"max":{"1min":7.7850000000000001,"5min":9.7370000000000001,"15min":95.259},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:06:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4330000000000001,"5min":1.347,"15min":1.0580000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":7.7619999999999996,"5min":12.035,"15min":19.786999999999999},"last":1.5620000000000001},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.661,"15min":1.292},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":8.3059999999999992,"5min":15.763999999999999,"15min":15.763999999999999},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4129999999999998,"5min":1.4750000000000001,"15min":1.351},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.182},"max":{"1min":14.243,"5min":14.243,"15min":14.243},"last":0.72599999999999998},{"interface":"front","average":{"1min":2.6059999999999999,"5min":1.325,"15min":1.3420000000000001},"min":{"1min":0.34399999999999997,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":7.8120000000000003,"5min":9.7140000000000004,"15min":19.754999999999999},"last":0.751}]},{"osd":4,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4980000000000002,"5min":2.5470000000000002,"15min":2.2559999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.222},"max":{"1min":38.600000000000001,"5min":70.728999999999999,"15min":111.36},"last":0.59699999999999998},{"interface":"front","average":{"1min":3.7919999999999998,"5min":2.4529999999999998,"15min":2.0539999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.252},"max":{"1min":38.524000000000001,"5min":70.712000000000003,"15min":112.473},"last":0.70499999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":1.575,"15min":1.581},"min":{"1min":0.41199999999999998,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":36.488999999999997,"5min":36.488999999999997,"15min":64.528000000000006},"last":6.8079999999999998},{"interface":"front","average":{"1min":3.5110000000000001,"5min":1.6879999999999999,"15min":1.647},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":36.295000000000002,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.3740000000000001,"15min":1.623},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.312},"max":{"1min":10.827999999999999,"5min":10.827999999999999,"15min":19.568000000000001},"last":0.624},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.448,"15min":1.5029999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":7.6749999999999998,"5min":14.356,"15min":15.573},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.3599999999999999,"15min":2.5510000000000002},"min":{"1min":0.5,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":7.7370000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":1.5980000000000001},{"interface":"front","average":{"1min":2.2549999999999999,"5min":2.3530000000000002,"15min":2.8170000000000002},"min":{"1min":0.48099999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":7.4249999999999998,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.64500000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986661,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1294052,"kb_used_data":733468,"kb_used_omap":0,"kb_used_meta":560576,"kb_avail":92451100,"statfs":{"total":95995035648,"available":94669926400,"internally_reserved":0,"allocated":751071232,"data_stored":749862288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":574029824},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.4770000000000001,"15min":2.6949999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":12.869,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.88200000000000001},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.2849999999999999,"15min":2.6920000000000002},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.223},"max":{"1min":12.848000000000001,"5min":12.848000000000001,"15min":144.77000000000001},"last":3.7269999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":12.298,"5min":8.5619999999999994,"15min":3.3940000000000001},"min":{"1min":0.34899999999999998,"5min":0.26900000000000002,"15min":0.23200000000000001},"max":{"1min":127.411,"5min":127.411,"15min":127.411},"last":0.26700000000000002},{"interface":"front","average":{"1min":4.8810000000000002,"5min":6.7430000000000003,"15min":2.758},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":65.688999999999993,"5min":118.075,"15min":118.075},"last":0.71999999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":6.7910000000000004,"15min":3.5299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":71.454999999999998,"5min":164.517,"15min":164.517},"last":3.8330000000000002},{"interface":"front","average":{"1min":5.343,"5min":6.282,"15min":3.5099999999999998},"min":{"1min":0.34899999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":71.191999999999993,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.379}]},{"osd":3,"last update":"Sat Sep 23 21:07:08 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":1.2170000000000001,"15min":2.1280000000000001},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.27300000000000002},"max":{"1min":2.2879999999999998,"5min":8.7420000000000009,"15min":142.28399999999999},"last":1.972},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.177,"15min":2.3029999999999999},"min":{"1min":0.35299999999999998,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":8.2899999999999991,"5min":8.2899999999999991,"15min":142.34700000000001},"last":2.0859999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.81999999999999995,"15min":1.052},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":2.2629999999999999,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.042,"5min":0.96799999999999997,"15min":1.1220000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.21099999999999999},"max":{"1min":4.282,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.1779999999999999,"15min":0.98199999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":4.8399999999999999,"5min":11.800000000000001,"15min":11.887},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.2410000000000001,"15min":0.93700000000000006},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0419999999999998,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.601,"15min":1.6040000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":4.3129999999999997,"5min":145.721,"15min":145.721},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.347,"5min":2.9489999999999998,"15min":1.776},"min":{"1min":0.38500000000000001,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":4.7969999999999997,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.40000000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725025,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986320,"kb_used_data":371656,"kb_used_omap":0,"kb_used_meta":614656,"kb_avail":92758832,"statfs":{"total":95995035648,"available":94985043968,"internally_reserved":0,"allocated":380575744,"data_stored":379383000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":629407744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":0.86399999999999999,"15min":1.325},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":10.488,"5min":10.488,"15min":80.105999999999995},"last":2.3639999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.90700000000000003,"15min":1.446},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":4.75,"5min":8.2759999999999998,"15min":80.152000000000001},"last":2.145}]},{"osd":1,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":13.016999999999999,"5min":6.4710000000000001,"15min":2.512},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":233.77099999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":13.031000000000001,"5min":6.1769999999999996,"15min":2.4220000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.247},"max":{"1min":233.75200000000001,"5min":233.75200000000001,"15min":233.75200000000001},"last":2.3220000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":15.840999999999999,"5min":3.7839999999999998,"15min":2.4689999999999999},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":231.34700000000001,"5min":231.34700000000001,"15min":231.34700000000001},"last":2.3900000000000001},{"interface":"front","average":{"1min":15.951000000000001,"5min":4.008,"15min":2.6110000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":231.381,"5min":231.381,"15min":231.381},"last":2.8340000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":1.3260000000000001,"15min":2.7959999999999998},"min":{"1min":0.38500000000000001,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":7.0300000000000002,"5min":10.670999999999999,"15min":153.60300000000001},"last":2.1930000000000001},{"interface":"front","average":{"1min":1.653,"5min":1.0580000000000001,"15min":2.6880000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":4.1779999999999999,"5min":7.3810000000000002,"15min":153.774},"last":2.4159999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59999999999999998,"15min":0.77600000000000002},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":2.2320000000000002,"5min":3.3029999999999999,"15min":12.739000000000001},"last":2.4529999999999998},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.88900000000000001,"15min":0.98299999999999998},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.247},"max":{"1min":3.645,"5min":8.3740000000000006,"15min":12.539},"last":2.222}]},{"osd":6,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.1160000000000001,"15min":1.1539999999999999},"min":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":3.6760000000000002,"5min":9.8480000000000008,"15min":71.230000000000004},"last":2.29},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.25,"15min":1.286},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":4.0880000000000001,"5min":12.673999999999999,"15min":70.266000000000005},"last":2.3399999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.143,"15min":1.2010000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":5.0119999999999996,"5min":8.4109999999999996,"15min":31.091999999999999},"last":2.1789999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65800000000000003,"15min":1.054},"min":{"1min":0.38500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.5339999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":2.4750000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":421724160,"data_stored":421724160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2350252032,"data_stored":2350252032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2159874048,"data_stored":2159869976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1572864,"data_stored":1572864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":742264832,"data_stored":742260760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":375521280,"data_stored":375521280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2100695040,"data_stored":2100690968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3286761472,"data_stored":3286761472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:08:01.808 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:08:02.064 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:02.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:02.173 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:02.203 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 27356/132375 objects degraded (20.666%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:02.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 27356/132375 objects degraded (20.666%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:02.670 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 27356/132375 objects degraded (20.666%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:02.854 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:02.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:02.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:02.960 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:03.002 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:03.073 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:03.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:03.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:03.182 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:03.213 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:03.352 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:08:03.466 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:03 smithi121 ceph-mon[211526]: pgmap v1306: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 27356/132375 objects degraded (20.666%); 34508/132375 objects misplaced (26.068%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:03.466 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:03 smithi121 ceph-mon[219709]: pgmap v1306: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 27356/132375 objects degraded (20.666%); 34508/132375 objects misplaced (26.068%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:03 smithi123 ceph-mon[193903]: pgmap v1306: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 27356/132375 objects degraded (20.666%); 34508/132375 objects misplaced (26.068%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:03.555 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:03.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:03.667 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:03.820 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:03.967 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:03.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:04.074 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:04.192 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:04.208 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:04.423 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:04.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:04.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:04.534 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:04.606 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:110359 10.16 10:6ce1aa55:::benchmark_data_smithi121_358375_object25614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 473.75885787099998, 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7173976529999999, 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.607 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 110359 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832684+0000", 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832682+0000", 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832691+0000", 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0270000000000001e-06 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832711+0000", 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.999e-05 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832721+0000", 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0033e-05 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832788+0000", 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7268999999999996e-05 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:10.832802+0000", 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3319000000000001e-05 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:11.299916+0000", 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.467114749 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:11.299930+0000", 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3368e-05 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.542785+0000", 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2428551299999999 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.542973+0000", 2023-09-23T21:08:04.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018853699999999999 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.543153+0000", 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000180065 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548645+0000", 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054919060000000004 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548699+0000", 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4026000000000001e-05 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548751+0000", 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2011000000000001e-05 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.548821+0000", 2023-09-23T21:08:04.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.97e-05 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:00:12.550081+0000", 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001260401 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151000 10.16 10:6fb6daa8:::benchmark_data_smithi121_358375_object66255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T21:08:04.615 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 365.18187082899999, 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.019180497, 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151000 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409669+0000", 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409680+0000", 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1647e-05 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409699+0000", 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8763e-05 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409711+0000", 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409930+0000", 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021934400000000001 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410095+0000", 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016493700000000001 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410262+0000", 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016674600000000001 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.415289+0000", 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050267660000000002 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428299+0000", 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013010763 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428313+0000", 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4064e-05 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428836+0000", 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0005229959999999 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428851+0000", 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4960000000000001e-05 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151024 10.6 10.8ded8286 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:08:04.622 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 365.09261010799997, 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.930075294, 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151024 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:08:04.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498930+0000", 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498938+0000", 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3890000000000005e-06 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498955+0000", 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6996000000000001e-05 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498965+0000", 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0629999999999995e-06 2023-09-23T21:08:04.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499015+0000", 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0868000000000002e-05 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499026+0000", 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0883999999999999e-05 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428989+0000", 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92996320600000004 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429007+0000", 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7419e-05 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.627 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151064 10.16 10.14ca33f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.955844345, 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79318550200000004, 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151064 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635696+0000", 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635703+0000", 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8220000000000001e-06 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635720+0000", 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7116999999999999e-05 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635729+0000", 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1929999999999999e-06 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635796+0000", 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7163999999999996e-05 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635810+0000", 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4469e-05 2023-09-23T21:08:04.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428874+0000", 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79306313900000003 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428883+0000", 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2390000000000008e-06 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.632 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151070 10.6 10.e7155d86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.90515864299999, 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74265465799999997, 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151070 2023-09-23T21:08:04.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686381+0000", 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686389+0000", 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.204e-06 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686407+0000", 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8130000000000001e-05 2023-09-23T21:08:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686416+0000", 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2369999999999998e-06 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686477+0000", 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1432000000000004e-05 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686488+0000", 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1026000000000001e-05 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429028+0000", 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74253917700000005 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429038+0000", 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.007e-05 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.637 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151071 10.16 10.6fc596f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.89792270200002, 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73529290700000005, 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.638 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151071 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693617+0000", 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693627+0000", 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4120000000000007e-06 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693644+0000", 2023-09-23T21:08:04.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7669999999999999e-05 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693654+0000", 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1980000000000007e-06 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693712+0000", 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8579999999999998e-05 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693726+0000", 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3847e-05 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428903+0000", 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73517710400000003 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428912+0000", 2023-09-23T21:08:04.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7469999999999994e-06 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151082 10.16 10.dd3bf356 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.87790460899998, 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71530539400000004, 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.643 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151082 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:08:04.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713635+0000", 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713645+0000", 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0113e-05 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713670+0000", 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4507000000000001e-05 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713679+0000", 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1710000000000008e-06 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713749+0000", 2023-09-23T21:08:04.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0607999999999999e-05 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713761+0000", 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1841999999999999e-05 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428934+0000", 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71517288000000001 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428942+0000", 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3040000000000005e-06 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151084 10.6 10.dfef7e86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.87115667400002, 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70867917800000002, 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.648 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151084 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720383+0000", 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720392+0000", 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1699999999999997e-06 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720411+0000", 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9681e-05 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720422+0000", 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0587999999999999e-05 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720488+0000", 2023-09-23T21:08:04.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.648e-05 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720504+0000", 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6172999999999999e-05 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429056+0000", 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70855175100000001 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429064+0000", 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0150000000000003e-06 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151096 10.16 10.c63710d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.80108649599998, 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63851427199999999, 2023-09-23T21:08:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151096 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790453+0000", 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790471+0000", 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7876999999999998e-05 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790488+0000", 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6642000000000001e-05 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790498+0000", 2023-09-23T21:08:04.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6569999999999992e-06 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790553+0000", 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5707e-05 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790563+0000", 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6520000000000001e-06 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428961+0000", 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63839818400000004 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428969+0000", 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3799999999999994e-06 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.658 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151153 10.6 10.b41dbb46 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 364.550014154, 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.387565296, 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151153 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041526+0000", 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041533+0000", 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1250000000000004e-06 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041546+0000", 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2289e-05 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041552+0000", 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041613+0000", 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0640000000000002e-05 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041623+0000", 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0309000000000001e-05 2023-09-23T21:08:04.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429084+0000", 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.38746082799999998 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429093+0000", 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6910000000000003e-06 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:102329 11.b e406/380 11:d9bc96de:::benchmark_data_smithi121_477989_object14140:head v 406'6390, mlcod=0'0)", 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:01.136486+0000", 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 63.455055594999997, 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17147641799999999, 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136486+0000", 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136486+0000", 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136485+0000", 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136498+0000", 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2951e-05 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136515+0000", 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7788e-05 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136526+0000", 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0813e-05 2023-09-23T21:08:04.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136597+0000", 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0481000000000005e-05 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.136623+0000", 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5897999999999998e-05 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.142913+0000", 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062904290000000002 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.307963+0000", 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16504951700000001 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:102330 11.f e406/397 11:f13df89c:::benchmark_data_smithi121_477989_object14141:head v 406'6350, mlcod=406'6348)", 2023-09-23T21:08:04.668 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:01.138765+0000", 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 63.452776938, 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17288621800000001, 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138765+0000", 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138765+0000", 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138764+0000", 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138772+0000", 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4249999999999998e-06 2023-09-23T21:08:04.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138785+0000", 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2486999999999999e-05 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138793+0000", 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7160000000000003e-06 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138834+0000", 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1726e-05 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.138856+0000", 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1330000000000001e-05 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.311618+0000", 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17276265499999999 2023-09-23T21:08:04.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.311651+0000", 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2697000000000002e-05 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:102302 11.6 e406/375 11:6f1629c5:::benchmark_data_smithi121_477989_object14113:head v 406'6263, mlcod=406'6261)", 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:01.142535+0000", 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 63.449006222999998, 2023-09-23T21:08:04.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17176377500000001, 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.142535+0000", 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.142535+0000", 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.142534+0000", 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.142545+0000", 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0730999999999999e-05 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.307854+0000", 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16530926400000001 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.307869+0000", 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4352000000000001e-05 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.308021+0000", 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015282300000000001 2023-09-23T21:08:04.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.308046+0000", 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4584e-05 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.314241+0000", 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061950690000000001 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.314299+0000", 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8242000000000002e-05 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:102335 11.1 e406/374 11:887774db:::benchmark_data_smithi121_477989_object14146:head v 406'6288, mlcod=406'6286)", 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:01.147313+0000", 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 63.444228567000003, 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16458046700000001, 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.678 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.147313+0000", 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.147313+0000", 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.147312+0000", 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.147321+0000", 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9129999999999992e-06 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.307885+0000", 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.16056403699999999 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.307892+0000", 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4409999999999998e-06 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.307984+0000", 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2838999999999997e-05 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.308009+0000", 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4394999999999999e-05 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.311858+0000", 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0038492209999999999 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:01.311894+0000", 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5635e-05 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.682 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132679 11.9 e411/374 11:977e897c:::benchmark_data_smithi121_477989_object44490:head v 411'8407, mlcod=411'8405)", 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.125736+0000", 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.4658059720000001, 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.093655253999999993, 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125736+0000", 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125736+0000", 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125735+0000", 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:04.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125743+0000", 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5439999999999999e-06 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125755+0000", 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1591e-05 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125763+0000", 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4850000000000003e-06 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125808+0000", 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5821000000000001e-05 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125831+0000", 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2789000000000002e-05 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.219357+0000", 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.093526317999999997 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.219391+0000", 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3540000000000001e-05 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132676 11.b e411/380 11:d3209208:::benchmark_data_smithi121_477989_object44487:head v 411'8324, mlcod=353'3180)", 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.125776+0000", 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.4657657139999998, 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.091437769000000002, 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125776+0000", 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125776+0000", 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125776+0000", 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125780+0000", 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8879999999999999e-06 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125786+0000", 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8680000000000004e-06 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125791+0000", 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1030000000000001e-06 2023-09-23T21:08:04.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125833+0000", 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2691000000000002e-05 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.125867+0000", 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3797999999999997e-05 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132365+0000", 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064983280000000003 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217214+0000", 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.084848340999999994 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132681 11.1 e411/374 11:833c5ce2:::benchmark_data_smithi121_477989_object44492:head v 411'8201, mlcod=411'8199)", 2023-09-23T21:08:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.127928+0000", 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.4636135060000002, 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.094949895000000006, 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.127928+0000", 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.127928+0000", 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.127927+0000", 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.127938+0000", 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0399e-05 2023-09-23T21:08:04.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.127947+0000", 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6730000000000004e-06 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.127956+0000", 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.901e-06 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217212+0000", 2023-09-23T21:08:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089255781000000006 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217231+0000", 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8831e-05 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.222840+0000", 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0056095750000000003 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.222878+0000", 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7817000000000001e-05 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132682 11.6 e411/375 11:677079b4:::benchmark_data_smithi121_477989_object44493:head v 411'8085, mlcod=411'8083)", 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.128582+0000", 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.4629599579999999, 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.092789165000000007, 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.697 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128582+0000", 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128582+0000", 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128581+0000", 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128588+0000", 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9229999999999996e-06 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128599+0000", 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0939e-05 2023-09-23T21:08:04.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128606+0000", 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2150000000000004e-06 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128650+0000", 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3615000000000002e-05 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.128671+0000", 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1543999999999999e-05 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.221332+0000", 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.092660302999999999 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.221371+0000", 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9366999999999998e-05 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132686 11.1 e411/374 11:80874bfe:::benchmark_data_smithi121_477989_object44497:head v 411'8202, mlcod=411'8199)", 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.137062+0000", 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.4544796639999999, 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.091674729999999996, 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.137062+0000", 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.137062+0000", 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.137061+0000", 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.137071+0000", 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0322e-05 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217167+0000", 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.080095395999999999 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217182+0000", 2023-09-23T21:08:04.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5308999999999999e-05 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.219299+0000", 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021170749999999999 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.219317+0000", 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8236e-05 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.228684+0000", 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0093670969999999996 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.228737+0000", 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2533e-05 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.706 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132687 11.f e411/397 11:fa0f0f78:::benchmark_data_smithi121_477989_object44498:head v 411'8243, mlcod=411'8241)", 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.139348+0000", 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.452194129, 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.086699916000000002, 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139348+0000", 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:04.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139348+0000", 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139346+0000", 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139357+0000", 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0699e-05 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.218901+0000", 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.079543645999999996 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.218911+0000", 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0159999999999999e-05 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.219217+0000", 2023-09-23T21:08:04.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00030570800000000002 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.219236+0000", 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9133000000000001e-05 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.225993+0000", 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067567599999999997 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:04.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.226047+0000", 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4895000000000002e-05 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:08:04.711 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:04.748 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:08:04.866 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:04.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2052457594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:04.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:04.982 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:05.060 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:08:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:08:04] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.43.0" 2023-09-23T21:08:05.061 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2052457594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:05.061 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:05.061 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2052457594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:05.061 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:05.075 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:05.855 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:05.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:05.962 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:05 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:05 smithi123 ceph-mon[193903]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T21:08:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:05 smithi123 ceph-mon[193903]: failed to encode map e412 with expected crc 2023-09-23T21:08:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:05 smithi123 ceph-mon[193903]: failed to encode map e412 with expected crc 2023-09-23T21:08:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:05 smithi123 ceph-mon[193903]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 27356/130911 objects degraded (20.897%); 34048/130911 objects misplaced (26.009%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:05.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:05.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:05.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[219709]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T21:08:05.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[219709]: failed to encode map e412 with expected crc 2023-09-23T21:08:05.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[219709]: failed to encode map e412 with expected crc 2023-09-23T21:08:05.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[219709]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 27356/130911 objects degraded (20.897%); 34048/130911 objects misplaced (26.009%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:05.997 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:05.998 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:05.998 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:05.998 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[211526]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T21:08:05.998 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[211526]: failed to encode map e412 with expected crc 2023-09-23T21:08:05.998 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[211526]: failed to encode map e412 with expected crc 2023-09-23T21:08:05.999 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:05 smithi121 ceph-mon[211526]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 27356/130911 objects degraded (20.897%); 34048/130911 objects misplaced (26.009%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:06.010 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:08:06.057 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:06.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:06.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:06.168 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:06.276 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:07.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 27356/130911 objects degraded (20.897%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:07.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 27356/130911 objects degraded (20.897%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 27356/130911 objects degraded (20.897%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:07.329 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:07.359 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:07.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:07.467 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:07.500 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:07.571 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:07.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:07.677 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:07.752 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:07.927 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:07 smithi121 ceph-mon[211526]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 26725/124986 objects degraded (21.382%); 32244/124986 objects misplaced (25.798%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:08:07.929 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:07 smithi121 ceph-mon[219709]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 26725/124986 objects degraded (21.382%); 32244/124986 objects misplaced (25.798%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:08:08.019 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:08.028 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:08.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:08.147 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:08.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:07 smithi123 ceph-mon[193903]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 26725/124986 objects degraded (21.382%); 32244/124986 objects misplaced (25.798%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:08:08.234 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:09.078 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:08:09.078 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 26725/124986 objects degraded (21.382%), 7 pgs degraded, 6 pgs undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1745,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":412,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+remapped+backfill_wait","count":6},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":41662,"data_bytes":2729318328,"bytes_used":11673665536,"bytes_avail":660291584000,"bytes_total":671965249536,"degraded_objects":26725,"degraded_total":124986,"degraded_ratio":0.2138239482822076,"misplaced_objects":32244,"misplaced_total":124986,"misplaced_ratio":0.25798089386011236,"recovering_objects_per_sec":144,"recovering_bytes_per_sec":9452568,"recovering_keys_per_sec":0,"num_objects_recovered":1443,"num_bytes_recovered":94568448,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":504},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":28,"modified":"2023-09-23T21:06:55.027429+0000","services":{}},"progress_events":{"2639f328-dcb0-4a01-b6c7-21332e7147ac":{"message":"Global Recovery Event (4m)\n [=======================.....] (remaining: 49s)","progress":0.84269660711288452,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:08:09.093 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:09.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:09.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:09.203 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:09.235 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:09.292 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:09.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:09.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:09.369 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:08:09.403 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:09.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:09 smithi121 ceph-mon[211526]: pgmap v1310: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 26725/124986 objects degraded (21.382%); 32244/124986 objects misplaced (25.798%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:08:09.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/432165777' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:08:09.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:09 smithi121 ceph-mon[219709]: pgmap v1310: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 26725/124986 objects degraded (21.382%); 32244/124986 objects misplaced (25.798%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:08:09.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/432165777' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:08:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:09 smithi123 ceph-mon[193903]: pgmap v1310: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 26725/124986 objects degraded (21.382%); 32244/124986 objects misplaced (25.798%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:08:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/432165777' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:08:09.537 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:10.234 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:08:10.235 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:08:10.250 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:10.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:10.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:10.358 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:10.501 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:10.920 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:10.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:10.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:11.026 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:11.110 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:11.353 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:11.358 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:11 smithi121 ceph-mon[211526]: pgmap v1311: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26068/117315 objects degraded (22.221%); 29983/117315 objects misplaced (25.558%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T21:08:11.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:11.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:11.359 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:11 smithi121 ceph-mon[219709]: pgmap v1311: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26068/117315 objects degraded (22.221%); 29983/117315 objects misplaced (25.558%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T21:08:11.460 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:11.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:11 smithi123 ceph-mon[193903]: pgmap v1311: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26068/117315 objects degraded (22.221%); 29983/117315 objects misplaced (25.558%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T21:08:11.490 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:11.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:08:11.549 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:11.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:11.563 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:08:11.580 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:11.581 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:11.608 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:08:11.657 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:11.673 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:08:11.717 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:08:11.755 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:08:11.756 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:11.807 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:08:11.856 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.884 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:08:11.926 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:11.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:11.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:12.050 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:12.096 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.173 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:12.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:12.241 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:12.241 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:12.280 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:12.317 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.372 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:12.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:12.381 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1793439714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:12.381 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 26068/117315 objects degraded (22.221%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:12.482 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:12.515 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.571 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:12.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:12.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:12.624 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1793439714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:12.624 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 26068/117315 objects degraded (22.221%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:12.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1793439714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:12.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 26068/117315 objects degraded (22.221%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:12.677 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:12.719 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.769 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:12.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:12.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:12.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:12.915 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:12.964 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:12.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:12.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:13.067 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:13.092 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:13.132 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:13.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:13.239 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:13.276 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:13.354 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:13.460 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:13.500 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:13.558 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:13.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:13.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:13.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:13.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:13 smithi121 ceph-mon[211526]: pgmap v1312: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26068/117315 objects degraded (22.221%); 29983/117315 objects misplaced (25.558%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T21:08:13.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:13.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:13.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:13 smithi121 ceph-mon[219709]: pgmap v1312: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26068/117315 objects degraded (22.221%); 29983/117315 objects misplaced (25.558%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T21:08:13.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:13.699 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:13.699 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:13.699 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:13 smithi123 ceph-mon[193903]: pgmap v1312: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 26068/117315 objects degraded (22.221%); 29983/117315 objects misplaced (25.558%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T21:08:13.705 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:13.767 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:13.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:13.874 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:13.907 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:13.948 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:13.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:14.060 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:14.098 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:14.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:08:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:08:14] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:08:15.079 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:08:15.151 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:15.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:15.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:15.255 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:15.280 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:15.323 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:15.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:15.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:15 smithi121 ceph-mon[211526]: pgmap v1313: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 26068/115851 objects degraded (22.501%); 29506/115851 objects misplaced (25.469%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:15.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:15 smithi121 ceph-mon[219709]: pgmap v1313: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 26068/115851 objects degraded (22.501%); 29506/115851 objects misplaced (25.469%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:15.430 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:15.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:15 smithi123 ceph-mon[193903]: pgmap v1313: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 26068/115851 objects degraded (22.501%); 29506/115851 objects misplaced (25.469%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:08:15.466 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:15.533 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:15.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:15.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:15.606 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:08:15.646 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:15.732 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:15.960 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:15.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:16.072 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:16.137 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:16.251 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:16.429 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:16.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:16.540 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:16.622 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:16.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 26068/115851 objects degraded (22.501%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:16.660 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 26068/115851 objects degraded (22.501%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:16.756 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:08:16.798 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:08:16.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 26068/115851 objects degraded (22.501%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:17.444 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:17.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:17.559 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:17.589 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:17.601 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:08:17.681 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:17.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:17.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:17.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[211526]: from='client.46559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:17.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:08:17.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[211526]: pgmap v1314: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 25444/109539 objects degraded (23.228%); 27610/109539 objects misplaced (25.206%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:08:17.754 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/378172790' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:17.754 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:17.756 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[219709]: from='client.46559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:17.756 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:08:17.757 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[219709]: pgmap v1314: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 25444/109539 objects degraded (23.228%); 27610/109539 objects misplaced (25.206%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:08:17.757 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/378172790' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:17.757 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:17 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:17.788 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:17.826 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:17 smithi123 ceph-mon[193903]: from='client.46559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:17.826 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:17 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:08:17.826 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:17 smithi123 ceph-mon[193903]: pgmap v1314: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 25444/109539 objects degraded (23.228%); 27610/109539 objects misplaced (25.206%); 11 MiB/s, 182 objects/s recovering 2023-09-23T21:08:17.826 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/378172790' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:17.826 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:17 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:08:17.829 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:17.964 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:17.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:17.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:18.072 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:18.130 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:18.330 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:18.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:18.441 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:18.489 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:18.564 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:18.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:18.672 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:18.722 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:18.861 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:18.868 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:18 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:18.868 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:18 smithi123 ceph-mon[193903]: failed to encode map e413 with expected crc 2023-09-23T21:08:18.868 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:18.868 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:18 smithi123 ceph-mon[193903]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T21:08:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:18.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:18.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:18.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[219709]: failed to encode map e413 with expected crc 2023-09-23T21:08:18.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:18.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[219709]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T21:08:18.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:18.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[211526]: failed to encode map e413 with expected crc 2023-09-23T21:08:18.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:08:18.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:18 smithi121 ceph-mon[211526]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T21:08:18.970 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:19.018 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:19.175 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:19.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:19.182 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:19.225 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1313,"stamp":"2023-09-23T21:08:15.075636+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2529761208,"num_objects":38617,"num_object_clones":0,"num_object_copies":115851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26068,"num_objects_misplaced":29506,"num_objects_unfound":0,"num_objects_dirty":38617,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":138081,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32801,"num_bytes_recovered":2144967948,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80996,"ondisk_log_size":80996,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10828888,"kb_used_data":6731424,"kb_used_omap":21,"kb_used_meta":4097322,"kb_avail":645387176,"statfs":{"total":671965249536,"available":660876468224,"internally_reserved":0,"allocated":6892978176,"data_stored":6881185217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4195658211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,4,3,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-328990720,"num_objects":-5020,"num_object_clones":0,"num_object_copies":-15060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1288,"num_objects_misplaced":-4542,"num_objects_unfound":0,"num_objects_dirty":-5020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5020,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1473,"num_bytes_recovered":96534528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.335206"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":183,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.205325+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:07:53.205325+0000","last_peered":"2023-09-23T21:07:53.205325+0000","last_clean":"2023-09-23T21:07:53.205325+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:07:53.205325+0000","last_undegraded":"2023-09-23T21:07:53.205325+0000","last_fullsized":"2023-09-23T21:07:53.205325+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":52,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.191954+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:07:53.191954+0000","last_peered":"2023-09-23T21:07:53.191954+0000","last_clean":"2023-09-23T21:07:53.191954+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:07:53.191954+0000","last_undegraded":"2023-09-23T21:07:53.191954+0000","last_fullsized":"2023-09-23T21:07:53.191954+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":325,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.204930+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:07:53.204930+0000","last_peered":"2023-09-23T21:07:53.204930+0000","last_clean":"2023-09-23T21:07:53.204930+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:07:53.204930+0000","last_undegraded":"2023-09-23T21:07:53.204930+0000","last_fullsized":"2023-09-23T21:07:53.204930+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":49,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.211086+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:07:53.211086+0000","last_peered":"2023-09-23T21:07:53.211086+0000","last_clean":"2023-09-23T21:07:53.211086+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:07:53.211086+0000","last_undegraded":"2023-09-23T21:07:53.211086+0000","last_fullsized":"2023-09-23T21:07:53.211086+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":70,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.191080+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:07:53.191080+0000","last_peered":"2023-09-23T21:07:53.191080+0000","last_clean":"2023-09-23T21:07:53.191080+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:07:53.191080+0000","last_undegraded":"2023-09-23T21:07:53.191080+0000","last_fullsized":"2023-09-23T21:07:53.191080+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":278,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.203681+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:07:53.203681+0000","last_peered":"2023-09-23T21:07:53.203681+0000","last_clean":"2023-09-23T21:07:53.203681+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:07:53.203681+0000","last_undegraded":"2023-09-23T21:07:53.203681+0000","last_fullsized":"2023-09-23T21:07:53.203681+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":49,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.210210+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:07:53.210210+0000","last_peered":"2023-09-23T21:07:53.210210+0000","last_clean":"2023-09-23T21:07:53.210210+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:07:53.210210+0000","last_undegraded":"2023-09-23T21:07:53.210210+0000","last_fullsized":"2023-09-23T21:07:53.210210+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":52,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.191004+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:07:53.191004+0000","last_peered":"2023-09-23T21:07:53.191004+0000","last_clean":"2023-09-23T21:07:53.191004+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:07:53.191004+0000","last_undegraded":"2023-09-23T21:07:53.191004+0000","last_fullsized":"2023-09-23T21:07:53.191004+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":35,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.216333+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:07:53.216333+0000","last_peered":"2023-09-23T21:07:53.216333+0000","last_clean":"2023-09-23T21:07:53.216333+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:07:53.216333+0000","last_undegraded":"2023-09-23T21:07:53.216333+0000","last_fullsized":"2023-09-23T21:07:53.216333+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":52,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.192628+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:07:53.192628+0000","last_peered":"2023-09-23T21:07:53.192628+0000","last_clean":"2023-09-23T21:07:53.192628+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:07:53.192628+0000","last_undegraded":"2023-09-23T21:07:53.192628+0000","last_fullsized":"2023-09-23T21:07:53.192628+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":68,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.192041+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:07:53.192041+0000","last_peered":"2023-09-23T21:07:53.192041+0000","last_clean":"2023-09-23T21:07:53.192041+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:07:53.192041+0000","last_undegraded":"2023-09-23T21:07:53.192041+0000","last_fullsized":"2023-09-23T21:07:53.192041+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.204721+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:07:53.204721+0000","last_peered":"2023-09-23T21:07:53.204721+0000","last_clean":"2023-09-23T21:07:53.204721+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:07:53.204721+0000","last_undegraded":"2023-09-23T21:07:53.204721+0000","last_fullsized":"2023-09-23T21:07:53.204721+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":358,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.205417+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:07:53.205417+0000","last_peered":"2023-09-23T21:07:53.205417+0000","last_clean":"2023-09-23T21:07:53.205417+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:07:53.205417+0000","last_undegraded":"2023-09-23T21:07:53.205417+0000","last_fullsized":"2023-09-23T21:07:53.205417+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":330,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209898+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:07:53.209898+0000","last_peered":"2023-09-23T21:07:53.209898+0000","last_clean":"2023-09-23T21:07:53.209898+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:07:53.209898+0000","last_undegraded":"2023-09-23T21:07:53.209898+0000","last_fullsized":"2023-09-23T21:07:53.209898+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":102,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.200697+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:07:53.200697+0000","last_peered":"2023-09-23T21:07:53.200697+0000","last_clean":"2023-09-23T21:07:53.200697+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:07:53.200697+0000","last_undegraded":"2023-09-23T21:07:53.200697+0000","last_fullsized":"2023-09-23T21:07:53.200697+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.190187+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:07:53.190187+0000","last_peered":"2023-09-23T21:07:53.190187+0000","last_clean":"2023-09-23T21:07:53.190187+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:07:53.190187+0000","last_undegraded":"2023-09-23T21:07:53.190187+0000","last_fullsized":"2023-09-23T21:07:53.190187+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":70,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.192451+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:07:53.192451+0000","last_peered":"2023-09-23T21:07:53.192451+0000","last_clean":"2023-09-23T21:07:53.192451+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:07:53.192451+0000","last_undegraded":"2023-09-23T21:07:53.192451+0000","last_fullsized":"2023-09-23T21:07:53.192451+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.192116+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:07:53.192116+0000","last_peered":"2023-09-23T21:07:53.192116+0000","last_clean":"2023-09-23T21:07:53.192116+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:07:53.192116+0000","last_undegraded":"2023-09-23T21:07:53.192116+0000","last_fullsized":"2023-09-23T21:07:53.192116+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":212,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209407+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:07:53.209407+0000","last_peered":"2023-09-23T21:07:53.209407+0000","last_clean":"2023-09-23T21:07:53.209407+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:07:53.209407+0000","last_undegraded":"2023-09-23T21:07:53.209407+0000","last_fullsized":"2023-09-23T21:07:53.209407+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":49,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.211164+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:07:53.211164+0000","last_peered":"2023-09-23T21:07:53.211164+0000","last_clean":"2023-09-23T21:07:53.211164+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:07:53.211164+0000","last_undegraded":"2023-09-23T21:07:53.211164+0000","last_fullsized":"2023-09-23T21:07:53.211164+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":408,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.203630+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:07:53.203630+0000","last_peered":"2023-09-23T21:07:53.203630+0000","last_clean":"2023-09-23T21:07:53.203630+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:07:53.203630+0000","last_undegraded":"2023-09-23T21:07:53.203630+0000","last_fullsized":"2023-09-23T21:07:53.203630+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":52,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.190917+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:07:53.190917+0000","last_peered":"2023-09-23T21:07:53.190917+0000","last_clean":"2023-09-23T21:07:53.190917+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:07:53.190917+0000","last_undegraded":"2023-09-23T21:07:53.190917+0000","last_fullsized":"2023-09-23T21:07:53.190917+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"412'8765","reported_seq":10134,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:10.631426+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:08:10.631426+0000","last_peered":"2023-09-23T21:08:10.631426+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:08:10.631426+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"359'4028","ondisk_log_start":"359'4028","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4737,"log_dups_size":0,"ondisk_log_size":4737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161546240,"num_objects":2465,"num_object_clones":0,"num_object_copies":7395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8765,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-3150},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":2464}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":116,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.208723+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:07:53.208723+0000","last_peered":"2023-09-23T21:07:53.208723+0000","last_clean":"2023-09-23T21:07:53.208723+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:07:53.208723+0000","last_undegraded":"2023-09-23T21:07:53.208723+0000","last_fullsized":"2023-09-23T21:07:53.208723+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":35,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.215223+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:07:53.215223+0000","last_peered":"2023-09-23T21:07:53.215223+0000","last_clean":"2023-09-23T21:07:53.215223+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:07:53.215223+0000","last_undegraded":"2023-09-23T21:07:53.215223+0000","last_fullsized":"2023-09-23T21:07:53.215223+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"412'8375","reported_seq":8457,"reported_epoch":412,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:10.579950+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:08:10.579950+0000","last_peered":"2023-09-23T21:08:10.579950+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:08:10.579950+0000","last_undegraded":"2023-09-23T21:08:10.579950+0000","last_fullsized":"2023-09-23T21:08:10.579950+0000","mapping_epoch":375,"log_start":"359'3612","ondisk_log_start":"359'3612","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4763,"log_dups_size":0,"ondisk_log_size":4763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158793728,"num_objects":2423,"num_object_clones":0,"num_object_copies":7269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2423,"num_objects_unfound":0,"num_objects_dirty":2423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8375,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":2423}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":110,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209922+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:07:53.209922+0000","last_peered":"2023-09-23T21:07:53.209922+0000","last_clean":"2023-09-23T21:07:53.209922+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:07:53.209922+0000","last_undegraded":"2023-09-23T21:07:53.209922+0000","last_fullsized":"2023-09-23T21:07:53.209922+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":372,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.204856+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:07:53.204856+0000","last_peered":"2023-09-23T21:07:53.204856+0000","last_clean":"2023-09-23T21:07:53.204856+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:07:53.204856+0000","last_undegraded":"2023-09-23T21:07:53.204856+0000","last_fullsized":"2023-09-23T21:07:53.204856+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"412'8504","reported_seq":19852,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:10.742155+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:08:10.742155+0000","last_peered":"2023-09-23T21:08:10.742155+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:08:10.742155+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"362'4411","ondisk_log_start":"362'4411","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4093,"log_dups_size":0,"ondisk_log_size":4093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4858,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8504,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4108,"num_bytes_recovered":269221888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":48},{"shards":"2,6,7","objects":2381},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":39,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.216042+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:07:53.216042+0000","last_peered":"2023-09-23T21:07:53.216042+0000","last_clean":"2023-09-23T21:07:53.216042+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:07:53.216042+0000","last_undegraded":"2023-09-23T21:07:53.216042+0000","last_fullsized":"2023-09-23T21:07:53.216042+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":300,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.204509+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:07:53.204509+0000","last_peered":"2023-09-23T21:07:53.204509+0000","last_clean":"2023-09-23T21:07:53.204509+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:07:53.204509+0000","last_undegraded":"2023-09-23T21:07:53.204509+0000","last_fullsized":"2023-09-23T21:07:53.204509+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"412'8616","reported_seq":9245,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:10.778305+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:08:10.778305+0000","last_peered":"2023-09-23T21:08:10.778305+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:08:10.778305+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"363'4468","ondisk_log_start":"363'4468","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4148,"log_dups_size":0,"ondisk_log_size":4148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8616,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-2819},{"shards":"2,6,7","objects":2431}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":52,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.190302+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:07:53.190302+0000","last_peered":"2023-09-23T21:07:53.190302+0000","last_clean":"2023-09-23T21:07:53.190302+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:07:53.190302+0000","last_undegraded":"2023-09-23T21:07:53.190302+0000","last_fullsized":"2023-09-23T21:07:53.190302+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":49,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.210667+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:07:53.210667+0000","last_peered":"2023-09-23T21:07:53.210667+0000","last_clean":"2023-09-23T21:07:53.210667+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:07:53.210667+0000","last_undegraded":"2023-09-23T21:07:53.210667+0000","last_fullsized":"2023-09-23T21:07:53.210667+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"412'8799","reported_seq":20300,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:10.736848+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:08:10.736848+0000","last_peered":"2023-09-23T21:08:10.736848+0000","last_clean":"2023-09-23T21:08:10.736848+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:08:10.736848+0000","last_undegraded":"2023-09-23T21:08:10.736848+0000","last_fullsized":"2023-09-23T21:08:10.736848+0000","mapping_epoch":400,"log_start":"365'4699","ondisk_log_start":"365'4699","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4100,"log_dups_size":0,"ondisk_log_size":4100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164429824,"num_objects":2509,"num_object_clones":0,"num_object_copies":7527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8799,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":69,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.191492+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:07:53.191492+0000","last_peered":"2023-09-23T21:07:53.191492+0000","last_clean":"2023-09-23T21:07:53.191492+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:07:53.191492+0000","last_undegraded":"2023-09-23T21:07:53.191492+0000","last_fullsized":"2023-09-23T21:07:53.191492+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":52,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.190112+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:07:53.190112+0000","last_peered":"2023-09-23T21:07:53.190112+0000","last_clean":"2023-09-23T21:07:53.190112+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:07:53.190112+0000","last_undegraded":"2023-09-23T21:07:53.190112+0000","last_fullsized":"2023-09-23T21:07:53.190112+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"412'8682","reported_seq":19796,"reported_epoch":412,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:14.198275+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:08:14.198275+0000","last_peered":"2023-09-23T21:08:14.198275+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:08:14.198275+0000","last_undegraded":"2023-09-23T21:08:14.198275+0000","last_fullsized":"2023-09-23T21:08:14.198275+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150732800,"num_objects":2300,"num_object_clones":0,"num_object_copies":6900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2300,"num_objects_unfound":0,"num_objects_dirty":2300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8682,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":2300}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":338,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.204577+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:07:53.204577+0000","last_peered":"2023-09-23T21:07:53.204577+0000","last_clean":"2023-09-23T21:07:53.204577+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:07:53.204577+0000","last_undegraded":"2023-09-23T21:07:53.204577+0000","last_fullsized":"2023-09-23T21:07:53.204577+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":423,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.741958+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:08:04.741958+0000","last_peered":"2023-09-23T21:08:04.741958+0000","last_clean":"2023-09-23T21:08:04.741958+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:08:04.741958+0000","last_undegraded":"2023-09-23T21:08:04.741958+0000","last_fullsized":"2023-09-23T21:08:04.741958+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"412'8481","reported_seq":8579,"reported_epoch":412,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:10.694278+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:08:10.694278+0000","last_peered":"2023-09-23T21:08:10.694278+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:08:10.694278+0000","last_undegraded":"2023-09-23T21:08:10.694278+0000","last_fullsized":"2023-09-23T21:08:10.694278+0000","mapping_epoch":374,"log_start":"362'4394","ondisk_log_start":"362'4394","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4087,"log_dups_size":0,"ondisk_log_size":4087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158793728,"num_objects":2423,"num_object_clones":0,"num_object_copies":7269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4846,"num_objects_unfound":0,"num_objects_dirty":2423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8481,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":2423}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":39,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209297+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:07:53.209297+0000","last_peered":"2023-09-23T21:07:53.209297+0000","last_clean":"2023-09-23T21:07:53.209297+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:07:53.209297+0000","last_undegraded":"2023-09-23T21:07:53.209297+0000","last_fullsized":"2023-09-23T21:07:53.209297+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":451,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.204218+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:07:53.204218+0000","last_peered":"2023-09-23T21:07:53.204218+0000","last_clean":"2023-09-23T21:07:53.204218+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:07:53.204218+0000","last_undegraded":"2023-09-23T21:07:53.204218+0000","last_fullsized":"2023-09-23T21:07:53.204218+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"412'8513","reported_seq":8698,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:10.632094+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:08:10.632094+0000","last_peered":"2023-09-23T21:08:10.632094+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:08:10.632094+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"359'3794","ondisk_log_start":"359'3794","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160497664,"num_objects":2449,"num_object_clones":0,"num_object_copies":7347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8513,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-2990},{"shards":"1,6,7","objects":2449}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":61,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.191208+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:07:53.191208+0000","last_peered":"2023-09-23T21:07:53.191208+0000","last_clean":"2023-09-23T21:07:53.191208+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:07:53.191208+0000","last_undegraded":"2023-09-23T21:07:53.191208+0000","last_fullsized":"2023-09-23T21:07:53.191208+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":200,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.216941+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:07:53.216941+0000","last_peered":"2023-09-23T21:07:53.216941+0000","last_clean":"2023-09-23T21:07:53.216941+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:07:53.216941+0000","last_undegraded":"2023-09-23T21:07:53.216941+0000","last_fullsized":"2023-09-23T21:07:53.216941+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"412'8710","reported_seq":14207,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:14.303751+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:08:14.303751+0000","last_peered":"2023-09-23T21:08:14.303751+0000","last_clean":"2023-09-23T21:08:14.303751+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:08:14.303751+0000","last_undegraded":"2023-09-23T21:08:14.303751+0000","last_fullsized":"2023-09-23T21:08:14.303751+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8201,"log_dups_size":0,"ondisk_log_size":8201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151978008,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8710,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":307,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.746294+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:08:04.746294+0000","last_peered":"2023-09-23T21:08:04.746294+0000","last_clean":"2023-09-23T21:08:04.746294+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:08:04.746294+0000","last_undegraded":"2023-09-23T21:08:04.746294+0000","last_fullsized":"2023-09-23T21:08:04.746294+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.216112+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:07:53.216112+0000","last_peered":"2023-09-23T21:07:53.216112+0000","last_clean":"2023-09-23T21:07:53.216112+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:07:53.216112+0000","last_undegraded":"2023-09-23T21:07:53.216112+0000","last_fullsized":"2023-09-23T21:07:53.216112+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":62,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.205039+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:07:53.205039+0000","last_peered":"2023-09-23T21:07:53.205039+0000","last_clean":"2023-09-23T21:07:53.205039+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:07:53.205039+0000","last_undegraded":"2023-09-23T21:07:53.205039+0000","last_fullsized":"2023-09-23T21:07:53.205039+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"412'8650","reported_seq":16119,"reported_epoch":412,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:08:10.775567+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:08:10.775567+0000","last_peered":"2023-09-23T21:08:10.775567+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:08:10.775567+0000","last_undegraded":"2023-09-23T21:08:10.775567+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"365'4557","ondisk_log_start":"365'4557","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4093,"log_dups_size":0,"ondisk_log_size":4093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155713536,"num_objects":2376,"num_object_clones":0,"num_object_copies":7128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5535,"num_objects_unfound":0,"num_objects_dirty":2376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8650,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3668,"num_bytes_recovered":240386048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":531},{"shards":"2,6,7","objects":1845}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":388,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.746034+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:08:04.746034+0000","last_peered":"2023-09-23T21:08:04.746034+0000","last_clean":"2023-09-23T21:08:04.746034+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:08:04.746034+0000","last_undegraded":"2023-09-23T21:08:04.746034+0000","last_fullsized":"2023-09-23T21:08:04.746034+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":362,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.208158+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:07:53.208158+0000","last_peered":"2023-09-23T21:07:53.208158+0000","last_clean":"2023-09-23T21:07:53.208158+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:07:53.208158+0000","last_undegraded":"2023-09-23T21:07:53.208158+0000","last_fullsized":"2023-09-23T21:07:53.208158+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":44,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.205095+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:07:53.205095+0000","last_peered":"2023-09-23T21:07:53.205095+0000","last_clean":"2023-09-23T21:07:53.205095+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:07:53.205095+0000","last_undegraded":"2023-09-23T21:07:53.205095+0000","last_fullsized":"2023-09-23T21:07:53.205095+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"412'8733","reported_seq":8913,"reported_epoch":412,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:14.295551+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:08:14.295551+0000","last_peered":"2023-09-23T21:08:14.295551+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:08:14.295551+0000","last_undegraded":"2023-09-23T21:08:14.295551+0000","last_fullsized":"2023-09-23T21:08:14.295551+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":7260,"log_dups_size":0,"ondisk_log_size":7260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152633344,"num_objects":2329,"num_object_clones":0,"num_object_copies":6987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4644,"num_objects_unfound":0,"num_objects_dirty":2329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8733,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2329}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":389,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.197996+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:07:53.197996+0000","last_peered":"2023-09-23T21:07:53.197996+0000","last_clean":"2023-09-23T21:07:53.197996+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:07:53.197996+0000","last_undegraded":"2023-09-23T21:07:53.197996+0000","last_fullsized":"2023-09-23T21:07:53.197996+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":307,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.205291+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:07:53.205291+0000","last_peered":"2023-09-23T21:07:53.205291+0000","last_clean":"2023-09-23T21:07:53.205291+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:07:53.205291+0000","last_undegraded":"2023-09-23T21:07:53.205291+0000","last_fullsized":"2023-09-23T21:07:53.205291+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":44,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.216831+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:07:53.216831+0000","last_peered":"2023-09-23T21:07:53.216831+0000","last_clean":"2023-09-23T21:07:53.216831+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:07:53.216831+0000","last_undegraded":"2023-09-23T21:07:53.216831+0000","last_fullsized":"2023-09-23T21:07:53.216831+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"412'8464","reported_seq":14722,"reported_epoch":412,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:10.724564+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:08:10.724564+0000","last_peered":"2023-09-23T21:08:10.724564+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:08:10.724564+0000","last_undegraded":"2023-09-23T21:08:10.724564+0000","last_fullsized":"2023-09-23T21:08:10.724564+0000","mapping_epoch":387,"log_start":"362'4380","ondisk_log_start":"362'4380","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4084,"log_dups_size":0,"ondisk_log_size":4084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157941760,"num_objects":2410,"num_object_clones":0,"num_object_copies":7230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2410,"num_objects_unfound":0,"num_objects_dirty":2410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8464,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","1","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":2410}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":227,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.200804+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:07:53.200804+0000","last_peered":"2023-09-23T21:07:53.200804+0000","last_clean":"2023-09-23T21:07:53.200804+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:07:53.200804+0000","last_undegraded":"2023-09-23T21:07:53.200804+0000","last_fullsized":"2023-09-23T21:07:53.200804+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":709,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:07:53.204867+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:07:53.204867+0000","last_peered":"2023-09-23T21:07:53.204867+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:07:53.204867+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":61,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.192380+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:07:53.192380+0000","last_peered":"2023-09-23T21:07:53.192380+0000","last_clean":"2023-09-23T21:07:53.192380+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:07:53.192380+0000","last_undegraded":"2023-09-23T21:07:53.192380+0000","last_fullsized":"2023-09-23T21:07:53.192380+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":300,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.205014+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:07:53.205014+0000","last_peered":"2023-09-23T21:07:53.205014+0000","last_clean":"2023-09-23T21:07:53.205014+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:07:53.205014+0000","last_undegraded":"2023-09-23T21:07:53.205014+0000","last_fullsized":"2023-09-23T21:07:53.205014+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"412'8680","reported_seq":8793,"reported_epoch":412,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:10.773686+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:08:10.773686+0000","last_peered":"2023-09-23T21:08:10.773686+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:08:10.773686+0000","last_undegraded":"2023-09-23T21:08:10.773686+0000","last_fullsized":"2023-09-23T21:08:10.773686+0000","mapping_epoch":374,"log_start":"365'4605","ondisk_log_start":"365'4605","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4075,"log_dups_size":0,"ondisk_log_size":4075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160038912,"num_objects":2442,"num_object_clones":0,"num_object_copies":7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4884,"num_objects_unfound":0,"num_objects_dirty":2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8680,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":2442}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.204763+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:07:53.204763+0000","last_peered":"2023-09-23T21:07:53.204763+0000","last_clean":"2023-09-23T21:07:53.204763+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:07:53.204763+0000","last_undegraded":"2023-09-23T21:07:53.204763+0000","last_fullsized":"2023-09-23T21:07:53.204763+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":58,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209235+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:07:53.209235+0000","last_peered":"2023-09-23T21:07:53.209235+0000","last_clean":"2023-09-23T21:07:53.209235+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:07:53.209235+0000","last_undegraded":"2023-09-23T21:07:53.209235+0000","last_fullsized":"2023-09-23T21:07:53.209235+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":49,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.210760+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:07:53.210760+0000","last_peered":"2023-09-23T21:07:53.210760+0000","last_clean":"2023-09-23T21:07:53.210760+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:07:53.210760+0000","last_undegraded":"2023-09-23T21:07:53.210760+0000","last_fullsized":"2023-09-23T21:07:53.210760+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"412'8575","reported_seq":14802,"reported_epoch":412,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:08:10.633186+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:08:10.633186+0000","last_peered":"2023-09-23T21:08:10.633186+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:08:10.633186+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:08:10.633186+0000","mapping_epoch":383,"log_start":"359'3826","ondisk_log_start":"359'3826","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4749,"log_dups_size":0,"ondisk_log_size":4749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160628736,"num_objects":2451,"num_object_clones":0,"num_object_copies":7353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2447,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":2451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8575,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3063,"num_bytes_recovered":200736768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":4},{"shards":"1,2,7","objects":2447}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209836+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:07:53.209836+0000","last_peered":"2023-09-23T21:07:53.209836+0000","last_clean":"2023-09-23T21:07:53.209836+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:07:53.209836+0000","last_undegraded":"2023-09-23T21:07:53.209836+0000","last_fullsized":"2023-09-23T21:07:53.209836+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":309,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.749439+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:08:04.749439+0000","last_peered":"2023-09-23T21:08:04.749439+0000","last_clean":"2023-09-23T21:08:04.749439+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:08:04.749439+0000","last_undegraded":"2023-09-23T21:08:04.749439+0000","last_fullsized":"2023-09-23T21:08:04.749439+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":87,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.215859+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:07:53.215859+0000","last_peered":"2023-09-23T21:07:53.215859+0000","last_clean":"2023-09-23T21:07:53.215859+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:07:53.215859+0000","last_undegraded":"2023-09-23T21:07:53.215859+0000","last_fullsized":"2023-09-23T21:07:53.215859+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"412'8493","reported_seq":21384,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:10.567666+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:08:10.567666+0000","last_peered":"2023-09-23T21:08:10.567666+0000","last_clean":"2023-09-23T21:08:10.567666+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:08:10.567666+0000","last_undegraded":"2023-09-23T21:08:10.567666+0000","last_fullsized":"2023-09-23T21:08:10.567666+0000","mapping_epoch":397,"log_start":"359'3745","ondisk_log_start":"359'3745","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4748,"log_dups_size":0,"ondisk_log_size":4748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155385856,"num_objects":2371,"num_object_clones":0,"num_object_copies":7113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8493,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":69,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.190997+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:07:53.190997+0000","last_peered":"2023-09-23T21:07:53.190997+0000","last_clean":"2023-09-23T21:07:53.190997+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:07:53.190997+0000","last_undegraded":"2023-09-23T21:07:53.190997+0000","last_fullsized":"2023-09-23T21:07:53.190997+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":44,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209426+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:07:53.209426+0000","last_peered":"2023-09-23T21:07:53.209426+0000","last_clean":"2023-09-23T21:07:53.209426+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:07:53.209426+0000","last_undegraded":"2023-09-23T21:07:53.209426+0000","last_fullsized":"2023-09-23T21:07:53.209426+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":61,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.190865+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:07:53.190865+0000","last_peered":"2023-09-23T21:07:53.190865+0000","last_clean":"2023-09-23T21:07:53.190865+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:07:53.190865+0000","last_undegraded":"2023-09-23T21:07:53.190865+0000","last_fullsized":"2023-09-23T21:07:53.190865+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"412'8741","reported_seq":15143,"reported_epoch":412,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:08:10.609542+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:08:10.609542+0000","last_peered":"2023-09-23T21:08:10.609542+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:08:10.609542+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:08:10.609542+0000","mapping_epoch":382,"log_start":"359'4012","ondisk_log_start":"359'4012","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4729,"log_dups_size":0,"ondisk_log_size":4729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161021952,"num_objects":2457,"num_object_clones":0,"num_object_copies":7371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2454,"num_objects_misplaced":2460,"num_objects_unfound":0,"num_objects_dirty":2457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8741,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3142,"num_bytes_recovered":205914112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":2454}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":308,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.203758+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:07:53.203758+0000","last_peered":"2023-09-23T21:07:53.203758+0000","last_clean":"2023-09-23T21:07:53.203758+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:07:53.203758+0000","last_undegraded":"2023-09-23T21:07:53.203758+0000","last_fullsized":"2023-09-23T21:07:53.203758+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":53,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.210915+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:07:53.210915+0000","last_peered":"2023-09-23T21:07:53.210915+0000","last_clean":"2023-09-23T21:07:53.210915+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:07:53.210915+0000","last_undegraded":"2023-09-23T21:07:53.210915+0000","last_fullsized":"2023-09-23T21:07:53.210915+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":97,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.215269+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:07:53.215269+0000","last_peered":"2023-09-23T21:07:53.215269+0000","last_clean":"2023-09-23T21:07:53.215269+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:07:53.215269+0000","last_undegraded":"2023-09-23T21:07:53.215269+0000","last_fullsized":"2023-09-23T21:07:53.215269+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":58,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.210601+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:07:53.210601+0000","last_peered":"2023-09-23T21:07:53.210601+0000","last_clean":"2023-09-23T21:07:53.210601+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:07:53.210601+0000","last_undegraded":"2023-09-23T21:07:53.210601+0000","last_fullsized":"2023-09-23T21:07:53.210601+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":435,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.198526+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:07:53.198526+0000","last_peered":"2023-09-23T21:07:53.198526+0000","last_clean":"2023-09-23T21:07:53.198526+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:07:53.198526+0000","last_undegraded":"2023-09-23T21:07:53.198526+0000","last_fullsized":"2023-09-23T21:07:53.198526+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":74,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.208977+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:07:53.208977+0000","last_peered":"2023-09-23T21:07:53.208977+0000","last_clean":"2023-09-23T21:07:53.208977+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:07:53.208977+0000","last_undegraded":"2023-09-23T21:07:53.208977+0000","last_fullsized":"2023-09-23T21:07:53.208977+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.191560+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:07:53.191560+0000","last_peered":"2023-09-23T21:07:53.191560+0000","last_clean":"2023-09-23T21:07:53.191560+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:07:53.191560+0000","last_undegraded":"2023-09-23T21:07:53.191560+0000","last_fullsized":"2023-09-23T21:07:53.191560+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":293,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.209895+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:07:53.209895+0000","last_peered":"2023-09-23T21:07:53.209895+0000","last_clean":"2023-09-23T21:07:53.209895+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:07:53.209895+0000","last_undegraded":"2023-09-23T21:07:53.209895+0000","last_fullsized":"2023-09-23T21:07:53.209895+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":70,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.192519+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:07:53.192519+0000","last_peered":"2023-09-23T21:07:53.192519+0000","last_clean":"2023-09-23T21:07:53.192519+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:07:53.192519+0000","last_undegraded":"2023-09-23T21:07:53.192519+0000","last_fullsized":"2023-09-23T21:07:53.192519+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":52,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.190735+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:07:53.190735+0000","last_peered":"2023-09-23T21:07:53.190735+0000","last_clean":"2023-09-23T21:07:53.190735+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:07:53.190735+0000","last_undegraded":"2023-09-23T21:07:53.190735+0000","last_fullsized":"2023-09-23T21:07:53.190735+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":61,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.192438+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:07:53.192438+0000","last_peered":"2023-09-23T21:07:53.192438+0000","last_clean":"2023-09-23T21:07:53.192438+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:07:53.192438+0000","last_undegraded":"2023-09-23T21:07:53.192438+0000","last_fullsized":"2023-09-23T21:07:53.192438+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":55,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.210067+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:07:53.210067+0000","last_peered":"2023-09-23T21:07:53.210067+0000","last_clean":"2023-09-23T21:07:53.210067+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:07:53.210067+0000","last_undegraded":"2023-09-23T21:07:53.210067+0000","last_fullsized":"2023-09-23T21:07:53.210067+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":39,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-09-23T21:07:53.208422+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:07:53.208422+0000","last_peered":"2023-09-23T21:07:53.208422+0000","last_clean":"2023-09-23T21:07:53.208422+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:07:53.208422+0000","last_undegraded":"2023-09-23T21:07:53.208422+0000","last_fullsized":"2023-09-23T21:07:53.208422+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2529165336,"num_objects":38593,"num_object_clones":0,"num_object_copies":115779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26066,"num_objects_misplaced":29506,"num_objects_unfound":0,"num_objects_dirty":38593,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137781,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32684,"num_bytes_recovered":2141978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10002968576,"data_stored":10002956360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80780,"ondisk_log_size":80780,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201754,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463374,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2266352,"kb_used_data":1741032,"kb_used_omap":0,"kb_used_meta":525248,"kb_avail":91478800,"statfs":{"total":95995035648,"available":93674291200,"internally_reserved":0,"allocated":1782816768,"data_stored":1781604605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":537853952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":6.6479999999999997,"5min":2.0830000000000002,"15min":2.2610000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":99.533000000000001,"5min":99.533000000000001,"15min":102.242},"last":0.26100000000000001},{"interface":"front","average":{"1min":6.8070000000000004,"5min":2.1949999999999998,"15min":2.0470000000000002},"min":{"1min":0.248,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":99.325000000000003,"5min":99.325000000000003,"15min":102.502},"last":3.2200000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":3.9460000000000002,"15min":2.379},"min":{"1min":0.251,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":11.743,"5min":119.077,"15min":119.077},"last":3.4790000000000001},{"interface":"front","average":{"1min":1.901,"5min":3.7730000000000001,"15min":2.452},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24199999999999999},"max":{"1min":11.936,"5min":119.313,"15min":119.313},"last":5.4699999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":13.304,"5min":6.1219999999999999,"15min":3.726},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":199.23400000000001,"5min":199.23400000000001,"15min":199.23400000000001},"last":0.622},{"interface":"front","average":{"1min":12.943,"5min":6.1870000000000003,"15min":3.8719999999999999},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.08000000000001},"last":6.3019999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":0.98599999999999999,"15min":4.0119999999999996},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":3.194,"5min":5.9420000000000002,"15min":250.804},"last":5.4260000000000002},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.45,"15min":4.0099999999999998},"min":{"1min":0.371,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":13.319000000000001,"5min":13.319000000000001,"15min":250.446},"last":2.9849999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.353,"15min":0.97099999999999997},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.24299999999999999},"max":{"1min":6.3689999999999998,"5min":13.1,"15min":13.1},"last":3.153},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.98699999999999999,"15min":0.95199999999999996},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":4.3849999999999998,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":1.76}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.333,"15min":1.105},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":6.343,"5min":11.281000000000001,"15min":18.962},"last":3.1819999999999999},{"interface":"front","average":{"1min":1.325,"5min":1.286,"15min":1.212},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":6.6890000000000001,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":1.7829999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:12 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.2569999999999999,"15min":1.4710000000000001},"min":{"1min":0.34899999999999998,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.8079999999999998,"5min":7.8079999999999998,"15min":101.04600000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.2509999999999999,"15min":1.5029999999999999},"min":{"1min":0.39200000000000002,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":4.407,"5min":7.1849999999999996,"15min":101.081},"last":1.7170000000000001}]}]},{"osd":1,"up_from":276,"seq":1185410973820,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2446152,"kb_used_data":2013952,"kb_used_omap":5,"kb_used_meta":432186,"kb_avail":91299000,"statfs":{"total":95995035648,"available":93490176000,"internally_reserved":0,"allocated":2062286848,"data_stored":2060231440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":442559203},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,3,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.738,"15min":0.86599999999999999},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":18.097999999999999,"5min":18.097999999999999,"15min":18.097999999999999},"last":23.486000000000001},{"interface":"front","average":{"1min":3.5459999999999998,"5min":1.407,"15min":1.0089999999999999},"min":{"1min":0.29999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":11.228,"5min":11.228,"15min":18.047999999999998},"last":4.2480000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0300000000000002,"5min":1.7010000000000001,"15min":1.4379999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.874000000000001,"5min":13.874000000000001,"15min":78.409000000000006},"last":3.6080000000000001},{"interface":"front","average":{"1min":5.5030000000000001,"5min":1.829,"15min":1.319},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.22},"max":{"1min":18.288,"5min":18.288,"15min":76.245000000000005},"last":23.539000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.415,"5min":1.716,"15min":2.782},"min":{"1min":0.28699999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":18.344999999999999,"5min":18.344999999999999,"15min":172.44399999999999},"last":23.407},{"interface":"front","average":{"1min":3.4009999999999998,"5min":1.627,"15min":2.8799999999999999},"min":{"1min":0.36399999999999999,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":11.311999999999999,"5min":11.311999999999999,"15min":172.58500000000001},"last":3.714}]},{"osd":4,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.446,"15min":1.0389999999999999},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6500000000000004,"5min":12.238,"15min":17.006},"last":0.38600000000000001},{"interface":"front","average":{"1min":3.9769999999999999,"5min":1.583,"15min":0.93100000000000005},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":18.173999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":23.332999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":1.397,"15min":1.05},"min":{"1min":0.34499999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.340999999999999,"5min":11.340999999999999,"15min":17.155000000000001},"last":3.411},{"interface":"front","average":{"1min":2.758,"5min":1.55,"15min":1.149},"min":{"1min":0.35799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.949999999999999,"5min":12.345000000000001,"15min":17.076000000000001},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":1.47,"15min":1.534},"min":{"1min":0.42699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":11.375,"5min":11.666,"15min":127.995},"last":3.5470000000000002},{"interface":"front","average":{"1min":2.7890000000000001,"5min":1.633,"15min":1.6180000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.012,"5min":12.276999999999999,"15min":126.836},"last":0.42099999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9609999999999999,"5min":3.0670000000000002,"15min":1.645},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":11.17,"5min":151.791,"15min":151.791},"last":0.46800000000000003},{"interface":"front","average":{"1min":4.173,"5min":3.0800000000000001,"15min":1.47},"min":{"1min":0.41799999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":18.260000000000002,"5min":151.809,"15min":151.809},"last":23.376000000000001}]}]},{"osd":0,"up_from":92,"seq":395136991561,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081156,"kb_used_data":340668,"kb_used_omap":5,"kb_used_meta":740474,"kb_avail":92663996,"statfs":{"total":95995035648,"available":94887931904,"internally_reserved":0,"allocated":348844032,"data_stored":346906456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":758246010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.423,"15min":1.841},"min":{"1min":0.315,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":41.149999999999999,"5min":61.341999999999999,"15min":61.341999999999999},"last":0.315},{"interface":"front","average":{"1min":6.5499999999999998,"5min":3.5920000000000001,"15min":1.845},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.217},"max":{"1min":38.936,"5min":61.299999999999997,"15min":61.299999999999997},"last":2.5619999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":5.468,"5min":2.4609999999999999,"15min":2.4790000000000001},"min":{"1min":0.24099999999999999,"5min":0.217,"15min":0.161},"max":{"1min":45.912999999999997,"5min":45.912999999999997,"15min":183.77099999999999},"last":2.2450000000000001},{"interface":"front","average":{"1min":5.2930000000000001,"5min":2.5070000000000001,"15min":2.6949999999999998},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":46.234000000000002,"5min":46.234000000000002,"15min":183.78700000000001},"last":2.6030000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.276,"15min":3.1070000000000002},"min":{"1min":0.28999999999999998,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":12.113,"5min":12.113,"15min":186.143},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.1759999999999999,"15min":3.1629999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":12.348000000000001,"5min":12.348000000000001,"15min":186.97399999999999},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":1.2789999999999999,"15min":1.3600000000000001},"min":{"1min":0.376,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":10.814,"5min":10.814,"15min":10.814},"last":0.37},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.03,"15min":0.96799999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.586,"5min":10.586,"15min":11.493},"last":0.496}]},{"osd":5,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.454,"15min":2.0590000000000002},"min":{"1min":0.34399999999999997,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":10.763,"5min":10.763,"15min":154.30600000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.129,"5min":1.147,"15min":1.5189999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28699999999999998},"max":{"1min":10.618,"5min":11.478,"15min":154.42400000000001},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":5.9500000000000002,"15min":3.214},"min":{"1min":0.35199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.654,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.494,"5min":5.9000000000000004,"15min":3.3380000000000001},"min":{"1min":0.39900000000000002,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":5.7750000000000004,"5min":169.767,"15min":169.767},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.347,"15min":2.2509999999999999},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.74,"5min":10.74,"15min":164.26400000000001},"last":2.5859999999999999},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.0720000000000001,"15min":2.2549999999999999},"min":{"1min":0.44,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":5.8159999999999998,"5min":5.8159999999999998,"15min":164.23099999999999},"last":2.637}]}]},{"osd":2,"up_from":129,"seq":554050781452,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2361480,"kb_used_data":1708416,"kb_used_omap":7,"kb_used_meta":653048,"kb_avail":91383672,"statfs":{"total":95995035648,"available":93576880128,"internally_reserved":0,"allocated":1749417984,"data_stored":1747418866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":668721233},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.4450000000000001,"15min":1.3959999999999999},"min":{"1min":0.20499999999999999,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":6.8440000000000003,"5min":15.667,"15min":26.356000000000002},"last":6.5960000000000001},{"interface":"front","average":{"1min":6.4569999999999999,"5min":2.0830000000000002,"15min":1.8320000000000001},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":23.963999999999999,"5min":23.963999999999999,"15min":26.341000000000001},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4109999999999996,"5min":5.375,"15min":2.6240000000000001},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.188},"max":{"1min":14.057,"5min":212.12,"15min":212.12},"last":2.1749999999999998},{"interface":"front","average":{"1min":2.5760000000000001,"5min":4.8449999999999998,"15min":2.3330000000000002},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":6.5709999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":6.6470000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:07:47 2023","interfaces":[{"interface":"back","average":{"1min":10.468999999999999,"5min":2.8500000000000001,"15min":3.2440000000000002},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":138.90299999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":1.4950000000000001},{"interface":"front","average":{"1min":10.629,"5min":2.8900000000000001,"15min":3.2090000000000001},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":138.87700000000001,"5min":138.87700000000001,"15min":138.87700000000001},"last":6.6219999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":1.153,"15min":1.113},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":6.383,"5min":12.586,"15min":12.586},"last":6.548},{"interface":"front","average":{"1min":4.1820000000000004,"5min":1.5700000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":14.672000000000001,"5min":14.672000000000001,"15min":14.672000000000001},"last":7.5590000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":4.367,"5min":1.7050000000000001,"15min":2.129},"min":{"1min":0.36799999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":14.710000000000001,"5min":14.710000000000001,"15min":173.34},"last":7.5209999999999999},{"interface":"front","average":{"1min":2.097,"5min":1.173,"15min":1.796},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":8.9740000000000002,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":3.3940000000000001,"15min":1.946},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":9.0329999999999995,"5min":175.72499999999999,"15min":175.72499999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.901,"5min":3.375,"15min":1.9550000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.5049999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":6.5019999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.655,"15min":2.081},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":6.4809999999999999,"5min":12.609,"15min":177.30799999999999},"last":6.5739999999999998},{"interface":"front","average":{"1min":4.2240000000000002,"5min":1.645,"15min":2.1989999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33600000000000002},"max":{"1min":14.736000000000001,"5min":14.736000000000001,"15min":177.40899999999999},"last":7.5830000000000002}]}]},{"osd":3,"up_from":148,"seq":635655160046,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":577608,"kb_used_data":6656,"kb_used_omap":2,"kb_used_meta":570941,"kb_avail":93167544,"statfs":{"total":95995035648,"available":95403565056,"internally_reserved":0,"allocated":6815744,"data_stored":4636408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":584643637},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":1.4910000000000001,"15min":1.774},"min":{"1min":0.249,"5min":0.20999999999999999,"15min":0.184},"max":{"1min":8.8629999999999995,"5min":44.963000000000001,"15min":95.507999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":1.8919999999999999,"5min":0.94799999999999995,"15min":1.6679999999999999},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.17199999999999999},"max":{"1min":7.7850000000000001,"5min":9.7370000000000001,"15min":95.259},"last":0.222}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":12.473000000000001,"5min":3.4580000000000002,"15min":1.8380000000000001},"min":{"1min":0.64300000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":109.51300000000001,"5min":109.51300000000001,"15min":109.51300000000001},"last":47.173999999999999},{"interface":"front","average":{"1min":13.468,"5min":3.6669999999999998,"15min":2.121},"min":{"1min":0.26500000000000001,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":111.13500000000001,"5min":111.13500000000001,"15min":111.13500000000001},"last":47.517000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4129999999999998,"5min":1.4750000000000001,"15min":1.351},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.182},"max":{"1min":14.243,"5min":14.243,"15min":14.243},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.6059999999999999,"5min":1.325,"15min":1.3420000000000001},"min":{"1min":0.34399999999999997,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":7.8120000000000003,"5min":9.7140000000000004,"15min":19.754999999999999},"last":0.26200000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4980000000000002,"5min":2.5470000000000002,"15min":2.2559999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.222},"max":{"1min":38.600000000000001,"5min":70.728999999999999,"15min":111.36},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.7919999999999998,"5min":2.4529999999999998,"15min":2.0539999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.252},"max":{"1min":38.524000000000001,"5min":70.712000000000003,"15min":112.473},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":1.575,"15min":1.581},"min":{"1min":0.41199999999999998,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":36.488999999999997,"5min":36.488999999999997,"15min":64.528000000000006},"last":0.46300000000000002},{"interface":"front","average":{"1min":3.5110000000000001,"5min":1.6879999999999999,"15min":1.647},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":36.295000000000002,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.40699999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.3740000000000001,"15min":1.623},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.312},"max":{"1min":10.827999999999999,"5min":10.827999999999999,"15min":19.568000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.448,"15min":1.5029999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":7.6749999999999998,"5min":14.356,"15min":15.573},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.3599999999999999,"15min":2.5510000000000002},"min":{"1min":0.5,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":7.7370000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.2549999999999999,"5min":2.3530000000000002,"15min":2.8170000000000002},"min":{"1min":0.48099999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":7.4249999999999998,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.439}]}]},{"osd":4,"up_from":31,"seq":133143986664,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1166192,"kb_used_data":610536,"kb_used_omap":0,"kb_used_meta":555648,"kb_avail":92578960,"statfs":{"total":95995035648,"available":94800855040,"internally_reserved":0,"allocated":625188864,"data_stored":623976085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":568983552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.4770000000000001,"15min":2.6949999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":12.869,"5min":47.476999999999997,"15min":146.25299999999999},"last":10.273999999999999},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.2849999999999999,"15min":2.6920000000000002},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.223},"max":{"1min":12.848000000000001,"5min":12.848000000000001,"15min":144.77000000000001},"last":2.1890000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.782,"5min":7.7530000000000001,"15min":3.5259999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":14.523999999999999,"5min":127.411,"15min":127.411},"last":1.6040000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":5.7130000000000001,"15min":2.835},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":5.8609999999999998,"5min":118.075,"15min":118.075},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":6.7910000000000004,"15min":3.5299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":71.454999999999998,"5min":164.517,"15min":164.517},"last":6.2359999999999998},{"interface":"front","average":{"1min":5.343,"5min":6.282,"15min":3.5099999999999998},"min":{"1min":0.34899999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":71.191999999999993,"5min":164.46799999999999,"15min":164.46799999999999},"last":1.0069999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":1.2909999999999999,"15min":2.0510000000000002},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":6.8120000000000003,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.878},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.2150000000000001,"15min":2.149},"min":{"1min":0.40600000000000003,"5min":0.32400000000000001,"15min":0.25600000000000001},"max":{"1min":6.181,"5min":8.2899999999999991,"15min":142.34700000000001},"last":1.821}]},{"osd":5,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.81999999999999995,"15min":1.052},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":2.2629999999999999,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.042,"5min":0.96799999999999997,"15min":1.1220000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.21099999999999999},"max":{"1min":4.282,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.32800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.1779999999999999,"15min":0.98199999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":4.8399999999999999,"5min":11.800000000000001,"15min":11.887},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.2410000000000001,"15min":0.93700000000000006},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0419999999999998,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.71899999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:07:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.601,"15min":1.6040000000000001},"min":{"1min":0.33300000000000002,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":4.3129999999999997,"5min":145.721,"15min":145.721},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.347,"5min":2.9489999999999998,"15min":1.776},"min":{"1min":0.38500000000000001,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":4.7969999999999997,"5min":148.45699999999999,"15min":148.45699999999999},"last":1.153}]}]},{"osd":5,"up_from":39,"seq":167503725028,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":929948,"kb_used_data":310164,"kb_used_omap":0,"kb_used_meta":619776,"kb_avail":92815204,"statfs":{"total":95995035648,"available":95042768896,"internally_reserved":0,"allocated":317607936,"data_stored":316411357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":634650624},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":0.86399999999999999,"15min":1.325},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":10.488,"5min":10.488,"15min":80.105999999999995},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.90700000000000003,"15min":1.446},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":4.75,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":13.016999999999999,"5min":6.4710000000000001,"15min":2.512},"min":{"1min":0.32300000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":233.77099999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":0.375},{"interface":"front","average":{"1min":13.031000000000001,"5min":6.1769999999999996,"15min":2.4220000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.247},"max":{"1min":233.75200000000001,"5min":233.75200000000001,"15min":233.75200000000001},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":15.840999999999999,"5min":3.7839999999999998,"15min":2.4689999999999999},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":231.34700000000001,"5min":231.34700000000001,"15min":231.34700000000001},"last":6.0369999999999999},{"interface":"front","average":{"1min":15.951000000000001,"5min":4.008,"15min":2.6110000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":231.381,"5min":231.381,"15min":231.381},"last":2.5659999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":1.3260000000000001,"15min":2.7959999999999998},"min":{"1min":0.38500000000000001,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":7.0300000000000002,"5min":10.670999999999999,"15min":153.60300000000001},"last":1.607},{"interface":"front","average":{"1min":1.653,"5min":1.0580000000000001,"15min":2.6880000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":4.1779999999999999,"5min":7.3810000000000002,"15min":153.774},"last":1.6859999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59999999999999998,"15min":0.77600000000000002},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":2.2320000000000002,"5min":3.3029999999999999,"15min":12.739000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.88900000000000001,"15min":0.98299999999999998},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.247},"max":{"1min":3.645,"5min":8.3740000000000006,"15min":12.539},"last":0.56399999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.1160000000000001,"15min":1.1539999999999999},"min":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":3.6760000000000002,"5min":9.8480000000000008,"15min":71.230000000000004},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.25,"15min":1.286},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":4.0880000000000001,"5min":12.673999999999999,"15min":70.266000000000005},"last":0.313}]},{"osd":7,"last update":"Sat Sep 23 21:07:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.143,"15min":1.2010000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":5.0119999999999996,"5min":8.4109999999999996,"15min":31.091999999999999},"last":0.624},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65800000000000003,"15min":1.054},"min":{"1min":0.38500000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.5339999999999998,"5min":2.6280000000000001,"15min":31.780000000000001},"last":0.35999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":340197376,"data_stored":340197376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2038104064,"data_stored":2038104064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1737494528,"data_stored":1737490456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1376256,"data_stored":1376256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":618532864,"data_stored":618528792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":312213504,"data_stored":312213504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1757024256,"data_stored":1757020184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3198025728,"data_stored":3198025728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:08:19.281 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:19.320 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:19.382 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:19.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:19.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:19.489 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:19.523 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:19.617 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:19.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:19.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:19.733 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:19.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:19 smithi121 ceph-mon[219709]: failed to encode map e413 with expected crc 2023-09-23T21:08:19.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:19 smithi121 ceph-mon[219709]: failed to encode map e413 with expected crc 2023-09-23T21:08:19.745 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:19 smithi121 ceph-mon[219709]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 25444/109539 objects degraded (23.228%); 27610/109539 objects misplaced (25.206%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T21:08:19.745 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:19 smithi121 ceph-mon[211526]: failed to encode map e413 with expected crc 2023-09-23T21:08:19.745 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:19 smithi121 ceph-mon[211526]: failed to encode map e413 with expected crc 2023-09-23T21:08:19.745 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:19 smithi121 ceph-mon[211526]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 25444/109539 objects degraded (23.228%); 27610/109539 objects misplaced (25.206%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T21:08:19.771 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:19 smithi123 ceph-mon[193903]: failed to encode map e413 with expected crc 2023-09-23T21:08:19.772 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:19 smithi123 ceph-mon[193903]: failed to encode map e413 with expected crc 2023-09-23T21:08:19.772 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:19 smithi123 ceph-mon[193903]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 25444/109539 objects degraded (23.228%); 27610/109539 objects misplaced (25.206%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T21:08:19.773 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:19.837 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:19.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:19.896 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:19.896 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:19.941 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:19.991 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:20.075 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:20.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:20.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:20.187 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:20.277 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:20.998 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:08:21.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:21 smithi121 ceph-mon[211526]: pgmap v1317: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 24773/101538 objects degraded (24.398%); 25141/101538 objects misplaced (24.760%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:21.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:21 smithi121 ceph-mon[219709]: pgmap v1317: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 24773/101538 objects degraded (24.398%); 25141/101538 objects misplaced (24.760%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:21 smithi123 ceph-mon[193903]: pgmap v1317: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 24773/101538 objects degraded (24.398%); 25141/101538 objects misplaced (24.760%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:21.583 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:21.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:21.692 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:21.735 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:21.795 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:21.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:21.906 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:22.015 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:08:22.093 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:22.197 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:22.233 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:08:22.282 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:22.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:22.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:22.394 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:22.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:22 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:22.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:22 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:22.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 24773/101538 objects degraded (24.398%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:22.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:22 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:22.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:22 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:22.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 24773/101538 objects degraded (24.398%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:22.613 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:22.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:22 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:22.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:22 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:22.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 24773/101538 objects degraded (24.398%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:22.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:08:22.806 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:22.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:22.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:22.830 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:08:22.894 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:08:22.929 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:22.961 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:08:22.965 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:23.005 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:08:23.026 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:23.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:23.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:23.041 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:08:23.091 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:08:23.132 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:23.139 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.152 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:08:23.240 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:23.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2931812100' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:23.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:23 smithi121 ceph-mon[219709]: pgmap v1318: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 24773/101538 objects degraded (24.398%); 25141/101538 objects misplaced (24.760%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2931812100' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:23.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:23 smithi121 ceph-mon[211526]: pgmap v1318: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 24773/101538 objects degraded (24.398%); 25141/101538 objects misplaced (24.760%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:23.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2931812100' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:23 smithi123 ceph-mon[193903]: pgmap v1318: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 24773/101538 objects degraded (24.398%); 25141/101538 objects misplaced (24.760%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:24.609 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:24.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:24.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:24.714 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:24.744 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:24.781 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:24.802 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:24.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:24.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:24.908 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:24.919 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:08:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:08:24] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:08:24.932 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:24.936 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:24.996 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:25.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:25.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:25.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:25.197 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:25.454 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:25.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:25.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:25 smithi121 ceph-mon[211526]: pgmap v1319: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 24773/100215 objects degraded (24.720%); 24676/100215 objects misplaced (24.623%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:25.465 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:25 smithi121 ceph-mon[219709]: pgmap v1319: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 24773/100215 objects degraded (24.720%); 24676/100215 objects misplaced (24.623%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:25 smithi123 ceph-mon[193903]: pgmap v1319: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 2.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 24773/100215 objects degraded (24.720%); 24676/100215 objects misplaced (24.623%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:08:25.562 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:25.652 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:25.788 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:25.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:25.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:25.902 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:25.938 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:26.000 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:26.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:26.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:26.110 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:26.148 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:26.209 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:26.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:26.315 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:26.343 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:08:26.371 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/372388369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:26.371 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/372388369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:26.421 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:26.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/372388369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:26.743 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:08:27.388 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:27.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:27.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:27.516 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:27.548 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[211526]: failed to encode map e414 with expected crc 2023-09-23T21:08:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/372388369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[211526]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T21:08:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[211526]: failed to encode map e414 with expected crc 2023-09-23T21:08:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 24773/100215 objects degraded (24.720%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[211526]: pgmap v1321: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 24136/94359 objects degraded (25.579%); 22915/94359 objects misplaced (24.285%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[219709]: failed to encode map e414 with expected crc 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/372388369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[219709]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[219709]: failed to encode map e414 with expected crc 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 24773/100215 objects degraded (24.720%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:27.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:27 smithi121 ceph-mon[219709]: pgmap v1321: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 24136/94359 objects degraded (25.579%); 22915/94359 objects misplaced (24.285%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:27 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:27 smithi123 ceph-mon[193903]: failed to encode map e414 with expected crc 2023-09-23T21:08:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/372388369' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:27 smithi123 ceph-mon[193903]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T21:08:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:27 smithi123 ceph-mon[193903]: failed to encode map e414 with expected crc 2023-09-23T21:08:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:27 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 24773/100215 objects degraded (24.720%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:27 smithi123 ceph-mon[193903]: pgmap v1321: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 24136/94359 objects degraded (25.579%); 22915/94359 objects misplaced (24.285%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:28.246 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:28.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:28.355 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:28.413 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:28.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:28.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:28.511 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:28.511 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:28.565 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:28.677 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:28.705 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:28.705 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:28.711 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:28.764 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:28.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:28.871 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:28.908 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:28.976 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:28.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:28.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:29.086 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:29.184 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:29.202 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:29.234 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50084 11.2 11:41af7975:::benchmark_data_smithi121_477989_object50083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.400207+0000", 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.82996468699997, 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1853260130000001, 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.235 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50084 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400207+0000", 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.236 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400207+0000", 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400206+0000", 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400237+0000", 2023-09-23T21:08:29.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1195999999999997e-05 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.401315+0000", 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0010784480000000001 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.401329+0000", 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3125000000000001e-05 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.405972+0000", 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046429330000000001 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.405982+0000", 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0834e-05 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393675+0000", 2023-09-23T21:08:29.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98769265399999995 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393689+0000", 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4436e-05 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.573878+0000", 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.180188276 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.574056+0000", 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000178381 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.574262+0000", 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020565500000000001 2023-09-23T21:08:29.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580224+0000", 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005962686 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580239+0000", 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4504e-05 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.583594+0000", 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033555109999999998 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.583631+0000", 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6523000000000003e-05 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585533+0000", 2023-09-23T21:08:29.243 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001902321 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50093 11.2 11:4ca61590:::benchmark_data_smithi121_477989_object50092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.417151+0000", 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.81302133200001, 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.174274069, 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.244 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50093 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417151+0000", 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417151+0000", 2023-09-23T21:08:29.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417149+0000", 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417171+0000", 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1328999999999999e-05 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417182+0000", 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.134e-05 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417190+0000", 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9400000000000002e-06 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417234+0000", 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4304000000000001e-05 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417244+0000", 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5000000000000005e-06 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393853+0000", 2023-09-23T21:08:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.97660954799999999 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393862+0000", 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0089999999999996e-06 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.578019+0000", 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.184156268 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.249 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.578132+0000", 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011379000000000001 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.578284+0000", 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000151621 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.583554+0000", 2023-09-23T21:08:29.250 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052696110000000004 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585553+0000", 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001998948 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.589246+0000", 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003693174 2023-09-23T21:08:29.251 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.589295+0000", 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9301000000000001e-05 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.591425+0000", 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021296150000000001 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.252 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50094 11.2 11:49734ba9:::benchmark_data_smithi121_477989_object50093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.422439+0000", 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.80773339199999, 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1746549339999999, 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50094 2023-09-23T21:08:29.253 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422439+0000", 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422439+0000", 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422437+0000", 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422463+0000", 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6129e-05 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.255 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422479+0000", 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6099e-05 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422488+0000", 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6589999999999996e-06 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422554+0000", 2023-09-23T21:08:29.256 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6344e-05 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422567+0000", 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2478e-05 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394015+0000", 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.97144863199999998 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.257 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394025+0000", 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7990000000000005e-06 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580262+0000", 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1862366040000001 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580377+0000", 2023-09-23T21:08:29.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115528 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580486+0000", 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000108835 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585561+0000", 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050745069999999998 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585596+0000", 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5036000000000001e-05 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.595075+0000", 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094792490000000004 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.595175+0000", 2023-09-23T21:08:29.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000100534 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.597094+0000", 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001918169 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50111 11.2 11:45a36be7:::benchmark_data_smithi121_477989_object50110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:29.261 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.437062+0000", 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.79310999400002, 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1613549390000002, 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50111 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.262 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437062+0000", 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437062+0000", 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437061+0000", 2023-09-23T21:08:29.263 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437081+0000", 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0035999999999999e-05 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437091+0000", 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8660000000000001e-06 2023-09-23T21:08:29.264 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437096+0000", 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7989999999999999e-06 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437159+0000", 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2572999999999999e-05 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.265 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437172+0000", 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3294e-05 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394055+0000", 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.956883068 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394061+0000", 2023-09-23T21:08:29.266 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0079999999999999e-06 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.581905+0000", 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1878437550000001 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.582005+0000", 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9733000000000002e-05 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.582122+0000", 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116845 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585626+0000", 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035039440000000002 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585635+0000", 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3479999999999993e-06 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.597148+0000", 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011512992 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.597185+0000", 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7446e-05 2023-09-23T21:08:29.269 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.598417+0000", 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012315150000000001 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50125 11.2 11:4350bada:::benchmark_data_smithi121_477989_object50124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.628739+0000", 2023-09-23T21:08:29.270 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 291.60143248899999, 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9714031510000001, 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50125 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628739+0000", 2023-09-23T21:08:29.271 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628739+0000", 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628738+0000", 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628757+0000", 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8709999999999999e-05 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628774+0000", 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7088e-05 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628782+0000", 2023-09-23T21:08:29.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4679999999999992e-06 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628843+0000", 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0961999999999997e-05 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628853+0000", 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6399999999999992e-06 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.274 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394173+0000", 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.765319466 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394181+0000", 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6770000000000001e-06 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.275 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585661+0000", 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1914793539999999 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585773+0000", 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112244 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585937+0000", 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016426900000000001 2023-09-23T21:08:29.276 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.589209+0000", 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032715309999999998 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.591450+0000", 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022412119999999998 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.277 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.598481+0000", 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070309209999999999 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.598543+0000", 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1839000000000002e-05 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.600143+0000", 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016000369999999999 2023-09-23T21:08:29.278 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94323 11.2 11:4f2271be:::benchmark_data_smithi121_477989_object6134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.497029+0000", 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.73314308499999, 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0223154210000001, 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.279 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94323 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497029+0000", 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497029+0000", 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497027+0000", 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497032+0000", 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7949999999999998e-06 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.281 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497040+0000", 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.875e-06 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497045+0000", 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3820000000000003e-06 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497093+0000", 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7811e-05 2023-09-23T21:08:29.282 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497105+0000", 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2347000000000001e-05 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.513818+0000", 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.016712711 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.283 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.513924+0000", 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010610799999999999 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.514008+0000", 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3659999999999995e-05 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516509+0000", 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025014540000000002 2023-09-23T21:08:29.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516558+0000", 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8269999999999997e-05 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519004+0000", 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024466309999999999 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519173+0000", 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168226 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519218+0000", 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5862000000000002e-05 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519344+0000", 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012590999999999999 2023-09-23T21:08:29.286 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94333 11.2 11:4e5950f9:::benchmark_data_smithi121_477989_object6144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.516562+0000", 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.71361044, 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0030305859999999, 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.287 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94333 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516562+0000", 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516562+0000", 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516560+0000", 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516567+0000", 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1690000000000003e-06 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516579+0000", 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2296000000000001e-05 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516587+0000", 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7689999999999996e-06 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516624+0000", 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7214e-05 2023-09-23T21:08:29.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516634+0000", 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3919999999999992e-06 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516590+0000", 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99995588499999999 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516719+0000", 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000129018 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516798+0000", 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9468000000000006e-05 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519416+0000", 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026183119999999998 2023-09-23T21:08:29.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519446+0000", 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9918000000000001e-05 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519454+0000", 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.413e-06 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519491+0000", 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6854999999999999e-05 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519526+0000", 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5567e-05 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519592+0000", 2023-09-23T21:08:29.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6006000000000005e-05 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94386 11.2 11:4212ad5d:::benchmark_data_smithi121_477989_object6197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.644675+0000", 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.585497164, 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.88700020899999998, 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.295 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94386 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644675+0000", 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644675+0000", 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644673+0000", 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644683+0000", 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0200999999999999e-05 2023-09-23T21:08:29.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644702+0000", 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9069e-05 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644713+0000", 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0939e-05 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644765+0000", 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1374e-05 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644776+0000", 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1126e-05 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519633+0000", 2023-09-23T21:08:29.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.87485688800000005 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519721+0000", 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.886e-05 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519799+0000", 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7966999999999994e-05 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528429+0000", 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086300349999999994 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531481+0000", 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003051144 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531505+0000", 2023-09-23T21:08:29.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4156e-05 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531538+0000", 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3365999999999999e-05 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531597+0000", 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9021000000000003e-05 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531675+0000", 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7866000000000003e-05 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94417 11.2 11:44af34fd:::benchmark_data_smithi121_477989_object6228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.699432+0000", 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.53074041000001, 2023-09-23T21:08:29.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.83896963999999996, 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94417 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699432+0000", 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699432+0000", 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699431+0000", 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699435+0000", 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.49e-06 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699453+0000", 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8845e-05 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699460+0000", 2023-09-23T21:08:29.306 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4030000000000004e-06 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.701531+0000", 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020708800000000002 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.701540+0000", 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3440000000000007e-06 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.307 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.522132+0000", 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.82059189200000004 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.522247+0000", 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115277 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.522377+0000", 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012988 2023-09-23T21:08:29.308 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531714+0000", 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0093365370000000007 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531733+0000", 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9267999999999999e-05 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.309 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531751+0000", 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7833e-05 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538195+0000", 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064444460000000004 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538248+0000", 2023-09-23T21:08:29.310 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2831999999999998e-05 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538401+0000", 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015315399999999999 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94430 11.2 11:4c4574b1:::benchmark_data_smithi121_477989_object6241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.311 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.727165+0000", 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.503007282, 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.81163672499999995, 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94430 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.313 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727165+0000", 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727165+0000", 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.314 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727163+0000", 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727171+0000", 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4919999999999996e-06 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727187+0000", 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6348e-05 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.315 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727196+0000", 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9809999999999992e-06 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727246+0000", 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9808999999999997e-05 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.316 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727257+0000", 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.147e-05 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.525318+0000", 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79806034599999998 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.525430+0000", 2023-09-23T21:08:29.317 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112805 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.525546+0000", 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011565000000000001 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531773+0000", 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062273459999999999 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.318 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538463+0000", 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0066899999999999998 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538530+0000", 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6587999999999993e-05 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538622+0000", 2023-09-23T21:08:29.319 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1810000000000004e-05 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538674+0000", 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2438999999999997e-05 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538801+0000", 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000127193 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.320 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94445 11.2 11:4d352ece:::benchmark_data_smithi121_477989_object6256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.481821702, 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79675490599999998, 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94445 2023-09-23T21:08:29.321 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.322 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748353+0000", 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7010000000000001e-06 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.323 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748357+0000", 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8940000000000001e-06 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748361+0000", 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2219999999999998e-06 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748402+0000", 2023-09-23T21:08:29.324 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1729e-05 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748413+0000", 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0552e-05 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528479+0000", 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.78006554900000002 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.325 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528635+0000", 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000156188 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528773+0000", 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013783900000000001 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531810+0000", 2023-09-23T21:08:29.326 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030374569999999999 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538539+0000", 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067294110000000002 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538578+0000", 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8931999999999999e-05 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.327 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.544961+0000", 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063821549999999996 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545014+0000", 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3165999999999999e-05 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545105+0000", 2023-09-23T21:08:29.328 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1481e-05 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94484 11.2 11:43d1d258:::benchmark_data_smithi121_477989_object6295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.841431+0000", 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.388740912, 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70393903800000002, 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.329 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94484 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841431+0000", 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.330 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841431+0000", 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841430+0000", 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841436+0000", 2023-09-23T21:08:29.331 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2360000000000001e-06 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841448+0000", 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1894e-05 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841456+0000", 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0299999999999994e-06 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.332 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841516+0000", 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0090000000000002e-05 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841529+0000", 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3399e-05 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531837+0000", 2023-09-23T21:08:29.333 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.69030767299999996 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531992+0000", 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015467200000000001 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.532149+0000", 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015686199999999999 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.334 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538518+0000", 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063691370000000004 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538548+0000", 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9923000000000001e-05 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538889+0000", 2023-09-23T21:08:29.335 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00034130500000000003 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545165+0000", 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062759620000000004 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545224+0000", 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8542000000000003e-05 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.336 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545370+0000", 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014655500000000001 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94488 11.2 11:46589f20:::benchmark_data_smithi121_477989_object6299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.846779+0000", 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 104.38339290099999, 2023-09-23T21:08:29.337 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.69881891100000004, 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94488 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846779+0000", 2023-09-23T21:08:29.338 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846779+0000", 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846778+0000", 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.339 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846785+0000", 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1930000000000004e-06 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846797+0000", 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.254e-05 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846805+0000", 2023-09-23T21:08:29.340 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1310000000000006e-06 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.847172+0000", 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00036666999999999998 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.847182+0000", 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8889999999999997e-06 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.341 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.534969+0000", 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.68778722800000003 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.535105+0000", 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0001362 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.342 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.535236+0000", 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013034900000000001 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538556+0000", 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033200830000000001 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538849+0000", 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00029309900000000001 2023-09-23T21:08:29.343 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538929+0000", 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9872999999999999e-05 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545451+0000", 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065226119999999997 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.344 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545484+0000", 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2274999999999998e-05 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545598+0000", 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114301 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:108794 11.2 11:4a1ad7fb:::benchmark_data_smithi121_477989_object20605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-09-23T21:08:29.345 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:13.984453+0000", 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 75.245718969999999, 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25625654599999997, 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 108794 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.346 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984453+0000", 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984453+0000", 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984452+0000", 2023-09-23T21:08:29.347 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984459+0000", 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.622e-06 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984473+0000", 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3522e-05 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.348 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984481+0000", 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3019999999999995e-06 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984530+0000", 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8823999999999999e-05 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984654+0000", 2023-09-23T21:08:29.349 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012446399999999999 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.984767+0000", 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112617 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.993344+0000", 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0085772720000000004 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.350 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.993391+0000", 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7323000000000001e-05 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:13.995530+0000", 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021388639999999999 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.351 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:14.240557+0000", 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24502650100000001 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:14.240602+0000", 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5167000000000003e-05 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:14.240710+0000", 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107543 2023-09-23T21:08:29.352 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:112730 11.2 11:4690fad5:::benchmark_data_smithi121_477989_object24541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:21.562763+0000", 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 67.667409340999995, 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25248654500000001, 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.353 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 112730 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562763+0000", 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.354 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562763+0000", 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562762+0000", 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562768+0000", 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1589999999999998e-06 2023-09-23T21:08:29.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562779+0000", 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1277000000000001e-05 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562786+0000", 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9399999999999996e-06 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.356 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562834+0000", 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7580999999999997e-05 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.562962+0000", 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012875399999999999 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.563074+0000", 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000111282 2023-09-23T21:08:29.357 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.567373+0000", 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042992409999999997 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814918+0000", 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24754522500000001 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.358 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.815044+0000", 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012580100000000001 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.815076+0000", 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2492e-05 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.815120+0000", 2023-09-23T21:08:29.359 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3965999999999999e-05 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.815249+0000", 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012882900000000001 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:112727 11.f e408/397 11:f0db793a:::benchmark_data_smithi121_477989_object24538:head v 408'6991, mlcod=408'6988)", 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:21.568701+0000", 2023-09-23T21:08:29.360 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 67.661471402999993, 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.246007535, 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568701+0000", 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568701+0000", 2023-09-23T21:08:29.361 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568700+0000", 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568706+0000", 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4799999999999998e-06 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.362 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568718+0000", 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1440999999999999e-05 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568725+0000", 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4170000000000003e-06 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.363 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568770+0000", 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4527999999999998e-05 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.568793+0000", 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2940999999999999e-05 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814672+0000", 2023-09-23T21:08:29.364 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.245879605 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814708+0000", 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5828999999999999e-05 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPush(11.b 408/380 [PushOp(11:d8f2c940:::benchmark_data_smithi121_477989_object25005:head, version: 349'1553, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:d8f2c940:::benchmark_data_smithi121_477989_object25005:head@349'1553, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:21.569748+0000", 2023-09-23T21:08:29.365 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 67.660424273000004, 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24722485599999999, 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.569748+0000", 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.366 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.569748+0000", 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.569747+0000", 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.569793+0000", 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6198000000000001e-05 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.367 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.569812+0000", 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8651000000000002e-05 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.569827+0000", 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5299999999999999e-05 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814717+0000", 2023-09-23T21:08:29.368 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.244889731 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814731+0000", 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3711e-05 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.816973+0000", 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022420420000000001 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.369 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:112731 11.4 e408/369 11:205ea7fa:::benchmark_data_smithi121_477989_object24542:head v 408'7087, mlcod=0'0)", 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:21.571336+0000", 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 67.658835890999995, 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.247579877, 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.370 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.571336+0000", 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.571336+0000", 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.571336+0000", 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.371 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.571341+0000", 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.729e-06 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.571346+0000", 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6230000000000002e-06 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.571351+0000", 2023-09-23T21:08:29.372 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3810000000000001e-06 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814685+0000", 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24333495199999999 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814718+0000", 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2910000000000002e-05 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.373 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.818891+0000", 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041730099999999996 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.818916+0000", 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4564000000000001e-05 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.374 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:112726 11.6 e408/375 11:6b644b87:::benchmark_data_smithi121_477989_object24537:head v 408'6913, mlcod=408'6910)", 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:21.574192+0000", 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 67.655979654000006, 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25003646800000001, 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.574192+0000", 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.375 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.574192+0000", 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.574192+0000", 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.376 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.574197+0000", 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2959999999999998e-06 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814676+0000", 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24047905999999999 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814683+0000", 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1799999999999999e-06 2023-09-23T21:08:29.377 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814769+0000", 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5468000000000002e-05 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814792+0000", 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3187e-05 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.378 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.824196+0000", 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.009404417 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.824229+0000", 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2518000000000003e-05 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.379 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:112738 11.d 11:ba8f225b:::benchmark_data_smithi121_477989_object24549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:21.575141+0000", 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 67.655030640000007, 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24690894699999999, 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 112738 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:29.380 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.575141+0000", 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.575141+0000", 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:29.381 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.575140+0000", 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.575148+0000", 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0630000000000006e-06 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814662+0000", 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23951367700000001 2023-09-23T21:08:29.382 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814680+0000", 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8445e-05 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814832+0000", 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015250300000000001 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.383 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.814972+0000", 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013907000000000001 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,6", 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.815094+0000", 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000122216 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.821926+0000", 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0068325240000000004 2023-09-23T21:08:29.384 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.821959+0000", 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2261999999999997e-05 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.821966+0000", 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1550000000000001e-06 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.385 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.822011+0000", 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5679000000000001e-05 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:21.822050+0000", 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8909999999999998e-05 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:29.386 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:29.387 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:08:29.538 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:29.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:29.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:29.698 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:29.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:29 smithi123 ceph-mon[193903]: pgmap v1322: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 24136/94359 objects degraded (25.579%); 22915/94359 objects misplaced (24.285%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:08:29.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:29 smithi121 ceph-mon[211526]: pgmap v1322: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 24136/94359 objects degraded (25.579%); 22915/94359 objects misplaced (24.285%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:08:29.749 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:29 smithi121 ceph-mon[219709]: pgmap v1322: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 505 op/s; 24136/94359 objects degraded (25.579%); 22915/94359 objects misplaced (24.285%); 9.4 MiB/s, 149 objects/s recovering 2023-09-23T21:08:29.751 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:29.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:29.857 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:29.889 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:29.949 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:29.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:30.057 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:30.084 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:08:30.086 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:30.113 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:08:30.148 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:30.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:30.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:30.247 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:08:30.254 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:30.392 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:30.710 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:30.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:30.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:30.770 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:30 smithi121 ceph-mon[211526]: from='client.46568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:30.770 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:30 smithi121 ceph-mon[219709]: from='client.46568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:30.829 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:30.861 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:30 smithi123 ceph-mon[193903]: from='client.46568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:30.863 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:30.947 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:30.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:31.053 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:31.160 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:31.378 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:31.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:31.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:31.487 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:31.522 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:31.595 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:31.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:31.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:31.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:31 smithi121 ceph-mon[211526]: pgmap v1323: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 23435/86253 objects degraded (27.170%); 20234/86253 objects misplaced (23.459%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:31.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:31 smithi121 ceph-mon[219709]: pgmap v1323: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 23435/86253 objects degraded (27.170%); 20234/86253 objects misplaced (23.459%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:31.639 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1322,"stamp":"2023-09-23T21:08:29.086814+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2060261304,"num_objects":31453,"num_object_clones":0,"num_object_copies":94359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24136,"num_objects_misplaced":22915,"num_objects_unfound":0,"num_objects_dirty":31453,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":145245,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35024,"num_bytes_recovered":2290654476,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81760,"ondisk_log_size":81760,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9808080,"kb_used_data":5670104,"kb_used_omap":21,"kb_used_meta":4137770,"kb_avail":646407984,"statfs":{"total":671965249536,"available":661921775616,"internally_reserved":0,"allocated":5806186496,"data_stored":5794347466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4237076963},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-331612160,"num_objects":-5060,"num_object_clones":0,"num_object_copies":-15180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1308,"num_objects_misplaced":-4695,"num_objects_unfound":0,"num_objects_dirty":-5060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5060,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1498,"num_bytes_recovered":98172928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009772"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":184,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.745798+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:08:04.745798+0000","last_peered":"2023-09-23T21:08:04.745798+0000","last_clean":"2023-09-23T21:08:04.745798+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:08:04.745798+0000","last_undegraded":"2023-09-23T21:08:04.745798+0000","last_fullsized":"2023-09-23T21:08:04.745798+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":53,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737540+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:08:04.737540+0000","last_peered":"2023-09-23T21:08:04.737540+0000","last_clean":"2023-09-23T21:08:04.737540+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:08:04.737540+0000","last_undegraded":"2023-09-23T21:08:04.737540+0000","last_fullsized":"2023-09-23T21:08:04.737540+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":326,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.742362+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:08:04.742362+0000","last_peered":"2023-09-23T21:08:04.742362+0000","last_clean":"2023-09-23T21:08:04.742362+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:08:04.742362+0000","last_undegraded":"2023-09-23T21:08:04.742362+0000","last_fullsized":"2023-09-23T21:08:04.742362+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":50,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.767311+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:08:04.767311+0000","last_peered":"2023-09-23T21:08:04.767311+0000","last_clean":"2023-09-23T21:08:04.767311+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:08:04.767311+0000","last_undegraded":"2023-09-23T21:08:04.767311+0000","last_fullsized":"2023-09-23T21:08:04.767311+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":71,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.757299+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:08:04.757299+0000","last_peered":"2023-09-23T21:08:04.757299+0000","last_clean":"2023-09-23T21:08:04.757299+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:08:04.757299+0000","last_undegraded":"2023-09-23T21:08:04.757299+0000","last_fullsized":"2023-09-23T21:08:04.757299+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":279,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.740727+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:08:04.740727+0000","last_peered":"2023-09-23T21:08:04.740727+0000","last_clean":"2023-09-23T21:08:04.740727+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:08:04.740727+0000","last_undegraded":"2023-09-23T21:08:04.740727+0000","last_fullsized":"2023-09-23T21:08:04.740727+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.766849+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:08:04.766849+0000","last_peered":"2023-09-23T21:08:04.766849+0000","last_clean":"2023-09-23T21:08:04.766849+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:08:04.766849+0000","last_undegraded":"2023-09-23T21:08:04.766849+0000","last_fullsized":"2023-09-23T21:08:04.766849+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":53,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.757197+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:08:04.757197+0000","last_peered":"2023-09-23T21:08:04.757197+0000","last_clean":"2023-09-23T21:08:04.757197+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:08:04.757197+0000","last_undegraded":"2023-09-23T21:08:04.757197+0000","last_fullsized":"2023-09-23T21:08:04.757197+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":37,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.558518+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:08:17.558518+0000","last_peered":"2023-09-23T21:08:17.558518+0000","last_clean":"2023-09-23T21:08:17.558518+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:08:17.558518+0000","last_undegraded":"2023-09-23T21:08:17.558518+0000","last_fullsized":"2023-09-23T21:08:17.558518+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":53,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737228+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:08:04.737228+0000","last_peered":"2023-09-23T21:08:04.737228+0000","last_clean":"2023-09-23T21:08:04.737228+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:08:04.737228+0000","last_undegraded":"2023-09-23T21:08:04.737228+0000","last_fullsized":"2023-09-23T21:08:04.737228+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":69,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737624+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:08:04.737624+0000","last_peered":"2023-09-23T21:08:04.737624+0000","last_clean":"2023-09-23T21:08:04.737624+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:08:04.737624+0000","last_undegraded":"2023-09-23T21:08:04.737624+0000","last_fullsized":"2023-09-23T21:08:04.737624+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":45,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.745415+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:08:04.745415+0000","last_peered":"2023-09-23T21:08:04.745415+0000","last_clean":"2023-09-23T21:08:04.745415+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:08:04.745415+0000","last_undegraded":"2023-09-23T21:08:04.745415+0000","last_fullsized":"2023-09-23T21:08:04.745415+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":359,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.740316+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:08:04.740316+0000","last_peered":"2023-09-23T21:08:04.740316+0000","last_clean":"2023-09-23T21:08:04.740316+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:08:04.740316+0000","last_undegraded":"2023-09-23T21:08:04.740316+0000","last_fullsized":"2023-09-23T21:08:04.740316+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":332,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.626222+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:08:17.626222+0000","last_peered":"2023-09-23T21:08:17.626222+0000","last_clean":"2023-09-23T21:08:17.626222+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:08:17.626222+0000","last_undegraded":"2023-09-23T21:08:17.626222+0000","last_fullsized":"2023-09-23T21:08:17.626222+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":103,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.741740+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:08:04.741740+0000","last_peered":"2023-09-23T21:08:04.741740+0000","last_clean":"2023-09-23T21:08:04.741740+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:08:04.741740+0000","last_undegraded":"2023-09-23T21:08:04.741740+0000","last_fullsized":"2023-09-23T21:08:04.741740+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737345+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:08:04.737345+0000","last_peered":"2023-09-23T21:08:04.737345+0000","last_clean":"2023-09-23T21:08:04.737345+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:08:04.737345+0000","last_undegraded":"2023-09-23T21:08:04.737345+0000","last_fullsized":"2023-09-23T21:08:04.737345+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.752622+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:08:04.752622+0000","last_peered":"2023-09-23T21:08:04.752622+0000","last_clean":"2023-09-23T21:08:04.752622+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:08:04.752622+0000","last_undegraded":"2023-09-23T21:08:04.752622+0000","last_fullsized":"2023-09-23T21:08:04.752622+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":71,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737694+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:08:04.737694+0000","last_peered":"2023-09-23T21:08:04.737694+0000","last_clean":"2023-09-23T21:08:04.737694+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:08:04.737694+0000","last_undegraded":"2023-09-23T21:08:04.737694+0000","last_fullsized":"2023-09-23T21:08:04.737694+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":214,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.626628+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:08:17.626628+0000","last_peered":"2023-09-23T21:08:17.626628+0000","last_clean":"2023-09-23T21:08:17.626628+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:08:17.626628+0000","last_undegraded":"2023-09-23T21:08:17.626628+0000","last_fullsized":"2023-09-23T21:08:17.626628+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":50,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.767345+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:08:04.767345+0000","last_peered":"2023-09-23T21:08:04.767345+0000","last_clean":"2023-09-23T21:08:04.767345+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:08:04.767345+0000","last_undegraded":"2023-09-23T21:08:04.767345+0000","last_fullsized":"2023-09-23T21:08:04.767345+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":409,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.739163+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:08:04.739163+0000","last_peered":"2023-09-23T21:08:04.739163+0000","last_clean":"2023-09-23T21:08:04.739163+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:08:04.739163+0000","last_undegraded":"2023-09-23T21:08:04.739163+0000","last_fullsized":"2023-09-23T21:08:04.739163+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.738006+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:08:04.738006+0000","last_peered":"2023-09-23T21:08:04.738006+0000","last_clean":"2023-09-23T21:08:04.738006+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:08:04.738006+0000","last_undegraded":"2023-09-23T21:08:04.738006+0000","last_fullsized":"2023-09-23T21:08:04.738006+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"413'9258","reported_seq":10628,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:25.637280+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:08:25.637280+0000","last_peered":"2023-09-23T21:08:25.637280+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:08:25.637280+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"362'4528","ondisk_log_start":"362'4528","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4730,"log_dups_size":0,"ondisk_log_size":4730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129236992,"num_objects":1972,"num_object_clones":0,"num_object_copies":5916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9258,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-3643},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":1971}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":118,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.626693+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:08:17.626693+0000","last_peered":"2023-09-23T21:08:17.626693+0000","last_clean":"2023-09-23T21:08:17.626693+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:08:17.626693+0000","last_undegraded":"2023-09-23T21:08:17.626693+0000","last_fullsized":"2023-09-23T21:08:17.626693+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":37,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.558434+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:08:17.558434+0000","last_peered":"2023-09-23T21:08:17.558434+0000","last_clean":"2023-09-23T21:08:17.558434+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:08:17.558434+0000","last_undegraded":"2023-09-23T21:08:17.558434+0000","last_fullsized":"2023-09-23T21:08:17.558434+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"413'8869","reported_seq":8952,"reported_epoch":413,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:25.637027+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:08:25.637027+0000","last_peered":"2023-09-23T21:08:25.637027+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:08:25.637027+0000","last_undegraded":"2023-09-23T21:08:25.637027+0000","last_fullsized":"2023-09-23T21:08:25.637027+0000","mapping_epoch":375,"log_start":"360'4112","ondisk_log_start":"360'4112","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4757,"log_dups_size":0,"ondisk_log_size":4757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126418944,"num_objects":1929,"num_object_clones":0,"num_object_copies":5787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1929,"num_objects_unfound":0,"num_objects_dirty":1929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8869,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":1929}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":111,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.744309+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:08:04.744309+0000","last_peered":"2023-09-23T21:08:04.744309+0000","last_clean":"2023-09-23T21:08:04.744309+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:08:04.744309+0000","last_undegraded":"2023-09-23T21:08:04.744309+0000","last_fullsized":"2023-09-23T21:08:04.744309+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":373,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.742326+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:08:04.742326+0000","last_peered":"2023-09-23T21:08:04.742326+0000","last_clean":"2023-09-23T21:08:04.742326+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:08:04.742326+0000","last_undegraded":"2023-09-23T21:08:04.742326+0000","last_fullsized":"2023-09-23T21:08:04.742326+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"413'8990","reported_seq":21797,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:25.771332+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:08:25.771332+0000","last_peered":"2023-09-23T21:08:25.771332+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:08:25.771332+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"366'4911","ondisk_log_start":"366'4911","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4079,"log_dups_size":0,"ondisk_log_size":4079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127664128,"num_objects":1948,"num_object_clones":0,"num_object_copies":5844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3886,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8990,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4594,"num_bytes_recovered":301072384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"2,6,7","objects":1943},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":41,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.559043+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:08:17.559043+0000","last_peered":"2023-09-23T21:08:17.559043+0000","last_clean":"2023-09-23T21:08:17.559043+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:08:17.559043+0000","last_undegraded":"2023-09-23T21:08:17.559043+0000","last_fullsized":"2023-09-23T21:08:17.559043+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":301,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.739795+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:08:04.739795+0000","last_peered":"2023-09-23T21:08:04.739795+0000","last_clean":"2023-09-23T21:08:04.739795+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:08:04.739795+0000","last_undegraded":"2023-09-23T21:08:04.739795+0000","last_fullsized":"2023-09-23T21:08:04.739795+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"413'9099","reported_seq":9729,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:25.578478+0000","last_change":"2023-09-23T21:05:24.146336+0000","last_active":"2023-09-23T21:08:25.578478+0000","last_peered":"2023-09-23T21:08:25.578478+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:08:25.578478+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"366'4968","ondisk_log_start":"366'4968","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4131,"log_dups_size":0,"ondisk_log_size":4131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127860736,"num_objects":1951,"num_object_clones":0,"num_object_copies":5853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9099,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-3302},{"shards":"2,6,7","objects":1948}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":53,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737420+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:08:04.737420+0000","last_peered":"2023-09-23T21:08:04.737420+0000","last_clean":"2023-09-23T21:08:04.737420+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:08:04.737420+0000","last_undegraded":"2023-09-23T21:08:04.737420+0000","last_fullsized":"2023-09-23T21:08:04.737420+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":50,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.766973+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:08:04.766973+0000","last_peered":"2023-09-23T21:08:04.766973+0000","last_clean":"2023-09-23T21:08:04.766973+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:08:04.766973+0000","last_undegraded":"2023-09-23T21:08:04.766973+0000","last_fullsized":"2023-09-23T21:08:04.766973+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"413'9286","reported_seq":20788,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:25.697708+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:08:25.697708+0000","last_peered":"2023-09-23T21:08:25.697708+0000","last_clean":"2023-09-23T21:08:25.697708+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:08:25.697708+0000","last_undegraded":"2023-09-23T21:08:25.697708+0000","last_fullsized":"2023-09-23T21:08:25.697708+0000","mapping_epoch":400,"log_start":"366'5199","ondisk_log_start":"366'5199","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4087,"log_dups_size":0,"ondisk_log_size":4087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132513792,"num_objects":2022,"num_object_clones":0,"num_object_copies":6066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9286,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":70,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.738395+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:08:04.738395+0000","last_peered":"2023-09-23T21:08:04.738395+0000","last_clean":"2023-09-23T21:08:04.738395+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:08:04.738395+0000","last_undegraded":"2023-09-23T21:08:04.738395+0000","last_fullsized":"2023-09-23T21:08:04.738395+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":53,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737261+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:08:04.737261+0000","last_peered":"2023-09-23T21:08:04.737261+0000","last_clean":"2023-09-23T21:08:04.737261+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:08:04.737261+0000","last_undegraded":"2023-09-23T21:08:04.737261+0000","last_fullsized":"2023-09-23T21:08:04.737261+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"413'8991","reported_seq":20106,"reported_epoch":413,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:24.099380+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:08:24.099380+0000","last_peered":"2023-09-23T21:08:24.099380+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:08:24.099380+0000","last_undegraded":"2023-09-23T21:08:24.099380+0000","last_fullsized":"2023-09-23T21:08:24.099380+0000","mapping_epoch":403,"log_start":"347'488","ondisk_log_start":"347'488","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8503,"log_dups_size":0,"ondisk_log_size":8503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130482176,"num_objects":1991,"num_object_clones":0,"num_object_copies":5973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1991,"num_objects_unfound":0,"num_objects_dirty":1991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8991,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":1991}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":339,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.739859+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:08:04.739859+0000","last_peered":"2023-09-23T21:08:04.739859+0000","last_clean":"2023-09-23T21:08:04.739859+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:08:04.739859+0000","last_undegraded":"2023-09-23T21:08:04.739859+0000","last_fullsized":"2023-09-23T21:08:04.739859+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":424,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:09.310290+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:08:09.310290+0000","last_peered":"2023-09-23T21:08:09.310290+0000","last_clean":"2023-09-23T21:08:09.310290+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:08:09.310290+0000","last_undegraded":"2023-09-23T21:08:09.310290+0000","last_fullsized":"2023-09-23T21:08:09.310290+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"413'8954","reported_seq":9053,"reported_epoch":413,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:25.730772+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:08:25.730772+0000","last_peered":"2023-09-23T21:08:25.730772+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:08:25.730772+0000","last_undegraded":"2023-09-23T21:08:25.730772+0000","last_fullsized":"2023-09-23T21:08:25.730772+0000","mapping_epoch":374,"log_start":"366'4894","ondisk_log_start":"366'4894","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4060,"log_dups_size":0,"ondisk_log_size":4060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127795200,"num_objects":1950,"num_object_clones":0,"num_object_copies":5850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3900,"num_objects_unfound":0,"num_objects_dirty":1950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8954,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":1950}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":41,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.626453+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:08:17.626453+0000","last_peered":"2023-09-23T21:08:17.626453+0000","last_clean":"2023-09-23T21:08:17.626453+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:08:17.626453+0000","last_undegraded":"2023-09-23T21:08:17.626453+0000","last_fullsized":"2023-09-23T21:08:17.626453+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":452,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.740970+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:08:04.740970+0000","last_peered":"2023-09-23T21:08:04.740970+0000","last_clean":"2023-09-23T21:08:04.740970+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:08:04.740970+0000","last_undegraded":"2023-09-23T21:08:04.740970+0000","last_fullsized":"2023-09-23T21:08:04.740970+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"413'8991","reported_seq":9177,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:25.647467+0000","last_change":"2023-09-23T21:05:01.719967+0000","last_active":"2023-09-23T21:08:25.647467+0000","last_peered":"2023-09-23T21:08:25.647467+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:08:25.647467+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"360'4294","ondisk_log_start":"360'4294","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4697,"log_dups_size":0,"ondisk_log_size":4697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129171456,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8991,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2752512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-3468},{"shards":"1,6,7","objects":1971}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.757440+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:08:04.757440+0000","last_peered":"2023-09-23T21:08:04.757440+0000","last_clean":"2023-09-23T21:08:04.757440+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:08:04.757440+0000","last_undegraded":"2023-09-23T21:08:04.757440+0000","last_fullsized":"2023-09-23T21:08:04.757440+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":201,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.752198+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:08:04.752198+0000","last_peered":"2023-09-23T21:08:04.752198+0000","last_clean":"2023-09-23T21:08:04.752198+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:08:04.752198+0000","last_undegraded":"2023-09-23T21:08:04.752198+0000","last_fullsized":"2023-09-23T21:08:04.752198+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"413'9010","reported_seq":14507,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:24.334352+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:08:24.334352+0000","last_peered":"2023-09-23T21:08:24.334352+0000","last_clean":"2023-09-23T21:08:24.334352+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:08:24.334352+0000","last_undegraded":"2023-09-23T21:08:24.334352+0000","last_fullsized":"2023-09-23T21:08:24.334352+0000","mapping_epoch":364,"log_start":"347'509","ondisk_log_start":"347'509","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8501,"log_dups_size":0,"ondisk_log_size":8501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132317208,"num_objects":2020,"num_object_clones":0,"num_object_copies":6060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2020,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9010,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":308,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:10.656952+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:08:10.656952+0000","last_peered":"2023-09-23T21:08:10.656952+0000","last_clean":"2023-09-23T21:08:10.656952+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:08:10.656952+0000","last_undegraded":"2023-09-23T21:08:10.656952+0000","last_fullsized":"2023-09-23T21:08:10.656952+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":79,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.559113+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:08:17.559113+0000","last_peered":"2023-09-23T21:08:17.559113+0000","last_clean":"2023-09-23T21:08:17.559113+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:08:17.559113+0000","last_undegraded":"2023-09-23T21:08:17.559113+0000","last_fullsized":"2023-09-23T21:08:17.559113+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.745493+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:08:04.745493+0000","last_peered":"2023-09-23T21:08:04.745493+0000","last_clean":"2023-09-23T21:08:04.745493+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:08:04.745493+0000","last_undegraded":"2023-09-23T21:08:04.745493+0000","last_fullsized":"2023-09-23T21:08:04.745493+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"413'9137","reported_seq":18161,"reported_epoch":413,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:08:25.779990+0000","last_change":"2023-09-23T21:06:44.241880+0000","last_active":"2023-09-23T21:08:25.779990+0000","last_peered":"2023-09-23T21:08:25.779990+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:05:12.774149+0000","last_became_peered":"2023-09-23T21:05:12.774149+0000","last_unstale":"2023-09-23T21:08:25.779990+0000","last_undegraded":"2023-09-23T21:08:25.779990+0000","last_fullsized":"2023-09-23T21:05:12.415742+0000","mapping_epoch":380,"log_start":"366'5057","ondisk_log_start":"366'5057","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4080,"log_dups_size":0,"ondisk_log_size":4080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:20.943505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123797504,"num_objects":1889,"num_object_clones":0,"num_object_copies":5667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3204,"num_objects_unfound":0,"num_objects_dirty":1889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9137,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4445,"num_bytes_recovered":291307520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":821},{"shards":"2,6,7","objects":1068}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":389,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:10.656923+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:08:10.656923+0000","last_peered":"2023-09-23T21:08:10.656923+0000","last_clean":"2023-09-23T21:08:10.656923+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:08:10.656923+0000","last_undegraded":"2023-09-23T21:08:10.656923+0000","last_fullsized":"2023-09-23T21:08:10.656923+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":363,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.740221+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:08:04.740221+0000","last_peered":"2023-09-23T21:08:04.740221+0000","last_clean":"2023-09-23T21:08:04.740221+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:08:04.740221+0000","last_undegraded":"2023-09-23T21:08:04.740221+0000","last_fullsized":"2023-09-23T21:08:04.740221+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":45,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.746160+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:08:04.746160+0000","last_peered":"2023-09-23T21:08:04.746160+0000","last_clean":"2023-09-23T21:08:04.746160+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:08:04.746160+0000","last_undegraded":"2023-09-23T21:08:04.746160+0000","last_fullsized":"2023-09-23T21:08:04.746160+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"413'9045","reported_seq":9227,"reported_epoch":413,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:24.312259+0000","last_change":"2023-09-23T21:05:34.237405+0000","last_active":"2023-09-23T21:08:24.312259+0000","last_peered":"2023-09-23T21:08:24.312259+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:08:24.312259+0000","last_undegraded":"2023-09-23T21:08:24.312259+0000","last_fullsized":"2023-09-23T21:08:24.312259+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132186112,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4021,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9045,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2017}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":390,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.741600+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:08:04.741600+0000","last_peered":"2023-09-23T21:08:04.741600+0000","last_clean":"2023-09-23T21:08:04.741600+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:08:04.741600+0000","last_undegraded":"2023-09-23T21:08:04.741600+0000","last_fullsized":"2023-09-23T21:08:04.741600+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":308,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.744669+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:08:04.744669+0000","last_peered":"2023-09-23T21:08:04.744669+0000","last_clean":"2023-09-23T21:08:04.744669+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:08:04.744669+0000","last_undegraded":"2023-09-23T21:08:04.744669+0000","last_fullsized":"2023-09-23T21:08:04.744669+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":45,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.752075+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:08:04.752075+0000","last_peered":"2023-09-23T21:08:04.752075+0000","last_clean":"2023-09-23T21:08:04.752075+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:08:04.752075+0000","last_undegraded":"2023-09-23T21:08:04.752075+0000","last_fullsized":"2023-09-23T21:08:04.752075+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"413'8937","reported_seq":15196,"reported_epoch":413,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:25.681645+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:08:25.681645+0000","last_peered":"2023-09-23T21:08:25.681645+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:08:25.681645+0000","last_undegraded":"2023-09-23T21:08:25.681645+0000","last_fullsized":"2023-09-23T21:08:25.681645+0000","mapping_epoch":387,"log_start":"366'4880","ondisk_log_start":"366'4880","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4057,"log_dups_size":0,"ondisk_log_size":4057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126943232,"num_objects":1937,"num_object_clones":0,"num_object_copies":5811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1937,"num_objects_unfound":0,"num_objects_dirty":1937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8937,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","1","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":1937}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":228,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.741818+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:08:04.741818+0000","last_peered":"2023-09-23T21:08:04.741818+0000","last_clean":"2023-09-23T21:08:04.741818+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:08:04.741818+0000","last_undegraded":"2023-09-23T21:08:04.741818+0000","last_fullsized":"2023-09-23T21:08:04.741818+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":710,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:04.741552+0000","last_change":"2023-09-23T21:04:59.516091+0000","last_active":"2023-09-23T21:08:04.741552+0000","last_peered":"2023-09-23T21:08:04.741552+0000","last_clean":"2023-09-23T20:58:24.950697+0000","last_became_active":"2023-09-23T21:04:59.515745+0000","last_became_peered":"2023-09-23T21:04:59.515745+0000","last_unstale":"2023-09-23T21:08:04.741552+0000","last_undegraded":"2023-09-23T21:04:59.503738+0000","last_fullsized":"2023-09-23T21:04:59.411121+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:08.324929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.752542+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:08:04.752542+0000","last_peered":"2023-09-23T21:08:04.752542+0000","last_clean":"2023-09-23T21:08:04.752542+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:08:04.752542+0000","last_undegraded":"2023-09-23T21:08:04.752542+0000","last_fullsized":"2023-09-23T21:08:04.752542+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":301,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.744078+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:08:04.744078+0000","last_peered":"2023-09-23T21:08:04.744078+0000","last_clean":"2023-09-23T21:08:04.744078+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:08:04.744078+0000","last_undegraded":"2023-09-23T21:08:04.744078+0000","last_fullsized":"2023-09-23T21:08:04.744078+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"413'9141","reported_seq":9256,"reported_epoch":413,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:25.761507+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:08:25.761507+0000","last_peered":"2023-09-23T21:08:25.761507+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:08:25.761507+0000","last_undegraded":"2023-09-23T21:08:25.761507+0000","last_fullsized":"2023-09-23T21:08:25.761507+0000","mapping_epoch":374,"log_start":"366'5005","ondisk_log_start":"366'5005","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4136,"log_dups_size":0,"ondisk_log_size":4136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129826816,"num_objects":1981,"num_object_clones":0,"num_object_copies":5943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3962,"num_objects_unfound":0,"num_objects_dirty":1981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9141,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":1981}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":45,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.741422+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:08:04.741422+0000","last_peered":"2023-09-23T21:08:04.741422+0000","last_clean":"2023-09-23T21:08:04.741422+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:08:04.741422+0000","last_undegraded":"2023-09-23T21:08:04.741422+0000","last_fullsized":"2023-09-23T21:08:04.741422+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.766563+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:08:04.766563+0000","last_peered":"2023-09-23T21:08:04.766563+0000","last_clean":"2023-09-23T21:08:04.766563+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:08:04.766563+0000","last_undegraded":"2023-09-23T21:08:04.766563+0000","last_fullsized":"2023-09-23T21:08:04.766563+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":50,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.766894+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:08:04.766894+0000","last_peered":"2023-09-23T21:08:04.766894+0000","last_clean":"2023-09-23T21:08:04.766894+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:08:04.766894+0000","last_undegraded":"2023-09-23T21:08:04.766894+0000","last_fullsized":"2023-09-23T21:08:04.766894+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"413'9042","reported_seq":16206,"reported_epoch":413,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:08:25.698053+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:08:25.698053+0000","last_peered":"2023-09-23T21:08:25.698053+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:08:25.698053+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:08:25.698053+0000","mapping_epoch":383,"log_start":"361'4326","ondisk_log_start":"361'4326","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4716,"log_dups_size":0,"ondisk_log_size":4716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130023424,"num_objects":1984,"num_object_clones":0,"num_object_copies":5952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1980,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":1984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9042,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3530,"num_bytes_recovered":231342080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":4},{"shards":"1,2,7","objects":1980}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":50,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.766795+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:08:04.766795+0000","last_peered":"2023-09-23T21:08:04.766795+0000","last_clean":"2023-09-23T21:08:04.766795+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:08:04.766795+0000","last_undegraded":"2023-09-23T21:08:04.766795+0000","last_fullsized":"2023-09-23T21:08:04.766795+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":310,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:10.656980+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:08:10.656980+0000","last_peered":"2023-09-23T21:08:10.656980+0000","last_clean":"2023-09-23T21:08:10.656980+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:08:10.656980+0000","last_undegraded":"2023-09-23T21:08:10.656980+0000","last_fullsized":"2023-09-23T21:08:10.656980+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":89,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.558688+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:08:17.558688+0000","last_peered":"2023-09-23T21:08:17.558688+0000","last_clean":"2023-09-23T21:08:17.558688+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:08:17.558688+0000","last_undegraded":"2023-09-23T21:08:17.558688+0000","last_fullsized":"2023-09-23T21:08:17.558688+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"413'8961","reported_seq":21853,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:25.687387+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:08:25.687387+0000","last_peered":"2023-09-23T21:08:25.687387+0000","last_clean":"2023-09-23T21:08:25.687387+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:08:25.687387+0000","last_undegraded":"2023-09-23T21:08:25.687387+0000","last_fullsized":"2023-09-23T21:08:25.687387+0000","mapping_epoch":397,"log_start":"360'4245","ondisk_log_start":"360'4245","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4716,"log_dups_size":0,"ondisk_log_size":4716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124715008,"num_objects":1903,"num_object_clones":0,"num_object_copies":5709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8961,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":70,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.738090+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:08:04.738090+0000","last_peered":"2023-09-23T21:08:04.738090+0000","last_clean":"2023-09-23T21:08:04.738090+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:08:04.738090+0000","last_undegraded":"2023-09-23T21:08:04.738090+0000","last_fullsized":"2023-09-23T21:08:04.738090+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":45,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.743937+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:08:04.743937+0000","last_peered":"2023-09-23T21:08:04.743937+0000","last_clean":"2023-09-23T21:08:04.743937+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:08:04.743937+0000","last_undegraded":"2023-09-23T21:08:04.743937+0000","last_fullsized":"2023-09-23T21:08:04.743937+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":62,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.756975+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:08:04.756975+0000","last_peered":"2023-09-23T21:08:04.756975+0000","last_clean":"2023-09-23T21:08:04.756975+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:08:04.756975+0000","last_undegraded":"2023-09-23T21:08:04.756975+0000","last_fullsized":"2023-09-23T21:08:04.756975+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"413'9234","reported_seq":16623,"reported_epoch":413,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:08:25.711724+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:08:25.711724+0000","last_peered":"2023-09-23T21:08:25.711724+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:08:25.711724+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:08:25.711724+0000","mapping_epoch":382,"log_start":"362'4512","ondisk_log_start":"362'4512","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4722,"log_dups_size":0,"ondisk_log_size":4722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1961,"num_objects_misplaced":1967,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9234,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3635,"num_bytes_recovered":238223360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":1961}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":309,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.740791+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:08:04.740791+0000","last_peered":"2023-09-23T21:08:04.740791+0000","last_clean":"2023-09-23T21:08:04.740791+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:08:04.740791+0000","last_undegraded":"2023-09-23T21:08:04.740791+0000","last_fullsized":"2023-09-23T21:08:04.740791+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":54,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.767185+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:08:04.767185+0000","last_peered":"2023-09-23T21:08:04.767185+0000","last_clean":"2023-09-23T21:08:04.767185+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:08:04.767185+0000","last_undegraded":"2023-09-23T21:08:04.767185+0000","last_fullsized":"2023-09-23T21:08:04.767185+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":99,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.558386+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:08:17.558386+0000","last_peered":"2023-09-23T21:08:17.558386+0000","last_clean":"2023-09-23T21:08:17.558386+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:08:17.558386+0000","last_undegraded":"2023-09-23T21:08:17.558386+0000","last_fullsized":"2023-09-23T21:08:17.558386+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":59,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.766886+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:08:04.766886+0000","last_peered":"2023-09-23T21:08:04.766886+0000","last_clean":"2023-09-23T21:08:04.766886+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:08:04.766886+0000","last_undegraded":"2023-09-23T21:08:04.766886+0000","last_fullsized":"2023-09-23T21:08:04.766886+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":436,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.743358+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:08:04.743358+0000","last_peered":"2023-09-23T21:08:04.743358+0000","last_clean":"2023-09-23T21:08:04.743358+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:08:04.743358+0000","last_undegraded":"2023-09-23T21:08:04.743358+0000","last_fullsized":"2023-09-23T21:08:04.743358+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":76,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.626807+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:08:17.626807+0000","last_peered":"2023-09-23T21:08:17.626807+0000","last_clean":"2023-09-23T21:08:17.626807+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:08:17.626807+0000","last_undegraded":"2023-09-23T21:08:17.626807+0000","last_fullsized":"2023-09-23T21:08:17.626807+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":79,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.738468+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:08:04.738468+0000","last_peered":"2023-09-23T21:08:04.738468+0000","last_clean":"2023-09-23T21:08:04.738468+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:08:04.738468+0000","last_undegraded":"2023-09-23T21:08:04.738468+0000","last_fullsized":"2023-09-23T21:08:04.738468+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":294,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.744813+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:08:04.744813+0000","last_peered":"2023-09-23T21:08:04.744813+0000","last_clean":"2023-09-23T21:08:04.744813+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:08:04.744813+0000","last_undegraded":"2023-09-23T21:08:04.744813+0000","last_fullsized":"2023-09-23T21:08:04.744813+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":71,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.752698+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:08:04.752698+0000","last_peered":"2023-09-23T21:08:04.752698+0000","last_clean":"2023-09-23T21:08:04.752698+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:08:04.752698+0000","last_undegraded":"2023-09-23T21:08:04.752698+0000","last_fullsized":"2023-09-23T21:08:04.752698+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":53,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737797+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:08:04.737797+0000","last_peered":"2023-09-23T21:08:04.737797+0000","last_clean":"2023-09-23T21:08:04.737797+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:08:04.737797+0000","last_undegraded":"2023-09-23T21:08:04.737797+0000","last_fullsized":"2023-09-23T21:08:04.737797+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":62,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.737030+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:08:04.737030+0000","last_peered":"2023-09-23T21:08:04.737030+0000","last_clean":"2023-09-23T21:08:04.737030+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:08:04.737030+0000","last_undegraded":"2023-09-23T21:08:04.737030+0000","last_fullsized":"2023-09-23T21:08:04.737030+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-09-23T21:08:04.766700+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:08:04.766700+0000","last_peered":"2023-09-23T21:08:04.766700+0000","last_clean":"2023-09-23T21:08:04.766700+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:08:04.766700+0000","last_undegraded":"2023-09-23T21:08:04.766700+0000","last_fullsized":"2023-09-23T21:08:04.766700+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":41,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-09-23T21:08:17.625969+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:08:17.625969+0000","last_peered":"2023-09-23T21:08:17.625969+0000","last_clean":"2023-09-23T21:08:17.625969+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:08:17.625969+0000","last_undegraded":"2023-09-23T21:08:17.625969+0000","last_fullsized":"2023-09-23T21:08:17.625969+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2059665432,"num_objects":31429,"num_object_clones":0,"num_object_copies":94287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24134,"num_objects_misplaced":22915,"num_objects_unfound":0,"num_objects_dirty":31429,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144945,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34907,"num_bytes_recovered":2287665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8856350720,"data_stored":8856338504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81544,"ondisk_log_size":81544,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201757,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463377,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1931708,"kb_used_data":1400180,"kb_used_omap":0,"kb_used_meta":531456,"kb_avail":91813444,"statfs":{"total":95995035648,"available":94016966656,"internally_reserved":0,"allocated":1433784320,"data_stored":1432568322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":544210944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":6.6479999999999997,"5min":2.0830000000000002,"15min":2.2610000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":99.533000000000001,"5min":99.533000000000001,"15min":102.242},"last":1.613},{"interface":"front","average":{"1min":6.8070000000000004,"5min":2.1949999999999998,"15min":2.0470000000000002},"min":{"1min":0.248,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":99.325000000000003,"5min":99.325000000000003,"15min":102.502},"last":0.73099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:08:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":4.3150000000000004,"15min":2.5110000000000001},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":30.526,"5min":119.077,"15min":119.077},"last":7.915},{"interface":"front","average":{"1min":4.2009999999999996,"5min":4.2759999999999998,"15min":2.5990000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":30.564,"5min":119.313,"15min":119.313},"last":2.71}]},{"osd":2,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":13.304,"5min":6.1219999999999999,"15min":3.726},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":199.23400000000001,"5min":199.23400000000001,"15min":199.23400000000001},"last":0.92000000000000004},{"interface":"front","average":{"1min":12.943,"5min":6.1870000000000003,"15min":3.8719999999999999},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.878}]},{"osd":3,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":0.98599999999999999,"15min":4.0119999999999996},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":3.194,"5min":5.9420000000000002,"15min":250.804},"last":0.83599999999999997},{"interface":"front","average":{"1min":2.6269999999999998,"5min":1.45,"15min":4.0099999999999998},"min":{"1min":0.371,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":13.319000000000001,"5min":13.319000000000001,"15min":250.446},"last":1.415}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.353,"15min":0.97099999999999997},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.24299999999999999},"max":{"1min":6.3689999999999998,"5min":13.1,"15min":13.1},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.98699999999999999,"15min":0.95199999999999996},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":4.3849999999999998,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.333,"15min":1.105},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.28999999999999998},"max":{"1min":6.343,"5min":11.281000000000001,"15min":18.962},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.325,"5min":1.286,"15min":1.212},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":6.6890000000000001,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":0.32200000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.2929999999999999,"15min":1.0740000000000001},"min":{"1min":0.379,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":3.1139999999999999,"5min":7.8079999999999998,"15min":11.388999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.276,"15min":1.1080000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30099999999999999},"max":{"1min":3.4830000000000001,"5min":7.1849999999999996,"15min":8.8010000000000002},"last":0.37}]}]},{"osd":1,"up_from":276,"seq":1185410973823,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2174104,"kb_used_data":1720272,"kb_used_omap":5,"kb_used_meta":453754,"kb_avail":91571048,"statfs":{"total":95995035648,"available":93768753152,"internally_reserved":0,"allocated":1761558528,"data_stored":1759495433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":464644835},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.738,"15min":0.86599999999999999},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":18.097999999999999,"5min":18.097999999999999,"15min":18.097999999999999},"last":13.537000000000001},{"interface":"front","average":{"1min":3.5459999999999998,"5min":1.407,"15min":1.0089999999999999},"min":{"1min":0.29999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":11.228,"5min":11.228,"15min":18.047999999999998},"last":16.148}]},{"osd":2,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0300000000000002,"5min":1.7010000000000001,"15min":1.4379999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.874000000000001,"5min":13.874000000000001,"15min":78.409000000000006},"last":21.695},{"interface":"front","average":{"1min":5.5030000000000001,"5min":1.829,"15min":1.319},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.22},"max":{"1min":18.288,"5min":18.288,"15min":76.245000000000005},"last":13.41}]},{"osd":3,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.415,"5min":1.716,"15min":2.782},"min":{"1min":0.28699999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":18.344999999999999,"5min":18.344999999999999,"15min":172.44399999999999},"last":13.558999999999999},{"interface":"front","average":{"1min":3.4009999999999998,"5min":1.627,"15min":2.8799999999999999},"min":{"1min":0.36399999999999999,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":11.311999999999999,"5min":11.311999999999999,"15min":172.58500000000001},"last":17.033999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.446,"15min":1.0389999999999999},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6500000000000004,"5min":12.238,"15min":17.006},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.9769999999999999,"5min":1.583,"15min":0.93100000000000005},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":18.173999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":13.442}]},{"osd":5,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":1.397,"15min":1.05},"min":{"1min":0.34499999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.340999999999999,"5min":11.340999999999999,"15min":17.155000000000001},"last":16.196000000000002},{"interface":"front","average":{"1min":2.758,"5min":1.55,"15min":1.149},"min":{"1min":0.35799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.949999999999999,"5min":12.345000000000001,"15min":17.076000000000001},"last":0.29199999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":1.47,"15min":1.534},"min":{"1min":0.42699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":11.375,"5min":11.666,"15min":127.995},"last":16.222000000000001},{"interface":"front","average":{"1min":2.7890000000000001,"5min":1.633,"15min":1.6180000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.012,"5min":12.276999999999999,"15min":126.836},"last":0.35499999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9609999999999999,"5min":3.0670000000000002,"15min":1.645},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":11.17,"5min":151.791,"15min":151.791},"last":0.38100000000000001},{"interface":"front","average":{"1min":4.173,"5min":3.0800000000000001,"15min":1.47},"min":{"1min":0.41799999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":18.260000000000002,"5min":151.809,"15min":151.809},"last":13.486000000000001}]}]},{"osd":0,"up_from":92,"seq":395136991563,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1050000,"kb_used_data":301320,"kb_used_omap":5,"kb_used_meta":748666,"kb_avail":92695152,"statfs":{"total":95995035648,"available":94919835648,"internally_reserved":0,"allocated":308551680,"data_stored":306610269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":766634618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.423,"15min":1.841},"min":{"1min":0.315,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":41.149999999999999,"5min":61.341999999999999,"15min":61.341999999999999},"last":4.9400000000000004},{"interface":"front","average":{"1min":6.5499999999999998,"5min":3.5920000000000001,"15min":1.845},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.217},"max":{"1min":38.936,"5min":61.299999999999997,"15min":61.299999999999997},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":5.468,"5min":2.4609999999999999,"15min":2.4790000000000001},"min":{"1min":0.24099999999999999,"5min":0.217,"15min":0.161},"max":{"1min":45.912999999999997,"5min":45.912999999999997,"15min":183.77099999999999},"last":4.7869999999999999},{"interface":"front","average":{"1min":5.2930000000000001,"5min":2.5070000000000001,"15min":2.6949999999999998},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":46.234000000000002,"5min":46.234000000000002,"15min":183.78700000000001},"last":0.33500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.276,"15min":3.1070000000000002},"min":{"1min":0.28999999999999998,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":12.113,"5min":12.113,"15min":186.143},"last":4.9630000000000001},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.1759999999999999,"15min":3.1629999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":12.348000000000001,"5min":12.348000000000001,"15min":186.97399999999999},"last":1.528}]},{"osd":4,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":1.2789999999999999,"15min":1.3600000000000001},"min":{"1min":0.376,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":10.814,"5min":10.814,"15min":10.814},"last":2.552},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.03,"15min":0.96799999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.586,"5min":10.586,"15min":11.493},"last":4.8380000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.454,"15min":2.0590000000000002},"min":{"1min":0.34399999999999997,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":10.763,"5min":10.763,"15min":154.30600000000001},"last":2.593},{"interface":"front","average":{"1min":2.129,"5min":1.147,"15min":1.5189999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28699999999999998},"max":{"1min":10.618,"5min":11.478,"15min":154.42400000000001},"last":4.891}]},{"osd":6,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":5.9500000000000002,"15min":3.214},"min":{"1min":0.35199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.654,"5min":170.12700000000001,"15min":170.12700000000001},"last":4.8659999999999997},{"interface":"front","average":{"1min":1.494,"5min":5.9000000000000004,"15min":3.3380000000000001},"min":{"1min":0.39900000000000002,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":5.7750000000000004,"5min":169.767,"15min":169.767},"last":0.53600000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.347,"15min":2.2509999999999999},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.74,"5min":10.74,"15min":164.26400000000001},"last":4.9160000000000004},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.0720000000000001,"15min":2.2549999999999999},"min":{"1min":0.44,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":5.8159999999999998,"5min":5.8159999999999998,"15min":164.23099999999999},"last":0.63200000000000001}]}]},{"osd":2,"up_from":129,"seq":554050781454,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2153944,"kb_used_data":1504976,"kb_used_omap":7,"kb_used_meta":648952,"kb_avail":91591208,"statfs":{"total":95995035648,"available":93789396992,"internally_reserved":0,"allocated":1541095424,"data_stored":1539088619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":664526929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.4450000000000001,"15min":1.3959999999999999},"min":{"1min":0.20499999999999999,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":6.8440000000000003,"5min":15.667,"15min":26.356000000000002},"last":11.699999999999999},{"interface":"front","average":{"1min":6.4569999999999999,"5min":2.0830000000000002,"15min":1.8320000000000001},"min":{"1min":0.28799999999999998,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":23.963999999999999,"5min":23.963999999999999,"15min":26.341000000000001},"last":9.0860000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4109999999999996,"5min":5.375,"15min":2.6240000000000001},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.188},"max":{"1min":14.057,"5min":212.12,"15min":212.12},"last":10.403},{"interface":"front","average":{"1min":2.5760000000000001,"5min":4.8449999999999998,"15min":2.3330000000000002},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":6.5709999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":11.840999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:47 2023","interfaces":[{"interface":"back","average":{"1min":10.468999999999999,"5min":2.8500000000000001,"15min":3.2440000000000002},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":138.90299999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":15.327999999999999},{"interface":"front","average":{"1min":10.629,"5min":2.8900000000000001,"15min":3.2090000000000001},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":138.87700000000001,"5min":138.87700000000001,"15min":138.87700000000001},"last":33.889000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":1.153,"15min":1.113},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":6.383,"5min":12.586,"15min":12.586},"last":11.744},{"interface":"front","average":{"1min":4.1820000000000004,"5min":1.5700000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":14.672000000000001,"5min":14.672000000000001,"15min":14.672000000000001},"last":8.9800000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":4.367,"5min":1.7050000000000001,"15min":2.129},"min":{"1min":0.36799999999999999,"5min":0.29999999999999999,"15min":0.27100000000000002},"max":{"1min":14.710000000000001,"5min":14.710000000000001,"15min":173.34},"last":9.0169999999999995},{"interface":"front","average":{"1min":2.097,"5min":1.173,"15min":1.796},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":8.9740000000000002,"5min":13.701000000000001,"15min":173.15000000000001},"last":5.4560000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":3.3940000000000001,"15min":1.946},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":9.0329999999999995,"5min":175.72499999999999,"15min":175.72499999999999},"last":5.423},{"interface":"front","average":{"1min":1.901,"5min":3.375,"15min":1.9550000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.5049999999999999,"5min":176.20400000000001,"15min":176.20400000000001},"last":11.782}]},{"osd":7,"last update":"Sat Sep 23 21:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9490000000000001,"5min":1.655,"15min":2.081},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":6.4809999999999999,"5min":12.609,"15min":177.30799999999999},"last":11.811999999999999},{"interface":"front","average":{"1min":4.2240000000000002,"5min":1.645,"15min":2.1989999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33600000000000002},"max":{"1min":14.736000000000001,"5min":14.736000000000001,"15min":177.40899999999999},"last":9.0559999999999992}]}]},{"osd":3,"up_from":148,"seq":635655160048,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":569368,"kb_used_data":6608,"kb_used_omap":2,"kb_used_meta":562749,"kb_avail":93175784,"statfs":{"total":95995035648,"available":95412002816,"internally_reserved":0,"allocated":6766592,"data_stored":4579569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":576255029},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":1.4910000000000001,"15min":1.774},"min":{"1min":0.249,"5min":0.20999999999999999,"15min":0.184},"max":{"1min":8.8629999999999995,"5min":44.963000000000001,"15min":95.507999999999996},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.8919999999999999,"5min":0.94799999999999995,"15min":1.6679999999999999},"min":{"1min":0.247,"5min":0.20300000000000001,"15min":0.17199999999999999},"max":{"1min":7.7850000000000001,"5min":9.7370000000000001,"15min":95.259},"last":0.79100000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":12.473000000000001,"5min":3.4580000000000002,"15min":1.8380000000000001},"min":{"1min":0.64300000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":109.51300000000001,"5min":109.51300000000001,"15min":109.51300000000001},"last":1.8879999999999999},{"interface":"front","average":{"1min":13.468,"5min":3.6669999999999998,"15min":2.121},"min":{"1min":0.26500000000000001,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":111.13500000000001,"5min":111.13500000000001,"15min":111.13500000000001},"last":1.071}]},{"osd":2,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4129999999999998,"5min":1.4750000000000001,"15min":1.351},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.182},"max":{"1min":14.243,"5min":14.243,"15min":14.243},"last":1.72},{"interface":"front","average":{"1min":2.6059999999999999,"5min":1.325,"15min":1.3420000000000001},"min":{"1min":0.34399999999999997,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":7.8120000000000003,"5min":9.7140000000000004,"15min":19.754999999999999},"last":0.879}]},{"osd":4,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4980000000000002,"5min":2.5470000000000002,"15min":2.2559999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.222},"max":{"1min":38.600000000000001,"5min":70.728999999999999,"15min":111.36},"last":0.57299999999999995},{"interface":"front","average":{"1min":3.7919999999999998,"5min":2.4529999999999998,"15min":2.0539999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.252},"max":{"1min":38.524000000000001,"5min":70.712000000000003,"15min":112.473},"last":0.81899999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":3.7730000000000001,"5min":1.575,"15min":1.581},"min":{"1min":0.41199999999999998,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":36.488999999999997,"5min":36.488999999999997,"15min":64.528000000000006},"last":0.54100000000000004},{"interface":"front","average":{"1min":3.5110000000000001,"5min":1.6879999999999999,"15min":1.647},"min":{"1min":0.374,"5min":0.36499999999999999,"15min":0.249},"max":{"1min":36.295000000000002,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.3740000000000001,"15min":1.623},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.312},"max":{"1min":10.827999999999999,"5min":10.827999999999999,"15min":19.568000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.2450000000000001,"5min":1.448,"15min":1.5029999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":7.6749999999999998,"5min":14.356,"15min":15.573},"last":1.7669999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:07:38 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":2.3599999999999999,"15min":2.5510000000000002},"min":{"1min":0.5,"5min":0.38100000000000001,"15min":0.30499999999999999},"max":{"1min":7.7370000000000001,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.2549999999999999,"5min":2.3530000000000002,"15min":2.8170000000000002},"min":{"1min":0.48099999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":7.4249999999999998,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.67400000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986667,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1059400,"kb_used_data":488192,"kb_used_omap":0,"kb_used_meta":571200,"kb_avail":92685752,"statfs":{"total":95995035648,"available":94910210048,"internally_reserved":0,"allocated":499908608,"data_stored":498688159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":584908800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.4770000000000001,"15min":2.6949999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":12.869,"5min":47.476999999999997,"15min":146.25299999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.2849999999999999,"15min":2.6920000000000002},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.223},"max":{"1min":12.848000000000001,"5min":12.848000000000001,"15min":144.77000000000001},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.782,"5min":7.7530000000000001,"15min":3.5259999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":14.523999999999999,"5min":127.411,"15min":127.411},"last":31.065000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":5.7130000000000001,"15min":2.835},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":5.8609999999999998,"5min":118.075,"15min":118.075},"last":30.948}]},{"osd":2,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":6.6639999999999997,"5min":6.7910000000000004,"15min":3.5299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":71.454999999999998,"5min":164.517,"15min":164.517},"last":0.48299999999999998},{"interface":"front","average":{"1min":5.343,"5min":6.282,"15min":3.5099999999999998},"min":{"1min":0.34899999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":71.191999999999993,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":1.2909999999999999,"15min":2.0510000000000002},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":6.8120000000000003,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.2150000000000001,"15min":2.149},"min":{"1min":0.40600000000000003,"5min":0.32400000000000001,"15min":0.25600000000000001},"max":{"1min":6.181,"5min":8.2899999999999991,"15min":142.34700000000001},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":0.95999999999999996,"15min":1.02},"min":{"1min":0.309,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":6.0069999999999997,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.754,"5min":1.0149999999999999,"15min":1.032},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.21099999999999999},"max":{"1min":4.0259999999999998,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:07:34 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.1779999999999999,"15min":0.98199999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":4.8399999999999999,"5min":11.800000000000001,"15min":11.887},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.2410000000000001,"15min":0.93700000000000006},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0419999999999998,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":2.6829999999999998,"15min":1.544},"min":{"1min":0.34300000000000003,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":3.726,"5min":145.721,"15min":145.721},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":3.0049999999999999,"15min":1.7},"min":{"1min":0.375,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":3.383,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.442}]}]},{"osd":5,"up_from":39,"seq":167503725031,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":869556,"kb_used_data":248556,"kb_used_omap":0,"kb_used_meta":620992,"kb_avail":92875596,"statfs":{"total":95995035648,"available":95104610304,"internally_reserved":0,"allocated":254521344,"data_stored":253317095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":635895808},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":0.86399999999999999,"15min":1.325},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":10.488,"5min":10.488,"15min":80.105999999999995},"last":1.4970000000000001},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.90700000000000003,"15min":1.446},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":4.75,"5min":8.2759999999999998,"15min":80.152000000000001},"last":1.3009999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":9.4730000000000008,"5min":7.8259999999999996,"15min":3.117},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":125.23399999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":7.7610000000000001},{"interface":"front","average":{"1min":9.9860000000000007,"5min":7.6059999999999999,"15min":3.0600000000000001},"min":{"1min":0.375,"5min":0.251,"15min":0.247},"max":{"1min":125.249,"5min":233.75200000000001,"15min":233.75200000000001},"last":9.4019999999999992}]},{"osd":2,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":15.840999999999999,"5min":3.7839999999999998,"15min":2.4689999999999999},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.22500000000000001},"max":{"1min":231.34700000000001,"5min":231.34700000000001,"15min":231.34700000000001},"last":1.429},{"interface":"front","average":{"1min":15.951000000000001,"5min":4.008,"15min":2.6110000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":231.381,"5min":231.381,"15min":231.381},"last":2.891}]},{"osd":3,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":1.3260000000000001,"15min":2.7959999999999998},"min":{"1min":0.38500000000000001,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":7.0300000000000002,"5min":10.670999999999999,"15min":153.60300000000001},"last":3.0219999999999998},{"interface":"front","average":{"1min":1.653,"5min":1.0580000000000001,"15min":2.6880000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":4.1779999999999999,"5min":7.3810000000000002,"15min":153.774},"last":1.6499999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.72399999999999998,"15min":0.71399999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":6.3940000000000001,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.0129999999999999,"15min":0.89200000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":5.8490000000000002,"5min":8.3740000000000006,"15min":12.539},"last":0.40000000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.1160000000000001,"15min":1.1539999999999999},"min":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.27100000000000002},"max":{"1min":3.6760000000000002,"5min":9.8480000000000008,"15min":71.230000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.25,"15min":1.286},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":4.0880000000000001,"5min":12.673999999999999,"15min":70.266000000000005},"last":0.375}]},{"osd":7,"last update":"Sat Sep 23 21:08:14 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.284,"15min":1.129},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.312},"max":{"1min":9.6329999999999991,"5min":9.6329999999999991,"15min":31.091999999999999},"last":1.202},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.70799999999999996,"15min":0.93500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35899999999999999,"15min":0.28899999999999998},"max":{"1min":4.29,"5min":4.29,"15min":31.780000000000001},"last":0.36099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":299827200,"data_stored":299827200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1742667776,"data_stored":1742667776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1530138624,"data_stored":1530134552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1245184,"data_stored":1245184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":490999808,"data_stored":490995736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":247922688,"data_stored":247922688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1411518464,"data_stored":1411514392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3132030976,"data_stored":3132030976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:08:31.645 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:08:31.703 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:31.768 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:08:31.823 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:31.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:31 smithi123 ceph-mon[193903]: pgmap v1323: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 23435/86253 objects degraded (27.170%); 20234/86253 objects misplaced (23.459%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:32.126 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:32.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:32.249 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:32.298 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:32.369 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:32.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:32.477 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:32.526 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:32.587 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:32.594 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 23435/86253 objects degraded (27.170%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:32.594 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:32 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:32.594 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:32 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:32.696 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:32.732 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:32.794 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:32.897 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 23435/86253 objects degraded (27.170%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:32 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:32 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:32.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 23435/86253 objects degraded (27.170%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:32.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:32 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:32.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:32 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:32.928 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:32.980 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:32.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:32.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:33.028 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:33.089 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:33.186 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:33.332 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:33.430 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:33.453 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:33.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:33.569 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:33.649 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:33.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:08:33.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:08:33.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:33.769 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[211526]: pgmap v1324: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 23435/86253 objects degraded (27.170%); 20234/86253 objects misplaced (23.459%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:33.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:08:33.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:08:33.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:33.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:33 smithi121 ceph-mon[219709]: pgmap v1324: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 23435/86253 objects degraded (27.170%); 20234/86253 objects misplaced (23.459%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:33 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:08:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:33 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:08:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:33 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:33.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:33 smithi123 ceph-mon[193903]: pgmap v1324: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 23435/86253 objects degraded (27.170%); 20234/86253 objects misplaced (23.459%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:34.276 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:08:34.277 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:08:34.277 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:08:34.277 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:08:34.308 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:08:34.339 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:08:34.393 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:08:34.425 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:08:34.443 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.444 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.444 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.444 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.444 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.444 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.444 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.444 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:08:34.494 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:34.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:34.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:34.605 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:34.642 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:34.699 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:34.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:34.804 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:34.893 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:35.067 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1058166905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:35.067 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1058166905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:35.068 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:08:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:08:34] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T21:08:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1058166905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:36.046 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:36.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:36.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:36.152 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:36.234 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:36.343 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:36 smithi121 ceph-mon[211526]: pgmap v1325: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 23435/84711 objects degraded (27.665%); 19724/84711 objects misplaced (23.284%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:36.343 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:36 smithi121 ceph-mon[219709]: pgmap v1325: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 23435/84711 objects degraded (27.665%); 19724/84711 objects misplaced (23.284%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:36.466 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:36.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:36.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:36.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:36 smithi123 ceph-mon[193903]: pgmap v1325: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 23435/84711 objects degraded (27.665%); 19724/84711 objects misplaced (23.284%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T21:08:36.574 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:36.694 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:37.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 23435/84711 objects degraded (27.665%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:37.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 23435/84711 objects degraded (27.665%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:37.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 23435/84711 objects degraded (27.665%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:37.508 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:37.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:37.623 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:37.672 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:37.726 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:37.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:37.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:37.833 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:37.860 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:38.115 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:38 smithi121 ceph-mon[219709]: pgmap v1326: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 22776/78363 objects degraded (29.065%); 17603/78363 objects misplaced (22.463%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:38.374 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:08:38.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:38 smithi121 ceph-mon[211526]: pgmap v1326: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 22776/78363 objects degraded (29.065%); 17603/78363 objects misplaced (22.463%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:38.455 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:38.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:38.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:38 smithi123 ceph-mon[193903]: pgmap v1326: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 22776/78363 objects degraded (29.065%); 17603/78363 objects misplaced (22.463%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:38.571 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:38.606 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:38.668 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:38.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:38.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:38.775 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:38.843 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:39.090 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:39.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:39.191 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:39.280 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:39.451 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:39.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:39.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:39.479 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:08:39.479 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:08:39.507 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:39.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:39.641 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:39.801 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:08:39.802 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22776/78363 objects degraded (29.065%), 7 pgs degraded, 6 pgs undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1776,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":414,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":75},{"state_name":"active+remapped+backfill_wait","count":6},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":26121,"data_bytes":1710823352,"bytes_used":9235808256,"bytes_avail":662729441280,"bytes_total":671965249536,"degraded_objects":22776,"degraded_total":78363,"degraded_ratio":0.2906473718464071,"misplaced_objects":17603,"misplaced_total":78363,"misplaced_ratio":0.22463407475466737,"recovering_objects_per_sec":139,"recovering_bytes_per_sec":9140455,"recovering_keys_per_sec":0,"num_objects_recovered":1675,"num_bytes_recovered":109772800,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":443},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":29,"modified":"2023-09-23T21:08:15.075906+0000","services":{}},"progress_events":{"2639f328-dcb0-4a01-b6c7-21332e7147ac":{"message":"Global Recovery Event (5m)\n [=======================.....] (remaining: 56s)","progress":0.84269660711288452,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:08:40.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[211526]: pgmap v1327: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 22776/78363 objects degraded (29.065%); 17603/78363 objects misplaced (22.463%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:08:40.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1060332181' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:40.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:40.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3436718713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:08:40.158 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:08:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[219709]: pgmap v1327: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 22776/78363 objects degraded (29.065%); 17603/78363 objects misplaced (22.463%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:08:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1060332181' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3436718713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:08:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:40 smithi123 ceph-mon[193903]: pgmap v1327: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 22776/78363 objects degraded (29.065%); 17603/78363 objects misplaced (22.463%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:08:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1060332181' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:08:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3436718713' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:08:40.602 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:40.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:40.711 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:40.753 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:40.810 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:40.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:40.919 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:40.944 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:40.989 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:40.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:41.095 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:41.121 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:41.196 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:41.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:41.303 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:41.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:41.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[211526]: failed to encode map e415 with expected crc 2023-09-23T21:08:41.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:08:41.391 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[211526]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T21:08:41.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:41.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[219709]: failed to encode map e415 with expected crc 2023-09-23T21:08:41.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:08:41.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:41 smithi121 ceph-mon[219709]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T21:08:41.391 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:41 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:08:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:41 smithi123 ceph-mon[193903]: failed to encode map e415 with expected crc 2023-09-23T21:08:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:08:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:41 smithi123 ceph-mon[193903]: osdmap e415: 8 total, 8 up, 7 in 2023-09-23T21:08:41.973 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:08:42.325 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e415 with expected crc 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: pgmap v1329: 89 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 19198/71100 objects degraded (27.001%); 20727/71100 objects misplaced (29.152%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 19198/71100 objects degraded (27.001%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.326 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[211526]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e415 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: pgmap v1329: 89 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 19198/71100 objects degraded (27.001%); 20727/71100 objects misplaced (29.152%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.327 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 19198/71100 objects degraded (27.001%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:42.328 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T21:08:42.328 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.328 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.328 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.328 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.328 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:42 smithi121 ceph-mon[219709]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.328 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:42.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:42.434 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e415 with expected crc 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: pgmap v1329: 89 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 19198/71100 objects degraded (27.001%); 20727/71100 objects misplaced (29.152%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: osdmap e416: 8 total, 8 up, 7 in 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e416 with expected crc 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 19198/71100 objects degraded (27.001%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: osdmap e417: 8 total, 8 up, 7 in 2023-09-23T21:08:42.458 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.459 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.459 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.459 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.459 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:42 smithi123 ceph-mon[193903]: failed to encode map e417 with expected crc 2023-09-23T21:08:42.460 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:42.505 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:42.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:42.612 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:42.654 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:08:42.654 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:08:42.737 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:42.885 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:42.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:42.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:42.997 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:43.027 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:43.091 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:43.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:43.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:43.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:43.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[219709]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[219709]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[219709]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T21:08:43.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[219709]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[219709]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[211526]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[211526]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[211526]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T21:08:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[211526]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:43 smithi121 ceph-mon[211526]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.195 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:43.266 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:43 smithi123 ceph-mon[193903]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:43 smithi123 ceph-mon[193903]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:43 smithi123 ceph-mon[193903]: osdmap e418: 8 total, 8 up, 7 in 2023-09-23T21:08:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:43 smithi123 ceph-mon[193903]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:43 smithi123 ceph-mon[193903]: failed to encode map e418 with expected crc 2023-09-23T21:08:43.812 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:43.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:43.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:43.918 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:43.947 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:43.959 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:44.006 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:44.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:44.087 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:44.115 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:44.148 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:44 smithi121 ceph-mon[211526]: 11.b continuing backfill to osd.0 from (367'5457,415'9577] MIN to 415'9577 2023-09-23T21:08:44.148 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:44 smithi121 ceph-mon[211526]: 11.b continuing backfill to osd.3 from (367'5457,415'9577] MIN to 415'9577 2023-09-23T21:08:44.148 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:44 smithi121 ceph-mon[211526]: pgmap v1333: 89 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19198/71100 objects degraded (27.001%); 20727/71100 objects misplaced (29.152%); 14 MiB/s, 220 objects/s recovering 2023-09-23T21:08:44.150 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:44.210 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:44.215 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:44 smithi123 ceph-mon[193903]: 11.b continuing backfill to osd.0 from (367'5457,415'9577] MIN to 415'9577 2023-09-23T21:08:44.215 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:44 smithi123 ceph-mon[193903]: 11.b continuing backfill to osd.3 from (367'5457,415'9577] MIN to 415'9577 2023-09-23T21:08:44.215 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:44 smithi123 ceph-mon[193903]: pgmap v1333: 89 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19198/71100 objects degraded (27.001%); 20727/71100 objects misplaced (29.152%); 14 MiB/s, 220 objects/s recovering 2023-09-23T21:08:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:44.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:44.316 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:44.346 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:44.402 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:44.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:44 smithi121 ceph-mon[219709]: 11.b continuing backfill to osd.0 from (367'5457,415'9577] MIN to 415'9577 2023-09-23T21:08:44.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:44 smithi121 ceph-mon[219709]: 11.b continuing backfill to osd.3 from (367'5457,415'9577] MIN to 415'9577 2023-09-23T21:08:44.404 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:44 smithi121 ceph-mon[219709]: pgmap v1333: 89 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19198/71100 objects degraded (27.001%); 20727/71100 objects misplaced (29.152%); 14 MiB/s, 220 objects/s recovering 2023-09-23T21:08:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:44.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:44.414 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:08:44.511 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:44.606 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:45.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:45 smithi121 ceph-mon[211526]: pgmap v1334: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 19198/69810 objects degraded (27.500%); 20221/69810 objects misplaced (28.966%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:08:45.154 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:08:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:08:44] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.43.0" 2023-09-23T21:08:45.420 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:45 smithi121 ceph-mon[219709]: pgmap v1334: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 19198/69810 objects degraded (27.500%); 20221/69810 objects misplaced (28.966%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:45 smithi123 ceph-mon[193903]: pgmap v1334: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 19198/69810 objects degraded (27.500%); 20221/69810 objects misplaced (28.966%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:08:45.631 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:45.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:45.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:45.745 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:45.828 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:46.556 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 19198/69810 objects degraded (27.500%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:46.559 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 19198/69810 objects degraded (27.500%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:46.760 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:46.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:46.799 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:08:46.843 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:08:46.875 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:46.882 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:08:46.935 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:08:46.969 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:08:46.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 19198/69810 objects degraded (27.500%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:46.991 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:47.011 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:08:47.055 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:08:47.097 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:08:47.141 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.142 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:08:47.726 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:47.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:47.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:47.835 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:47.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/648352769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:47.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:47 smithi123 ceph-mon[193903]: pgmap v1335: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 18682/65205 objects degraded (28.651%); 18910/65205 objects misplaced (29.001%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:08:47.875 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:47.887 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/648352769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:47.887 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:47 smithi121 ceph-mon[211526]: pgmap v1335: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 18682/65205 objects degraded (28.651%); 18910/65205 objects misplaced (29.001%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:08:47.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/648352769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:08:47.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:47 smithi121 ceph-mon[219709]: pgmap v1335: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 327 op/s; 18682/65205 objects degraded (28.651%); 18910/65205 objects misplaced (29.001%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:08:47.941 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:47.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:48.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:48.082 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:48.139 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:48.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:48.245 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:48.303 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:49.015 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:49.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:49.138 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:49.176 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:49.249 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:49.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:49.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:49.307 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:49 smithi121 ceph-mon[211526]: pgmap v1336: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.3 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 247 op/s; 18682/65205 objects degraded (28.651%); 18910/65205 objects misplaced (29.001%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:08:49.308 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:49 smithi121 ceph-mon[219709]: pgmap v1336: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.3 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 247 op/s; 18682/65205 objects degraded (28.651%); 18910/65205 objects misplaced (29.001%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:08:49.358 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:49.378 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:49.441 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:49 smithi123 ceph-mon[193903]: pgmap v1336: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.3 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 247 op/s; 18682/65205 objects degraded (28.651%); 18910/65205 objects misplaced (29.001%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:08:49.480 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:08:49.480 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:08:49.480 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:08:49.480 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:08:49.481 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:08:50.045 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:08:50.071 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:08:50.226 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:50.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:50.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:50.383 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:50 smithi121 ceph-mon[219709]: from='client.36573 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:50.387 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:50 smithi121 ceph-mon[211526]: from='client.36573 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:50.403 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:50.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:50 smithi123 ceph-mon[193903]: from='client.36573 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:08:51.054 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:08:51.245 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:51.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:51.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:51.361 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:51.365 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:08:51.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:51 smithi121 ceph-mon[219709]: pgmap v1337: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 18056/57636 objects degraded (31.328%); 16703/57636 objects misplaced (28.980%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:08:51.381 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:51 smithi121 ceph-mon[211526]: pgmap v1337: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 18056/57636 objects degraded (31.328%); 16703/57636 objects misplaced (28.980%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:08:51.395 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:51.452 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:51.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:51.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:51.560 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:51.588 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:51 smithi123 ceph-mon[193903]: pgmap v1337: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 18056/57636 objects degraded (31.328%); 16703/57636 objects misplaced (28.980%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:08:51.600 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:51.666 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:51.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:51.771 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:08:51.800 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:51.866 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:08:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:51.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:08:51.976 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:52.067 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:52.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18056/57636 objects degraded (31.328%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:52.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18056/57636 objects degraded (31.328%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:52.625 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:52.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:52.715 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1336,"stamp":"2023-09-23T21:08:49.098894+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1423382456,"num_objects":21735,"num_object_clones":0,"num_object_copies":65205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18682,"num_objects_misplaced":18910,"num_objects_unfound":0,"num_objects_dirty":21735,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":154963,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38338,"num_bytes_recovered":2508300076,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82978,"ondisk_log_size":82978,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8396620,"kb_used_data":4186516,"kb_used_omap":21,"kb_used_meta":4210026,"kb_avail":647819444,"statfs":{"total":671965249536,"available":663367110656,"internally_reserved":0,"allocated":4286992384,"data_stored":4274980418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4311067107},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-128778240,"num_objects":-1965,"num_object_clones":0,"num_object_copies":-5895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-516,"num_objects_misplaced":-1817,"num_objects_unfound":0,"num_objects_dirty":-1965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1965,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":755,"num_bytes_recovered":49479680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.923919"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":189,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.508202+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:08:41.508202+0000","last_peered":"2023-09-23T21:08:41.508202+0000","last_clean":"2023-09-23T21:08:41.508202+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:08:41.508202+0000","last_undegraded":"2023-09-23T21:08:41.508202+0000","last_fullsized":"2023-09-23T21:08:41.508202+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":58,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509913+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:08:41.509913+0000","last_peered":"2023-09-23T21:08:41.509913+0000","last_clean":"2023-09-23T21:08:41.509913+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:08:41.509913+0000","last_undegraded":"2023-09-23T21:08:41.509913+0000","last_fullsized":"2023-09-23T21:08:41.509913+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":331,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.508057+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:08:41.508057+0000","last_peered":"2023-09-23T21:08:41.508057+0000","last_clean":"2023-09-23T21:08:41.508057+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:08:41.508057+0000","last_undegraded":"2023-09-23T21:08:41.508057+0000","last_fullsized":"2023-09-23T21:08:41.508057+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":55,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509670+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:08:41.509670+0000","last_peered":"2023-09-23T21:08:41.509670+0000","last_clean":"2023-09-23T21:08:41.509670+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:08:41.509670+0000","last_undegraded":"2023-09-23T21:08:41.509670+0000","last_fullsized":"2023-09-23T21:08:41.509670+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":76,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.508656+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:08:41.508656+0000","last_peered":"2023-09-23T21:08:41.508656+0000","last_clean":"2023-09-23T21:08:41.508656+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:08:41.508656+0000","last_undegraded":"2023-09-23T21:08:41.508656+0000","last_fullsized":"2023-09-23T21:08:41.508656+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":284,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.516212+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:08:41.516212+0000","last_peered":"2023-09-23T21:08:41.516212+0000","last_clean":"2023-09-23T21:08:41.516212+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:08:41.516212+0000","last_undegraded":"2023-09-23T21:08:41.516212+0000","last_fullsized":"2023-09-23T21:08:41.516212+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":55,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.510094+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:08:41.510094+0000","last_peered":"2023-09-23T21:08:41.510094+0000","last_clean":"2023-09-23T21:08:41.510094+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:08:41.510094+0000","last_undegraded":"2023-09-23T21:08:41.510094+0000","last_fullsized":"2023-09-23T21:08:41.510094+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":58,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.508583+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:08:41.508583+0000","last_peered":"2023-09-23T21:08:41.508583+0000","last_clean":"2023-09-23T21:08:41.508583+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:08:41.508583+0000","last_undegraded":"2023-09-23T21:08:41.508583+0000","last_fullsized":"2023-09-23T21:08:41.508583+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":41,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.518257+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:08:41.518257+0000","last_peered":"2023-09-23T21:08:41.518257+0000","last_clean":"2023-09-23T21:08:41.518257+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:08:41.518257+0000","last_undegraded":"2023-09-23T21:08:41.518257+0000","last_fullsized":"2023-09-23T21:08:41.518257+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":58,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509032+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:08:41.509032+0000","last_peered":"2023-09-23T21:08:41.509032+0000","last_clean":"2023-09-23T21:08:41.509032+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:08:41.509032+0000","last_undegraded":"2023-09-23T21:08:41.509032+0000","last_fullsized":"2023-09-23T21:08:41.509032+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":74,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509989+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:08:41.509989+0000","last_peered":"2023-09-23T21:08:41.509989+0000","last_clean":"2023-09-23T21:08:41.509989+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:08:41.509989+0000","last_undegraded":"2023-09-23T21:08:41.509989+0000","last_fullsized":"2023-09-23T21:08:41.509989+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":50,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.507844+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:08:41.507844+0000","last_peered":"2023-09-23T21:08:41.507844+0000","last_clean":"2023-09-23T21:08:41.507844+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:08:41.507844+0000","last_undegraded":"2023-09-23T21:08:41.507844+0000","last_fullsized":"2023-09-23T21:08:41.507844+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":364,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.516386+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:08:41.516386+0000","last_peered":"2023-09-23T21:08:41.516386+0000","last_clean":"2023-09-23T21:08:41.516386+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:08:41.516386+0000","last_undegraded":"2023-09-23T21:08:41.516386+0000","last_fullsized":"2023-09-23T21:08:41.516386+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":336,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513738+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:08:41.513738+0000","last_peered":"2023-09-23T21:08:41.513738+0000","last_clean":"2023-09-23T21:08:41.513738+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:08:41.513738+0000","last_undegraded":"2023-09-23T21:08:41.513738+0000","last_fullsized":"2023-09-23T21:08:41.513738+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":108,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.518893+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:08:41.518893+0000","last_peered":"2023-09-23T21:08:41.518893+0000","last_clean":"2023-09-23T21:08:41.518893+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:08:41.518893+0000","last_undegraded":"2023-09-23T21:08:41.518893+0000","last_fullsized":"2023-09-23T21:08:41.518893+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":67,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509587+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:08:41.509587+0000","last_peered":"2023-09-23T21:08:41.509587+0000","last_clean":"2023-09-23T21:08:41.509587+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:08:41.509587+0000","last_undegraded":"2023-09-23T21:08:41.509587+0000","last_fullsized":"2023-09-23T21:08:41.509587+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":76,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.510840+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:08:41.510840+0000","last_peered":"2023-09-23T21:08:41.510840+0000","last_clean":"2023-09-23T21:08:41.510840+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:08:41.510840+0000","last_undegraded":"2023-09-23T21:08:41.510840+0000","last_fullsized":"2023-09-23T21:08:41.510840+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":76,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.510095+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:08:41.510095+0000","last_peered":"2023-09-23T21:08:41.510095+0000","last_clean":"2023-09-23T21:08:41.510095+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:08:41.510095+0000","last_undegraded":"2023-09-23T21:08:41.510095+0000","last_fullsized":"2023-09-23T21:08:41.510095+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":218,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513496+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:08:41.513496+0000","last_peered":"2023-09-23T21:08:41.513496+0000","last_clean":"2023-09-23T21:08:41.513496+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:08:41.513496+0000","last_undegraded":"2023-09-23T21:08:41.513496+0000","last_fullsized":"2023-09-23T21:08:41.513496+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":55,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509751+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:08:41.509751+0000","last_peered":"2023-09-23T21:08:41.509751+0000","last_clean":"2023-09-23T21:08:41.509751+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:08:41.509751+0000","last_undegraded":"2023-09-23T21:08:41.509751+0000","last_fullsized":"2023-09-23T21:08:41.509751+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":414,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.515410+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:08:41.515410+0000","last_peered":"2023-09-23T21:08:41.515410+0000","last_clean":"2023-09-23T21:08:41.515410+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:08:41.515410+0000","last_undegraded":"2023-09-23T21:08:41.515410+0000","last_fullsized":"2023-09-23T21:08:41.515410+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":58,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.508758+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:08:41.508758+0000","last_peered":"2023-09-23T21:08:41.508758+0000","last_clean":"2023-09-23T21:08:41.508758+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:08:41.508758+0000","last_undegraded":"2023-09-23T21:08:41.508758+0000","last_fullsized":"2023-09-23T21:08:41.508758+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"418'9877","reported_seq":11252,"reported_epoch":418,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:45.785154+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:08:45.785154+0000","last_peered":"2023-09-23T21:08:45.785154+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:08:45.785154+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"366'5128","ondisk_log_start":"366'5128","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4749,"log_dups_size":0,"ondisk_log_size":4749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9877,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-4262},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":1352}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":122,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.514116+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:08:41.514116+0000","last_peered":"2023-09-23T21:08:41.514116+0000","last_clean":"2023-09-23T21:08:41.514116+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:08:41.514116+0000","last_undegraded":"2023-09-23T21:08:41.514116+0000","last_fullsized":"2023-09-23T21:08:41.514116+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":41,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.516770+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:08:41.516770+0000","last_peered":"2023-09-23T21:08:41.516770+0000","last_clean":"2023-09-23T21:08:41.516770+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:08:41.516770+0000","last_undegraded":"2023-09-23T21:08:41.516770+0000","last_fullsized":"2023-09-23T21:08:41.516770+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"418'9488","reported_seq":9576,"reported_epoch":418,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:45.759004+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:08:45.759004+0000","last_peered":"2023-09-23T21:08:45.759004+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:08:45.759004+0000","last_undegraded":"2023-09-23T21:08:45.759004+0000","last_fullsized":"2023-09-23T21:08:45.759004+0000","mapping_epoch":375,"log_start":"366'4712","ondisk_log_start":"366'4712","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4776,"log_dups_size":0,"ondisk_log_size":4776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85852160,"num_objects":1310,"num_object_clones":0,"num_object_copies":3930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1310,"num_objects_unfound":0,"num_objects_dirty":1310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9488,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":1310}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":116,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513717+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:08:41.513717+0000","last_peered":"2023-09-23T21:08:41.513717+0000","last_clean":"2023-09-23T21:08:41.513717+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:08:41.513717+0000","last_undegraded":"2023-09-23T21:08:41.513717+0000","last_fullsized":"2023-09-23T21:08:41.513717+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":379,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.507981+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:08:41.507981+0000","last_peered":"2023-09-23T21:08:41.507981+0000","last_clean":"2023-09-23T21:08:41.507981+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:08:41.507981+0000","last_undegraded":"2023-09-23T21:08:41.507981+0000","last_fullsized":"2023-09-23T21:08:41.507981+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"418'9604","reported_seq":24257,"reported_epoch":418,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:45.800769+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:08:45.800769+0000","last_peered":"2023-09-23T21:08:45.800769+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:08:45.800769+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"401'5511","ondisk_log_start":"401'5511","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4093,"log_dups_size":0,"ondisk_log_size":4093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87425024,"num_objects":1334,"num_object_clones":0,"num_object_copies":4002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2658,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9604,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5208,"num_bytes_recovered":341311488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"2,6,7","objects":1329},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":45,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.517391+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:08:41.517391+0000","last_peered":"2023-09-23T21:08:41.517391+0000","last_clean":"2023-09-23T21:08:41.517391+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:08:41.517391+0000","last_undegraded":"2023-09-23T21:08:41.517391+0000","last_fullsized":"2023-09-23T21:08:41.517391+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":306,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.515671+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:08:41.515671+0000","last_peered":"2023-09-23T21:08:41.515671+0000","last_clean":"2023-09-23T21:08:41.515671+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:08:41.515671+0000","last_undegraded":"2023-09-23T21:08:41.515671+0000","last_fullsized":"2023-09-23T21:08:41.515671+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"418'9665","reported_seq":10727,"reported_epoch":418,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:45.787996+0000","last_change":"2023-09-23T21:08:40.299610+0000","last_active":"2023-09-23T21:08:45.787996+0000","last_peered":"2023-09-23T21:08:45.787996+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:08:45.787996+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"401'5568","ondisk_log_start":"401'5568","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4097,"log_dups_size":0,"ondisk_log_size":4097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90767360,"num_objects":1385,"num_object_clones":0,"num_object_copies":4155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3749,"num_objects_misplaced":1291,"num_objects_unfound":0,"num_objects_dirty":1385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9665,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":482,"num_bytes_recovered":31588352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-3655},{"shards":"2,6,7","objects":1291}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":58,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509659+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:08:41.509659+0000","last_peered":"2023-09-23T21:08:41.509659+0000","last_clean":"2023-09-23T21:08:41.509659+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:08:41.509659+0000","last_undegraded":"2023-09-23T21:08:41.509659+0000","last_fullsized":"2023-09-23T21:08:41.509659+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":55,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509977+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:08:41.509977+0000","last_peered":"2023-09-23T21:08:41.509977+0000","last_clean":"2023-09-23T21:08:41.509977+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:08:41.509977+0000","last_undegraded":"2023-09-23T21:08:41.509977+0000","last_fullsized":"2023-09-23T21:08:41.509977+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"418'9923","reported_seq":21430,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:45.676066+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:08:45.676066+0000","last_peered":"2023-09-23T21:08:45.676066+0000","last_clean":"2023-09-23T21:08:45.676066+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:08:45.676066+0000","last_undegraded":"2023-09-23T21:08:45.676066+0000","last_fullsized":"2023-09-23T21:08:45.676066+0000","mapping_epoch":400,"log_start":"401'5799","ondisk_log_start":"401'5799","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4124,"log_dups_size":0,"ondisk_log_size":4124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90767360,"num_objects":1385,"num_object_clones":0,"num_object_copies":4155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9923,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":75,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509193+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:08:41.509193+0000","last_peered":"2023-09-23T21:08:41.509193+0000","last_clean":"2023-09-23T21:08:41.509193+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:08:41.509193+0000","last_undegraded":"2023-09-23T21:08:41.509193+0000","last_fullsized":"2023-09-23T21:08:41.509193+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509519+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:08:41.509519+0000","last_peered":"2023-09-23T21:08:41.509519+0000","last_clean":"2023-09-23T21:08:41.509519+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:08:41.509519+0000","last_undegraded":"2023-09-23T21:08:41.509519+0000","last_fullsized":"2023-09-23T21:08:41.509519+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"418'9569","reported_seq":20689,"reported_epoch":418,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:44.305730+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:08:44.305730+0000","last_peered":"2023-09-23T21:08:44.305730+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:08:44.305730+0000","last_undegraded":"2023-09-23T21:08:44.305730+0000","last_fullsized":"2023-09-23T21:08:44.305730+0000","mapping_epoch":403,"log_start":"347'888","ondisk_log_start":"347'888","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8681,"log_dups_size":0,"ondisk_log_size":8681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92602368,"num_objects":1413,"num_object_clones":0,"num_object_copies":4239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1413,"num_objects_unfound":0,"num_objects_dirty":1413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9569,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":1413}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":344,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.515744+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:08:41.515744+0000","last_peered":"2023-09-23T21:08:41.515744+0000","last_clean":"2023-09-23T21:08:41.515744+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:08:41.515744+0000","last_undegraded":"2023-09-23T21:08:41.515744+0000","last_fullsized":"2023-09-23T21:08:41.515744+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":429,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.518572+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:08:41.518572+0000","last_peered":"2023-09-23T21:08:41.518572+0000","last_clean":"2023-09-23T21:08:41.518572+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:08:41.518572+0000","last_undegraded":"2023-09-23T21:08:41.518572+0000","last_fullsized":"2023-09-23T21:08:41.518572+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"418'9597","reported_seq":9700,"reported_epoch":418,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:45.803755+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:08:45.803755+0000","last_peered":"2023-09-23T21:08:45.803755+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:08:45.803755+0000","last_undegraded":"2023-09-23T21:08:45.803755+0000","last_fullsized":"2023-09-23T21:08:45.803755+0000","mapping_epoch":374,"log_start":"401'5494","ondisk_log_start":"401'5494","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4103,"log_dups_size":0,"ondisk_log_size":4103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85655552,"num_objects":1307,"num_object_clones":0,"num_object_copies":3921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2614,"num_objects_unfound":0,"num_objects_dirty":1307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9597,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":1307}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513423+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:08:41.513423+0000","last_peered":"2023-09-23T21:08:41.513423+0000","last_clean":"2023-09-23T21:08:41.513423+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:08:41.513423+0000","last_undegraded":"2023-09-23T21:08:41.513423+0000","last_fullsized":"2023-09-23T21:08:41.513423+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":457,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.516046+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:08:41.516046+0000","last_peered":"2023-09-23T21:08:41.516046+0000","last_clean":"2023-09-23T21:08:41.516046+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:08:41.516046+0000","last_undegraded":"2023-09-23T21:08:41.516046+0000","last_fullsized":"2023-09-23T21:08:41.516046+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"418'9613","reported_seq":10157,"reported_epoch":418,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:08:45.798236+0000","last_change":"2023-09-23T21:08:40.308232+0000","last_active":"2023-09-23T21:08:45.798236+0000","last_peered":"2023-09-23T21:08:45.798236+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:08:45.798236+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"366'4894","ondisk_log_start":"366'4894","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4719,"log_dups_size":0,"ondisk_log_size":4719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88408064,"num_objects":1349,"num_object_clones":0,"num_object_copies":4047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3957,"num_objects_misplaced":1306,"num_objects_unfound":0,"num_objects_dirty":1349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9613,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":14286848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-3914},{"shards":"1,6,7","objects":1306}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":67,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.508803+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:08:41.508803+0000","last_peered":"2023-09-23T21:08:41.508803+0000","last_clean":"2023-09-23T21:08:41.508803+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:08:41.508803+0000","last_undegraded":"2023-09-23T21:08:41.508803+0000","last_fullsized":"2023-09-23T21:08:41.508803+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":206,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.514551+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:08:41.514551+0000","last_peered":"2023-09-23T21:08:41.514551+0000","last_clean":"2023-09-23T21:08:41.514551+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:08:41.514551+0000","last_undegraded":"2023-09-23T21:08:41.514551+0000","last_fullsized":"2023-09-23T21:08:41.514551+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"418'9636","reported_seq":15139,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:44.098930+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:08:44.098930+0000","last_peered":"2023-09-23T21:08:44.098930+0000","last_clean":"2023-09-23T21:08:44.098930+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:08:44.098930+0000","last_undegraded":"2023-09-23T21:08:44.098930+0000","last_fullsized":"2023-09-23T21:08:44.098930+0000","mapping_epoch":364,"log_start":"347'909","ondisk_log_start":"347'909","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8727,"log_dups_size":0,"ondisk_log_size":8727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91291672,"num_objects":1394,"num_object_clones":0,"num_object_copies":4182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1394,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9636,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":313,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.514647+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:08:41.514647+0000","last_peered":"2023-09-23T21:08:41.514647+0000","last_clean":"2023-09-23T21:08:41.514647+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:08:41.514647+0000","last_undegraded":"2023-09-23T21:08:41.514647+0000","last_fullsized":"2023-09-23T21:08:41.514647+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":83,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.517439+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:08:41.517439+0000","last_peered":"2023-09-23T21:08:41.517439+0000","last_clean":"2023-09-23T21:08:41.517439+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:08:41.517439+0000","last_undegraded":"2023-09-23T21:08:41.517439+0000","last_fullsized":"2023-09-23T21:08:41.517439+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.507929+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:08:41.507929+0000","last_peered":"2023-09-23T21:08:41.507929+0000","last_clean":"2023-09-23T21:08:41.507929+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:08:41.507929+0000","last_undegraded":"2023-09-23T21:08:41.507929+0000","last_fullsized":"2023-09-23T21:08:41.507929+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"418'9649","reported_seq":20823,"reported_epoch":418,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:44.336219+0000","last_change":"2023-09-23T21:08:42.553079+0000","last_active":"2023-09-23T21:08:44.336219+0000","last_peered":"2023-09-23T21:08:44.336219+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:08:42.521706+0000","last_became_peered":"2023-09-23T21:08:42.521706+0000","last_unstale":"2023-09-23T21:08:44.336219+0000","last_undegraded":"2023-09-23T21:08:44.336219+0000","last_fullsized":"2023-09-23T21:08:44.336219+0000","mapping_epoch":417,"log_start":"367'5457","ondisk_log_start":"367'5457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4192,"log_dups_size":0,"ondisk_log_size":4192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.707578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90243072,"num_objects":1377,"num_object_clones":0,"num_object_copies":4131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2754,"num_objects_unfound":0,"num_objects_dirty":1377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9649,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5513,"num_bytes_recovered":361299968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[2,1,6],"avail_no_missing":["2","1","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":1377}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":394,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.511376+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:08:41.511376+0000","last_peered":"2023-09-23T21:08:41.511376+0000","last_clean":"2023-09-23T21:08:41.511376+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:08:41.511376+0000","last_undegraded":"2023-09-23T21:08:41.511376+0000","last_fullsized":"2023-09-23T21:08:41.511376+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":368,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.516344+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:08:41.516344+0000","last_peered":"2023-09-23T21:08:41.516344+0000","last_clean":"2023-09-23T21:08:41.516344+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:08:41.516344+0000","last_undegraded":"2023-09-23T21:08:41.516344+0000","last_fullsized":"2023-09-23T21:08:41.516344+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":50,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513421+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:08:41.513421+0000","last_peered":"2023-09-23T21:08:41.513421+0000","last_clean":"2023-09-23T21:08:41.513421+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:08:41.513421+0000","last_undegraded":"2023-09-23T21:08:41.513421+0000","last_fullsized":"2023-09-23T21:08:41.513421+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"418'9643","reported_seq":9839,"reported_epoch":418,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:44.252524+0000","last_change":"2023-09-23T21:08:40.306414+0000","last_active":"2023-09-23T21:08:44.252524+0000","last_peered":"2023-09-23T21:08:44.252524+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:08:44.252524+0000","last_undegraded":"2023-09-23T21:08:44.252524+0000","last_fullsized":"2023-09-23T21:08:44.252524+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8170,"log_dups_size":0,"ondisk_log_size":8170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92995584,"num_objects":1419,"num_object_clones":0,"num_object_copies":4257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2825,"num_objects_unfound":0,"num_objects_dirty":1419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9643,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1419}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":395,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.517946+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:08:41.517946+0000","last_peered":"2023-09-23T21:08:41.517946+0000","last_clean":"2023-09-23T21:08:41.517946+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:08:41.517946+0000","last_undegraded":"2023-09-23T21:08:41.517946+0000","last_fullsized":"2023-09-23T21:08:41.517946+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":313,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.524680+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:08:41.524680+0000","last_peered":"2023-09-23T21:08:41.524680+0000","last_clean":"2023-09-23T21:08:41.524680+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:08:41.524680+0000","last_undegraded":"2023-09-23T21:08:41.524680+0000","last_fullsized":"2023-09-23T21:08:41.524680+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":50,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.514447+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:08:41.514447+0000","last_peered":"2023-09-23T21:08:41.514447+0000","last_clean":"2023-09-23T21:08:41.514447+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:08:41.514447+0000","last_undegraded":"2023-09-23T21:08:41.514447+0000","last_fullsized":"2023-09-23T21:08:41.514447+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"418'9541","reported_seq":15805,"reported_epoch":418,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:45.741061+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:08:45.741061+0000","last_peered":"2023-09-23T21:08:45.741061+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:08:45.741061+0000","last_undegraded":"2023-09-23T21:08:45.741061+0000","last_fullsized":"2023-09-23T21:08:45.741061+0000","mapping_epoch":387,"log_start":"401'5480","ondisk_log_start":"401'5480","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4061,"log_dups_size":0,"ondisk_log_size":4061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87359488,"num_objects":1333,"num_object_clones":0,"num_object_copies":3999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1333,"num_objects_unfound":0,"num_objects_dirty":1333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9541,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","1","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":1333}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":233,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.518976+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:08:41.518976+0000","last_peered":"2023-09-23T21:08:41.518976+0000","last_clean":"2023-09-23T21:08:41.518976+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:08:41.518976+0000","last_undegraded":"2023-09-23T21:08:41.518976+0000","last_fullsized":"2023-09-23T21:08:41.518976+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":678,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.515806+0000","last_change":"2023-09-23T21:08:41.538586+0000","last_active":"2023-09-23T21:08:42.515806+0000","last_peered":"2023-09-23T21:08:42.515806+0000","last_clean":"2023-09-23T21:08:42.515806+0000","last_became_active":"2023-09-23T21:08:41.538107+0000","last_became_peered":"2023-09-23T21:08:41.538107+0000","last_unstale":"2023-09-23T21:08:42.515806+0000","last_undegraded":"2023-09-23T21:08:42.515806+0000","last_fullsized":"2023-09-23T21:08:42.515806+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:48.268707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":67,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.510797+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:08:41.510797+0000","last_peered":"2023-09-23T21:08:41.510797+0000","last_clean":"2023-09-23T21:08:41.510797+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:08:41.510797+0000","last_undegraded":"2023-09-23T21:08:41.510797+0000","last_fullsized":"2023-09-23T21:08:41.510797+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":306,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.517375+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:08:41.517375+0000","last_peered":"2023-09-23T21:08:41.517375+0000","last_clean":"2023-09-23T21:08:41.517375+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:08:41.517375+0000","last_undegraded":"2023-09-23T21:08:41.517375+0000","last_fullsized":"2023-09-23T21:08:41.517375+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"418'9768","reported_seq":9888,"reported_epoch":418,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:45.788002+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:08:45.788002+0000","last_peered":"2023-09-23T21:08:45.788002+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:08:45.788002+0000","last_undegraded":"2023-09-23T21:08:45.788002+0000","last_fullsized":"2023-09-23T21:08:45.788002+0000","mapping_epoch":374,"log_start":"401'5705","ondisk_log_start":"401'5705","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4063,"log_dups_size":0,"ondisk_log_size":4063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88735744,"num_objects":1354,"num_object_clones":0,"num_object_copies":4062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2708,"num_objects_unfound":0,"num_objects_dirty":1354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9768,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":1354}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513563+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:08:41.513563+0000","last_peered":"2023-09-23T21:08:41.513563+0000","last_clean":"2023-09-23T21:08:41.513563+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:08:41.513563+0000","last_undegraded":"2023-09-23T21:08:41.513563+0000","last_fullsized":"2023-09-23T21:08:41.513563+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":64,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.507437+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:08:41.507437+0000","last_peered":"2023-09-23T21:08:41.507437+0000","last_clean":"2023-09-23T21:08:41.507437+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:08:41.507437+0000","last_undegraded":"2023-09-23T21:08:41.507437+0000","last_fullsized":"2023-09-23T21:08:41.507437+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":55,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509245+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:08:41.509245+0000","last_peered":"2023-09-23T21:08:41.509245+0000","last_clean":"2023-09-23T21:08:41.509245+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:08:41.509245+0000","last_undegraded":"2023-09-23T21:08:41.509245+0000","last_fullsized":"2023-09-23T21:08:41.509245+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"418'9665","reported_seq":18078,"reported_epoch":418,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:08:45.785519+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:08:45.785519+0000","last_peered":"2023-09-23T21:08:45.785519+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:08:45.785519+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:08:45.785519+0000","mapping_epoch":383,"log_start":"366'4926","ondisk_log_start":"366'4926","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4739,"log_dups_size":0,"ondisk_log_size":4739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89194496,"num_objects":1361,"num_object_clones":0,"num_object_copies":4083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1358,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":1361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9665,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4152,"num_bytes_recovered":272105472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":3},{"shards":"1,2,7","objects":1358}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":55,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509592+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:08:41.509592+0000","last_peered":"2023-09-23T21:08:41.509592+0000","last_clean":"2023-09-23T21:08:41.509592+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:08:41.509592+0000","last_undegraded":"2023-09-23T21:08:41.509592+0000","last_fullsized":"2023-09-23T21:08:41.509592+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":315,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.514767+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:08:41.514767+0000","last_peered":"2023-09-23T21:08:41.514767+0000","last_clean":"2023-09-23T21:08:41.514767+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:08:41.514767+0000","last_undegraded":"2023-09-23T21:08:41.514767+0000","last_fullsized":"2023-09-23T21:08:41.514767+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":93,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.517323+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:08:41.517323+0000","last_peered":"2023-09-23T21:08:41.517323+0000","last_clean":"2023-09-23T21:08:41.517323+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:08:41.517323+0000","last_undegraded":"2023-09-23T21:08:41.517323+0000","last_fullsized":"2023-09-23T21:08:41.517323+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"418'9576","reported_seq":22473,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:45.796192+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:08:45.796192+0000","last_peered":"2023-09-23T21:08:45.796192+0000","last_clean":"2023-09-23T21:08:45.796192+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:08:45.796192+0000","last_undegraded":"2023-09-23T21:08:45.796192+0000","last_fullsized":"2023-09-23T21:08:45.796192+0000","mapping_epoch":397,"log_start":"366'4845","ondisk_log_start":"366'4845","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4731,"log_dups_size":0,"ondisk_log_size":4731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84410368,"num_objects":1288,"num_object_clones":0,"num_object_copies":3864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9576,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":75,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.508845+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:08:41.508845+0000","last_peered":"2023-09-23T21:08:41.508845+0000","last_clean":"2023-09-23T21:08:41.508845+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:08:41.508845+0000","last_undegraded":"2023-09-23T21:08:41.508845+0000","last_fullsized":"2023-09-23T21:08:41.508845+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":50,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513305+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:08:41.513305+0000","last_peered":"2023-09-23T21:08:41.513305+0000","last_clean":"2023-09-23T21:08:41.513305+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:08:41.513305+0000","last_undegraded":"2023-09-23T21:08:41.513305+0000","last_fullsized":"2023-09-23T21:08:41.513305+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.520641+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:08:42.520641+0000","last_peered":"2023-09-23T21:08:42.520641+0000","last_clean":"2023-09-23T21:08:42.520641+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:08:42.520641+0000","last_undegraded":"2023-09-23T21:08:42.520641+0000","last_fullsized":"2023-09-23T21:08:42.520641+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"418'9849","reported_seq":18475,"reported_epoch":418,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:08:45.807652+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:08:45.807652+0000","last_peered":"2023-09-23T21:08:45.807652+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:08:45.807652+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:08:45.807652+0000","mapping_epoch":382,"log_start":"366'5112","ondisk_log_start":"366'5112","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4737,"log_dups_size":0,"ondisk_log_size":4737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88408064,"num_objects":1349,"num_object_clones":0,"num_object_copies":4047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1345,"num_objects_misplaced":1353,"num_objects_unfound":0,"num_objects_dirty":1349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9849,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4251,"num_bytes_recovered":278593536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":4},{"shards":"1,6,7","objects":1345}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":314,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.516215+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:08:41.516215+0000","last_peered":"2023-09-23T21:08:41.516215+0000","last_clean":"2023-09-23T21:08:41.516215+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:08:41.516215+0000","last_undegraded":"2023-09-23T21:08:41.516215+0000","last_fullsized":"2023-09-23T21:08:41.516215+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":59,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509428+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:08:41.509428+0000","last_peered":"2023-09-23T21:08:41.509428+0000","last_clean":"2023-09-23T21:08:41.509428+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:08:41.509428+0000","last_undegraded":"2023-09-23T21:08:41.509428+0000","last_fullsized":"2023-09-23T21:08:41.509428+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":103,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.516725+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:08:41.516725+0000","last_peered":"2023-09-23T21:08:41.516725+0000","last_clean":"2023-09-23T21:08:41.516725+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:08:41.516725+0000","last_undegraded":"2023-09-23T21:08:41.516725+0000","last_fullsized":"2023-09-23T21:08:41.516725+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":64,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509908+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:08:41.509908+0000","last_peered":"2023-09-23T21:08:41.509908+0000","last_clean":"2023-09-23T21:08:41.509908+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:08:41.509908+0000","last_undegraded":"2023-09-23T21:08:41.509908+0000","last_fullsized":"2023-09-23T21:08:41.509908+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":441,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.518241+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:08:41.518241+0000","last_peered":"2023-09-23T21:08:41.518241+0000","last_clean":"2023-09-23T21:08:41.518241+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:08:41.518241+0000","last_undegraded":"2023-09-23T21:08:41.518241+0000","last_fullsized":"2023-09-23T21:08:41.518241+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":80,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513961+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:08:41.513961+0000","last_peered":"2023-09-23T21:08:41.513961+0000","last_clean":"2023-09-23T21:08:41.513961+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:08:41.513961+0000","last_undegraded":"2023-09-23T21:08:41.513961+0000","last_fullsized":"2023-09-23T21:08:41.513961+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509317+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:08:41.509317+0000","last_peered":"2023-09-23T21:08:41.509317+0000","last_clean":"2023-09-23T21:08:41.509317+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:08:41.509317+0000","last_undegraded":"2023-09-23T21:08:41.509317+0000","last_fullsized":"2023-09-23T21:08:41.509317+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":299,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.517520+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:08:41.517520+0000","last_peered":"2023-09-23T21:08:41.517520+0000","last_clean":"2023-09-23T21:08:41.517520+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:08:41.517520+0000","last_undegraded":"2023-09-23T21:08:41.517520+0000","last_fullsized":"2023-09-23T21:08:41.517520+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.510878+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:08:41.510878+0000","last_peered":"2023-09-23T21:08:41.510878+0000","last_clean":"2023-09-23T21:08:41.510878+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:08:41.510878+0000","last_undegraded":"2023-09-23T21:08:41.510878+0000","last_fullsized":"2023-09-23T21:08:41.510878+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":59,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.521572+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:08:42.521572+0000","last_peered":"2023-09-23T21:08:42.521572+0000","last_clean":"2023-09-23T21:08:42.521572+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:08:42.521572+0000","last_undegraded":"2023-09-23T21:08:42.521572+0000","last_fullsized":"2023-09-23T21:08:42.521572+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":67,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509435+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:08:41.509435+0000","last_peered":"2023-09-23T21:08:41.509435+0000","last_clean":"2023-09-23T21:08:41.509435+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:08:41.509435+0000","last_undegraded":"2023-09-23T21:08:41.509435+0000","last_fullsized":"2023-09-23T21:08:41.509435+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":61,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.509972+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:08:41.509972+0000","last_peered":"2023-09-23T21:08:41.509972+0000","last_clean":"2023-09-23T21:08:41.509972+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:08:41.509972+0000","last_undegraded":"2023-09-23T21:08:41.509972+0000","last_fullsized":"2023-09-23T21:08:41.509972+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":45,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T21:08:41.513307+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:08:41.513307+0000","last_peered":"2023-09-23T21:08:41.513307+0000","last_clean":"2023-09-23T21:08:41.513307+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:08:41.513307+0000","last_undegraded":"2023-09-23T21:08:41.513307+0000","last_fullsized":"2023-09-23T21:08:41.513307+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1422786584,"num_objects":21711,"num_object_clones":0,"num_object_copies":65133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18682,"num_objects_misplaced":18910,"num_objects_unfound":0,"num_objects_dirty":21711,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":154663,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38219,"num_bytes_recovered":2504720384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7263498240,"data_stored":7263486024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82762,"ondisk_log_size":82762,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201761,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463381,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518648,"kb_used_data":966960,"kb_used_omap":0,"kb_used_meta":551680,"kb_avail":92226504,"statfs":{"total":95995035648,"available":94439940096,"internally_reserved":0,"allocated":990167040,"data_stored":988931792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":564920320},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.181,"15min":1.958},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.3849999999999998,"5min":99.533000000000001,"15min":102.242},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.7809999999999999,"5min":2.3439999999999999,"15min":1.7769999999999999},"min":{"1min":0.34300000000000003,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":6.04,"5min":99.325000000000003,"15min":102.502},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:08:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":4.3150000000000004,"15min":2.5110000000000001},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":30.526,"5min":119.077,"15min":119.077},"last":0.64700000000000002},{"interface":"front","average":{"1min":4.2009999999999996,"5min":4.2759999999999998,"15min":2.5990000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":30.564,"5min":119.313,"15min":119.313},"last":2.6019999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":5.1950000000000003,"15min":3.7799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.5470000000000006,"5min":199.23400000000001,"15min":199.23400000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.0870000000000002,"5min":5.1689999999999996,"15min":3.8319999999999999},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.21199999999999999},"max":{"1min":6.3019999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":1.627}]},{"osd":3,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.339,"15min":2.7229999999999999},"min":{"1min":0.31,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":13.340999999999999,"5min":13.340999999999999,"15min":178.40000000000001},"last":7.2130000000000001},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.7509999999999999,"15min":2.7040000000000002},"min":{"1min":0.53100000000000003,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":7.7759999999999998,"5min":13.319000000000001,"15min":178.26599999999999},"last":3.4289999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.423,"15min":0.90700000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":6.4989999999999997,"5min":13.1,"15min":13.1},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0269999999999999,"15min":0.871},"min":{"1min":0.31,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":5.9580000000000002,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":2.1259999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.333,"15min":0.94799999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.1819999999999999,"5min":11.281000000000001,"15min":11.281000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.208,"15min":1.05},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9279999999999999,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":2.1629999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.2929999999999999,"15min":1.0740000000000001},"min":{"1min":0.379,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":3.1139999999999999,"5min":7.8079999999999998,"15min":11.388999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.276,"15min":1.1080000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30099999999999999},"max":{"1min":3.4830000000000001,"5min":7.1849999999999996,"15min":8.8010000000000002},"last":2.194}]}]},{"osd":1,"up_from":276,"seq":1185410973827,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1811936,"kb_used_data":1366360,"kb_used_omap":5,"kb_used_meta":445562,"kb_avail":91933216,"statfs":{"total":95995035648,"available":94139613184,"internally_reserved":0,"allocated":1399152640,"data_stored":1397058866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":456256227},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.738,"15min":0.86599999999999999},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":18.097999999999999,"5min":18.097999999999999,"15min":18.097999999999999},"last":5.1619999999999999},{"interface":"front","average":{"1min":3.5459999999999998,"5min":1.407,"15min":1.0089999999999999},"min":{"1min":0.29999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":11.228,"5min":11.228,"15min":18.047999999999998},"last":0.77200000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0300000000000002,"5min":1.7010000000000001,"15min":1.4379999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.874000000000001,"5min":13.874000000000001,"15min":78.409000000000006},"last":0.375},{"interface":"front","average":{"1min":5.5030000000000001,"5min":1.829,"15min":1.319},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.22},"max":{"1min":18.288,"5min":18.288,"15min":76.245000000000005},"last":5.2050000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.415,"5min":1.716,"15min":2.782},"min":{"1min":0.28699999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":18.344999999999999,"5min":18.344999999999999,"15min":172.44399999999999},"last":2.6699999999999999},{"interface":"front","average":{"1min":3.4009999999999998,"5min":1.627,"15min":2.8799999999999999},"min":{"1min":0.36399999999999999,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":11.311999999999999,"5min":11.311999999999999,"15min":172.58500000000001},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.446,"15min":1.0389999999999999},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6500000000000004,"5min":12.238,"15min":17.006},"last":2.4870000000000001},{"interface":"front","average":{"1min":3.9769999999999999,"5min":1.583,"15min":0.93100000000000005},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":18.173999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":2.7269999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":1.397,"15min":1.05},"min":{"1min":0.34499999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.340999999999999,"5min":11.340999999999999,"15min":17.155000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.758,"5min":1.55,"15min":1.149},"min":{"1min":0.35799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.949999999999999,"5min":12.345000000000001,"15min":17.076000000000001},"last":2.5209999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":1.47,"15min":1.534},"min":{"1min":0.42699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":11.375,"5min":11.666,"15min":127.995},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.7890000000000001,"5min":1.633,"15min":1.6180000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.012,"5min":12.276999999999999,"15min":126.836},"last":2.5640000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9609999999999999,"5min":3.0670000000000002,"15min":1.645},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":11.17,"5min":151.791,"15min":151.791},"last":2.589},{"interface":"front","average":{"1min":4.173,"5min":3.0800000000000001,"15min":1.47},"min":{"1min":0.41799999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":18.260000000000002,"5min":151.809,"15min":151.809},"last":2.7029999999999998}]}]},{"osd":0,"up_from":92,"seq":395136991567,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":974616,"kb_used_data":225936,"kb_used_omap":5,"kb_used_meta":748666,"kb_avail":92770536,"statfs":{"total":95995035648,"available":94997028864,"internally_reserved":0,"allocated":231358464,"data_stored":229386604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":766634618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.423,"15min":1.841},"min":{"1min":0.315,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":41.149999999999999,"5min":61.341999999999999,"15min":61.341999999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":6.5499999999999998,"5min":3.5920000000000001,"15min":1.845},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.217},"max":{"1min":38.936,"5min":61.299999999999997,"15min":61.299999999999997},"last":0.28599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":5.468,"5min":2.4609999999999999,"15min":2.4790000000000001},"min":{"1min":0.24099999999999999,"5min":0.217,"15min":0.161},"max":{"1min":45.912999999999997,"5min":45.912999999999997,"15min":183.77099999999999},"last":0.378},{"interface":"front","average":{"1min":5.2930000000000001,"5min":2.5070000000000001,"15min":2.6949999999999998},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":46.234000000000002,"5min":46.234000000000002,"15min":183.78700000000001},"last":0.36699999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.276,"15min":3.1070000000000002},"min":{"1min":0.28999999999999998,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":12.113,"5min":12.113,"15min":186.143},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.1759999999999999,"15min":3.1629999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":12.348000000000001,"5min":12.348000000000001,"15min":186.97399999999999},"last":0.40699999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7429999999999999,"5min":1.2789999999999999,"15min":1.3600000000000001},"min":{"1min":0.376,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":10.814,"5min":10.814,"15min":10.814},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.03,"15min":0.96799999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.586,"5min":10.586,"15min":11.493},"last":0.60999999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.454,"15min":2.0590000000000002},"min":{"1min":0.34399999999999997,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":10.763,"5min":10.763,"15min":154.30600000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.129,"5min":1.147,"15min":1.5189999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28699999999999998},"max":{"1min":10.618,"5min":11.478,"15min":154.42400000000001},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":5.9500000000000002,"15min":3.214},"min":{"1min":0.35199999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.654,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.497},{"interface":"front","average":{"1min":1.494,"5min":5.9000000000000004,"15min":3.3380000000000001},"min":{"1min":0.39900000000000002,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":5.7750000000000004,"5min":169.767,"15min":169.767},"last":0.621}]},{"osd":7,"last update":"Sat Sep 23 21:07:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.347,"15min":2.2509999999999999},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.74,"5min":10.74,"15min":164.26400000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.0720000000000001,"15min":2.2549999999999999},"min":{"1min":0.44,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":5.8159999999999998,"5min":5.8159999999999998,"15min":164.23099999999999},"last":0.59099999999999997}]}]},{"osd":2,"up_from":129,"seq":554050781458,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804504,"kb_used_data":1114320,"kb_used_omap":7,"kb_used_meta":690168,"kb_avail":91940648,"statfs":{"total":95995035648,"available":94147223552,"internally_reserved":0,"allocated":1141063680,"data_stored":1139033530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":706732113},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7930000000000001,"5min":1.5169999999999999,"15min":1.423},"min":{"1min":0.222,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":11.699999999999999,"5min":11.699999999999999,"15min":26.356000000000002},"last":9.8149999999999995},{"interface":"front","average":{"1min":3.6800000000000002,"5min":2.444,"15min":1.8560000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":16.974,"5min":23.963999999999999,"15min":26.341000000000001},"last":9.9480000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4109999999999996,"5min":5.375,"15min":2.6240000000000001},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.188},"max":{"1min":14.057,"5min":212.12,"15min":212.12},"last":9.8659999999999997},{"interface":"front","average":{"1min":2.5760000000000001,"5min":4.8449999999999998,"15min":2.3330000000000002},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":6.5709999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":0.73799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:07:47 2023","interfaces":[{"interface":"back","average":{"1min":10.468999999999999,"5min":2.8500000000000001,"15min":3.2440000000000002},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":138.90299999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":9.9060000000000006},{"interface":"front","average":{"1min":10.629,"5min":2.8900000000000001,"15min":3.2090000000000001},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.215},"max":{"1min":138.87700000000001,"5min":138.87700000000001,"15min":138.87700000000001},"last":0.84299999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.169,"15min":1.139},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26200000000000001},"max":{"1min":11.744,"5min":11.744,"15min":12.586},"last":0.80000000000000004},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.028,"15min":1.46},"min":{"1min":0.35699999999999998,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":16.725999999999999,"5min":16.725999999999999,"15min":16.725999999999999},"last":1.3}]},{"osd":5,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":3.8119999999999998,"5min":2.222,"15min":2.2130000000000001},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":16.673999999999999,"5min":16.673999999999999,"15min":173.34},"last":1.345},{"interface":"front","average":{"1min":2.2480000000000002,"5min":1.4770000000000001,"15min":1.8380000000000001},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":12.368,"5min":13.701000000000001,"15min":173.15000000000001},"last":2.113}]},{"osd":6,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":3.6800000000000002,"15min":1.9590000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.323,"5min":175.72499999999999,"15min":175.72499999999999},"last":2.3159999999999998},{"interface":"front","average":{"1min":2.0230000000000001,"5min":3.3780000000000001,"15min":1.915},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":11.782,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.77100000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.665,"15min":2.0579999999999998},"min":{"1min":0.42399999999999999,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":11.811999999999999,"5min":11.811999999999999,"15min":177.30799999999999},"last":0.82099999999999995},{"interface":"front","average":{"1min":3.6339999999999999,"5min":2.109,"15min":2.2410000000000001},"min":{"1min":0.38200000000000001,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":16.875,"5min":16.875,"15min":177.40899999999999},"last":1.373}]}]},{"osd":3,"up_from":148,"seq":635655160052,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576232,"kb_used_data":9376,"kb_used_omap":2,"kb_used_meta":566845,"kb_avail":93168920,"statfs":{"total":95995035648,"available":95404974080,"internally_reserved":0,"allocated":9601024,"data_stored":7375618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":580449333},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":1.984,"15min":1.4970000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.184},"max":{"1min":17.425999999999998,"5min":44.963000000000001,"15min":44.963000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.629,"5min":1.1080000000000001,"15min":1.302},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.19500000000000001},"max":{"1min":7.9699999999999998,"5min":9.7370000000000001,"15min":37.634999999999998},"last":3.9750000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":12.473000000000001,"5min":3.4580000000000002,"15min":1.8380000000000001},"min":{"1min":0.64300000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":109.51300000000001,"5min":109.51300000000001,"15min":109.51300000000001},"last":3.8330000000000002},{"interface":"front","average":{"1min":13.468,"5min":3.6669999999999998,"15min":2.121},"min":{"1min":0.26500000000000001,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":111.13500000000001,"5min":111.13500000000001,"15min":111.13500000000001},"last":2.0979999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":9.6780000000000008,"5min":3.2469999999999999,"15min":1.9259999999999999},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":88.825999999999993,"5min":88.825999999999993,"15min":88.825999999999993},"last":0.48899999999999999},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.5790000000000002,"15min":1.732},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":85.626000000000005,"5min":85.626000000000005,"15min":85.626000000000005},"last":8.9339999999999993}]},{"osd":4,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":2.8900000000000001,"15min":2.347},"min":{"1min":0.374,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":70.728999999999999,"15min":111.36},"last":1.7949999999999999},{"interface":"front","average":{"1min":2.605,"5min":2.7679999999999998,"15min":2.1429999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":17.286999999999999,"5min":70.712000000000003,"15min":112.473},"last":0.42899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.7989999999999999,"15min":1.5920000000000001},"min":{"1min":0.39400000000000002,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":7.9050000000000002,"5min":36.488999999999997,"15min":64.528000000000006},"last":3.8759999999999999},{"interface":"front","average":{"1min":3.278,"5min":2.101,"15min":1.724},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":12.382999999999999,"5min":36.295000000000002,"15min":36.295000000000002},"last":1.8360000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.3220000000000001,"5min":1.8080000000000001,"15min":1.7050000000000001},"min":{"1min":0.36099999999999999,"5min":0.318,"15min":0.312},"max":{"1min":12.317,"5min":12.317,"15min":19.568000000000001},"last":3.0590000000000002},{"interface":"front","average":{"1min":2.3570000000000002,"5min":1.722,"15min":1.522},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":17.324999999999999,"5min":17.324999999999999,"15min":17.324999999999999},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.427,"15min":2.488},"min":{"1min":0.377,"5min":0.377,"15min":0.30499999999999999},"max":{"1min":7.9459999999999997,"5min":74.019000000000005,"15min":215.70099999999999},"last":3.8999999999999999},{"interface":"front","average":{"1min":3.46,"5min":2.7970000000000002,"15min":2.8879999999999999},"min":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":12.407,"5min":73.364999999999995,"15min":222.11699999999999},"last":1.9359999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986671,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908600,"kb_used_data":333168,"kb_used_omap":0,"kb_used_meta":575424,"kb_avail":92836552,"statfs":{"total":95995035648,"available":95064629248,"internally_reserved":0,"allocated":341164032,"data_stored":339928168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":589234176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3560000000000001,"15min":2.2829999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":10.273999999999999,"5min":12.869,"15min":102.358},"last":4.6020000000000003},{"interface":"front","average":{"1min":1.883,"5min":1.4790000000000001,"15min":2.29},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":8.3190000000000008,"5min":12.848000000000001,"15min":102.58199999999999},"last":4.5279999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.782,"5min":7.7530000000000001,"15min":3.5259999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":14.523999999999999,"5min":127.411,"15min":127.411},"last":1.8799999999999999},{"interface":"front","average":{"1min":1.7629999999999999,"5min":5.7130000000000001,"15min":2.835},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":5.8609999999999998,"5min":118.075,"15min":118.075},"last":1.857}]},{"osd":2,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":5.6589999999999998,"15min":3.6699999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":7.4480000000000004,"5min":164.517,"15min":164.517},"last":1.972},{"interface":"front","average":{"1min":1.4419999999999999,"5min":4.7889999999999997,"15min":3.4769999999999999},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":5.9400000000000004,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":1.2909999999999999,"15min":2.0510000000000002},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":6.8120000000000003,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.44},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.2150000000000001,"15min":2.149},"min":{"1min":0.40600000000000003,"5min":0.32400000000000001,"15min":0.25600000000000001},"max":{"1min":6.181,"5min":8.2899999999999991,"15min":142.34700000000001},"last":5.1299999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":0.95999999999999996,"15min":1.02},"min":{"1min":0.309,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":6.0069999999999997,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.754,"5min":1.0149999999999999,"15min":1.032},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.21099999999999999},"max":{"1min":4.0259999999999998,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.36199999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.2949999999999999,"15min":0.95299999999999996},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":6.4269999999999996,"5min":11.800000000000001,"15min":11.887},"last":0.373},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.341,"15min":0.92900000000000005},"min":{"1min":0.39300000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":4.5449999999999999,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":2.6829999999999998,"15min":1.544},"min":{"1min":0.34300000000000003,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":3.726,"5min":145.721,"15min":145.721},"last":0.502},{"interface":"front","average":{"1min":0.98699999999999999,"5min":3.0049999999999999,"15min":1.7},"min":{"1min":0.375,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":3.383,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.46400000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725035,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":802084,"kb_used_data":170396,"kb_used_omap":0,"kb_used_meta":631680,"kb_avail":92943068,"statfs":{"total":95995035648,"available":95173701632,"internally_reserved":0,"allocated":174485504,"data_stored":173265840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":646840320},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.0660000000000001,"15min":1.319},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.2640000000000002,"5min":10.488,"15min":80.105999999999995},"last":4.7969999999999997},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.1799999999999999,"15min":1.4590000000000001},"min":{"1min":0.27000000000000002,"5min":0.222,"15min":0.222},"max":{"1min":6.6619999999999999,"5min":8.2759999999999998,"15min":80.152000000000001},"last":1.073}]},{"osd":1,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":9.4730000000000008,"5min":7.8259999999999996,"15min":3.117},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":125.23399999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":2.698},{"interface":"front","average":{"1min":9.9860000000000007,"5min":7.6059999999999999,"15min":3.0600000000000001},"min":{"1min":0.375,"5min":0.251,"15min":0.247},"max":{"1min":125.249,"5min":233.75200000000001,"15min":233.75200000000001},"last":1.6890000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":4.9640000000000004,"15min":2.794},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.254},"max":{"1min":70.810000000000002,"5min":231.34700000000001,"15min":231.34700000000001},"last":11.538},{"interface":"front","average":{"1min":8.1370000000000005,"5min":5.4619999999999997,"15min":3.0289999999999999},"min":{"1min":0.40200000000000002,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":70.507000000000005,"5min":231.381,"15min":231.381},"last":1.024}]},{"osd":3,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.6519999999999999,"15min":2.786},"min":{"1min":0.33000000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":12.765000000000001,"5min":12.765000000000001,"15min":153.60300000000001},"last":2.7589999999999999},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.3959999999999999,"15min":2.7130000000000001},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":9.6120000000000001,"5min":9.6120000000000001,"15min":153.774},"last":4.0910000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.72399999999999998,"15min":0.71399999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":6.3940000000000001,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.0129999999999999,"15min":0.89200000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":5.8490000000000002,"5min":8.3740000000000006,"15min":12.539},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":1.1990000000000001,"15min":1.1259999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.1130000000000004,"5min":9.8480000000000008,"15min":71.230000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.252,"15min":1.163},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":3.4359999999999999,"5min":12.673999999999999,"15min":70.266000000000005},"last":2.734}]},{"osd":7,"last update":"Sat Sep 23 21:08:14 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.284,"15min":1.129},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.312},"max":{"1min":9.6329999999999991,"5min":9.6329999999999991,"15min":31.091999999999999},"last":2.8769999999999998},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.70799999999999996,"15min":0.93500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35899999999999999,"15min":0.28899999999999998},"max":{"1min":4.29,"5min":4.29,"15min":31.780000000000001},"last":0.51000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":221052928,"data_stored":221052928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1376321536,"data_stored":1376321536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1133907968,"data_stored":1133903896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3932160,"data_stored":3932160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":332992512,"data_stored":332988440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":168624128,"data_stored":168624128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":974393344,"data_stored":974389272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3052273664,"data_stored":3052273664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:08:52.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18056/57636 objects degraded (31.328%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:52.734 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:52.817 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:52.948 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:52.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:52.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:53.065 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:53.090 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.131 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:53.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:53.238 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:53.269 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.283 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:53 smithi121 ceph-mon[211526]: pgmap v1338: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 18056/57636 objects degraded (31.328%); 16703/57636 objects misplaced (28.980%); 9.1 MiB/s, 144 objects/s recovering 2023-09-23T21:08:53.323 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:53.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:53.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:53.431 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:53.467 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:53 smithi123 ceph-mon[193903]: pgmap v1338: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 18056/57636 objects degraded (31.328%); 16703/57636 objects misplaced (28.980%); 9.1 MiB/s, 144 objects/s recovering 2023-09-23T21:08:53.472 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.541 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:53.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:53.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:53.645 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:53.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:53 smithi121 ceph-mon[219709]: pgmap v1338: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 18056/57636 objects degraded (31.328%); 16703/57636 objects misplaced (28.980%); 9.1 MiB/s, 144 objects/s recovering 2023-09-23T21:08:53.684 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.745 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:53.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:53.847 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:53.851 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:08:53.880 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:53.932 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:08:53.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:53.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:08:53.975 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:54.037 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:54.122 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:54.505 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:54.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:54.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:54.609 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:54.709 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:55.030 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:08:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:08:54] "GET /metrics HTTP/1.1" 200 36704 "" "Prometheus/2.43.0" 2023-09-23T21:08:55.030 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1993688915' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:55.030 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:55.065 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:08:55.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1993688915' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:55.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1993688915' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:08:55.722 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:08:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[211526]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T21:08:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[211526]: failed to encode map e419 with expected crc 2023-09-23T21:08:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[211526]: failed to encode map e419 with expected crc 2023-09-23T21:08:56.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[211526]: pgmap v1340: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 18056/55695 objects degraded (32.419%); 15875/55695 objects misplaced (28.503%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T21:08:56.166 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:56.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:56.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:56.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[219709]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T21:08:56.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[219709]: failed to encode map e419 with expected crc 2023-09-23T21:08:56.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[219709]: failed to encode map e419 with expected crc 2023-09-23T21:08:56.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:56 smithi121 ceph-mon[219709]: pgmap v1340: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 18056/55695 objects degraded (32.419%); 15875/55695 objects misplaced (28.503%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T21:08:56.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:56.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:56.272 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:08:56.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:56 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:08:56.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:56 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:08:56.310 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:56 smithi123 ceph-mon[193903]: osdmap e419: 8 total, 8 up, 7 in 2023-09-23T21:08:56.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:56 smithi123 ceph-mon[193903]: failed to encode map e419 with expected crc 2023-09-23T21:08:56.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:56 smithi123 ceph-mon[193903]: failed to encode map e419 with expected crc 2023-09-23T21:08:56.311 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:56 smithi123 ceph-mon[193903]: pgmap v1340: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 18056/55695 objects degraded (32.419%); 15875/55695 objects misplaced (28.503%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T21:08:56.311 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:56.393 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:08:56.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:56.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:08:56.500 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:08:56.599 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:57.320 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:57 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18056/55695 objects degraded (32.419%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:57.320 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:57 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18056/55695 objects degraded (32.419%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:57.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:57 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18056/55695 objects degraded (32.419%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:08:57.780 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:08:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:57.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:08:57.883 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:08:57.932 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:58.070 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:58 smithi121 ceph-mon[211526]: pgmap v1341: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 17420/50166 objects degraded (34.725%); 14544/50166 objects misplaced (28.992%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:58.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:58.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:58.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:58 smithi121 ceph-mon[219709]: pgmap v1341: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 17420/50166 objects degraded (34.725%); 14544/50166 objects misplaced (28.992%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:58.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:58.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:58 smithi123 ceph-mon[193903]: pgmap v1341: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 17420/50166 objects degraded (34.725%); 14544/50166 objects misplaced (28.992%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:08:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:08:58.490 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:08:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:58.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:08:58.598 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:08:58.730 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:58.738 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:08:58.885 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:120247 11.5 e409/389 11:afe7d819:::benchmark_data_smithi121_477989_object32058:head v 409'7457, mlcod=0'0)", 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:36.572123+0000", 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 82.304897787000002, 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12892674100000001, 2023-09-23T21:08:58.886 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572123+0000", 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572123+0000", 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572122+0000", 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572130+0000", 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6200000000000005e-06 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572142+0000", 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1761e-05 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572150+0000", 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1769999999999998e-06 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572197+0000", 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6672999999999999e-05 2023-09-23T21:08:58.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572225+0000", 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8133999999999999e-05 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.701006+0000", 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12878052500000001 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.701050+0000", 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3881000000000002e-05 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:120269 11.9 e409/374 11:91fb775b:::benchmark_data_smithi121_477989_object32080:head v 409'7604, mlcod=409'7602)", 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:36.572167+0000", 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 82.304854075999998, 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.11784984699999999, 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.891 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572167+0000", 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572167+0000", 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572166+0000", 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572170+0000", 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8979999999999998e-06 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572176+0000", 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.195e-06 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572181+0000", 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8539999999999996e-06 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572223+0000", 2023-09-23T21:08:58.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1551999999999998e-05 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.572243+0000", 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0250000000000001e-05 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.574258+0000", 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020147110000000002 2023-09-23T21:08:58.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.690016+0000", 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.115758677 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:120270 11.1 e409/374 11:847a2e93:::benchmark_data_smithi121_477989_object32081:head v 409'7426, mlcod=409'7424)", 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:36.575959+0000", 2023-09-23T21:08:58.896 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 82.301061609000001, 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.116248723, 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.575959+0000", 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.575959+0000", 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.575958+0000", 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.575967+0000", 2023-09-23T21:08:58.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4659999999999999e-06 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687113+0000", 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.111146965 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687130+0000", 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6453e-05 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687559+0000", 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00042898300000000002 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687583+0000", 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3822e-05 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.692173+0000", 2023-09-23T21:08:58.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0045898390000000001 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.692208+0000", 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5165e-05 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:120272 11.8 e409/386 11:1d5ab60c:::benchmark_data_smithi121_477989_object32083:head v 409'7538, mlcod=409'7536)", 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:36.578200+0000", 2023-09-23T21:08:58.901 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 82.298820289000005, 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.11307403100000001, 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.578200+0000", 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.578200+0000", 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.578199+0000", 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.578209+0000", 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.007e-05 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687726+0000", 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.109517055 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687735+0000", 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1580000000000005e-06 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687808+0000", 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2976000000000001e-05 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.687828+0000", 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0234e-05 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.691228+0000", 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034001320000000002 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:36.691274+0000", 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6434000000000003e-05 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:121287 11.b e409/380 11:d89fa918:::benchmark_data_smithi121_477989_object33098:head v 409'7603, mlcod=0'0)", 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:38.356039+0000", 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 80.520981208999999, 2023-09-23T21:08:58.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14354399000000001, 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356039+0000", 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356039+0000", 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356039+0000", 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356047+0000", 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0690000000000002e-06 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356059+0000", 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2315e-05 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356068+0000", 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8149999999999995e-06 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356115+0000", 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6811000000000003e-05 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.356130+0000", 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5571000000000001e-05 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.499543+0000", 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.143413019 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.499583+0000", 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0311000000000001e-05 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:121294 11.a e409/387 11:55c49412:::benchmark_data_smithi121_477989_object33105:head v 409'7448, mlcod=409'7446)", 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:38.358608+0000", 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 80.518412768999994, 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14317017400000001, 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.358608+0000", 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.358608+0000", 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.358607+0000", 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.358617+0000", 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0042e-05 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496113+0000", 2023-09-23T21:08:58.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.13749641500000001 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496139+0000", 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5850000000000002e-05 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496230+0000", 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1052999999999997e-05 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496257+0000", 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6757000000000001e-05 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.501745+0000", 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005488667 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.501778+0000", 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2592999999999998e-05 2023-09-23T21:08:58.915 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:121286 11.2 e409/403 11:498c532f:::benchmark_data_smithi121_477989_object33097:head v 409'7551, mlcod=409'7548)", 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:38.360762+0000", 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 80.516258993999998, 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14104582800000001, 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.360762+0000", 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.360762+0000", 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.360760+0000", 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.360771+0000", 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0433e-05 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496123+0000", 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.13535180199999999 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496148+0000", 2023-09-23T21:08:58.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5023000000000001e-05 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496244+0000", 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6691e-05 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496429+0000", 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018419000000000001 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.501782+0000", 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053530690000000002 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.501807+0000", 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5809e-05 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:121298 11.0 e409/371 11:080a51dc:::benchmark_data_smithi121_477989_object33109:head v 409'7524, mlcod=346'42)", 2023-09-23T21:08:58.920 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:07:38.362762+0000", 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 80.514258204000001, 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.144400426, 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.362762+0000", 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.362762+0000", 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.362758+0000", 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999952 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.362770+0000", 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2334000000000001e-05 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496103+0000", 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.13333268700000001 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496133+0000", 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9683e-05 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496243+0000", 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110055 2023-09-23T21:08:58.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.496277+0000", 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4622999999999997e-05 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.507134+0000", 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010856308 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:07:38.507163+0000", 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9351000000000001e-05 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132677 11.8 e411/386 11:1a1877d8:::benchmark_data_smithi121_477989_object44488:head v 411'8333, mlcod=411'8331)", 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.132318+0000", 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 57.744702363000002, 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.088780843999999998, 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132318+0000", 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132318+0000", 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132317+0000", 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132326+0000", 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3590000000000008e-06 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132337+0000", 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1005000000000001e-05 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132344+0000", 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7839999999999998e-06 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132409+0000", 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5029000000000004e-05 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132431+0000", 2023-09-23T21:08:58.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2512999999999999e-05 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.221043+0000", 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.088612148000000002 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.221099+0000", 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5858e-05 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.929 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132680 11.8 e411/386 11:1ea9b53b:::benchmark_data_smithi121_477989_object44491:head v 411'8334, mlcod=411'8331)", 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.132359+0000", 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 57.744661344000001, 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.088768506999999996, 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132359+0000", 2023-09-23T21:08:58.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132359+0000", 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132359+0000", 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132363+0000", 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.771e-06 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132368+0000", 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3709999999999999e-06 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.132372+0000", 2023-09-23T21:08:58.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.16e-06 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.135533+0000", 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003161058 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.135561+0000", 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7345e-05 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.221106+0000", 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.085544998999999997 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.221128+0000", 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2062999999999999e-05 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132684 11.a e411/387 11:514a28c2:::benchmark_data_smithi121_477989_object44495:head v 411'8160, mlcod=411'8158)", 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.136486+0000", 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 57.740534617999998, 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089238986000000006, 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136486+0000", 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136486+0000", 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136485+0000", 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136492+0000", 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1709999999999996e-06 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136504+0000", 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1625e-05 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136512+0000", 2023-09-23T21:08:58.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9289999999999998e-06 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136660+0000", 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014761400000000001 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136680+0000", 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0134999999999999e-05 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.225690+0000", 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089009916999999994 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.225725+0000", 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5420999999999998e-05 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132686 11.1 e411/374 11:80874bfe:::benchmark_data_smithi121_477989_object44497:head v 411'8202, mlcod=411'8199)", 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.136528+0000", 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 57.740492482000001, 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087048018000000005, 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136528+0000", 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136528+0000", 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136528+0000", 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136532+0000", 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0029999999999996e-06 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136538+0000", 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3999999999999997e-06 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136543+0000", 2023-09-23T21:08:58.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9030000000000003e-06 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217291+0000", 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.080748035999999995 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217317+0000", 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5718e-05 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.223540+0000", 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062233410000000003 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.223576+0000", 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5880000000000002e-05 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132688 11.a e411/387 11:51b74649:::benchmark_data_smithi121_477989_object44499:head v 411'8161, mlcod=411'8158)", 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.136844+0000", 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 57.740176806000001, 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.090774911999999999, 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136844+0000", 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136844+0000", 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136843+0000", 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.136849+0000", 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.711e-06 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217237+0000", 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.080388058999999998 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217248+0000", 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0676000000000001e-05 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.223651+0000", 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064036900000000001 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.223681+0000", 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9260000000000001e-05 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.227587+0000", 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0039064440000000002 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.227619+0000", 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1628999999999998e-05 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.949 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:132689 11.0 e411/371 11:0f64e0d9:::benchmark_data_smithi121_477989_object44500:head v 411'8207, mlcod=346'42)", 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:01.139753+0000", 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 57.737267322000001, 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.084910534999999995, 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139753+0000", 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139753+0000", 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139752+0000", 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.139760+0000", 2023-09-23T21:08:58.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8339999999999999e-06 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217230+0000", 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.077469892999999998 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217244+0000", 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3482000000000001e-05 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217313+0000", 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9261e-05 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.217339+0000", 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6057000000000001e-05 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.224619+0000", 2023-09-23T21:08:58.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0072805550000000002 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:01.224664+0000", 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4362e-05 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:159859 11.5 e419/389 11:ad29955f:::benchmark_data_smithi121_477989_object71670:head v 419'9911, mlcod=0'0)", 2023-09-23T21:08:58.954 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:55.786914+0000", 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.090106794, 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.075912684999999994, 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.786914+0000", 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.786914+0000", 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.786913+0000", 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.786922+0000", 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9299999999999992e-06 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.786933+0000", 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1544e-05 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.786940+0000", 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0790000000000003e-06 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.787838+0000", 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089705799999999995 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.787857+0000", 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9522999999999999e-05 2023-09-23T21:08:58.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.862775+0000", 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.074918055999999997 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.862827+0000", 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1430000000000001e-05 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.960 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:159870 11.b e419/417 11:d0fcb628:::benchmark_data_smithi121_477989_object71681:head v 419'10003, mlcod=419'10000)", 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:55.793550+0000", 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.0834703139999999, 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.080580837000000002, 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.793550+0000", 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.793550+0000", 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.793549+0000", 2023-09-23T21:08:58.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.793560+0000", 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1e-05 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.860466+0000", 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.066905667000000002 2023-09-23T21:08:58.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.860478+0000", 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1712999999999999e-05 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.860857+0000", 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00037943500000000002 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.860879+0000", 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1715000000000001e-05 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.874104+0000", 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013224672999999999 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:55.874131+0000", 2023-09-23T21:08:58.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7521999999999998e-05 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:160670 11.2 e419/403 11:4e14c82d:::benchmark_data_smithi121_477989_object72481:head v 419'10011, mlcod=419'10009)", 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:57.390356+0000", 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.486664542, 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087390359000000001, 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.966 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390356+0000", 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390356+0000", 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390355+0000", 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390361+0000", 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6829999999999996e-06 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390373+0000", 2023-09-23T21:08:58.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1647e-05 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390380+0000", 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8820000000000003e-06 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390447+0000", 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7612000000000003e-05 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.390473+0000", 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5720000000000001e-05 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.407791+0000", 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017318508 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.477746+0000", 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.069955028000000002 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:160676 11.1 e419/374 11:89322b33:::benchmark_data_smithi121_477989_object72487:head v 419'9963, mlcod=419'9957)", 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:57.403051+0000", 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.4739696659999999, 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.077695687999999999, 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403051+0000", 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403051+0000", 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403051+0000", 2023-09-23T21:08:58.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403054+0000", 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2200000000000001e-06 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403057+0000", 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2100000000000002e-06 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403060+0000", 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5019999999999999e-06 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.405232+0000", 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021718150000000001 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.405252+0000", 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0860999999999999e-05 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.480709+0000", 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.075457062000000005 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.480747+0000", 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7218999999999997e-05 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.975 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:160677 11.1 e419/374 11:8420ed01:::benchmark_data_smithi121_477989_object72488:head v 419'9964, mlcod=419'9957)", 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:57.403363+0000", 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.4736581200000001, 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.077407924000000003, 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403363+0000", 2023-09-23T21:08:58.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403363+0000", 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403362+0000", 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403368+0000", 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.127e-06 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403378+0000", 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.048e-05 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.403385+0000", 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5350000000000001e-06 2023-09-23T21:08:58.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.407688+0000", 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0043034630000000004 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.407722+0000", 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3800000000000002e-05 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.480752+0000", 2023-09-23T21:08:58.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.073029431000000006 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.480770+0000", 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8715999999999998e-05 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36195.0:160681 11.a e419/387 11:5a876f03:::benchmark_data_smithi121_477989_object72492:head v 419'9894, mlcod=419'9892)", 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:08:57.409023+0000", 2023-09-23T21:08:58.980 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 1.4679980079999999, 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087333117000000002, 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.409023+0000", 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.409023+0000", 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:08:58.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.409022+0000", 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.409029+0000", 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7680000000000003e-06 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:08:58.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.477360+0000", 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.068331875 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.477371+0000", 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0511e-05 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.482863+0000", 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054920710000000003 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.482884+0000", 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1356999999999999e-05 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.496331+0000", 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013446296 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:08:57.496356+0000", 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5014000000000001e-05 2023-09-23T21:08:58.984 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.985 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.985 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.985 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:08:58.985 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:08:58.985 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:08:58.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:58.986 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:08:58.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:08:59.087 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:08:59.213 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:08:59 smithi123 ceph-mon[193903]: pgmap v1342: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 17420/50166 objects degraded (34.725%); 14544/50166 objects misplaced (28.992%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:59.627 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:08:59.627 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:08:59.627 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:08:59.628 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:08:59.628 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:08:59.628 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:08:59.628 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:08:59.637 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:08:59.650 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.650 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.650 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.651 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.651 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.651 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.651 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.651 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:08:59.662 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:08:59.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:08:59 smithi121 ceph-mon[211526]: pgmap v1342: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 17420/50166 objects degraded (34.725%); 14544/50166 objects misplaced (28.992%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:59.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:08:59 smithi121 ceph-mon[219709]: pgmap v1342: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 17420/50166 objects degraded (34.725%); 14544/50166 objects misplaced (28.992%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T21:08:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:59.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:08:59.773 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:08:59.811 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:08:59.885 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:08:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:08:59.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:08:59.993 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:00.024 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:00.092 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:00.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:00.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:00.282 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:00.311 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:09:00.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/815724345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:00.486 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/815724345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:00.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/815724345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:01.410 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:01.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:01.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:01 smithi123 ceph-mon[193903]: pgmap v1343: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 882 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 16786/42378 objects degraded (39.610%); 12147/42378 objects misplaced (28.663%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:09:01.520 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:01.552 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:01.608 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:01.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:01.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:01.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:01 smithi121 ceph-mon[219709]: pgmap v1343: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 882 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 16786/42378 objects degraded (39.610%); 12147/42378 objects misplaced (28.663%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:09:01.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:01 smithi121 ceph-mon[211526]: pgmap v1343: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 882 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 16786/42378 objects degraded (39.610%); 12147/42378 objects misplaced (28.663%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:09:01.715 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:01.747 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:09:01.814 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:02.238 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:02.595 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16786/42378 objects degraded (39.610%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:02.595 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16786/42378 objects degraded (39.610%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:02.639 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:02.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:02.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:02.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16786/42378 objects degraded (39.610%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:02.751 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:02.839 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:02.888 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:02.981 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:02.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:02.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:03.092 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:03.126 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:03.241 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:03 smithi121 ceph-mon[211526]: pgmap v1344: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 882 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 16786/42378 objects degraded (39.610%); 12147/42378 objects misplaced (28.663%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:09:03.408 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:03 smithi123 ceph-mon[193903]: pgmap v1344: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 882 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 16786/42378 objects degraded (39.610%); 12147/42378 objects misplaced (28.663%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:09:03.453 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:03.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:03.558 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:03.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:03 smithi121 ceph-mon[219709]: pgmap v1344: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 882 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 16786/42378 objects degraded (39.610%); 12147/42378 objects misplaced (28.663%); 10 MiB/s, 161 objects/s recovering 2023-09-23T21:09:03.658 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:03.906 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:03.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:03.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:03.937 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:09:03.961 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:09:04.013 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:04.089 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:04.608 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:09:04.631 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:04 smithi121 ceph-mon[219709]: from='client.36585 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:04.632 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:04 smithi121 ceph-mon[211526]: from='client.36585 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:04.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:04 smithi123 ceph-mon[193903]: from='client.36585 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:04.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:09:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:09:04] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.43.0" 2023-09-23T21:09:05.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:05 smithi121 ceph-mon[211526]: pgmap v1345: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 843 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 16786/40494 objects degraded (41.453%); 11372/40494 objects misplaced (28.083%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:05.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:05 smithi121 ceph-mon[219709]: pgmap v1345: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 843 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 16786/40494 objects degraded (41.453%); 11372/40494 objects misplaced (28.083%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:05 smithi123 ceph-mon[193903]: pgmap v1345: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 843 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 16786/40494 objects degraded (41.453%); 11372/40494 objects misplaced (28.083%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:06.231 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:06.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:06.275 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1344,"stamp":"2023-09-23T21:09:03.114430+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":924719032,"num_objects":14126,"num_object_clones":0,"num_object_copies":42378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16786,"num_objects_misplaced":12147,"num_objects_unfound":0,"num_objects_dirty":14126,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":162572,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40733,"num_bytes_recovered":2665258796,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84387,"ondisk_log_size":84387,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7273572,"kb_used_data":3038572,"kb_used_omap":21,"kb_used_meta":4234794,"kb_avail":648942492,"statfs":{"total":671965249536,"available":664517111808,"internally_reserved":0,"allocated":3111497728,"data_stored":3099454890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4336429539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,4,3,3,5,3,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-333316096,"num_objects":-5086,"num_object_clones":0,"num_object_copies":-15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1270,"num_objects_misplaced":-4556,"num_objects_unfound":0,"num_objects_dirty":-5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5086,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1615,"num_bytes_recovered":105840640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011337"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":190,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516849+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:08:42.516849+0000","last_peered":"2023-09-23T21:08:42.516849+0000","last_clean":"2023-09-23T21:08:42.516849+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:08:42.516849+0000","last_undegraded":"2023-09-23T21:08:42.516849+0000","last_fullsized":"2023-09-23T21:08:42.516849+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":59,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.522357+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:08:42.522357+0000","last_peered":"2023-09-23T21:08:42.522357+0000","last_clean":"2023-09-23T21:08:42.522357+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:08:42.522357+0000","last_undegraded":"2023-09-23T21:08:42.522357+0000","last_fullsized":"2023-09-23T21:08:42.522357+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":332,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516221+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:08:42.516221+0000","last_peered":"2023-09-23T21:08:42.516221+0000","last_clean":"2023-09-23T21:08:42.516221+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:08:42.516221+0000","last_undegraded":"2023-09-23T21:08:42.516221+0000","last_fullsized":"2023-09-23T21:08:42.516221+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":57,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.088248+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:08:55.088248+0000","last_peered":"2023-09-23T21:08:55.088248+0000","last_clean":"2023-09-23T21:08:55.088248+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:08:55.088248+0000","last_undegraded":"2023-09-23T21:08:55.088248+0000","last_fullsized":"2023-09-23T21:08:55.088248+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":77,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.520813+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:08:42.520813+0000","last_peered":"2023-09-23T21:08:42.520813+0000","last_clean":"2023-09-23T21:08:42.520813+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:08:42.520813+0000","last_undegraded":"2023-09-23T21:08:42.520813+0000","last_fullsized":"2023-09-23T21:08:42.520813+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":285,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518568+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:08:42.518568+0000","last_peered":"2023-09-23T21:08:42.518568+0000","last_clean":"2023-09-23T21:08:42.518568+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:08:42.518568+0000","last_undegraded":"2023-09-23T21:08:42.518568+0000","last_fullsized":"2023-09-23T21:08:42.518568+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":56,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517411+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:08:42.517411+0000","last_peered":"2023-09-23T21:08:42.517411+0000","last_clean":"2023-09-23T21:08:42.517411+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:08:42.517411+0000","last_undegraded":"2023-09-23T21:08:42.517411+0000","last_fullsized":"2023-09-23T21:08:42.517411+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":59,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.520810+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:08:42.520810+0000","last_peered":"2023-09-23T21:08:42.520810+0000","last_clean":"2023-09-23T21:08:42.520810+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:08:42.520810+0000","last_undegraded":"2023-09-23T21:08:42.520810+0000","last_fullsized":"2023-09-23T21:08:42.520810+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":42,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517145+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:08:42.517145+0000","last_peered":"2023-09-23T21:08:42.517145+0000","last_clean":"2023-09-23T21:08:42.517145+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:08:42.517145+0000","last_undegraded":"2023-09-23T21:08:42.517145+0000","last_fullsized":"2023-09-23T21:08:42.517145+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":59,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.521041+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:08:42.521041+0000","last_peered":"2023-09-23T21:08:42.521041+0000","last_clean":"2023-09-23T21:08:42.521041+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:08:42.521041+0000","last_undegraded":"2023-09-23T21:08:42.521041+0000","last_fullsized":"2023-09-23T21:08:42.521041+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":75,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.522353+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:08:42.522353+0000","last_peered":"2023-09-23T21:08:42.522353+0000","last_clean":"2023-09-23T21:08:42.522353+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:08:42.522353+0000","last_undegraded":"2023-09-23T21:08:42.522353+0000","last_fullsized":"2023-09-23T21:08:42.522353+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":51,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516512+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:08:42.516512+0000","last_peered":"2023-09-23T21:08:42.516512+0000","last_clean":"2023-09-23T21:08:42.516512+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:08:42.516512+0000","last_undegraded":"2023-09-23T21:08:42.516512+0000","last_fullsized":"2023-09-23T21:08:42.516512+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":365,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.523969+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:08:42.523969+0000","last_peered":"2023-09-23T21:08:42.523969+0000","last_clean":"2023-09-23T21:08:42.523969+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:08:42.523969+0000","last_undegraded":"2023-09-23T21:08:42.523969+0000","last_fullsized":"2023-09-23T21:08:42.523969+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":337,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.568084+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:08:42.568084+0000","last_peered":"2023-09-23T21:08:42.568084+0000","last_clean":"2023-09-23T21:08:42.568084+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:08:42.568084+0000","last_undegraded":"2023-09-23T21:08:42.568084+0000","last_fullsized":"2023-09-23T21:08:42.568084+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":109,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.513689+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:08:42.513689+0000","last_peered":"2023-09-23T21:08:42.513689+0000","last_clean":"2023-09-23T21:08:42.513689+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:08:42.513689+0000","last_undegraded":"2023-09-23T21:08:42.513689+0000","last_fullsized":"2023-09-23T21:08:42.513689+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":68,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.521182+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:08:42.521182+0000","last_peered":"2023-09-23T21:08:42.521182+0000","last_clean":"2023-09-23T21:08:42.521182+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:08:42.521182+0000","last_undegraded":"2023-09-23T21:08:42.521182+0000","last_fullsized":"2023-09-23T21:08:42.521182+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":77,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.522073+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:08:42.522073+0000","last_peered":"2023-09-23T21:08:42.522073+0000","last_clean":"2023-09-23T21:08:42.522073+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:08:42.522073+0000","last_undegraded":"2023-09-23T21:08:42.522073+0000","last_fullsized":"2023-09-23T21:08:42.522073+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.522401+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:08:42.522401+0000","last_peered":"2023-09-23T21:08:42.522401+0000","last_clean":"2023-09-23T21:08:42.522401+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:08:42.522401+0000","last_undegraded":"2023-09-23T21:08:42.522401+0000","last_fullsized":"2023-09-23T21:08:42.522401+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":219,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.567967+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:08:42.567967+0000","last_peered":"2023-09-23T21:08:42.567967+0000","last_clean":"2023-09-23T21:08:42.567967+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:08:42.567967+0000","last_undegraded":"2023-09-23T21:08:42.567967+0000","last_fullsized":"2023-09-23T21:08:42.567967+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":56,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517082+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:08:42.517082+0000","last_peered":"2023-09-23T21:08:42.517082+0000","last_clean":"2023-09-23T21:08:42.517082+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:08:42.517082+0000","last_undegraded":"2023-09-23T21:08:42.517082+0000","last_fullsized":"2023-09-23T21:08:42.517082+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":415,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517282+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:08:42.517282+0000","last_peered":"2023-09-23T21:08:42.517282+0000","last_clean":"2023-09-23T21:08:42.517282+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:08:42.517282+0000","last_undegraded":"2023-09-23T21:08:42.517282+0000","last_fullsized":"2023-09-23T21:08:42.517282+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":59,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.521740+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:08:42.521740+0000","last_peered":"2023-09-23T21:08:42.521740+0000","last_clean":"2023-09-23T21:08:42.521740+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:08:42.521740+0000","last_undegraded":"2023-09-23T21:08:42.521740+0000","last_fullsized":"2023-09-23T21:08:42.521740+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"419'10345","reported_seq":11721,"reported_epoch":419,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:00.876397+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:09:00.876397+0000","last_peered":"2023-09-23T21:09:00.876397+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:09:00.876397+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"401'5628","ondisk_log_start":"401'5628","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4717,"log_dups_size":0,"ondisk_log_size":4717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57999360,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10345,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-4730},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":884}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":123,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.568371+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:08:42.568371+0000","last_peered":"2023-09-23T21:08:42.568371+0000","last_clean":"2023-09-23T21:08:42.568371+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:08:42.568371+0000","last_undegraded":"2023-09-23T21:08:42.568371+0000","last_fullsized":"2023-09-23T21:08:42.568371+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516598+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:08:42.516598+0000","last_peered":"2023-09-23T21:08:42.516598+0000","last_clean":"2023-09-23T21:08:42.516598+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:08:42.516598+0000","last_undegraded":"2023-09-23T21:08:42.516598+0000","last_fullsized":"2023-09-23T21:08:42.516598+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"419'9957","reported_seq":10046,"reported_epoch":419,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:00.880053+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:09:00.880053+0000","last_peered":"2023-09-23T21:09:00.880053+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:09:00.880053+0000","last_undegraded":"2023-09-23T21:09:00.880053+0000","last_fullsized":"2023-09-23T21:09:00.880053+0000","mapping_epoch":375,"log_start":"367'5212","ondisk_log_start":"367'5212","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4745,"log_dups_size":0,"ondisk_log_size":4745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55115776,"num_objects":841,"num_object_clones":0,"num_object_copies":2523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":841,"num_objects_unfound":0,"num_objects_dirty":841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9957,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":841}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":117,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517611+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:08:42.517611+0000","last_peered":"2023-09-23T21:08:42.517611+0000","last_clean":"2023-09-23T21:08:42.517611+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:08:42.517611+0000","last_undegraded":"2023-09-23T21:08:42.517611+0000","last_fullsized":"2023-09-23T21:08:42.517611+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":380,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516140+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:08:42.516140+0000","last_peered":"2023-09-23T21:08:42.516140+0000","last_clean":"2023-09-23T21:08:42.516140+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:08:42.516140+0000","last_undegraded":"2023-09-23T21:08:42.516140+0000","last_fullsized":"2023-09-23T21:08:42.516140+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"419'10059","reported_seq":26081,"reported_epoch":419,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:00.817316+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:09:00.817316+0000","last_peered":"2023-09-23T21:09:00.817316+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:09:00.817316+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"404'5911","ondisk_log_start":"404'5911","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4148,"log_dups_size":0,"ondisk_log_size":4148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57606144,"num_objects":879,"num_object_clones":0,"num_object_copies":2637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1746,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10059,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5663,"num_bytes_recovered":371130368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2},{"shards":"2,6,7","objects":872},{"shards":"3,6","objects":5}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":46,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517630+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:08:42.517630+0000","last_peered":"2023-09-23T21:08:42.517630+0000","last_clean":"2023-09-23T21:08:42.517630+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:08:42.517630+0000","last_undegraded":"2023-09-23T21:08:42.517630+0000","last_fullsized":"2023-09-23T21:08:42.517630+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":307,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518058+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:08:42.518058+0000","last_peered":"2023-09-23T21:08:42.518058+0000","last_clean":"2023-09-23T21:08:42.518058+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:08:42.518058+0000","last_undegraded":"2023-09-23T21:08:42.518058+0000","last_fullsized":"2023-09-23T21:08:42.518058+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"419'10155","reported_seq":12260,"reported_epoch":419,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:00.804858+0000","last_change":"2023-09-23T21:08:40.299610+0000","last_active":"2023-09-23T21:09:00.804858+0000","last_peered":"2023-09-23T21:09:00.804858+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:09:00.804858+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"404'6068","ondisk_log_start":"404'6068","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4087,"log_dups_size":0,"ondisk_log_size":4087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58654720,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3749,"num_objects_misplaced":770,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10155,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1003,"num_bytes_recovered":65732608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-3624},{"shards":"2,6,7","objects":770}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.521272+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:08:42.521272+0000","last_peered":"2023-09-23T21:08:42.521272+0000","last_clean":"2023-09-23T21:08:42.521272+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:08:42.521272+0000","last_undegraded":"2023-09-23T21:08:42.521272+0000","last_fullsized":"2023-09-23T21:08:42.521272+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":56,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516661+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:08:42.516661+0000","last_peered":"2023-09-23T21:08:42.516661+0000","last_clean":"2023-09-23T21:08:42.516661+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:08:42.516661+0000","last_undegraded":"2023-09-23T21:08:42.516661+0000","last_fullsized":"2023-09-23T21:08:42.516661+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"419'10415","reported_seq":21923,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:09:00.755550+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:09:00.755550+0000","last_peered":"2023-09-23T21:09:00.755550+0000","last_clean":"2023-09-23T21:09:00.755550+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:09:00.755550+0000","last_undegraded":"2023-09-23T21:09:00.755550+0000","last_fullsized":"2023-09-23T21:09:00.755550+0000","mapping_epoch":400,"log_start":"404'6299","ondisk_log_start":"404'6299","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4116,"log_dups_size":0,"ondisk_log_size":4116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58523648,"num_objects":893,"num_object_clones":0,"num_object_copies":2679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10415,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":76,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.520920+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:08:42.520920+0000","last_peered":"2023-09-23T21:08:42.520920+0000","last_clean":"2023-09-23T21:08:42.520920+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:08:42.520920+0000","last_undegraded":"2023-09-23T21:08:42.520920+0000","last_fullsized":"2023-09-23T21:08:42.520920+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.521114+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:08:42.521114+0000","last_peered":"2023-09-23T21:08:42.521114+0000","last_clean":"2023-09-23T21:08:42.521114+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:08:42.521114+0000","last_undegraded":"2023-09-23T21:08:42.521114+0000","last_fullsized":"2023-09-23T21:08:42.521114+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"419'10074","reported_seq":21195,"reported_epoch":419,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:59.376341+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:08:59.376341+0000","last_peered":"2023-09-23T21:08:59.376341+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:08:59.376341+0000","last_undegraded":"2023-09-23T21:08:59.376341+0000","last_fullsized":"2023-09-23T21:08:59.376341+0000","mapping_epoch":403,"log_start":"347'1188","ondisk_log_start":"347'1188","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8886,"log_dups_size":0,"ondisk_log_size":8886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59506688,"num_objects":908,"num_object_clones":0,"num_object_copies":2724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":908,"num_objects_unfound":0,"num_objects_dirty":908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10074,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":908}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":345,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518090+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:08:42.518090+0000","last_peered":"2023-09-23T21:08:42.518090+0000","last_clean":"2023-09-23T21:08:42.518090+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:08:42.518090+0000","last_undegraded":"2023-09-23T21:08:42.518090+0000","last_fullsized":"2023-09-23T21:08:42.518090+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":430,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.513682+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:08:42.513682+0000","last_peered":"2023-09-23T21:08:42.513682+0000","last_clean":"2023-09-23T21:08:42.513682+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:08:42.513682+0000","last_undegraded":"2023-09-23T21:08:42.513682+0000","last_fullsized":"2023-09-23T21:08:42.513682+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"419'10070","reported_seq":10175,"reported_epoch":419,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:00.736971+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:09:00.736971+0000","last_peered":"2023-09-23T21:09:00.736971+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:09:00.736971+0000","last_undegraded":"2023-09-23T21:09:00.736971+0000","last_fullsized":"2023-09-23T21:09:00.736971+0000","mapping_epoch":374,"log_start":"404'5994","ondisk_log_start":"404'5994","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4076,"log_dups_size":0,"ondisk_log_size":4076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54657024,"num_objects":834,"num_object_clones":0,"num_object_copies":2502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1668,"num_objects_unfound":0,"num_objects_dirty":834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10070,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":834}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":46,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.567884+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:08:42.567884+0000","last_peered":"2023-09-23T21:08:42.567884+0000","last_clean":"2023-09-23T21:08:42.567884+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:08:42.567884+0000","last_undegraded":"2023-09-23T21:08:42.567884+0000","last_fullsized":"2023-09-23T21:08:42.567884+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":458,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518299+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:08:42.518299+0000","last_peered":"2023-09-23T21:08:42.518299+0000","last_clean":"2023-09-23T21:08:42.518299+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:08:42.518299+0000","last_undegraded":"2023-09-23T21:08:42.518299+0000","last_fullsized":"2023-09-23T21:08:42.518299+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"419'10089","reported_seq":11453,"reported_epoch":419,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:00.894980+0000","last_change":"2023-09-23T21:08:40.308232+0000","last_active":"2023-09-23T21:09:00.894980+0000","last_peered":"2023-09-23T21:09:00.894980+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:09:00.894980+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"367'5394","ondisk_log_start":"367'5394","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3982,"num_objects_misplaced":871,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10089,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":628,"num_bytes_recovered":41156608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-3980},{"shards":"1,6,7","objects":871}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":68,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.520912+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:08:42.520912+0000","last_peered":"2023-09-23T21:08:42.520912+0000","last_clean":"2023-09-23T21:08:42.520912+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:08:42.520912+0000","last_undegraded":"2023-09-23T21:08:42.520912+0000","last_fullsized":"2023-09-23T21:08:42.520912+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":207,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518556+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:08:42.518556+0000","last_peered":"2023-09-23T21:08:42.518556+0000","last_clean":"2023-09-23T21:08:42.518556+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:08:42.518556+0000","last_undegraded":"2023-09-23T21:08:42.518556+0000","last_fullsized":"2023-09-23T21:08:42.518556+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"419'10081","reported_seq":15585,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:59.301236+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:08:59.301236+0000","last_peered":"2023-09-23T21:08:59.301236+0000","last_clean":"2023-09-23T21:08:59.301236+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:08:59.301236+0000","last_undegraded":"2023-09-23T21:08:59.301236+0000","last_fullsized":"2023-09-23T21:08:59.301236+0000","mapping_epoch":364,"log_start":"347'1209","ondisk_log_start":"347'1209","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8872,"log_dups_size":0,"ondisk_log_size":8872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62128152,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10081,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":314,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517612+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:08:42.517612+0000","last_peered":"2023-09-23T21:08:42.517612+0000","last_clean":"2023-09-23T21:08:42.517612+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:08:42.517612+0000","last_undegraded":"2023-09-23T21:08:42.517612+0000","last_fullsized":"2023-09-23T21:08:42.517612+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":84,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517646+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:08:42.517646+0000","last_peered":"2023-09-23T21:08:42.517646+0000","last_clean":"2023-09-23T21:08:42.517646+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:08:42.517646+0000","last_undegraded":"2023-09-23T21:08:42.517646+0000","last_fullsized":"2023-09-23T21:08:42.517646+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":69,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516584+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:08:42.516584+0000","last_peered":"2023-09-23T21:08:42.516584+0000","last_clean":"2023-09-23T21:08:42.516584+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:08:42.516584+0000","last_undegraded":"2023-09-23T21:08:42.516584+0000","last_fullsized":"2023-09-23T21:08:42.516584+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"419'10118","reported_seq":21293,"reported_epoch":419,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:59.223640+0000","last_change":"2023-09-23T21:08:42.553079+0000","last_active":"2023-09-23T21:08:59.223640+0000","last_peered":"2023-09-23T21:08:59.223640+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:08:42.521706+0000","last_became_peered":"2023-09-23T21:08:42.521706+0000","last_unstale":"2023-09-23T21:08:59.223640+0000","last_undegraded":"2023-09-23T21:08:59.223640+0000","last_fullsized":"2023-09-23T21:08:59.223640+0000","mapping_epoch":417,"log_start":"367'5457","ondisk_log_start":"367'5457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4661,"log_dups_size":0,"ondisk_log_size":4661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.707578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59506688,"num_objects":908,"num_object_clones":0,"num_object_copies":2724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1816,"num_objects_unfound":0,"num_objects_dirty":908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10118,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5513,"num_bytes_recovered":361299968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[2,1,6],"avail_no_missing":["2","1","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":908}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":395,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517005+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:08:42.517005+0000","last_peered":"2023-09-23T21:08:42.517005+0000","last_clean":"2023-09-23T21:08:42.517005+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:08:42.517005+0000","last_undegraded":"2023-09-23T21:08:42.517005+0000","last_fullsized":"2023-09-23T21:08:42.517005+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":369,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.519542+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:08:42.519542+0000","last_peered":"2023-09-23T21:08:42.519542+0000","last_clean":"2023-09-23T21:08:42.519542+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:08:42.519542+0000","last_undegraded":"2023-09-23T21:08:42.519542+0000","last_fullsized":"2023-09-23T21:08:42.519542+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":51,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517459+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:08:42.517459+0000","last_peered":"2023-09-23T21:08:42.517459+0000","last_clean":"2023-09-23T21:08:42.517459+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:08:42.517459+0000","last_undegraded":"2023-09-23T21:08:42.517459+0000","last_fullsized":"2023-09-23T21:08:42.517459+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"419'10167","reported_seq":10364,"reported_epoch":419,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:08:59.325227+0000","last_change":"2023-09-23T21:08:40.306414+0000","last_active":"2023-09-23T21:08:59.325227+0000","last_peered":"2023-09-23T21:08:59.325227+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:08:59.325227+0000","last_undegraded":"2023-09-23T21:08:59.325227+0000","last_fullsized":"2023-09-23T21:08:59.325227+0000","mapping_epoch":386,"log_start":"349'1473","ondisk_log_start":"349'1473","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8694,"log_dups_size":0,"ondisk_log_size":8694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58654720,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1785,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10167,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":895}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":396,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.513468+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:08:42.513468+0000","last_peered":"2023-09-23T21:08:42.513468+0000","last_clean":"2023-09-23T21:08:42.513468+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:08:42.513468+0000","last_undegraded":"2023-09-23T21:08:42.513468+0000","last_fullsized":"2023-09-23T21:08:42.513468+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":314,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.523916+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:08:42.523916+0000","last_peered":"2023-09-23T21:08:42.523916+0000","last_clean":"2023-09-23T21:08:42.523916+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:08:42.523916+0000","last_undegraded":"2023-09-23T21:08:42.523916+0000","last_fullsized":"2023-09-23T21:08:42.523916+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":51,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518422+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:08:42.518422+0000","last_peered":"2023-09-23T21:08:42.518422+0000","last_clean":"2023-09-23T21:08:42.518422+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:08:42.518422+0000","last_undegraded":"2023-09-23T21:08:42.518422+0000","last_fullsized":"2023-09-23T21:08:42.518422+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"419'9993","reported_seq":16258,"reported_epoch":419,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:00.724513+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:09:00.724513+0000","last_peered":"2023-09-23T21:09:00.724513+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:09:00.724513+0000","last_undegraded":"2023-09-23T21:09:00.724513+0000","last_fullsized":"2023-09-23T21:09:00.724513+0000","mapping_epoch":387,"log_start":"404'5880","ondisk_log_start":"404'5880","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4113,"log_dups_size":0,"ondisk_log_size":4113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57737216,"num_objects":881,"num_object_clones":0,"num_object_copies":2643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":881,"num_objects_unfound":0,"num_objects_dirty":881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9993,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","1","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":881}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":234,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.513749+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:08:42.513749+0000","last_peered":"2023-09-23T21:08:42.513749+0000","last_clean":"2023-09-23T21:08:42.513749+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:08:42.513749+0000","last_undegraded":"2023-09-23T21:08:42.513749+0000","last_fullsized":"2023-09-23T21:08:42.513749+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":679,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:44.226037+0000","last_change":"2023-09-23T21:08:41.538586+0000","last_active":"2023-09-23T21:08:44.226037+0000","last_peered":"2023-09-23T21:08:44.226037+0000","last_clean":"2023-09-23T21:08:44.226037+0000","last_became_active":"2023-09-23T21:08:41.538107+0000","last_became_peered":"2023-09-23T21:08:41.538107+0000","last_unstale":"2023-09-23T21:08:44.226037+0000","last_undegraded":"2023-09-23T21:08:44.226037+0000","last_fullsized":"2023-09-23T21:08:44.226037+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:48.268707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.015372+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:08:55.015372+0000","last_peered":"2023-09-23T21:08:55.015372+0000","last_clean":"2023-09-23T21:08:55.015372+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:08:55.015372+0000","last_undegraded":"2023-09-23T21:08:55.015372+0000","last_fullsized":"2023-09-23T21:08:55.015372+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518203+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:08:42.518203+0000","last_peered":"2023-09-23T21:08:42.518203+0000","last_clean":"2023-09-23T21:08:42.518203+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:08:42.518203+0000","last_undegraded":"2023-09-23T21:08:42.518203+0000","last_fullsized":"2023-09-23T21:08:42.518203+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"419'10244","reported_seq":10365,"reported_epoch":419,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:00.771553+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:09:00.771553+0000","last_peered":"2023-09-23T21:09:00.771553+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:09:00.771553+0000","last_undegraded":"2023-09-23T21:09:00.771553+0000","last_fullsized":"2023-09-23T21:09:00.771553+0000","mapping_epoch":374,"log_start":"404'6105","ondisk_log_start":"404'6105","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4139,"log_dups_size":0,"ondisk_log_size":4139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1756,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10244,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":878}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":51,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518421+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:08:42.518421+0000","last_peered":"2023-09-23T21:08:42.518421+0000","last_clean":"2023-09-23T21:08:42.518421+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:08:42.518421+0000","last_undegraded":"2023-09-23T21:08:42.518421+0000","last_fullsized":"2023-09-23T21:08:42.518421+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":65,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517473+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:08:42.517473+0000","last_peered":"2023-09-23T21:08:42.517473+0000","last_clean":"2023-09-23T21:08:42.517473+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:08:42.517473+0000","last_undegraded":"2023-09-23T21:08:42.517473+0000","last_fullsized":"2023-09-23T21:08:42.517473+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":56,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516783+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:08:42.516783+0000","last_peered":"2023-09-23T21:08:42.516783+0000","last_clean":"2023-09-23T21:08:42.516783+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:08:42.516783+0000","last_undegraded":"2023-09-23T21:08:42.516783+0000","last_fullsized":"2023-09-23T21:08:42.516783+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"419'10167","reported_seq":19585,"reported_epoch":419,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:09:00.914156+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:09:00.914156+0000","last_peered":"2023-09-23T21:09:00.914156+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:09:00.914156+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:09:00.914156+0000","mapping_epoch":383,"log_start":"367'5426","ondisk_log_start":"367'5426","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4741,"log_dups_size":0,"ondisk_log_size":4741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295424,"num_objects":859,"num_object_clones":0,"num_object_copies":2577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":855,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10167,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4655,"num_bytes_recovered":305070080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":4},{"shards":"1,2,7","objects":855}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":56,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516159+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:08:42.516159+0000","last_peered":"2023-09-23T21:08:42.516159+0000","last_clean":"2023-09-23T21:08:42.516159+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:08:42.516159+0000","last_undegraded":"2023-09-23T21:08:42.516159+0000","last_fullsized":"2023-09-23T21:08:42.516159+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":316,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.519663+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:08:42.519663+0000","last_peered":"2023-09-23T21:08:42.519663+0000","last_clean":"2023-09-23T21:08:42.519663+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:08:42.519663+0000","last_undegraded":"2023-09-23T21:08:42.519663+0000","last_fullsized":"2023-09-23T21:08:42.519663+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":94,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517269+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:08:42.517269+0000","last_peered":"2023-09-23T21:08:42.517269+0000","last_clean":"2023-09-23T21:08:42.517269+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:08:42.517269+0000","last_undegraded":"2023-09-23T21:08:42.517269+0000","last_fullsized":"2023-09-23T21:08:42.517269+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"419'9983","reported_seq":22881,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:09:00.854071+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:09:00.854071+0000","last_peered":"2023-09-23T21:09:00.854071+0000","last_clean":"2023-09-23T21:09:00.854071+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:09:00.854071+0000","last_undegraded":"2023-09-23T21:09:00.854071+0000","last_fullsized":"2023-09-23T21:09:00.854071+0000","mapping_epoch":397,"log_start":"367'5245","ondisk_log_start":"367'5245","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4738,"log_dups_size":0,"ondisk_log_size":4738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57737216,"num_objects":881,"num_object_clones":0,"num_object_copies":2643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9983,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":76,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.521813+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:08:42.521813+0000","last_peered":"2023-09-23T21:08:42.521813+0000","last_clean":"2023-09-23T21:08:42.521813+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:08:42.521813+0000","last_undegraded":"2023-09-23T21:08:42.521813+0000","last_fullsized":"2023-09-23T21:08:42.521813+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":51,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517235+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:08:42.517235+0000","last_peered":"2023-09-23T21:08:42.517235+0000","last_clean":"2023-09-23T21:08:42.517235+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:08:42.517235+0000","last_undegraded":"2023-09-23T21:08:42.517235+0000","last_fullsized":"2023-09-23T21:08:42.517235+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:45.010800+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:08:45.010800+0000","last_peered":"2023-09-23T21:08:45.010800+0000","last_clean":"2023-09-23T21:08:45.010800+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:08:45.010800+0000","last_undegraded":"2023-09-23T21:08:45.010800+0000","last_fullsized":"2023-09-23T21:08:45.010800+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"419'10355","reported_seq":19991,"reported_epoch":419,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:09:00.923758+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:09:00.923758+0000","last_peered":"2023-09-23T21:09:00.923758+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:09:00.923758+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:09:00.923758+0000","mapping_epoch":382,"log_start":"401'5612","ondisk_log_start":"401'5612","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4743,"log_dups_size":0,"ondisk_log_size":4743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55246848,"num_objects":843,"num_object_clones":0,"num_object_copies":2529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":839,"num_objects_misplaced":847,"num_objects_unfound":0,"num_objects_dirty":843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10355,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4757,"num_bytes_recovered":311754752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":4},{"shards":"1,6,7","objects":839}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":315,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518606+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:08:42.518606+0000","last_peered":"2023-09-23T21:08:42.518606+0000","last_clean":"2023-09-23T21:08:42.518606+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:08:42.518606+0000","last_undegraded":"2023-09-23T21:08:42.518606+0000","last_fullsized":"2023-09-23T21:08:42.518606+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":60,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517077+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:08:42.517077+0000","last_peered":"2023-09-23T21:08:42.517077+0000","last_clean":"2023-09-23T21:08:42.517077+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:08:42.517077+0000","last_undegraded":"2023-09-23T21:08:42.517077+0000","last_fullsized":"2023-09-23T21:08:42.517077+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":104,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516616+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:08:42.516616+0000","last_peered":"2023-09-23T21:08:42.516616+0000","last_clean":"2023-09-23T21:08:42.516616+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:08:42.516616+0000","last_undegraded":"2023-09-23T21:08:42.516616+0000","last_fullsized":"2023-09-23T21:08:42.516616+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":65,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.516597+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:08:42.516597+0000","last_peered":"2023-09-23T21:08:42.516597+0000","last_clean":"2023-09-23T21:08:42.516597+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:08:42.516597+0000","last_undegraded":"2023-09-23T21:08:42.516597+0000","last_fullsized":"2023-09-23T21:08:42.516597+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":442,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.513898+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:08:42.513898+0000","last_peered":"2023-09-23T21:08:42.513898+0000","last_clean":"2023-09-23T21:08:42.513898+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:08:42.513898+0000","last_undegraded":"2023-09-23T21:08:42.513898+0000","last_fullsized":"2023-09-23T21:08:42.513898+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":81,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.568326+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:08:42.568326+0000","last_peered":"2023-09-23T21:08:42.568326+0000","last_clean":"2023-09-23T21:08:42.568326+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:08:42.568326+0000","last_undegraded":"2023-09-23T21:08:42.568326+0000","last_fullsized":"2023-09-23T21:08:42.568326+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.520952+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:08:42.520952+0000","last_peered":"2023-09-23T21:08:42.520952+0000","last_clean":"2023-09-23T21:08:42.520952+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:08:42.520952+0000","last_undegraded":"2023-09-23T21:08:42.520952+0000","last_fullsized":"2023-09-23T21:08:42.520952+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":300,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.518352+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:08:42.518352+0000","last_peered":"2023-09-23T21:08:42.518352+0000","last_clean":"2023-09-23T21:08:42.518352+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:08:42.518352+0000","last_undegraded":"2023-09-23T21:08:42.518352+0000","last_fullsized":"2023-09-23T21:08:42.518352+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.522131+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:08:42.522131+0000","last_peered":"2023-09-23T21:08:42.522131+0000","last_clean":"2023-09-23T21:08:42.522131+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:08:42.522131+0000","last_undegraded":"2023-09-23T21:08:42.522131+0000","last_fullsized":"2023-09-23T21:08:42.522131+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":60,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:45.010937+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:08:45.010937+0000","last_peered":"2023-09-23T21:08:45.010937+0000","last_clean":"2023-09-23T21:08:45.010937+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:08:45.010937+0000","last_undegraded":"2023-09-23T21:08:45.010937+0000","last_fullsized":"2023-09-23T21:08:45.010937+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":68,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.520836+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:08:42.520836+0000","last_peered":"2023-09-23T21:08:42.520836+0000","last_clean":"2023-09-23T21:08:42.520836+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:08:42.520836+0000","last_undegraded":"2023-09-23T21:08:42.520836+0000","last_fullsized":"2023-09-23T21:08:42.520836+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":62,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.517277+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:08:42.517277+0000","last_peered":"2023-09-23T21:08:42.517277+0000","last_clean":"2023-09-23T21:08:42.517277+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:08:42.517277+0000","last_undegraded":"2023-09-23T21:08:42.517277+0000","last_fullsized":"2023-09-23T21:08:42.517277+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":46,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T21:08:42.567724+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:08:42.567724+0000","last_peered":"2023-09-23T21:08:42.567724+0000","last_clean":"2023-09-23T21:08:42.567724+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:08:42.567724+0000","last_undegraded":"2023-09-23T21:08:42.567724+0000","last_fullsized":"2023-09-23T21:08:42.567724+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":924123160,"num_objects":14102,"num_object_clones":0,"num_object_copies":42306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16786,"num_objects_misplaced":12147,"num_objects_unfound":0,"num_objects_dirty":14102,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162272,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40614,"num_bytes_recovered":2661679104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6041448448,"data_stored":6041436232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84171,"ondisk_log_size":84171,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201763,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463384,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173508,"kb_used_data":632188,"kb_used_omap":0,"kb_used_meta":541312,"kb_avail":92571644,"statfs":{"total":95995035648,"available":94793363456,"internally_reserved":0,"allocated":647360512,"data_stored":646121404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":554303488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.181,"15min":1.958},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.3849999999999998,"5min":99.533000000000001,"15min":102.242},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.7809999999999999,"5min":2.3439999999999999,"15min":1.7769999999999999},"min":{"1min":0.34300000000000003,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":6.04,"5min":99.325000000000003,"15min":102.502},"last":2.5739999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:08:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7599999999999998,"5min":4.3150000000000004,"15min":2.5110000000000001},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":30.526,"5min":119.077,"15min":119.077},"last":2.6070000000000002},{"interface":"front","average":{"1min":4.2009999999999996,"5min":4.2759999999999998,"15min":2.5990000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.24199999999999999},"max":{"1min":30.564,"5min":119.313,"15min":119.313},"last":1.895}]},{"osd":2,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":5.1950000000000003,"15min":3.7799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.5470000000000006,"5min":199.23400000000001,"15min":199.23400000000001},"last":2.23},{"interface":"front","average":{"1min":2.0870000000000002,"5min":5.1689999999999996,"15min":3.8319999999999999},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.21199999999999999},"max":{"1min":6.3019999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":2.5259999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.339,"15min":2.7229999999999999},"min":{"1min":0.31,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":13.340999999999999,"5min":13.340999999999999,"15min":178.40000000000001},"last":1.9370000000000001},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.7509999999999999,"15min":2.7040000000000002},"min":{"1min":0.53100000000000003,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":7.7759999999999998,"5min":13.319000000000001,"15min":178.26599999999999},"last":0.97399999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.423,"15min":0.90700000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":6.4989999999999997,"5min":13.1,"15min":13.1},"last":2.6360000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0269999999999999,"15min":0.871},"min":{"1min":0.31,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":5.9580000000000002,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":6.4400000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.333,"15min":0.94799999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.1819999999999999,"5min":11.281000000000001,"15min":11.281000000000001},"last":4.4199999999999999},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.208,"15min":1.05},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9279999999999999,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":1.9179999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":1.2929999999999999,"15min":1.0740000000000001},"min":{"1min":0.379,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":3.1139999999999999,"5min":7.8079999999999998,"15min":11.388999999999999},"last":6.6299999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.276,"15min":1.1080000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30099999999999999},"max":{"1min":3.4830000000000001,"5min":7.1849999999999996,"15min":8.8010000000000002},"last":6.4749999999999996}]}]},{"osd":1,"up_from":276,"seq":1185410973830,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504556,"kb_used_data":1036708,"kb_used_omap":5,"kb_used_meta":467706,"kb_avail":92240596,"statfs":{"total":95995035648,"available":94454370304,"internally_reserved":0,"allocated":1061588992,"data_stored":1059491358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":478931683},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3,5,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0960000000000001,"5min":1.738,"15min":0.86599999999999999},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":18.097999999999999,"5min":18.097999999999999,"15min":18.097999999999999},"last":1.292},{"interface":"front","average":{"1min":3.5459999999999998,"5min":1.407,"15min":1.0089999999999999},"min":{"1min":0.29999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":11.228,"5min":11.228,"15min":18.047999999999998},"last":0.38700000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.0300000000000002,"5min":1.7010000000000001,"15min":1.4379999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.874000000000001,"5min":13.874000000000001,"15min":78.409000000000006},"last":10.294},{"interface":"front","average":{"1min":5.5030000000000001,"5min":1.829,"15min":1.319},"min":{"1min":0.32400000000000001,"5min":0.245,"15min":0.22},"max":{"1min":18.288,"5min":18.288,"15min":76.245000000000005},"last":2.415}]},{"osd":3,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":5.415,"5min":1.716,"15min":2.782},"min":{"1min":0.28699999999999998,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":18.344999999999999,"5min":18.344999999999999,"15min":172.44399999999999},"last":0.97699999999999998},{"interface":"front","average":{"1min":3.4009999999999998,"5min":1.627,"15min":2.8799999999999999},"min":{"1min":0.36399999999999999,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":11.311999999999999,"5min":11.311999999999999,"15min":172.58500000000001},"last":0.875}]},{"osd":4,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.446,"15min":1.0389999999999999},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.6500000000000004,"5min":12.238,"15min":17.006},"last":2.8799999999999999},{"interface":"front","average":{"1min":3.9769999999999999,"5min":1.583,"15min":0.93100000000000005},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":18.173999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.95599999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":1.397,"15min":1.05},"min":{"1min":0.34499999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.340999999999999,"5min":11.340999999999999,"15min":17.155000000000001},"last":0.92500000000000004},{"interface":"front","average":{"1min":2.758,"5min":1.55,"15min":1.149},"min":{"1min":0.35799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.949999999999999,"5min":12.345000000000001,"15min":17.076000000000001},"last":2.794}]},{"osd":6,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":1.47,"15min":1.534},"min":{"1min":0.42699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":11.375,"5min":11.666,"15min":127.995},"last":0.90300000000000002},{"interface":"front","average":{"1min":2.7890000000000001,"5min":1.633,"15min":1.6180000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.012,"5min":12.276999999999999,"15min":126.836},"last":2.8330000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:07:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9609999999999999,"5min":3.0670000000000002,"15min":1.645},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":11.17,"5min":151.791,"15min":151.791},"last":2.8559999999999999},{"interface":"front","average":{"1min":4.173,"5min":3.0800000000000001,"15min":1.47},"min":{"1min":0.41799999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":18.260000000000002,"5min":151.809,"15min":151.809},"last":1.0029999999999999}]}]},{"osd":0,"up_from":92,"seq":395136991570,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":926052,"kb_used_data":167516,"kb_used_omap":5,"kb_used_meta":758522,"kb_avail":92819100,"statfs":{"total":95995035648,"available":95046758400,"internally_reserved":0,"allocated":171536384,"data_stored":169560664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":776727162},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:08:01 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.423,"15min":1.841},"min":{"1min":0.315,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":41.149999999999999,"5min":61.341999999999999,"15min":61.341999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":6.5499999999999998,"5min":3.5920000000000001,"15min":1.845},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.217},"max":{"1min":38.936,"5min":61.299999999999997,"15min":61.299999999999997},"last":0.76300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":3.5539999999999998,"5min":3.0030000000000001,"15min":2.2130000000000001},"min":{"1min":0.35699999999999998,"5min":0.217,"15min":0.161},"max":{"1min":12.916,"5min":45.912999999999997,"15min":183.77099999999999},"last":0.93799999999999994},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8210000000000002,"15min":2.3719999999999999},"min":{"1min":0.33500000000000002,"5min":0.254,"15min":0.214},"max":{"1min":9.1370000000000005,"5min":46.234000000000002,"15min":183.78700000000001},"last":0.97799999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.613,"5min":1.655,"15min":2.4750000000000001},"min":{"1min":0.36099999999999999,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":10.755000000000001,"5min":12.113,"15min":186.143},"last":8.5299999999999994},{"interface":"front","average":{"1min":2.7719999999999998,"5min":1.5620000000000001,"15min":2.5099999999999998},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":7.6260000000000003,"5min":12.348000000000001,"15min":186.97399999999999},"last":8.5809999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.401,"15min":1.3009999999999999},"min":{"1min":0.34200000000000003,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.9450000000000003,"5min":10.814,"15min":10.814},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.171,"15min":1.012},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.629,"5min":10.629,"15min":11.493},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.5329999999999999,"15min":1.879},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.984,"5min":10.763,"15min":154.30600000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.373,"15min":1.5640000000000001},"min":{"1min":0.42099999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":10.685,"5min":11.478,"15min":154.42400000000001},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":6.0860000000000003,"15min":3.2360000000000002},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.710000000000001,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.105,"5min":6.0289999999999999,"15min":3.2810000000000001},"min":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":8.0269999999999992,"5min":169.767,"15min":169.767},"last":0.73899999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":1.605,"15min":2.3119999999999998},"min":{"1min":0.47399999999999998,"5min":0.35199999999999998,"15min":0.33600000000000002},"max":{"1min":10.661,"5min":10.74,"15min":164.26400000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.4830000000000001,"5min":1.3280000000000001,"15min":2.306},"min":{"1min":0.41199999999999998,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":164.23099999999999},"last":0.70699999999999996}]}]},{"osd":2,"up_from":129,"seq":554050781461,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504228,"kb_used_data":814044,"kb_used_omap":7,"kb_used_meta":690168,"kb_avail":92240924,"statfs":{"total":95995035648,"available":94454706176,"internally_reserved":0,"allocated":833581056,"data_stored":831547046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":706732113},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,4,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7930000000000001,"5min":1.5169999999999999,"15min":1.423},"min":{"1min":0.222,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":11.699999999999999,"5min":11.699999999999999,"15min":26.356000000000002},"last":1.8540000000000001},{"interface":"front","average":{"1min":3.6800000000000002,"5min":2.444,"15min":1.8560000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":16.974,"5min":23.963999999999999,"15min":26.341000000000001},"last":6.875}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":4.4109999999999996,"5min":5.375,"15min":2.6240000000000001},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.188},"max":{"1min":14.057,"5min":212.12,"15min":212.12},"last":6.2880000000000003},{"interface":"front","average":{"1min":2.5760000000000001,"5min":4.8449999999999998,"15min":2.3330000000000002},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":6.5709999999999997,"5min":213.02799999999999,"15min":213.02799999999999},"last":5.2240000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:08:48 2023","interfaces":[{"interface":"back","average":{"1min":4.4729999999999999,"5min":3.5760000000000001,"15min":3.3439999999999999},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":15.327999999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":4.1029999999999998},{"interface":"front","average":{"1min":4.1790000000000003,"5min":3.4279999999999999,"15min":3.2949999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.215},"max":{"1min":33.889000000000003,"5min":138.87700000000001,"15min":138.87700000000001},"last":5.2599999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.169,"15min":1.139},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26200000000000001},"max":{"1min":11.744,"5min":11.744,"15min":12.586},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.028,"15min":1.46},"min":{"1min":0.35699999999999998,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":16.725999999999999,"5min":16.725999999999999,"15min":16.725999999999999},"last":5.3230000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":3.8119999999999998,"5min":2.222,"15min":2.2130000000000001},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":16.673999999999999,"5min":16.673999999999999,"15min":173.34},"last":5.3540000000000001},{"interface":"front","average":{"1min":2.2480000000000002,"5min":1.4770000000000001,"15min":1.8380000000000001},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":12.368,"5min":13.701000000000001,"15min":173.15000000000001},"last":4.0369999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":3.6800000000000002,"15min":1.9590000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.323,"5min":175.72499999999999,"15min":175.72499999999999},"last":4.069},{"interface":"front","average":{"1min":2.0230000000000001,"5min":3.3780000000000001,"15min":1.915},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":11.782,"5min":176.20400000000001,"15min":176.20400000000001},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.665,"15min":2.0579999999999998},"min":{"1min":0.42399999999999999,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":11.811999999999999,"5min":11.811999999999999,"15min":177.30799999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.6339999999999999,"5min":2.109,"15min":2.2410000000000001},"min":{"1min":0.38200000000000001,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":16.875,"5min":16.875,"15min":177.40899999999999},"last":5.3840000000000003}]}]},{"osd":3,"up_from":148,"seq":635655160055,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":586936,"kb_used_data":6128,"kb_used_omap":2,"kb_used_meta":580797,"kb_avail":93158216,"statfs":{"total":95995035648,"available":95394013184,"internally_reserved":0,"allocated":6275072,"data_stored":4041954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":594736181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":1.984,"15min":1.4970000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.184},"max":{"1min":17.425999999999998,"5min":44.963000000000001,"15min":44.963000000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.629,"5min":1.1080000000000001,"15min":1.302},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.19500000000000001},"max":{"1min":7.9699999999999998,"5min":9.7370000000000001,"15min":37.634999999999998},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:08:00 2023","interfaces":[{"interface":"back","average":{"1min":12.473000000000001,"5min":3.4580000000000002,"15min":1.8380000000000001},"min":{"1min":0.64300000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":109.51300000000001,"5min":109.51300000000001,"15min":109.51300000000001},"last":0.313},{"interface":"front","average":{"1min":13.468,"5min":3.6669999999999998,"15min":2.121},"min":{"1min":0.26500000000000001,"5min":0.23100000000000001,"15min":0.182},"max":{"1min":111.13500000000001,"5min":111.13500000000001,"15min":111.13500000000001},"last":0.23000000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":9.6780000000000008,"5min":3.2469999999999999,"15min":1.9259999999999999},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":88.825999999999993,"5min":88.825999999999993,"15min":88.825999999999993},"last":17.373999999999999},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.5790000000000002,"15min":1.732},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":85.626000000000005,"5min":85.626000000000005,"15min":85.626000000000005},"last":17.145}]},{"osd":4,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":2.8900000000000001,"15min":2.347},"min":{"1min":0.374,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":70.728999999999999,"15min":111.36},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.605,"5min":2.7679999999999998,"15min":2.1429999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":17.286999999999999,"5min":70.712000000000003,"15min":112.473},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.7989999999999999,"15min":1.5920000000000001},"min":{"1min":0.39400000000000002,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":7.9050000000000002,"5min":36.488999999999997,"15min":64.528000000000006},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.278,"5min":2.101,"15min":1.724},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":12.382999999999999,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.3220000000000001,"5min":1.8080000000000001,"15min":1.7050000000000001},"min":{"1min":0.36099999999999999,"5min":0.318,"15min":0.312},"max":{"1min":12.317,"5min":12.317,"15min":19.568000000000001},"last":0.439},{"interface":"front","average":{"1min":2.3570000000000002,"5min":1.722,"15min":1.522},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":17.324999999999999,"5min":17.324999999999999,"15min":17.324999999999999},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.427,"15min":2.488},"min":{"1min":0.377,"5min":0.377,"15min":0.30499999999999999},"max":{"1min":7.9459999999999997,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.503},{"interface":"front","average":{"1min":3.46,"5min":2.7970000000000002,"15min":2.8879999999999999},"min":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":12.407,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.47799999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986673,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":817668,"kb_used_data":252412,"kb_used_omap":0,"kb_used_meta":565248,"kb_avail":92927484,"statfs":{"total":95995035648,"available":95157743616,"internally_reserved":0,"allocated":258469888,"data_stored":257230164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":578813952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3560000000000001,"15min":2.2829999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":10.273999999999999,"5min":12.869,"15min":102.358},"last":0.318},{"interface":"front","average":{"1min":1.883,"5min":1.4790000000000001,"15min":2.29},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":8.3190000000000008,"5min":12.848000000000001,"15min":102.58199999999999},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.782,"5min":7.7530000000000001,"15min":3.5259999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":14.523999999999999,"5min":127.411,"15min":127.411},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.7629999999999999,"5min":5.7130000000000001,"15min":2.835},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":5.8609999999999998,"5min":118.075,"15min":118.075},"last":3.4729999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":5.6589999999999998,"15min":3.6699999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":7.4480000000000004,"5min":164.517,"15min":164.517},"last":2.3820000000000001},{"interface":"front","average":{"1min":1.4419999999999999,"5min":4.7889999999999997,"15min":3.4769999999999999},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":5.9400000000000004,"5min":164.46799999999999,"15min":164.46799999999999},"last":0.38500000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:08:09 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":1.2909999999999999,"15min":2.0510000000000002},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.27300000000000002},"max":{"1min":6.8120000000000003,"5min":8.7420000000000009,"15min":142.28399999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.2150000000000001,"15min":2.149},"min":{"1min":0.40600000000000003,"5min":0.32400000000000001,"15min":0.25600000000000001},"max":{"1min":6.181,"5min":8.2899999999999991,"15min":142.34700000000001},"last":2.0600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":0.95999999999999996,"15min":1.02},"min":{"1min":0.309,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":6.0069999999999997,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.754,"5min":1.0149999999999999,"15min":1.032},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.21099999999999999},"max":{"1min":4.0259999999999998,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.64400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.2949999999999999,"15min":0.95299999999999996},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":6.4269999999999996,"5min":11.800000000000001,"15min":11.887},"last":0.373},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.341,"15min":0.92900000000000005},"min":{"1min":0.39300000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":4.5449999999999999,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":2.6829999999999998,"15min":1.544},"min":{"1min":0.34300000000000003,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":3.726,"5min":145.721,"15min":145.721},"last":1.6990000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":3.0049999999999999,"15min":1.7},"min":{"1min":0.375,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":3.383,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.53000000000000003}]}]},{"osd":5,"up_from":39,"seq":167503725037,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":760624,"kb_used_data":129576,"kb_used_omap":0,"kb_used_meta":631040,"kb_avail":92984528,"statfs":{"total":95995035648,"available":95216156672,"internally_reserved":0,"allocated":132685824,"data_stored":131462300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":646184960},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.0660000000000001,"15min":1.319},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.2640000000000002,"5min":10.488,"15min":80.105999999999995},"last":2.3420000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.1799999999999999,"15min":1.4590000000000001},"min":{"1min":0.27000000000000002,"5min":0.222,"15min":0.222},"max":{"1min":6.6619999999999999,"5min":8.2759999999999998,"15min":80.152000000000001},"last":2.0600000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":9.4730000000000008,"5min":7.8259999999999996,"15min":3.117},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":125.23399999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":2.1440000000000001},{"interface":"front","average":{"1min":9.9860000000000007,"5min":7.6059999999999999,"15min":3.0600000000000001},"min":{"1min":0.375,"5min":0.251,"15min":0.247},"max":{"1min":125.249,"5min":233.75200000000001,"15min":233.75200000000001},"last":1.6990000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":4.9640000000000004,"15min":2.794},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.254},"max":{"1min":70.810000000000002,"5min":231.34700000000001,"15min":231.34700000000001},"last":1.6519999999999999},{"interface":"front","average":{"1min":8.1370000000000005,"5min":5.4619999999999997,"15min":3.0289999999999999},"min":{"1min":0.40200000000000002,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":70.507000000000005,"5min":231.381,"15min":231.381},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.6519999999999999,"15min":2.786},"min":{"1min":0.33000000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":12.765000000000001,"5min":12.765000000000001,"15min":153.60300000000001},"last":2.177},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.3959999999999999,"15min":2.7130000000000001},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":9.6120000000000001,"5min":9.6120000000000001,"15min":153.774},"last":1.7230000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.72399999999999998,"15min":0.71399999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":6.3940000000000001,"5min":6.3940000000000001,"15min":12.739000000000001},"last":1.946},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.0129999999999999,"15min":0.89200000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":5.8490000000000002,"5min":8.3740000000000006,"15min":12.539},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":1.1990000000000001,"15min":1.1259999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.1130000000000004,"5min":9.8480000000000008,"15min":71.230000000000004},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.252,"15min":1.163},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":3.4359999999999999,"5min":12.673999999999999,"15min":70.266000000000005},"last":2.5049999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:08:14 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":1.284,"15min":1.129},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.312},"max":{"1min":9.6329999999999991,"5min":9.6329999999999991,"15min":31.091999999999999},"last":2.0910000000000002},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.70799999999999996,"15min":0.93500000000000005},"min":{"1min":0.35999999999999999,"5min":0.35899999999999999,"15min":0.28899999999999998},"max":{"1min":4.29,"5min":4.29,"15min":31.780000000000001},"last":2.0179999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":162594816,"data_stored":162594816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1033043968,"data_stored":1033043968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":826740736,"data_stored":826736664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":917504,"data_stored":917504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":249892864,"data_stored":249888792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":126746624,"data_stored":126746624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":628166656,"data_stored":628162584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3013345280,"data_stored":3013345280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:09:06.282 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:09:06.338 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:06.376 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.434 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:06.539 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:06.574 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.631 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:06.636 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16786/40494 objects degraded (41.453%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:06.719 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:06.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:06.773 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16786/40494 objects degraded (41.453%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:06.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16786/40494 objects degraded (41.453%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:06.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:06.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:06.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:07.082 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:07.187 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:07.316 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:07.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:07.377 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:07.430 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:07.492 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:07.584 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:07.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:07.604 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:07.604 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:07.648 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:09:07.690 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:07.736 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:07.790 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:07.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:07.893 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:07.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:07 smithi121 ceph-mon[211526]: pgmap v1346: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 721 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 16173/34677 objects degraded (46.639%); 9917/34677 objects misplaced (28.598%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:09:07.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1479472835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:07.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:07 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:07.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:07 smithi121 ceph-mon[219709]: pgmap v1346: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 721 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 16173/34677 objects degraded (46.639%); 9917/34677 objects misplaced (28.598%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:09:07.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1479472835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:07.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:07 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:07 smithi123 ceph-mon[193903]: pgmap v1346: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 721 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 16173/34677 objects degraded (46.639%); 9917/34677 objects misplaced (28.598%); 13 MiB/s, 203 objects/s recovering 2023-09-23T21:09:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1479472835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:07 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:07.986 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:08.773 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:09:08.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:08.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[211526]: failed to encode map e420 with expected crc 2023-09-23T21:09:08.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:08.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[211526]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T21:09:08.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[211526]: failed to encode map e420 with expected crc 2023-09-23T21:09:08.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:08.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[219709]: failed to encode map e420 with expected crc 2023-09-23T21:09:08.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:08.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[219709]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T21:09:08.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:08 smithi121 ceph-mon[219709]: failed to encode map e420 with expected crc 2023-09-23T21:09:08.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:08 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:08.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:08 smithi123 ceph-mon[193903]: failed to encode map e420 with expected crc 2023-09-23T21:09:08.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:08 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:08.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:08 smithi123 ceph-mon[193903]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T21:09:08.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:08 smithi123 ceph-mon[193903]: failed to encode map e420 with expected crc 2023-09-23T21:09:09.247 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:09.318 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:09.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:09.428 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:09.455 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:09.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:09 smithi123 ceph-mon[193903]: failed to encode map e420 with expected crc 2023-09-23T21:09:09.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:09 smithi123 ceph-mon[193903]: pgmap v1348: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 721 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 16173/34677 objects degraded (46.639%); 9917/34677 objects misplaced (28.598%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:09:09.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:09 smithi121 ceph-mon[219709]: failed to encode map e420 with expected crc 2023-09-23T21:09:09.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:09 smithi121 ceph-mon[219709]: pgmap v1348: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 721 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 16173/34677 objects degraded (46.639%); 9917/34677 objects misplaced (28.598%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:09:09.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:09 smithi121 ceph-mon[211526]: failed to encode map e420 with expected crc 2023-09-23T21:09:09.921 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:09 smithi121 ceph-mon[211526]: pgmap v1348: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 721 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 16173/34677 objects degraded (46.639%); 9917/34677 objects misplaced (28.598%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:09:09.962 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:09.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:10.073 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:10.184 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:10.751 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:10.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:10.863 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:10.891 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:10.985 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:10.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:10.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:11.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:11.147 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:11.209 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:11 smithi123 ceph-mon[193903]: pgmap v1349: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 631 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 15875/30315 objects degraded (52.367%); 8348/30315 objects misplaced (27.538%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:09:11.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:11 smithi121 ceph-mon[211526]: pgmap v1349: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 631 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 15875/30315 objects degraded (52.367%); 8348/30315 objects misplaced (27.538%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:09:11.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:11 smithi121 ceph-mon[219709]: pgmap v1349: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 631 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 15875/30315 objects degraded (52.367%); 8348/30315 objects misplaced (27.538%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:09:11.474 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:11.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:11.585 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:11.684 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:11.699 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:11.699 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:11.823 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:11.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:11.894 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:09:11.932 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:09:11.936 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:12.004 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:09:12.011 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:09:12.012 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:12.040 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:09:12.092 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:09:12.141 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:09:12.141 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:12.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:12.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:12.195 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:09:12.247 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.248 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:09:12.251 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:12.340 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15875/30315 objects degraded (52.367%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:12.341 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3125115596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:12.343 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15875/30315 objects degraded (52.367%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:12.343 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3125115596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:12.363 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15875/30315 objects degraded (52.367%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3125115596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:12.547 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:12.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:12.661 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:12.687 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:09:12.748 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:12.941 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:09:12.941 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15875/30315 objects degraded (52.367%), 6 pgs degraded, 4 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1809,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":420,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":76},{"state_name":"active+remapped+backfill_wait","count":7},{"state_name":"active+recovery_wait+degraded+remapped","count":2},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":89,"num_pools":5,"num_objects":10105,"data_bytes":661198776,"bytes_used":6792597504,"bytes_avail":665172652032,"bytes_total":671965249536,"degraded_objects":15875,"degraded_total":30315,"degraded_ratio":0.52366815108032327,"misplaced_objects":8348,"misplaced_total":30315,"misplaced_ratio":0.27537522678541976,"recovering_objects_per_sec":115,"recovering_bytes_per_sec":7541436,"recovering_keys_per_sec":0,"num_objects_recovered":1152,"num_bytes_recovered":75497472,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":401},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":29,"modified":"2023-09-23T21:08:15.075906+0000","services":{}},"progress_events":{"2639f328-dcb0-4a01-b6c7-21332e7147ac":{"message":"Global Recovery Event (5m)\n [=======================.....] (remaining: 56s)","progress":0.85393255949020386,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:09:13.112 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:13.655 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:13.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:13.722 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:13.722 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:13.722 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2507123778' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:09:13.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:13 smithi123 ceph-mon[193903]: pgmap v1350: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 578 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 15594/27780 objects degraded (56.134%); 7897/27780 objects misplaced (28.427%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:13.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:13.839 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:13.839 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:13.839 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2507123778' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:09:13.839 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[211526]: pgmap v1350: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 578 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 15594/27780 objects degraded (56.134%); 7897/27780 objects misplaced (28.427%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:13.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:13.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:13.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2507123778' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:09:13.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:13 smithi121 ceph-mon[219709]: pgmap v1350: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 578 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 15594/27780 objects degraded (56.134%); 7897/27780 objects misplaced (28.427%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:13.864 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:14.537 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:14.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:14.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:14.549 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:09:14.549 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:09:14.644 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:14.659 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:14.756 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:14.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:09:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:09:14] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2023-09-23T21:09:14.992 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:14.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:15.100 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:15.169 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(11.d 359/352 [PushOp(11:bbfdb74e:::benchmark_data_smithi121_477989_object23171:head, version: 349'1437, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:bbfdb74e:::benchmark_data_smithi121_477989_object23171:head@349'1437, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 322.99856067799999, 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.12563809300000001, 2023-09-23T21:09:15.220 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219323+0000", 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219386+0000", 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3328000000000001e-05 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219395+0000", 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4940000000000003e-06 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219400+0000", 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4770000000000002e-06 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219450+0000", 2023-09-23T21:09:15.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9388999999999998e-05 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.219454+0000", 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4379999999999997e-06 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:52.344961+0000", 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.12550729599999999 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.224 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:72126 11.d e365/364 11:bf1689fe:::benchmark_data_smithi121_477989_object72125:head v 365'4475, mlcod=365'4468)", 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:07.407031+0000", 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 307.81085320400001, 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.036833280000000003, 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.407031+0000", 2023-09-23T21:09:15.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.407031+0000", 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.407030+0000", 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.411069+0000", 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040389040000000003 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.411076+0000", 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9709999999999998e-06 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.411081+0000", 2023-09-23T21:09:15.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3510000000000001e-06 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.411121+0000", 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9496000000000003e-05 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.411142+0000", 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0996999999999999e-05 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.443818+0000", 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032675818000000002 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:07.443864+0000", 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6521999999999999e-05 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.229 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73768 11.6 e365/343 11:608fae37:::benchmark_data_smithi121_477989_object73767:head v 365'4518, mlcod=365'4515)", 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.430536+0000", 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 305.787348485, 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042828863000000002, 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430536+0000", 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430536+0000", 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430535+0000", 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430569+0000", 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3970999999999997e-05 2023-09-23T21:09:15.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430581+0000", 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2292e-05 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430587+0000", 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0299999999999999e-06 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430625+0000", 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8036e-05 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430642+0000", 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7461000000000001e-05 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.473297+0000", 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042654408999999997 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.473364+0000", 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7594000000000004e-05 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73771 11.7 e365/343 11:efacb03e:::benchmark_data_smithi121_477989_object73770:head v 365'4702, mlcod=365'4700)", 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.431350+0000", 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 305.78653443000002, 2023-09-23T21:09:15.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.047820113999999997, 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431350+0000", 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431350+0000", 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431349+0000", 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431439+0000", 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0464000000000007e-05 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431457+0000", 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7408999999999999e-05 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431463+0000", 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0599999999999996e-06 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431520+0000", 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6932000000000003e-05 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431538+0000", 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7935999999999999e-05 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479111+0000", 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.047573794000000003 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479170+0000", 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8316000000000002e-05 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73778 11.d e365/364 11:ba746c3a:::benchmark_data_smithi121_477989_object73777:head v 365'4584, mlcod=365'4582)", 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.435221+0000", 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 305.782663007, 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.038198362999999999, 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.239 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435221+0000", 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435221+0000", 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435220+0000", 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435261+0000", 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0533e-05 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435276+0000", 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4907e-05 2023-09-23T21:09:15.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435282+0000", 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9200000000000001e-06 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435439+0000", 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015761900000000001 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435463+0000", 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3439e-05 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.473389+0000", 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.037925759000000003 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.473419+0000", 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0905000000000003e-05 2023-09-23T21:09:15.243 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73776 11.6 e365/343 11:673e70e2:::benchmark_data_smithi121_477989_object73775:head v 365'4519, mlcod=365'4516)", 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.435295+0000", 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 305.782588852, 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.039010832000000002, 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.244 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435295+0000", 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435295+0000", 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435295+0000", 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435326+0000", 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1226000000000002e-05 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435332+0000", 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2990000000000002e-06 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.435337+0000", 2023-09-23T21:09:15.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9749999999999998e-06 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.460394+0000", 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025056228 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.460414+0000", 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0254999999999998e-05 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.474248+0000", 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01383382 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.474306+0000", 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8310999999999998e-05 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.248 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73777 11.7 e365/343 11:e307afbb:::benchmark_data_smithi121_477989_object73776:head v 365'4703, mlcod=365'4700)", 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.437194+0000", 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 305.78068969499998, 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042000286999999997, 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437194+0000", 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437194+0000", 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437193+0000", 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437310+0000", 2023-09-23T21:09:15.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117335 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437320+0000", 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0520000000000007e-06 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437324+0000", 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8960000000000002e-06 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.457544+0000", 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020220005999999999 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.457566+0000", 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1832999999999999e-05 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479174+0000", 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021607695 2023-09-23T21:09:15.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479195+0000", 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0701000000000001e-05 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75115 11.7 e365/343 11:e673a9a4:::benchmark_data_smithi121_477989_object75114:head v 365'4791, mlcod=365'4788)", 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.098286+0000", 2023-09-23T21:09:15.253 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.11959834599998, 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.079072325999999998, 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098286+0000", 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098286+0000", 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098284+0000", 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098331+0000", 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6903999999999997e-05 2023-09-23T21:09:15.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098348+0000", 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6639000000000001e-05 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098355+0000", 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0650000000000001e-06 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098401+0000", 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5549e-05 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098418+0000", 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7513999999999999e-05 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100712+0000", 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022936300000000001 2023-09-23T21:09:15.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177358+0000", 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.076646251999999998 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75121 11.8 e365/343 11:16e12dc2:::benchmark_data_smithi121_477989_object75120:head v 365'4741, mlcod=365'4739)", 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.100430+0000", 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.11745456099999, 2023-09-23T21:09:15.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.087722467999999998, 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100430+0000", 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100430+0000", 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100428+0000", 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100457+0000", 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8529000000000001e-05 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100473+0000", 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5502000000000002e-05 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100480+0000", 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4490000000000002e-06 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100559+0000", 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9246999999999995e-05 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100583+0000", 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3499e-05 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188104+0000", 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.087521579000000002 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188152+0000", 2023-09-23T21:09:15.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7685000000000002e-05 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75128 11.d e365/364 11:b4c8a7c0:::benchmark_data_smithi121_477989_object75127:head v 365'4669, mlcod=365'4667)", 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.110280+0000", 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.107603907, 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.069341358000000006, 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.263 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110280+0000", 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110280+0000", 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110279+0000", 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110313+0000", 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4057000000000003e-05 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176778+0000", 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.066465381000000004 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176793+0000", 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4304000000000001e-05 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177003+0000", 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021034100000000001 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177027+0000", 2023-09-23T21:09:15.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4105e-05 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179579+0000", 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025522380000000001 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179622+0000", 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2051000000000001e-05 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.267 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75143 11.8 e365/343 11:1870ba7c:::benchmark_data_smithi121_477989_object75142:head v 365'4742, mlcod=365'4739)", 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.127972+0000", 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.08991211199998, 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.060207736999999997, 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.127972+0000", 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.127972+0000", 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.127971+0000", 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.128003+0000", 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2271000000000003e-05 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176780+0000", 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.048777648 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176792+0000", 2023-09-23T21:09:15.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1612e-05 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179249+0000", 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024567289999999999 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179272+0000", 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3546000000000001e-05 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188155+0000", 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088824309999999997 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188180+0000", 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4926000000000002e-05 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75146 11.7 e365/343 11:e7f09222:::benchmark_data_smithi121_477989_object75145:head v 365'4792, mlcod=365'4789)", 2023-09-23T21:09:15.272 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.133924+0000", 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 304.08395968899998, 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.052426268999999998, 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133924+0000", 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133924+0000", 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133923+0000", 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133986+0000", 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2927000000000002e-05 2023-09-23T21:09:15.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176800+0000", 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042814011999999999 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176845+0000", 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4521999999999998e-05 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177391+0000", 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00054616599999999995 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177410+0000", 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9324999999999999e-05 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.186297+0000", 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088871019999999992 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.186351+0000", 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3195000000000001e-05 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:94174 11.6 e402/375 11:6b7068d0:::benchmark_data_smithi121_477989_object5985:head v 402'5748, mlcod=402'5745)", 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:44.144799+0000", 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 151.073085083, 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.126705347, 2023-09-23T21:09:15.277 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144799+0000", 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144799+0000", 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144798+0000", 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144807+0000", 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6500000000000008e-06 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144820+0000", 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2224e-05 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144826+0000", 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0399999999999998e-06 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144871+0000", 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5735000000000002e-05 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144898+0000", 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6435e-05 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.271480+0000", 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.12658245100000001 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.271504+0000", 2023-09-23T21:09:15.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4170999999999999e-05 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:94192 11.8 e402/386 11:1f65d4ab:::benchmark_data_smithi121_477989_object6003:head v 402'5957, mlcod=402'5955)", 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:44.188483+0000", 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 151.029401361, 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.077938652999999997, 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188483+0000", 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188483+0000", 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188482+0000", 2023-09-23T21:09:15.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188506+0000", 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3886000000000001e-05 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263882+0000", 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.075376705000000002 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263897+0000", 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4396000000000001e-05 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263939+0000", 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2661999999999999e-05 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263963+0000", 2023-09-23T21:09:15.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3879e-05 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.266378+0000", 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002415169 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.266421+0000", 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2985999999999998e-05 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:100873 11.6 e405/375 11:602c3a22:::benchmark_data_smithi121_477989_object12684:head v 405'6177, mlcod=405'6175)", 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:58.129773+0000", 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.08811135900001, 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040138322999999997, 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129773+0000", 2023-09-23T21:09:15.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129773+0000", 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129772+0000", 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129778+0000", 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0020000000000003e-06 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129787+0000", 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1230000000000006e-06 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129791+0000", 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2679999999999996e-06 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129842+0000", 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0748000000000003e-05 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129859+0000", 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7144999999999999e-05 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.169864+0000", 2023-09-23T21:09:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040004524999999999 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.169911+0000", 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7222000000000003e-05 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:100877 11.6 e405/375 11:667b60ba:::benchmark_data_smithi121_477989_object12688:head v 405'6178, mlcod=405'6175)", 2023-09-23T21:09:15.291 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:58.133492+0000", 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.08439205900001, 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.036867186000000003, 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.133492+0000", 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.133492+0000", 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.133491+0000", 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.133497+0000", 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5999999999999997e-06 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.165100+0000", 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031602542999999997 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.165113+0000", 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3675999999999999e-05 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.167380+0000", 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022671430000000001 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.167406+0000", 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5757000000000001e-05 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.170326+0000", 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029202970000000001 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.170359+0000", 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2774000000000002e-05 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102759 11.8 e406/386 11:1f479d3e:::benchmark_data_smithi121_477989_object14570:head v 406'6487, mlcod=406'6483)", 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:02.054425+0000", 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 133.16345898200001, 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040732209999999998, 2023-09-23T21:09:15.296 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054425+0000", 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054425+0000", 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054424+0000", 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054433+0000", 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0559999999999994e-06 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054448+0000", 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5193e-05 2023-09-23T21:09:15.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054456+0000", 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1189999999999999e-06 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054511+0000", 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5461999999999999e-05 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054540+0000", 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9091e-05 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095107+0000", 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040567021000000002 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095157+0000", 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0207999999999998e-05 2023-09-23T21:09:15.300 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:109584 11.d e407/364 11:bb1f75d3:::benchmark_data_smithi121_477989_object21395:head v 407'6837, mlcod=407'6834)", 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:15.609395+0000", 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 119.60848889, 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.071550276999999995, 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609395+0000", 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609395+0000", 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609394+0000", 2023-09-23T21:09:15.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609409+0000", 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4746000000000001e-05 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609425+0000", 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5625999999999999e-05 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609432+0000", 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1930000000000004e-06 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609479+0000", 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7236e-05 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609500+0000", 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0567000000000001e-05 2023-09-23T21:09:15.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612284+0000", 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027845119999999998 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680945+0000", 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.068661429999999996 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.305 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:109575 11.e e407/382 11:7af111bd:::benchmark_data_smithi121_477989_object21386:head v 407'6939, mlcod=0'0)", 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:15.612355+0000", 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 119.60552899299999, 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.075662072999999996, 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612355+0000", 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612355+0000", 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612354+0000", 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612362+0000", 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8080000000000005e-06 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680918+0000", 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.068555560000000001 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680928+0000", 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0455000000000001e-05 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.681061+0000", 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000133273 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.681082+0000", 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0496000000000002e-05 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.687981+0000", 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0068987909999999996 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.688017+0000", 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6429e-05 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:109588 11.d e407/364 11:be08a181:::benchmark_data_smithi121_477989_object21399:head v 407'6838, mlcod=407'6834)", 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:15.624386+0000", 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 119.593497922, 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.062509666000000005, 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624386+0000", 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624386+0000", 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624385+0000", 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:15.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624394+0000", 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4530000000000002e-06 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680913+0000", 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.056519579 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680926+0000", 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2864000000000001e-05 2023-09-23T21:09:15.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680975+0000", 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9079999999999998e-05 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680994+0000", 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9083e-05 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.686856+0000", 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058616989999999997 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.686896+0000", 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9870000000000003e-05 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:15.315 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:09:15.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:15 smithi121 ceph-mon[211526]: pgmap v1351: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 543 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 15594/26109 objects degraded (59.727%); 7203/26109 objects misplaced (27.588%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:15.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:15 smithi123 ceph-mon[193903]: pgmap v1351: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 543 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 15594/26109 objects degraded (59.727%); 7203/26109 objects misplaced (27.588%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:15.316 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:15 smithi121 ceph-mon[219709]: pgmap v1351: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 543 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 15594/26109 objects degraded (59.727%); 7203/26109 objects misplaced (27.588%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:09:15.454 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:15.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:15.567 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:15.609 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:15.696 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:15.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:15.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:15.800 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:15.829 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:15.878 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:15.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:15.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:15.981 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:16.010 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:16.254 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:09:16.255 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:16.297 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:09:16.363 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:16.426 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:16.580 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15594/26109 objects degraded (59.727%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:16.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15594/26109 objects degraded (59.727%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:16.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15594/26109 objects degraded (59.727%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:16.715 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:16.724 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:16.746 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:16.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:16.861 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:16.890 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.008 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:17.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:17.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:17.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:17.206 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.442 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:17.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:17.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:17.553 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:17.589 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.652 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:17.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:17.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:17 smithi121 ceph-mon[211526]: pgmap v1352: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 436 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 15056/20991 objects degraded (71.726%); 5885/20991 objects misplaced (28.036%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T21:09:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:17 smithi121 ceph-mon[219709]: pgmap v1352: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 436 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 15056/20991 objects degraded (71.726%); 5885/20991 objects misplaced (28.036%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T21:09:17.761 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:17.860 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:17.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:17 smithi123 ceph-mon[193903]: pgmap v1352: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 436 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 15056/20991 objects degraded (71.726%); 5885/20991 objects misplaced (28.036%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T21:09:18.005 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:18.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:18.116 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:18.159 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:18.287 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:18.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:18.393 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:18.434 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:18.598 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:18.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:18.666 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:09:18.704 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2506133857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:18.747 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:18.815 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:18.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:18.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:18.898 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:09:18.898 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:171152 11.8 e421/386 11:1d55d1bf:::benchmark_data_smithi121_477989_object82963:head v 421'10748, mlcod=421'10744)", 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:18.888744+0000", 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.002647974, 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026756190000000002, 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:09:18.899 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888744+0000", 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:18.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888744+0000", 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888743+0000", 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:18.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888751+0000", 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8709999999999997e-06 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888769+0000", 2023-09-23T21:09:18.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8737999999999999e-05 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888785+0000", 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6116e-05 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:18.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888824+0000", 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8479000000000002e-05 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:18.888844+0000", 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0469e-05 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:18.904 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:18.905 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:18.905 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:18.905 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T21:09:18.905 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 1 2023-09-23T21:09:18.905 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:09:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2506133857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:18.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2506133857' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:18.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:18.918 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:18.953 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:19.586 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:19.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:19.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:19.689 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:19.719 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:19.892 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:19.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:19.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[211526]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[211526]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:19.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[211526]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T21:09:19.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[211526]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[211526]: pgmap v1354: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 436 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 15056/20991 objects degraded (71.726%); 5885/20991 objects misplaced (28.036%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T21:09:19.906 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:19.906 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[219709]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.906 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[219709]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.906 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:19.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[219709]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T21:09:19.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[219709]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:19 smithi121 ceph-mon[219709]: pgmap v1354: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 436 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 15056/20991 objects degraded (71.726%); 5885/20991 objects misplaced (28.036%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T21:09:19.949 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:19.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:19 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:19.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:19 smithi123 ceph-mon[193903]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:19 smithi123 ceph-mon[193903]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:19 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:19.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:19 smithi123 ceph-mon[193903]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T21:09:19.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:19 smithi123 ceph-mon[193903]: failed to encode map e421 with expected crc 2023-09-23T21:09:19.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:19 smithi123 ceph-mon[193903]: pgmap v1354: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 436 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 15056/20991 objects degraded (71.726%); 5885/20991 objects misplaced (28.036%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T21:09:20.060 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:20.109 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:20.149 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:09:20.196 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:20.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:20.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:20.312 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:20.353 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:20.431 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:20.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:20.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:20.542 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:20.581 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:20.657 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:20.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:20.676 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:20 smithi121 ceph-mon[211526]: failed to encode map e421 with expected crc 2023-09-23T21:09:20.676 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:20 smithi121 ceph-mon[211526]: failed to encode map e421 with expected crc 2023-09-23T21:09:20.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:20 smithi121 ceph-mon[219709]: failed to encode map e421 with expected crc 2023-09-23T21:09:20.676 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:20 smithi121 ceph-mon[219709]: failed to encode map e421 with expected crc 2023-09-23T21:09:20.685 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:20 smithi123 ceph-mon[193903]: failed to encode map e421 with expected crc 2023-09-23T21:09:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:20 smithi123 ceph-mon[193903]: failed to encode map e421 with expected crc 2023-09-23T21:09:20.765 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:20.794 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:20.843 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:09:20.852 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:09:20.915 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:20.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:21.019 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:21.128 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:21.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:21 smithi121 ceph-mon[219709]: from='client.56686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:21 smithi121 ceph-mon[219709]: pgmap v1355: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 332 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 14718/15999 objects degraded (91.993%); 3994/15999 objects misplaced (24.964%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:09:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:21 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15056/20991 objects degraded (71.726%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:21 smithi121 ceph-mon[211526]: from='client.56686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:21 smithi121 ceph-mon[211526]: pgmap v1355: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 332 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 14718/15999 objects degraded (91.993%); 3994/15999 objects misplaced (24.964%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:09:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:21 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15056/20991 objects degraded (71.726%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:21.930 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:09:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:21 smithi123 ceph-mon[193903]: from='client.56686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:21 smithi123 ceph-mon[193903]: pgmap v1355: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 332 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 14718/15999 objects degraded (91.993%); 3994/15999 objects misplaced (24.964%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:09:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:21 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15056/20991 objects degraded (71.726%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:22.995 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:09:22.996 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:09:22.996 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:09:22.996 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:09:22.996 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:09:23.306 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:23.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:23.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:23.351 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1354,"stamp":"2023-09-23T21:09:19.130404+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":457512888,"num_objects":6997,"num_object_clones":0,"num_object_copies":20991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15056,"num_objects_misplaced":5885,"num_objects_unfound":0,"num_objects_dirty":6997,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":169701,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43069,"num_bytes_recovered":2818350892,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85016,"ondisk_log_size":85016,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6112388,"kb_used_data":1826508,"kb_used_omap":21,"kb_used_meta":4285674,"kb_avail":650103676,"statfs":{"total":671965249536,"available":665706164224,"internally_reserved":0,"allocated":1870344192,"data_stored":1858262778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4388530659},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,3,4,2,0,0,0,15],"upper_bound":512},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-298975232,"num_objects":-4562,"num_object_clones":0,"num_object_copies":-13686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1117,"num_objects_misplaced":-4032,"num_objects_unfound":0,"num_objects_dirty":-4562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4562,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1502,"num_bytes_recovered":98435072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007065"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":191,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065602+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:08:55.065602+0000","last_peered":"2023-09-23T21:08:55.065602+0000","last_clean":"2023-09-23T21:08:55.065602+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:08:55.065602+0000","last_undegraded":"2023-09-23T21:08:55.065602+0000","last_fullsized":"2023-09-23T21:08:55.065602+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":60,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.061298+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:08:55.061298+0000","last_peered":"2023-09-23T21:08:55.061298+0000","last_clean":"2023-09-23T21:08:55.061298+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:08:55.061298+0000","last_undegraded":"2023-09-23T21:08:55.061298+0000","last_fullsized":"2023-09-23T21:08:55.061298+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":333,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065346+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:08:55.065346+0000","last_peered":"2023-09-23T21:08:55.065346+0000","last_clean":"2023-09-23T21:08:55.065346+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:08:55.065346+0000","last_undegraded":"2023-09-23T21:08:55.065346+0000","last_fullsized":"2023-09-23T21:08:55.065346+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":58,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:59.238345+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:08:59.238345+0000","last_peered":"2023-09-23T21:08:59.238345+0000","last_clean":"2023-09-23T21:08:59.238345+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:08:59.238345+0000","last_undegraded":"2023-09-23T21:08:59.238345+0000","last_fullsized":"2023-09-23T21:08:59.238345+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.059722+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:08:55.059722+0000","last_peered":"2023-09-23T21:08:55.059722+0000","last_clean":"2023-09-23T21:08:55.059722+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:08:55.059722+0000","last_undegraded":"2023-09-23T21:08:55.059722+0000","last_fullsized":"2023-09-23T21:08:55.059722+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":286,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.057419+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:08:55.057419+0000","last_peered":"2023-09-23T21:08:55.057419+0000","last_clean":"2023-09-23T21:08:55.057419+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:08:55.057419+0000","last_undegraded":"2023-09-23T21:08:55.057419+0000","last_fullsized":"2023-09-23T21:08:55.057419+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":57,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.083149+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:08:55.083149+0000","last_peered":"2023-09-23T21:08:55.083149+0000","last_clean":"2023-09-23T21:08:55.083149+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:08:55.083149+0000","last_undegraded":"2023-09-23T21:08:55.083149+0000","last_fullsized":"2023-09-23T21:08:55.083149+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":60,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.059632+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:08:55.059632+0000","last_peered":"2023-09-23T21:08:55.059632+0000","last_clean":"2023-09-23T21:08:55.059632+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:08:55.059632+0000","last_undegraded":"2023-09-23T21:08:55.059632+0000","last_fullsized":"2023-09-23T21:08:55.059632+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":43,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.062502+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:08:55.062502+0000","last_peered":"2023-09-23T21:08:55.062502+0000","last_clean":"2023-09-23T21:08:55.062502+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:08:55.062502+0000","last_undegraded":"2023-09-23T21:08:55.062502+0000","last_fullsized":"2023-09-23T21:08:55.062502+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":60,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.069625+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:08:55.069625+0000","last_peered":"2023-09-23T21:08:55.069625+0000","last_clean":"2023-09-23T21:08:55.069625+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:08:55.069625+0000","last_undegraded":"2023-09-23T21:08:55.069625+0000","last_fullsized":"2023-09-23T21:08:55.069625+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":76,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.061394+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:08:55.061394+0000","last_peered":"2023-09-23T21:08:55.061394+0000","last_clean":"2023-09-23T21:08:55.061394+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:08:55.061394+0000","last_undegraded":"2023-09-23T21:08:55.061394+0000","last_fullsized":"2023-09-23T21:08:55.061394+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065297+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:08:55.065297+0000","last_peered":"2023-09-23T21:08:55.065297+0000","last_clean":"2023-09-23T21:08:55.065297+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:08:55.065297+0000","last_undegraded":"2023-09-23T21:08:55.065297+0000","last_fullsized":"2023-09-23T21:08:55.065297+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":366,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.060297+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:08:55.060297+0000","last_peered":"2023-09-23T21:08:55.060297+0000","last_clean":"2023-09-23T21:08:55.060297+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:08:55.060297+0000","last_undegraded":"2023-09-23T21:08:55.060297+0000","last_fullsized":"2023-09-23T21:08:55.060297+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":338,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.054774+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:08:55.054774+0000","last_peered":"2023-09-23T21:08:55.054774+0000","last_clean":"2023-09-23T21:08:55.054774+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:08:55.054774+0000","last_undegraded":"2023-09-23T21:08:55.054774+0000","last_fullsized":"2023-09-23T21:08:55.054774+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":111,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.633182+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:09:07.633182+0000","last_peered":"2023-09-23T21:09:07.633182+0000","last_clean":"2023-09-23T21:09:07.633182+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:09:07.633182+0000","last_undegraded":"2023-09-23T21:09:07.633182+0000","last_fullsized":"2023-09-23T21:09:07.633182+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.060939+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:08:55.060939+0000","last_peered":"2023-09-23T21:08:55.060939+0000","last_clean":"2023-09-23T21:08:55.060939+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:08:55.060939+0000","last_undegraded":"2023-09-23T21:08:55.060939+0000","last_fullsized":"2023-09-23T21:08:55.060939+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":78,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.067664+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:08:55.067664+0000","last_peered":"2023-09-23T21:08:55.067664+0000","last_clean":"2023-09-23T21:08:55.067664+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:08:55.067664+0000","last_undegraded":"2023-09-23T21:08:55.067664+0000","last_fullsized":"2023-09-23T21:08:55.067664+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":78,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.061477+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:08:55.061477+0000","last_peered":"2023-09-23T21:08:55.061477+0000","last_clean":"2023-09-23T21:08:55.061477+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:08:55.061477+0000","last_undegraded":"2023-09-23T21:08:55.061477+0000","last_fullsized":"2023-09-23T21:08:55.061477+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":220,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.054597+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:08:55.054597+0000","last_peered":"2023-09-23T21:08:55.054597+0000","last_clean":"2023-09-23T21:08:55.054597+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:08:55.054597+0000","last_undegraded":"2023-09-23T21:08:55.054597+0000","last_fullsized":"2023-09-23T21:08:55.054597+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":57,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.088320+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:08:55.088320+0000","last_peered":"2023-09-23T21:08:55.088320+0000","last_clean":"2023-09-23T21:08:55.088320+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:08:55.088320+0000","last_undegraded":"2023-09-23T21:08:55.088320+0000","last_fullsized":"2023-09-23T21:08:55.088320+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":416,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.057347+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:08:55.057347+0000","last_peered":"2023-09-23T21:08:55.057347+0000","last_clean":"2023-09-23T21:08:55.057347+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:08:55.057347+0000","last_undegraded":"2023-09-23T21:08:55.057347+0000","last_fullsized":"2023-09-23T21:08:55.057347+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":60,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065316+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:08:55.065316+0000","last_peered":"2023-09-23T21:08:55.065316+0000","last_clean":"2023-09-23T21:08:55.065316+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:08:55.065316+0000","last_undegraded":"2023-09-23T21:08:55.065316+0000","last_fullsized":"2023-09-23T21:08:55.065316+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"420'10823","reported_seq":12200,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:15.997040+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:09:15.997040+0000","last_peered":"2023-09-23T21:09:15.997040+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:09:15.997040+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"404'6128","ondisk_log_start":"404'6128","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26673152,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10823,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-5208},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":406}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":124,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.055186+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:08:55.055186+0000","last_peered":"2023-09-23T21:08:55.055186+0000","last_clean":"2023-09-23T21:08:55.055186+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:08:55.055186+0000","last_undegraded":"2023-09-23T21:08:55.055186+0000","last_fullsized":"2023-09-23T21:08:55.055186+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":43,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.061623+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:08:55.061623+0000","last_peered":"2023-09-23T21:08:55.061623+0000","last_clean":"2023-09-23T21:08:55.061623+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:08:55.061623+0000","last_undegraded":"2023-09-23T21:08:55.061623+0000","last_fullsized":"2023-09-23T21:08:55.061623+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"420'10383","reported_seq":10473,"reported_epoch":420,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:15.705253+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:09:15.705253+0000","last_peered":"2023-09-23T21:09:15.705253+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:09:15.705253+0000","last_undegraded":"2023-09-23T21:09:15.705253+0000","last_fullsized":"2023-09-23T21:09:15.705253+0000","mapping_epoch":375,"log_start":"402'5612","ondisk_log_start":"402'5612","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4771,"log_dups_size":0,"ondisk_log_size":4771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27197440,"num_objects":415,"num_object_clones":0,"num_object_copies":1245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":415,"num_objects_unfound":0,"num_objects_dirty":415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10383,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":415}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":119,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.621287+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:09:07.621287+0000","last_peered":"2023-09-23T21:09:07.621287+0000","last_clean":"2023-09-23T21:09:07.621287+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:09:07.621287+0000","last_undegraded":"2023-09-23T21:09:07.621287+0000","last_fullsized":"2023-09-23T21:09:07.621287+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":381,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065210+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:08:55.065210+0000","last_peered":"2023-09-23T21:08:55.065210+0000","last_clean":"2023-09-23T21:08:55.065210+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:08:55.065210+0000","last_undegraded":"2023-09-23T21:08:55.065210+0000","last_fullsized":"2023-09-23T21:08:55.065210+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"420'10523","reported_seq":27939,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:15.925113+0000","last_change":"2023-09-23T21:06:30.880782+0000","last_active":"2023-09-23T21:09:15.925113+0000","last_peered":"2023-09-23T21:09:15.925113+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:09:15.925113+0000","last_undegraded":"2023-09-23T21:05:54.927099+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"406'6411","ondisk_log_start":"406'6411","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4112,"log_dups_size":0,"ondisk_log_size":4112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27197440,"num_objects":415,"num_object_clones":0,"num_object_copies":1245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":818,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10523,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6128,"num_bytes_recovered":401604608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"2,6,7","objects":409},{"shards":"3,6","objects":6}],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":47,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.062821+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:08:55.062821+0000","last_peered":"2023-09-23T21:08:55.062821+0000","last_clean":"2023-09-23T21:08:55.062821+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:08:55.062821+0000","last_undegraded":"2023-09-23T21:08:55.062821+0000","last_fullsized":"2023-09-23T21:08:55.062821+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":308,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.058282+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:08:55.058282+0000","last_peered":"2023-09-23T21:08:55.058282+0000","last_clean":"2023-09-23T21:08:55.058282+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:08:55.058282+0000","last_undegraded":"2023-09-23T21:08:55.058282+0000","last_fullsized":"2023-09-23T21:08:55.058282+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"420'10622","reported_seq":13834,"reported_epoch":420,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:15.914674+0000","last_change":"2023-09-23T21:08:40.299610+0000","last_active":"2023-09-23T21:09:15.914674+0000","last_peered":"2023-09-23T21:09:15.914674+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:09:15.914674+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"406'6468","ondisk_log_start":"406'6468","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4154,"log_dups_size":0,"ondisk_log_size":4154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3749,"num_objects_misplaced":217,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10622,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1556,"num_bytes_recovered":101974016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-3538},{"shards":"2,6,7","objects":217}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.613443+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:09:07.613443+0000","last_peered":"2023-09-23T21:09:07.613443+0000","last_clean":"2023-09-23T21:09:07.613443+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:09:07.613443+0000","last_undegraded":"2023-09-23T21:09:07.613443+0000","last_fullsized":"2023-09-23T21:09:07.613443+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":57,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.083660+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:08:55.083660+0000","last_peered":"2023-09-23T21:08:55.083660+0000","last_clean":"2023-09-23T21:08:55.083660+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:08:55.083660+0000","last_undegraded":"2023-09-23T21:08:55.083660+0000","last_fullsized":"2023-09-23T21:08:55.083660+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"420'10885","reported_seq":22394,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:15.925005+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:09:15.925005+0000","last_peered":"2023-09-23T21:09:15.925005+0000","last_clean":"2023-09-23T21:09:15.925005+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:09:15.925005+0000","last_undegraded":"2023-09-23T21:09:15.925005+0000","last_fullsized":"2023-09-23T21:09:15.925005+0000","mapping_epoch":400,"log_start":"406'6799","ondisk_log_start":"406'6799","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4086,"log_dups_size":0,"ondisk_log_size":4086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27721728,"num_objects":423,"num_object_clones":0,"num_object_copies":1269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10885,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":78,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.614730+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:09:07.614730+0000","last_peered":"2023-09-23T21:09:07.614730+0000","last_clean":"2023-09-23T21:09:07.614730+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:09:07.614730+0000","last_undegraded":"2023-09-23T21:09:07.614730+0000","last_fullsized":"2023-09-23T21:09:07.614730+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":61,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.613294+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:09:07.613294+0000","last_peered":"2023-09-23T21:09:07.613294+0000","last_clean":"2023-09-23T21:09:07.613294+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:09:07.613294+0000","last_undegraded":"2023-09-23T21:09:07.613294+0000","last_fullsized":"2023-09-23T21:09:07.613294+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"420'10503","reported_seq":21625,"reported_epoch":420,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:14.262381+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:09:14.262381+0000","last_peered":"2023-09-23T21:09:14.262381+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:09:14.262381+0000","last_undegraded":"2023-09-23T21:09:14.262381+0000","last_fullsized":"2023-09-23T21:09:14.262381+0000","mapping_epoch":403,"log_start":"349'1588","ondisk_log_start":"349'1588","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8915,"log_dups_size":0,"ondisk_log_size":8915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31391744,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":479,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10503,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","7"],"object_location_counts":[{"shards":"0,1,2","objects":479}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":346,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.058365+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:08:55.058365+0000","last_peered":"2023-09-23T21:08:55.058365+0000","last_clean":"2023-09-23T21:08:55.058365+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:08:55.058365+0000","last_undegraded":"2023-09-23T21:08:55.058365+0000","last_fullsized":"2023-09-23T21:08:55.058365+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":431,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.074948+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:08:55.074948+0000","last_peered":"2023-09-23T21:08:55.074948+0000","last_clean":"2023-09-23T21:08:55.074948+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:08:55.074948+0000","last_undegraded":"2023-09-23T21:08:55.074948+0000","last_fullsized":"2023-09-23T21:08:55.074948+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"420'10488","reported_seq":10594,"reported_epoch":420,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:15.658936+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:09:15.658936+0000","last_peered":"2023-09-23T21:09:15.658936+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:09:15.658936+0000","last_undegraded":"2023-09-23T21:09:15.658936+0000","last_fullsized":"2023-09-23T21:09:15.658936+0000","mapping_epoch":374,"log_start":"406'6394","ondisk_log_start":"406'6394","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27262976,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":832,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10488,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":416}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":47,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.054516+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:08:55.054516+0000","last_peered":"2023-09-23T21:08:55.054516+0000","last_clean":"2023-09-23T21:08:55.054516+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:08:55.054516+0000","last_undegraded":"2023-09-23T21:08:55.054516+0000","last_fullsized":"2023-09-23T21:08:55.054516+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":459,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.058432+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:08:55.058432+0000","last_peered":"2023-09-23T21:08:55.058432+0000","last_clean":"2023-09-23T21:08:55.058432+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:08:55.058432+0000","last_undegraded":"2023-09-23T21:08:55.058432+0000","last_fullsized":"2023-09-23T21:08:55.058432+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"420'10549","reported_seq":12872,"reported_epoch":420,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:15.997263+0000","last_change":"2023-09-23T21:08:40.308232+0000","last_active":"2023-09-23T21:09:15.997263+0000","last_peered":"2023-09-23T21:09:15.997263+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:09:15.997263+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"402'5794","ondisk_log_start":"402'5794","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4755,"log_dups_size":0,"ondisk_log_size":4755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27066368,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4020,"num_objects_misplaced":355,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10549,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1106,"num_bytes_recovered":72482816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-3962},{"shards":"1,6,7","objects":355}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":70,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:09:05.020011+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:09:05.020011+0000","last_peered":"2023-09-23T21:09:05.020011+0000","last_clean":"2023-09-23T21:09:05.020011+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:09:05.020011+0000","last_undegraded":"2023-09-23T21:09:05.020011+0000","last_fullsized":"2023-09-23T21:09:05.020011+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":209,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.625642+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:09:07.625642+0000","last_peered":"2023-09-23T21:09:07.625642+0000","last_clean":"2023-09-23T21:09:07.625642+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:09:07.625642+0000","last_undegraded":"2023-09-23T21:09:07.625642+0000","last_fullsized":"2023-09-23T21:09:07.625642+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"420'10530","reported_seq":16034,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:14.377733+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:09:14.377733+0000","last_peered":"2023-09-23T21:09:14.377733+0000","last_clean":"2023-09-23T21:09:14.377733+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:09:14.377733+0000","last_undegraded":"2023-09-23T21:09:14.377733+0000","last_fullsized":"2023-09-23T21:09:14.377733+0000","mapping_epoch":364,"log_start":"349'1609","ondisk_log_start":"349'1609","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8921,"log_dups_size":0,"ondisk_log_size":8921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32702488,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10530,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":315,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.071097+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:08:55.071097+0000","last_peered":"2023-09-23T21:08:55.071097+0000","last_clean":"2023-09-23T21:08:55.071097+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:08:55.071097+0000","last_undegraded":"2023-09-23T21:08:55.071097+0000","last_fullsized":"2023-09-23T21:08:55.071097+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.062892+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:08:55.062892+0000","last_peered":"2023-09-23T21:08:55.062892+0000","last_clean":"2023-09-23T21:08:55.062892+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:08:55.062892+0000","last_undegraded":"2023-09-23T21:08:55.062892+0000","last_fullsized":"2023-09-23T21:08:55.062892+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065366+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:08:55.065366+0000","last_peered":"2023-09-23T21:08:55.065366+0000","last_clean":"2023-09-23T21:08:55.065366+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:08:55.065366+0000","last_undegraded":"2023-09-23T21:08:55.065366+0000","last_fullsized":"2023-09-23T21:08:55.065366+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"420'10557","reported_seq":21733,"reported_epoch":420,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:14.335696+0000","last_change":"2023-09-23T21:08:42.553079+0000","last_active":"2023-09-23T21:09:14.335696+0000","last_peered":"2023-09-23T21:09:14.335696+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:08:42.521706+0000","last_became_peered":"2023-09-23T21:08:42.521706+0000","last_unstale":"2023-09-23T21:09:14.335696+0000","last_undegraded":"2023-09-23T21:09:14.335696+0000","last_fullsized":"2023-09-23T21:09:14.335696+0000","mapping_epoch":417,"log_start":"367'5457","ondisk_log_start":"367'5457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.707578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30736384,"num_objects":469,"num_object_clones":0,"num_object_copies":1407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":938,"num_objects_unfound":0,"num_objects_dirty":469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10557,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5513,"num_bytes_recovered":361299968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[2,1,6],"avail_no_missing":["2","1","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":469}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":396,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065848+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:08:55.065848+0000","last_peered":"2023-09-23T21:08:55.065848+0000","last_clean":"2023-09-23T21:08:55.065848+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:08:55.065848+0000","last_undegraded":"2023-09-23T21:08:55.065848+0000","last_fullsized":"2023-09-23T21:08:55.065848+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":370,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.057762+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:08:55.057762+0000","last_peered":"2023-09-23T21:08:55.057762+0000","last_clean":"2023-09-23T21:08:55.057762+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:08:55.057762+0000","last_undegraded":"2023-09-23T21:08:55.057762+0000","last_fullsized":"2023-09-23T21:08:55.057762+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":52,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.070955+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:08:55.070955+0000","last_peered":"2023-09-23T21:08:55.070955+0000","last_clean":"2023-09-23T21:08:55.070955+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:08:55.070955+0000","last_undegraded":"2023-09-23T21:08:55.070955+0000","last_fullsized":"2023-09-23T21:08:55.070955+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"420'10610","reported_seq":10809,"reported_epoch":420,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:14.404325+0000","last_change":"2023-09-23T21:08:40.306414+0000","last_active":"2023-09-23T21:09:14.404325+0000","last_peered":"2023-09-23T21:09:14.404325+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:09:14.404325+0000","last_undegraded":"2023-09-23T21:09:14.404325+0000","last_fullsized":"2023-09-23T21:09:14.404325+0000","mapping_epoch":386,"log_start":"349'1773","ondisk_log_start":"349'1773","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8837,"log_dups_size":0,"ondisk_log_size":8837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29622272,"num_objects":452,"num_object_clones":0,"num_object_copies":1356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":903,"num_objects_unfound":0,"num_objects_dirty":452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10610,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":452}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":397,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.074093+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:08:55.074093+0000","last_peered":"2023-09-23T21:08:55.074093+0000","last_clean":"2023-09-23T21:08:55.074093+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:08:55.074093+0000","last_undegraded":"2023-09-23T21:08:55.074093+0000","last_fullsized":"2023-09-23T21:08:55.074093+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":315,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.070531+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:08:55.070531+0000","last_peered":"2023-09-23T21:08:55.070531+0000","last_clean":"2023-09-23T21:08:55.070531+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:08:55.070531+0000","last_undegraded":"2023-09-23T21:08:55.070531+0000","last_fullsized":"2023-09-23T21:08:55.070531+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":52,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.072388+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:08:55.072388+0000","last_peered":"2023-09-23T21:08:55.072388+0000","last_clean":"2023-09-23T21:08:55.072388+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:08:55.072388+0000","last_undegraded":"2023-09-23T21:08:55.072388+0000","last_fullsized":"2023-09-23T21:08:55.072388+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"420'10455","reported_seq":16721,"reported_epoch":420,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:15.683300+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:09:15.683300+0000","last_peered":"2023-09-23T21:09:15.683300+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:09:15.683300+0000","last_undegraded":"2023-09-23T21:09:15.683300+0000","last_fullsized":"2023-09-23T21:09:15.683300+0000","mapping_epoch":387,"log_start":"406'6380","ondisk_log_start":"406'6380","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4075,"log_dups_size":0,"ondisk_log_size":4075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27459584,"num_objects":419,"num_object_clones":0,"num_object_copies":1257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":419,"num_objects_unfound":0,"num_objects_dirty":419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10455,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","1","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":419}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":235,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.073857+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:08:55.073857+0000","last_peered":"2023-09-23T21:08:55.073857+0000","last_clean":"2023-09-23T21:08:55.073857+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:08:55.073857+0000","last_undegraded":"2023-09-23T21:08:55.073857+0000","last_fullsized":"2023-09-23T21:08:55.073857+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":680,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.082170+0000","last_change":"2023-09-23T21:08:41.538586+0000","last_active":"2023-09-23T21:08:55.082170+0000","last_peered":"2023-09-23T21:08:55.082170+0000","last_clean":"2023-09-23T21:08:55.082170+0000","last_became_active":"2023-09-23T21:08:41.538107+0000","last_became_peered":"2023-09-23T21:08:41.538107+0000","last_unstale":"2023-09-23T21:08:55.082170+0000","last_undegraded":"2023-09-23T21:08:55.082170+0000","last_fullsized":"2023-09-23T21:08:55.082170+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:48.268707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.067576+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:08:55.067576+0000","last_peered":"2023-09-23T21:08:55.067576+0000","last_clean":"2023-09-23T21:08:55.067576+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:08:55.067576+0000","last_undegraded":"2023-09-23T21:08:55.067576+0000","last_fullsized":"2023-09-23T21:08:55.067576+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":308,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.060896+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:08:55.060896+0000","last_peered":"2023-09-23T21:08:55.060896+0000","last_clean":"2023-09-23T21:08:55.060896+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:08:55.060896+0000","last_undegraded":"2023-09-23T21:08:55.060896+0000","last_fullsized":"2023-09-23T21:08:55.060896+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"420'10665","reported_seq":10787,"reported_epoch":420,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:15.913025+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:09:15.913025+0000","last_peered":"2023-09-23T21:09:15.913025+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:09:15.913025+0000","last_undegraded":"2023-09-23T21:09:15.913025+0000","last_fullsized":"2023-09-23T21:09:15.913025+0000","mapping_epoch":374,"log_start":"406'6605","ondisk_log_start":"406'6605","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4060,"log_dups_size":0,"ondisk_log_size":4060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29949952,"num_objects":457,"num_object_clones":0,"num_object_copies":1371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":914,"num_objects_unfound":0,"num_objects_dirty":457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10665,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":457}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":53,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.619893+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:09:07.619893+0000","last_peered":"2023-09-23T21:09:07.619893+0000","last_clean":"2023-09-23T21:09:07.619893+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:09:07.619893+0000","last_undegraded":"2023-09-23T21:09:07.619893+0000","last_fullsized":"2023-09-23T21:09:07.619893+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":67,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.632654+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:09:07.632654+0000","last_peered":"2023-09-23T21:09:07.632654+0000","last_clean":"2023-09-23T21:09:07.632654+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:09:07.632654+0000","last_undegraded":"2023-09-23T21:09:07.632654+0000","last_fullsized":"2023-09-23T21:09:07.632654+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":57,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.086377+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:08:55.086377+0000","last_peered":"2023-09-23T21:08:55.086377+0000","last_clean":"2023-09-23T21:08:55.086377+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:08:55.086377+0000","last_undegraded":"2023-09-23T21:08:55.086377+0000","last_fullsized":"2023-09-23T21:08:55.086377+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"420'10573","reported_seq":20804,"reported_epoch":420,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:09:15.732871+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:09:15.732871+0000","last_peered":"2023-09-23T21:09:15.732871+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:09:15.732871+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:09:15.732871+0000","mapping_epoch":383,"log_start":"402'5826","ondisk_log_start":"402'5826","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4747,"log_dups_size":0,"ondisk_log_size":4747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29687808,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":450,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10573,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5060,"num_bytes_recovered":331612160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":3},{"shards":"1,2,7","objects":450}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":58,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.632446+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:09:07.632446+0000","last_peered":"2023-09-23T21:09:07.632446+0000","last_clean":"2023-09-23T21:09:07.632446+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:09:07.632446+0000","last_undegraded":"2023-09-23T21:09:07.632446+0000","last_fullsized":"2023-09-23T21:09:07.632446+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":317,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.072767+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:08:55.072767+0000","last_peered":"2023-09-23T21:08:55.072767+0000","last_clean":"2023-09-23T21:08:55.072767+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:08:55.072767+0000","last_undegraded":"2023-09-23T21:08:55.072767+0000","last_fullsized":"2023-09-23T21:08:55.072767+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":95,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.063919+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:08:55.063919+0000","last_peered":"2023-09-23T21:08:55.063919+0000","last_clean":"2023-09-23T21:08:55.063919+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:08:55.063919+0000","last_undegraded":"2023-09-23T21:08:55.063919+0000","last_fullsized":"2023-09-23T21:08:55.063919+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"420'10444","reported_seq":23342,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:15.705325+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:09:15.705325+0000","last_peered":"2023-09-23T21:09:15.705325+0000","last_clean":"2023-09-23T21:09:15.705325+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:09:15.705325+0000","last_undegraded":"2023-09-23T21:09:15.705325+0000","last_fullsized":"2023-09-23T21:09:15.705325+0000","mapping_epoch":397,"log_start":"402'5745","ondisk_log_start":"402'5745","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4699,"log_dups_size":0,"ondisk_log_size":4699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27525120,"num_objects":420,"num_object_clones":0,"num_object_copies":1260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10444,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":78,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:09:05.020103+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:09:05.020103+0000","last_peered":"2023-09-23T21:09:05.020103+0000","last_clean":"2023-09-23T21:09:05.020103+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:09:05.020103+0000","last_undegraded":"2023-09-23T21:09:05.020103+0000","last_fullsized":"2023-09-23T21:09:05.020103+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":52,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.076786+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:08:55.076786+0000","last_peered":"2023-09-23T21:08:55.076786+0000","last_clean":"2023-09-23T21:08:55.076786+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:08:55.076786+0000","last_undegraded":"2023-09-23T21:08:55.076786+0000","last_fullsized":"2023-09-23T21:08:55.076786+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.059472+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:08:55.059472+0000","last_peered":"2023-09-23T21:08:55.059472+0000","last_clean":"2023-09-23T21:08:55.059472+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:08:55.059472+0000","last_undegraded":"2023-09-23T21:08:55.059472+0000","last_fullsized":"2023-09-23T21:08:55.059472+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"420'10791","reported_seq":21301,"reported_epoch":420,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:09:15.726803+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:09:15.726803+0000","last_peered":"2023-09-23T21:09:15.726803+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:09:15.726803+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:09:15.726803+0000","mapping_epoch":382,"log_start":"404'6012","ondisk_log_start":"404'6012","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4779,"log_dups_size":0,"ondisk_log_size":4779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26673152,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":404,"num_objects_misplaced":410,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10791,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5192,"num_bytes_recovered":340262912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":404}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":316,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.057491+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:08:55.057491+0000","last_peered":"2023-09-23T21:08:55.057491+0000","last_clean":"2023-09-23T21:08:55.057491+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:08:55.057491+0000","last_undegraded":"2023-09-23T21:08:55.057491+0000","last_fullsized":"2023-09-23T21:08:55.057491+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":61,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.086550+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:08:55.086550+0000","last_peered":"2023-09-23T21:08:55.086550+0000","last_clean":"2023-09-23T21:08:55.086550+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:08:55.086550+0000","last_undegraded":"2023-09-23T21:08:55.086550+0000","last_fullsized":"2023-09-23T21:08:55.086550+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":105,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.062405+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:08:55.062405+0000","last_peered":"2023-09-23T21:08:55.062405+0000","last_clean":"2023-09-23T21:08:55.062405+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:08:55.062405+0000","last_undegraded":"2023-09-23T21:08:55.062405+0000","last_fullsized":"2023-09-23T21:08:55.062405+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":66,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.083594+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:08:55.083594+0000","last_peered":"2023-09-23T21:08:55.083594+0000","last_clean":"2023-09-23T21:08:55.083594+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:08:55.083594+0000","last_undegraded":"2023-09-23T21:08:55.083594+0000","last_fullsized":"2023-09-23T21:08:55.083594+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":443,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.074434+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:08:55.074434+0000","last_peered":"2023-09-23T21:08:55.074434+0000","last_clean":"2023-09-23T21:08:55.074434+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:08:55.074434+0000","last_undegraded":"2023-09-23T21:08:55.074434+0000","last_fullsized":"2023-09-23T21:08:55.074434+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":82,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.054825+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:08:55.054825+0000","last_peered":"2023-09-23T21:08:55.054825+0000","last_clean":"2023-09-23T21:08:55.054825+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:08:55.054825+0000","last_undegraded":"2023-09-23T21:08:55.054825+0000","last_fullsized":"2023-09-23T21:08:55.054825+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.060702+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:08:55.060702+0000","last_peered":"2023-09-23T21:08:55.060702+0000","last_clean":"2023-09-23T21:08:55.060702+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:08:55.060702+0000","last_undegraded":"2023-09-23T21:08:55.060702+0000","last_fullsized":"2023-09-23T21:08:55.060702+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":301,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.062788+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:08:55.062788+0000","last_peered":"2023-09-23T21:08:55.062788+0000","last_clean":"2023-09-23T21:08:55.062788+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:08:55.062788+0000","last_undegraded":"2023-09-23T21:08:55.062788+0000","last_fullsized":"2023-09-23T21:08:55.062788+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.067744+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:08:55.067744+0000","last_peered":"2023-09-23T21:08:55.067744+0000","last_clean":"2023-09-23T21:08:55.067744+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:08:55.067744+0000","last_undegraded":"2023-09-23T21:08:55.067744+0000","last_fullsized":"2023-09-23T21:08:55.067744+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":61,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.065114+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:08:55.065114+0000","last_peered":"2023-09-23T21:08:55.065114+0000","last_clean":"2023-09-23T21:08:55.065114+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:08:55.065114+0000","last_undegraded":"2023-09-23T21:08:55.065114+0000","last_fullsized":"2023-09-23T21:08:55.065114+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":69,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.069442+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:08:55.069442+0000","last_peered":"2023-09-23T21:08:55.069442+0000","last_clean":"2023-09-23T21:08:55.069442+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:08:55.069442+0000","last_undegraded":"2023-09-23T21:08:55.069442+0000","last_fullsized":"2023-09-23T21:08:55.069442+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":64,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.643470+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:09:07.643470+0000","last_peered":"2023-09-23T21:09:07.643470+0000","last_clean":"2023-09-23T21:09:07.643470+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:09:07.643470+0000","last_undegraded":"2023-09-23T21:09:07.643470+0000","last_fullsized":"2023-09-23T21:09:07.643470+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":47,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T21:08:55.054385+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:08:55.054385+0000","last_peered":"2023-09-23T21:08:55.054385+0000","last_clean":"2023-09-23T21:08:55.054385+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:08:55.054385+0000","last_undegraded":"2023-09-23T21:08:55.054385+0000","last_fullsized":"2023-09-23T21:08:55.054385+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":456917016,"num_objects":6973,"num_object_clones":0,"num_object_copies":20919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15056,"num_objects_misplaced":5885,"num_objects_unfound":0,"num_objects_dirty":6973,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169401,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42950,"num_bytes_recovered":2814771200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4742852608,"data_stored":4742840392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84800,"ondisk_log_size":84800,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201767,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463387,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":855376,"kb_used_data":304072,"kb_used_omap":0,"kb_used_meta":551296,"kb_avail":92889776,"statfs":{"total":95995035648,"available":95119130624,"internally_reserved":0,"allocated":311369728,"data_stored":310126760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":564527104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.181,"15min":1.958},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.3849999999999998,"5min":99.533000000000001,"15min":102.242},"last":0.245},{"interface":"front","average":{"1min":1.7809999999999999,"5min":2.3439999999999999,"15min":1.7769999999999999},"min":{"1min":0.34300000000000003,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":6.04,"5min":99.325000000000003,"15min":102.502},"last":0.58199999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7440000000000002,"5min":3.3039999999999998,"15min":2.5760000000000001},"min":{"1min":0.29599999999999999,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":7.915,"5min":119.077,"15min":119.077},"last":3.661},{"interface":"front","average":{"1min":1.9319999999999999,"5min":3.1040000000000001,"15min":2.5939999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.0229999999999997,"5min":119.313,"15min":119.313},"last":3.0830000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":5.1950000000000003,"15min":3.7799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.5470000000000006,"5min":199.23400000000001,"15min":199.23400000000001},"last":2.714},{"interface":"front","average":{"1min":2.0870000000000002,"5min":5.1689999999999996,"15min":3.8319999999999999},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.21199999999999999},"max":{"1min":6.3019999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":7.274}]},{"osd":3,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.339,"15min":2.7229999999999999},"min":{"1min":0.31,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":13.340999999999999,"5min":13.340999999999999,"15min":178.40000000000001},"last":1.641},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.7509999999999999,"15min":2.7040000000000002},"min":{"1min":0.53100000000000003,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":7.7759999999999998,"5min":13.319000000000001,"15min":178.26599999999999},"last":0.29899999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.423,"15min":0.90700000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":6.4989999999999997,"5min":13.1,"15min":13.1},"last":0.37},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0269999999999999,"15min":0.871},"min":{"1min":0.31,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":5.9580000000000002,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":1.4350000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.333,"15min":0.94799999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.1819999999999999,"5min":11.281000000000001,"15min":11.281000000000001},"last":3.629},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.208,"15min":1.05},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9279999999999999,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":0.34100000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.1080000000000001,"15min":0.97499999999999998},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":6.6299999999999999,"5min":7.8079999999999998,"15min":11.388999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.0669999999999999,"15min":1.0760000000000001},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.30099999999999999},"max":{"1min":6.4749999999999996,"5min":7.1849999999999996,"15min":8.8010000000000002},"last":0.46500000000000002}]}]},{"osd":1,"up_from":276,"seq":1185410973833,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1192252,"kb_used_data":724468,"kb_used_omap":5,"kb_used_meta":467706,"kb_avail":92552900,"statfs":{"total":95995035648,"available":94774169600,"internally_reserved":0,"allocated":741855232,"data_stored":739749886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":478931683},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,15],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.274,"5min":2.4860000000000002,"15min":1.1259999999999999},"min":{"1min":0.27600000000000002,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":23.486000000000001,"5min":23.486000000000001,"15min":23.486000000000001},"last":0.98599999999999999},{"interface":"front","average":{"1min":3.9540000000000002,"5min":1.996,"15min":1.2310000000000001},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":16.148,"5min":16.148,"15min":18.047999999999998},"last":0.73499999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4459999999999997,"5min":2.556,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":21.695,"5min":21.695,"15min":78.409000000000006},"last":1.103},{"interface":"front","average":{"1min":4.8369999999999997,"5min":2.6619999999999999,"15min":1.6140000000000001},"min":{"1min":0.502,"5min":0.245,"15min":0.22},"max":{"1min":23.539000000000001,"5min":23.539000000000001,"15min":76.245000000000005},"last":1.228}]},{"osd":3,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":2.4750000000000001,"15min":3.0379999999999998},"min":{"1min":0.39500000000000002,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":23.407,"5min":23.407,"15min":172.44399999999999},"last":3.3380000000000001},{"interface":"front","average":{"1min":3.9260000000000002,"5min":2.173,"15min":3.1059999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":17.033999999999999,"5min":17.033999999999999,"15min":172.58500000000001},"last":0.89300000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.631,"15min":1.137},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.5389999999999997,"5min":12.238,"15min":17.006},"last":1.0329999999999999},{"interface":"front","average":{"1min":3.3530000000000002,"5min":2.0640000000000001,"15min":1.1180000000000001},"min":{"1min":0.42199999999999999,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":23.332999999999998,"5min":23.332999999999998,"15min":23.332999999999998},"last":1.175}]},{"osd":5,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":16.196000000000002,"5min":16.196000000000002,"15min":17.155000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":2.5339999999999998,"5min":1.7969999999999999,"15min":1.284},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.9660000000000002,"5min":12.345000000000001,"15min":17.076000000000001},"last":1.113}]},{"osd":6,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":1.8089999999999999,"15min":1.73},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.222000000000001,"5min":16.222000000000001,"15min":127.995},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.6160000000000001,"5min":1.962,"15min":1.758},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8959999999999999,"5min":12.276999999999999,"15min":126.836},"last":1.0900000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":3.2639999999999998,"15min":1.756},"min":{"1min":0.38100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.859,"5min":151.791,"15min":151.791},"last":1.137},{"interface":"front","average":{"1min":3.4039999999999999,"5min":3.552,"15min":1.6579999999999999},"min":{"1min":0.45000000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":23.376000000000001,"5min":151.809,"15min":151.809},"last":1.2}]}]},{"osd":0,"up_from":92,"seq":395136991573,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":881524,"kb_used_data":110636,"kb_used_omap":5,"kb_used_meta":770810,"kb_avail":92863628,"statfs":{"total":95995035648,"available":95092355072,"internally_reserved":0,"allocated":113291264,"data_stored":111307832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":789310074},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.3210000000000002,"15min":2.0219999999999998},"min":{"1min":0.26300000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":18.140999999999998,"5min":52.590000000000003,"15min":61.341999999999999},"last":1.014},{"interface":"front","average":{"1min":4.5570000000000004,"5min":3.754,"15min":2.1059999999999999},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.219999999999999,"5min":52.276000000000003,"15min":61.299999999999997},"last":2.9620000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":3.5539999999999998,"5min":3.0030000000000001,"15min":2.2130000000000001},"min":{"1min":0.35699999999999998,"5min":0.217,"15min":0.161},"max":{"1min":12.916,"5min":45.912999999999997,"15min":183.77099999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8210000000000002,"15min":2.3719999999999999},"min":{"1min":0.33500000000000002,"5min":0.254,"15min":0.214},"max":{"1min":9.1370000000000005,"5min":46.234000000000002,"15min":183.78700000000001},"last":3.835}]},{"osd":3,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.613,"5min":1.655,"15min":2.4750000000000001},"min":{"1min":0.36099999999999999,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":10.755000000000001,"5min":12.113,"15min":186.143},"last":3.9529999999999998},{"interface":"front","average":{"1min":2.7719999999999998,"5min":1.5620000000000001,"15min":2.5099999999999998},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":7.6260000000000003,"5min":12.348000000000001,"15min":186.97399999999999},"last":3.0539999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.401,"15min":1.3009999999999999},"min":{"1min":0.34200000000000003,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.9450000000000003,"5min":10.814,"15min":10.814},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.171,"15min":1.012},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.629,"5min":10.629,"15min":11.493},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.5329999999999999,"15min":1.879},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.984,"5min":10.763,"15min":154.30600000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.373,"15min":1.5640000000000001},"min":{"1min":0.42099999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":10.685,"5min":11.478,"15min":154.42400000000001},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":6.0860000000000003,"15min":3.2360000000000002},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.710000000000001,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.105,"5min":6.0289999999999999,"15min":3.2810000000000001},"min":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":8.0269999999999992,"5min":169.767,"15min":169.767},"last":3.004}]},{"osd":7,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":1.605,"15min":2.3119999999999998},"min":{"1min":0.47399999999999998,"5min":0.35199999999999998,"15min":0.33600000000000002},"max":{"1min":10.661,"5min":10.74,"15min":164.26400000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.4830000000000001,"5min":1.3280000000000001,"15min":2.306},"min":{"1min":0.41199999999999998,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":164.23099999999999},"last":3.0310000000000001}]}]},{"osd":2,"up_from":129,"seq":554050781464,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240304,"kb_used_data":532072,"kb_used_omap":7,"kb_used_meta":708216,"kb_avail":92504848,"statfs":{"total":95995035648,"available":94724964352,"internally_reserved":0,"allocated":544841728,"data_stored":542803858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":725213265},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7930000000000001,"5min":1.5169999999999999,"15min":1.423},"min":{"1min":0.222,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":11.699999999999999,"5min":11.699999999999999,"15min":26.356000000000002},"last":2.911},{"interface":"front","average":{"1min":3.6800000000000002,"5min":2.444,"15min":1.8560000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":16.974,"5min":23.963999999999999,"15min":26.341000000000001},"last":2.7269999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":3.3039999999999998,"15min":2.802},"min":{"1min":0.24299999999999999,"5min":0.19,"15min":0.188},"max":{"1min":11.271000000000001,"5min":67.296999999999997,"15min":212.12},"last":5.5810000000000004},{"interface":"front","average":{"1min":2.7069999999999999,"5min":2.5960000000000001,"15min":2.4860000000000002},"min":{"1min":0.34200000000000003,"5min":0.214,"15min":0.214},"max":{"1min":11.840999999999999,"5min":67.355000000000004,"15min":213.02799999999999},"last":5.6319999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:08:48 2023","interfaces":[{"interface":"back","average":{"1min":4.4729999999999999,"5min":3.5760000000000001,"15min":3.3439999999999999},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":15.327999999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":2.8290000000000002},{"interface":"front","average":{"1min":4.1790000000000003,"5min":3.4279999999999999,"15min":3.2949999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.215},"max":{"1min":33.889000000000003,"5min":138.87700000000001,"15min":138.87700000000001},"last":7.0099999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.169,"15min":1.139},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26200000000000001},"max":{"1min":11.744,"5min":11.744,"15min":12.586},"last":2.8809999999999998},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.028,"15min":1.46},"min":{"1min":0.35699999999999998,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":16.725999999999999,"5min":16.725999999999999,"15min":16.725999999999999},"last":2.6699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":3.8119999999999998,"5min":2.222,"15min":2.2130000000000001},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":16.673999999999999,"5min":16.673999999999999,"15min":173.34},"last":2.7519999999999998},{"interface":"front","average":{"1min":2.2480000000000002,"5min":1.4770000000000001,"15min":1.8380000000000001},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":12.368,"5min":13.701000000000001,"15min":173.15000000000001},"last":2.7400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":3.6800000000000002,"15min":1.9590000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.323,"5min":175.72499999999999,"15min":175.72499999999999},"last":2.77},{"interface":"front","average":{"1min":2.0230000000000001,"5min":3.3780000000000001,"15min":1.915},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":11.782,"5min":176.20400000000001,"15min":176.20400000000001},"last":5.6689999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.665,"15min":2.0579999999999998},"min":{"1min":0.42399999999999999,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":11.811999999999999,"5min":11.811999999999999,"15min":177.30799999999999},"last":2.9340000000000002},{"interface":"front","average":{"1min":3.6339999999999999,"5min":2.109,"15min":2.2410000000000001},"min":{"1min":0.38200000000000001,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":16.875,"5min":16.875,"15min":177.40899999999999},"last":2.706}]}]},{"osd":3,"up_from":148,"seq":635655160058,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588936,"kb_used_data":8128,"kb_used_omap":2,"kb_used_meta":580797,"kb_avail":93156216,"statfs":{"total":95995035648,"available":95391965184,"internally_reserved":0,"allocated":8323072,"data_stored":6082242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":594736181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":1.984,"15min":1.4970000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.184},"max":{"1min":17.425999999999998,"5min":44.963000000000001,"15min":44.963000000000001},"last":0.214},{"interface":"front","average":{"1min":1.629,"5min":1.1080000000000001,"15min":1.302},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.19500000000000001},"max":{"1min":7.9699999999999998,"5min":9.7370000000000001,"15min":37.634999999999998},"last":0.19500000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":19.158999999999999,"5min":7.0279999999999996,"15min":3.0649999999999999},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.20999999999999999},"max":{"1min":166.56999999999999,"5min":166.56999999999999,"15min":166.56999999999999},"last":237.072},{"interface":"front","average":{"1min":17.228999999999999,"5min":7.0049999999999999,"15min":3.202},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.182},"max":{"1min":166.291,"5min":166.291,"15min":166.291},"last":234.21899999999999}]},{"osd":2,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":9.6780000000000008,"5min":3.2469999999999999,"15min":1.9259999999999999},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":88.825999999999993,"5min":88.825999999999993,"15min":88.825999999999993},"last":0.29999999999999999},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.5790000000000002,"15min":1.732},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":85.626000000000005,"5min":85.626000000000005,"15min":85.626000000000005},"last":0.25}]},{"osd":4,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":2.8900000000000001,"15min":2.347},"min":{"1min":0.374,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":70.728999999999999,"15min":111.36},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.605,"5min":2.7679999999999998,"15min":2.1429999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":17.286999999999999,"5min":70.712000000000003,"15min":112.473},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.7989999999999999,"15min":1.5920000000000001},"min":{"1min":0.39400000000000002,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":7.9050000000000002,"5min":36.488999999999997,"15min":64.528000000000006},"last":0.27200000000000002},{"interface":"front","average":{"1min":3.278,"5min":2.101,"15min":1.724},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":12.382999999999999,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.315}]},{"osd":6,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.3220000000000001,"5min":1.8080000000000001,"15min":1.7050000000000001},"min":{"1min":0.36099999999999999,"5min":0.318,"15min":0.312},"max":{"1min":12.317,"5min":12.317,"15min":19.568000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.3570000000000002,"5min":1.722,"15min":1.522},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":17.324999999999999,"5min":17.324999999999999,"15min":17.324999999999999},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.427,"15min":2.488},"min":{"1min":0.377,"5min":0.377,"15min":0.30499999999999999},"max":{"1min":7.9459999999999997,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":3.46,"5min":2.7970000000000002,"15min":2.8879999999999999},"min":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":12.407,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.377}]}]},{"osd":4,"up_from":31,"seq":133143986677,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665168,"kb_used_data":95816,"kb_used_omap":0,"kb_used_meta":569344,"kb_avail":93079984,"statfs":{"total":95995035648,"available":95313903616,"internally_reserved":0,"allocated":98115584,"data_stored":96872000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":583008256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3560000000000001,"15min":2.2829999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":10.273999999999999,"5min":12.869,"15min":102.358},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.883,"5min":1.4790000000000001,"15min":2.29},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":8.3190000000000008,"5min":12.848000000000001,"15min":102.58199999999999},"last":1.841}]},{"osd":1,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":7.0010000000000003,"5min":6.1950000000000003,"15min":3.9399999999999999},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":42.887,"5min":127.411,"15min":127.411},"last":0.55000000000000004},{"interface":"front","average":{"1min":7.258,"5min":4.3179999999999996,"15min":3.2770000000000001},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.253},"max":{"1min":44.148000000000003,"5min":118.075,"15min":118.075},"last":1.488}]},{"osd":2,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":5.6589999999999998,"15min":3.6699999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":7.4480000000000004,"5min":164.517,"15min":164.517},"last":6.056},{"interface":"front","average":{"1min":1.4419999999999999,"5min":4.7889999999999997,"15min":3.4769999999999999},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":5.9400000000000004,"5min":164.46799999999999,"15min":164.46799999999999},"last":2.0329999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.3380000000000001,"15min":1.4359999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27300000000000002},"max":{"1min":6.9390000000000001,"5min":6.9390000000000001,"15min":60.720999999999997},"last":1.038},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.3859999999999999,"15min":1.4870000000000001},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.25600000000000001},"max":{"1min":6.4020000000000001,"5min":8.2899999999999991,"15min":60.631999999999998},"last":4.375}]},{"osd":5,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":0.95999999999999996,"15min":1.02},"min":{"1min":0.309,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":6.0069999999999997,"5min":8.7089999999999996,"15min":72.304000000000002},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.754,"5min":1.0149999999999999,"15min":1.032},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.21099999999999999},"max":{"1min":4.0259999999999998,"5min":7.0179999999999998,"15min":72.290000000000006},"last":0.309}]},{"osd":6,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.2949999999999999,"15min":0.95299999999999996},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":6.4269999999999996,"5min":11.800000000000001,"15min":11.887},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.341,"15min":0.92900000000000005},"min":{"1min":0.39300000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":4.5449999999999999,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.34399999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":2.6829999999999998,"15min":1.544},"min":{"1min":0.34300000000000003,"5min":0.309,"15min":0.30199999999999999},"max":{"1min":3.726,"5min":145.721,"15min":145.721},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":3.0049999999999999,"15min":1.7},"min":{"1min":0.375,"5min":0.35699999999999998,"15min":0.28000000000000003},"max":{"1min":3.383,"5min":148.45699999999999,"15min":148.45699999999999},"last":0.375}]}]},{"osd":5,"up_from":39,"seq":167503725041,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":688828,"kb_used_data":51316,"kb_used_omap":0,"kb_used_meta":637504,"kb_avail":93056324,"statfs":{"total":95995035648,"available":95289675776,"internally_reserved":0,"allocated":52547584,"data_stored":51320200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":652804096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.0660000000000001,"15min":1.319},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.2640000000000002,"5min":10.488,"15min":80.105999999999995},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.1799999999999999,"15min":1.4590000000000001},"min":{"1min":0.27000000000000002,"5min":0.222,"15min":0.222},"max":{"1min":6.6619999999999999,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":9.4730000000000008,"5min":7.8259999999999996,"15min":3.117},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":125.23399999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":0.96299999999999997},{"interface":"front","average":{"1min":9.9860000000000007,"5min":7.6059999999999999,"15min":3.0600000000000001},"min":{"1min":0.375,"5min":0.251,"15min":0.247},"max":{"1min":125.249,"5min":233.75200000000001,"15min":233.75200000000001},"last":0.97599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":4.9640000000000004,"15min":2.794},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.254},"max":{"1min":70.810000000000002,"5min":231.34700000000001,"15min":231.34700000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":8.1370000000000005,"5min":5.4619999999999997,"15min":3.0289999999999999},"min":{"1min":0.40200000000000002,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":70.507000000000005,"5min":231.381,"15min":231.381},"last":2.226}]},{"osd":3,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.6519999999999999,"15min":2.786},"min":{"1min":0.33000000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":12.765000000000001,"5min":12.765000000000001,"15min":153.60300000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.3959999999999999,"15min":2.7130000000000001},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":9.6120000000000001,"5min":9.6120000000000001,"15min":153.774},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.72399999999999998,"15min":0.71399999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.247},"max":{"1min":6.3940000000000001,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.0129999999999999,"15min":0.89200000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":5.8490000000000002,"5min":8.3740000000000006,"15min":12.539},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":1.1990000000000001,"15min":1.1259999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.1130000000000004,"5min":9.8480000000000008,"15min":71.230000000000004},"last":0.438},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.252,"15min":1.163},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":3.4359999999999999,"5min":12.673999999999999,"15min":70.266000000000005},"last":0.498}]},{"osd":7,"last update":"Sat Sep 23 21:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.262,"15min":1.0289999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.312},"max":{"1min":4.3399999999999999,"5min":9.6329999999999991,"15min":31.091999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.69899999999999995,"15min":0.80100000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":2.0179999999999998,"5min":4.29,"15min":31.780000000000001},"last":0.56599999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":104333312,"data_stored":104333312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":723714048,"data_stored":723714048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":537333760,"data_stored":537329688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3014656,"data_stored":3014656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":89722880,"data_stored":89718808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":46530560,"data_stored":46530560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":304091136,"data_stored":304087064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2934112256,"data_stored":2934112256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:09:23.412 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:23 smithi123 ceph-mon[193903]: pgmap v1356: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 267 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 14341/12840 objects degraded (111.690%); 3488/12840 objects misplaced (27.165%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:09:23.517 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:23 smithi121 ceph-mon[211526]: pgmap v1356: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 267 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 14341/12840 objects degraded (111.690%); 3488/12840 objects misplaced (27.165%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:09:23.518 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:23 smithi121 ceph-mon[219709]: pgmap v1356: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 267 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 14341/12840 objects degraded (111.690%); 3488/12840 objects misplaced (27.165%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:09:23.526 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:23.537 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:23.690 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:23.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:23.803 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:23.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:09:23.937 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:24.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:24.061 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:24.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:24.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:09:24.151 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:09:24.162 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:24.187 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:09:24.237 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:09:24.281 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:24.282 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:09:24.325 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:09:24.342 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:24.384 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:09:24.425 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:09:24.434 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.434 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.434 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.434 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.435 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.435 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.435 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.435 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:09:24.481 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3030628402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:24.482 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3030628402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:24.567 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3030628402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:24.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:09:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:09:24] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2023-09-23T21:09:25.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:25 smithi121 ceph-mon[219709]: pgmap v1357: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 226 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 14341/10893 objects degraded (131.653%); 2692/10893 objects misplaced (24.713%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:09:25.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:25 smithi121 ceph-mon[211526]: pgmap v1357: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 226 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 14341/10893 objects degraded (131.653%); 2692/10893 objects misplaced (24.713%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:09:25.663 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:25.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:25.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:25.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:25 smithi123 ceph-mon[193903]: pgmap v1357: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 226 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 14341/10893 objects degraded (131.653%); 2692/10893 objects misplaced (24.713%); 10 MiB/s, 163 objects/s recovering 2023-09-23T21:09:25.772 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:25.811 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:25.873 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:25.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:25.983 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:26.044 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:26.358 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:09:26.547 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:26.547 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:26.666 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:26.696 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14341/10893 objects degraded (131.653%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:26.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:26.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:26.697 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14341/10893 objects degraded (131.653%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:26.798 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:26.890 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:26.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14341/10893 objects degraded (131.653%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:27.095 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:27.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:27.202 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:27.286 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:27.449 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:27.464 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:27.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:27.575 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:27.668 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:27.705 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:09:27.821 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:27 smithi121 ceph-mon[211526]: pgmap v1358: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 102 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 13515/4935 objects degraded (273.860%); 1326/4935 objects misplaced (26.869%); 9.8 MiB/s, 177 objects/s recovering 2023-09-23T21:09:27.821 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:27.822 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:27 smithi121 ceph-mon[219709]: pgmap v1358: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 102 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 13515/4935 objects degraded (273.860%); 1326/4935 objects misplaced (26.869%); 9.8 MiB/s, 177 objects/s recovering 2023-09-23T21:09:27.822 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:27.892 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:27.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:27.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:27.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:27 smithi123 ceph-mon[193903]: pgmap v1358: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 102 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 13515/4935 objects degraded (273.860%); 1326/4935 objects misplaced (26.869%); 9.8 MiB/s, 177 objects/s recovering 2023-09-23T21:09:27.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:28.002 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:28.093 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:28.228 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:28.330 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:28.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:28.436 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:28.477 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:28.542 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 88186 objects 2023-09-23T21:09:28.542 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :177.282 2023-09-23T21:09:28.551 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-09-23T21:09:28.551 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-09-23T21:09:28.564 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:28.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:28.623 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/19975332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:28.623 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:28 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:28.633 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:09:28.638 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:09:28.668 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:28.696 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/19975332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:28 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:28.742 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:28.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:28.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:28.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:28.874 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/19975332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:28.874 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:28 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:09:28.875 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:29.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[211526]: failed to encode map e422 with expected crc 2023-09-23T21:09:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[211526]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T21:09:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[211526]: pgmap v1360: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 102 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 13515/4935 objects degraded (273.860%); 1326/4935 objects misplaced (26.869%); 9.8 MiB/s, 177 objects/s recovering 2023-09-23T21:09:29.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:29.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[219709]: failed to encode map e422 with expected crc 2023-09-23T21:09:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[219709]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T21:09:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:29 smithi121 ceph-mon[219709]: pgmap v1360: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 102 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 13515/4935 objects degraded (273.860%); 1326/4935 objects misplaced (26.869%); 9.8 MiB/s, 177 objects/s recovering 2023-09-23T21:09:29.927 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:29.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:29 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:29 smithi123 ceph-mon[193903]: failed to encode map e422 with expected crc 2023-09-23T21:09:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:29 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:09:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:29 smithi123 ceph-mon[193903]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T21:09:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:29 smithi123 ceph-mon[193903]: pgmap v1360: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 102 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 13515/4935 objects degraded (273.860%); 1326/4935 objects misplaced (26.869%); 9.8 MiB/s, 177 objects/s recovering 2023-09-23T21:09:30.032 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:30.060 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:30.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:30.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:30.219 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:30.247 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:30.566 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:30.566 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:30.942 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:30.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:31.047 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:31.076 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:31.120 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:31.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:31.133 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:31.224 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:31.250 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:31.293 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:31.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:31.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:31.359 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:31.397 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:31.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:31 smithi121 ceph-mon[211526]: pgmap v1361: 89 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 30 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 12833/1488 objects degraded (862.433%); 177/1488 objects misplaced (11.895%); 8.0 MiB/s, 198 objects/s recovering 2023-09-23T21:09:31.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:31 smithi121 ceph-mon[219709]: pgmap v1361: 89 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 30 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 12833/1488 objects degraded (862.433%); 177/1488 objects misplaced (11.895%); 8.0 MiB/s, 198 objects/s recovering 2023-09-23T21:09:31.421 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:31 smithi123 ceph-mon[193903]: pgmap v1361: 89 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 30 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 483 op/s; 12833/1488 objects degraded (862.433%); 177/1488 objects misplaced (11.895%); 8.0 MiB/s, 198 objects/s recovering 2023-09-23T21:09:31.424 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:31.469 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:31.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:31.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:31.572 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:31.608 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:31.726 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:09:31.729 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:09:32.269 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:09:32.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12833/1488 objects degraded (862.433%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:32.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:32 smithi123 ceph-mon[193903]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:32.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12833/1488 objects degraded (862.433%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:32.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:32 smithi121 ceph-mon[211526]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12833/1488 objects degraded (862.433%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:32 smithi121 ceph-mon[219709]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:32.665 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:33.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:09:33.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:33 smithi123 ceph-mon[193903]: pgmap v1362: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 76 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 12325/72 objects degraded (17118.056%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T21:09:33.524 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:33.526 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:33 smithi121 ceph-mon[219709]: pgmap v1362: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 76 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 12325/72 objects degraded (17118.056%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T21:09:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:33.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:33.528 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:33 smithi121 ceph-mon[211526]: pgmap v1362: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 76 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 12325/72 objects degraded (17118.056%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T21:09:33.629 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:33.657 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:34 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:09:34.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:34 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:09:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:34 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:09:34.860 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:34.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:34.964 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:34.996 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:35.130 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:09:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:09:34] "GET /metrics HTTP/1.1" 200 36710 "" "Prometheus/2.43.0" 2023-09-23T21:09:35.253 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:35.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:35.264 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1361,"stamp":"2023-09-23T21:09:31.139492+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":31528864,"num_objects":496,"num_object_clones":0,"num_object_copies":1488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12833,"num_objects_misplaced":177,"num_objects_unfound":0,"num_objects_dirty":496,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":176202,"num_write_kb":5646350,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45434,"num_bytes_recovered":2927206188,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85517,"ondisk_log_size":85517,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5091528,"kb_used_data":782096,"kb_used_omap":21,"kb_used_meta":4309354,"kb_avail":651124536,"statfs":{"total":671965249536,"available":666751524864,"internally_reserved":0,"allocated":800866304,"data_stored":788735174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4412778979},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-316932120,"num_objects":-4837,"num_object_clones":0,"num_object_copies":-14511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1885,"num_objects_misplaced":-3817,"num_objects_unfound":0,"num_objects_dirty":-4837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4837,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1987,"num_bytes_recovered":84082688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008612"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":192,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.618909+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:09:07.618909+0000","last_peered":"2023-09-23T21:09:07.618909+0000","last_clean":"2023-09-23T21:09:07.618909+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:09:07.618909+0000","last_undegraded":"2023-09-23T21:09:07.618909+0000","last_fullsized":"2023-09-23T21:09:07.618909+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.635812+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:09:18.635812+0000","last_peered":"2023-09-23T21:09:18.635812+0000","last_clean":"2023-09-23T21:09:18.635812+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:09:18.635812+0000","last_undegraded":"2023-09-23T21:09:18.635812+0000","last_fullsized":"2023-09-23T21:09:18.635812+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":334,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.618118+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:09:07.618118+0000","last_peered":"2023-09-23T21:09:07.618118+0000","last_clean":"2023-09-23T21:09:07.618118+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:09:07.618118+0000","last_undegraded":"2023-09-23T21:09:07.618118+0000","last_fullsized":"2023-09-23T21:09:07.618118+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":60,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.633670+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:09:18.633670+0000","last_peered":"2023-09-23T21:09:18.633670+0000","last_clean":"2023-09-23T21:09:18.633670+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:09:18.633670+0000","last_undegraded":"2023-09-23T21:09:18.633670+0000","last_fullsized":"2023-09-23T21:09:18.633670+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.637826+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:09:18.637826+0000","last_peered":"2023-09-23T21:09:18.637826+0000","last_clean":"2023-09-23T21:09:18.637826+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:09:18.637826+0000","last_undegraded":"2023-09-23T21:09:18.637826+0000","last_fullsized":"2023-09-23T21:09:18.637826+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":288,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.644663+0000","last_change":"2023-09-23T20:57:44.249390+0000","last_active":"2023-09-23T21:09:18.644663+0000","last_peered":"2023-09-23T21:09:18.644663+0000","last_clean":"2023-09-23T21:09:18.644663+0000","last_became_active":"2023-09-23T20:57:44.249085+0000","last_became_peered":"2023-09-23T20:57:44.249085+0000","last_unstale":"2023-09-23T21:09:18.644663+0000","last_undegraded":"2023-09-23T21:09:18.644663+0000","last_fullsized":"2023-09-23T21:09:18.644663+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:29.656602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":59,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.633839+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:09:18.633839+0000","last_peered":"2023-09-23T21:09:18.633839+0000","last_clean":"2023-09-23T21:09:18.633839+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:09:18.633839+0000","last_undegraded":"2023-09-23T21:09:18.633839+0000","last_fullsized":"2023-09-23T21:09:18.633839+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:25.029567+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:09:25.029567+0000","last_peered":"2023-09-23T21:09:25.029567+0000","last_clean":"2023-09-23T21:09:25.029567+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:09:25.029567+0000","last_undegraded":"2023-09-23T21:09:25.029567+0000","last_fullsized":"2023-09-23T21:09:25.029567+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":44,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.627821+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:09:07.627821+0000","last_peered":"2023-09-23T21:09:07.627821+0000","last_clean":"2023-09-23T21:09:07.627821+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:09:07.627821+0000","last_undegraded":"2023-09-23T21:09:07.627821+0000","last_fullsized":"2023-09-23T21:09:07.627821+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:25.029613+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:09:25.029613+0000","last_peered":"2023-09-23T21:09:25.029613+0000","last_clean":"2023-09-23T21:09:25.029613+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:09:25.029613+0000","last_undegraded":"2023-09-23T21:09:25.029613+0000","last_fullsized":"2023-09-23T21:09:25.029613+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":78,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.635896+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:09:18.635896+0000","last_peered":"2023-09-23T21:09:18.635896+0000","last_clean":"2023-09-23T21:09:18.635896+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:09:18.635896+0000","last_undegraded":"2023-09-23T21:09:18.635896+0000","last_fullsized":"2023-09-23T21:09:18.635896+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.618585+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:09:07.618585+0000","last_peered":"2023-09-23T21:09:07.618585+0000","last_clean":"2023-09-23T21:09:07.618585+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:09:07.618585+0000","last_undegraded":"2023-09-23T21:09:07.618585+0000","last_fullsized":"2023-09-23T21:09:07.618585+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":368,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.644393+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:09:18.644393+0000","last_peered":"2023-09-23T21:09:18.644393+0000","last_clean":"2023-09-23T21:09:18.644393+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:09:18.644393+0000","last_undegraded":"2023-09-23T21:09:18.644393+0000","last_fullsized":"2023-09-23T21:09:18.644393+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":339,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.635353+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:09:07.635353+0000","last_peered":"2023-09-23T21:09:07.635353+0000","last_clean":"2023-09-23T21:09:07.635353+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:09:07.635353+0000","last_undegraded":"2023-09-23T21:09:07.635353+0000","last_fullsized":"2023-09-23T21:09:07.635353+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":113,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.652048+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:09:18.652048+0000","last_peered":"2023-09-23T21:09:18.652048+0000","last_clean":"2023-09-23T21:09:18.652048+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:09:18.652048+0000","last_undegraded":"2023-09-23T21:09:18.652048+0000","last_fullsized":"2023-09-23T21:09:18.652048+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:25.029690+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:09:25.029690+0000","last_peered":"2023-09-23T21:09:25.029690+0000","last_clean":"2023-09-23T21:09:25.029690+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:09:25.029690+0000","last_undegraded":"2023-09-23T21:09:25.029690+0000","last_fullsized":"2023-09-23T21:09:25.029690+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":81,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:25.029639+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:09:25.029639+0000","last_peered":"2023-09-23T21:09:25.029639+0000","last_clean":"2023-09-23T21:09:25.029639+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:09:25.029639+0000","last_undegraded":"2023-09-23T21:09:25.029639+0000","last_fullsized":"2023-09-23T21:09:25.029639+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":80,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.635963+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:09:18.635963+0000","last_peered":"2023-09-23T21:09:18.635963+0000","last_clean":"2023-09-23T21:09:18.635963+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:09:18.635963+0000","last_undegraded":"2023-09-23T21:09:18.635963+0000","last_fullsized":"2023-09-23T21:09:18.635963+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":221,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.634988+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:09:07.634988+0000","last_peered":"2023-09-23T21:09:07.634988+0000","last_clean":"2023-09-23T21:09:07.634988+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:09:07.634988+0000","last_undegraded":"2023-09-23T21:09:07.634988+0000","last_fullsized":"2023-09-23T21:09:07.634988+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":59,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.633712+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:09:18.633712+0000","last_peered":"2023-09-23T21:09:18.633712+0000","last_clean":"2023-09-23T21:09:18.633712+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:09:18.633712+0000","last_undegraded":"2023-09-23T21:09:18.633712+0000","last_fullsized":"2023-09-23T21:09:18.633712+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":418,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.643071+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:09:18.643071+0000","last_peered":"2023-09-23T21:09:18.643071+0000","last_clean":"2023-09-23T21:09:18.643071+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:09:18.643071+0000","last_undegraded":"2023-09-23T21:09:18.643071+0000","last_fullsized":"2023-09-23T21:09:18.643071+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.634933+0000","last_change":"2023-09-23T21:04:45.260283+0000","last_active":"2023-09-23T21:09:18.634933+0000","last_peered":"2023-09-23T21:09:18.634933+0000","last_clean":"2023-09-23T21:09:18.634933+0000","last_became_active":"2023-09-23T21:04:45.259934+0000","last_became_peered":"2023-09-23T21:04:45.259934+0000","last_unstale":"2023-09-23T21:09:18.634933+0000","last_undegraded":"2023-09-23T21:09:18.634933+0000","last_fullsized":"2023-09-23T21:09:18.634933+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:13.239534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"421'11164","reported_seq":12542,"reported_epoch":421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:26.096630+0000","last_change":"2023-09-23T21:05:34.372322+0000","last_active":"2023-09-23T21:09:26.096630+0000","last_peered":"2023-09-23T21:09:26.096630+0000","last_clean":"2023-09-23T21:04:28.707759+0000","last_became_active":"2023-09-23T21:05:34.365237+0000","last_became_peered":"2023-09-23T21:05:34.365237+0000","last_unstale":"2023-09-23T21:09:26.096630+0000","last_undegraded":"2023-09-23T21:05:34.194730+0000","last_fullsized":"2023-09-23T21:05:34.163633+0000","mapping_epoch":386,"log_start":"405'6428","ondisk_log_start":"405'6428","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4736,"log_dups_size":0,"ondisk_log_size":4736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:34.627168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4325376,"num_objects":66,"num_object_clones":0,"num_object_copies":198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5615,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11164,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4","objects":-5549},{"shards":"1,4,6","objects":1},{"shards":"1,4,7","objects":65}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":125,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.636432+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:09:07.636432+0000","last_peered":"2023-09-23T21:09:07.636432+0000","last_clean":"2023-09-23T21:09:07.636432+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:09:07.636432+0000","last_undegraded":"2023-09-23T21:09:07.636432+0000","last_fullsized":"2023-09-23T21:09:07.636432+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":44,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.627638+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:09:07.627638+0000","last_peered":"2023-09-23T21:09:07.627638+0000","last_clean":"2023-09-23T21:09:07.627638+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:09:07.627638+0000","last_undegraded":"2023-09-23T21:09:07.627638+0000","last_fullsized":"2023-09-23T21:09:07.627638+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"421'10719","reported_seq":10811,"reported_epoch":421,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:26.131472+0000","last_change":"2023-09-23T21:04:59.580150+0000","last_active":"2023-09-23T21:09:26.131472+0000","last_peered":"2023-09-23T21:09:26.131472+0000","last_clean":"2023-09-23T21:04:31.032392+0000","last_became_active":"2023-09-23T21:04:59.576700+0000","last_became_peered":"2023-09-23T21:04:59.576700+0000","last_unstale":"2023-09-23T21:09:26.131472+0000","last_undegraded":"2023-09-23T21:09:26.131472+0000","last_fullsized":"2023-09-23T21:09:26.131472+0000","mapping_epoch":375,"log_start":"405'6012","ondisk_log_start":"405'6012","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4707,"log_dups_size":0,"ondisk_log_size":4707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:52.617948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5177344,"num_objects":79,"num_object_clones":0,"num_object_copies":237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":79,"num_objects_unfound":0,"num_objects_dirty":79,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10719,"num_write_kb":345536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":79}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":120,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:16.158964+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:09:16.158964+0000","last_peered":"2023-09-23T21:09:16.158964+0000","last_clean":"2023-09-23T21:09:16.158964+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:09:16.158964+0000","last_undegraded":"2023-09-23T21:09:16.158964+0000","last_fullsized":"2023-09-23T21:09:16.158964+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":382,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.618039+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:09:07.618039+0000","last_peered":"2023-09-23T21:09:07.618039+0000","last_clean":"2023-09-23T21:09:07.618039+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:09:07.618039+0000","last_undegraded":"2023-09-23T21:09:07.618039+0000","last_fullsized":"2023-09-23T21:09:07.618039+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"421'10938","reported_seq":29583,"reported_epoch":421,"state":"active+recovery_wait+undersized+remapped","last_fresh":"2023-09-23T21:09:28.519526+0000","last_change":"2023-09-23T21:09:28.414430+0000","last_active":"2023-09-23T21:09:28.519526+0000","last_peered":"2023-09-23T21:09:28.519526+0000","last_clean":"2023-09-23T21:03:37.385636+0000","last_became_active":"2023-09-23T21:06:30.668500+0000","last_became_peered":"2023-09-23T21:06:30.668500+0000","last_unstale":"2023-09-23T21:09:28.519526+0000","last_undegraded":"2023-09-23T21:09:28.519526+0000","last_fullsized":"2023-09-23T21:05:41.214730+0000","mapping_epoch":389,"log_start":"407'6811","ondisk_log_start":"407'6811","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4127,"log_dups_size":0,"ondisk_log_size":4127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:33.769807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10938,"num_write_kb":350016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6537,"num_bytes_recovered":428408832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[6,3],"avail_no_missing":["6","1","3","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":48,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.627943+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:09:07.627943+0000","last_peered":"2023-09-23T21:09:07.627943+0000","last_clean":"2023-09-23T21:09:07.627943+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:09:07.627943+0000","last_undegraded":"2023-09-23T21:09:07.627943+0000","last_fullsized":"2023-09-23T21:09:07.627943+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":310,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.643964+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:09:18.643964+0000","last_peered":"2023-09-23T21:09:18.643964+0000","last_clean":"2023-09-23T21:09:18.643964+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:09:18.643964+0000","last_undegraded":"2023-09-23T21:09:18.643964+0000","last_fullsized":"2023-09-23T21:09:18.643964+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"421'11050","reported_seq":16106,"reported_epoch":422,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:30.936755+0000","last_change":"2023-09-23T21:08:40.299610+0000","last_active":"2023-09-23T21:09:30.936755+0000","last_peered":"2023-09-23T21:09:30.936755+0000","last_clean":"2023-09-23T21:04:28.742301+0000","last_became_active":"2023-09-23T21:04:31.370892+0000","last_became_peered":"2023-09-23T21:04:31.370892+0000","last_unstale":"2023-09-23T21:09:30.936755+0000","last_undegraded":"2023-09-23T21:05:24.143631+0000","last_fullsized":"2023-09-23T21:04:30.768848+0000","mapping_epoch":369,"log_start":"407'6968","ondisk_log_start":"407'6968","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4082,"log_dups_size":0,"ondisk_log_size":4082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.554303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11050,"num_write_kb":353600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2477,"num_bytes_recovered":116195328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6","objects":-3045}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":63,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.634370+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:09:18.634370+0000","last_peered":"2023-09-23T21:09:18.634370+0000","last_clean":"2023-09-23T21:09:18.634370+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:09:18.634370+0000","last_undegraded":"2023-09-23T21:09:18.634370+0000","last_fullsized":"2023-09-23T21:09:18.634370+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":60,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T21:09:29.139965+0000","last_change":"2023-09-23T21:04:58.426933+0000","last_active":"2023-09-23T21:09:29.139965+0000","last_peered":"2023-09-23T21:09:29.139965+0000","last_clean":"2023-09-23T21:09:29.139965+0000","last_became_active":"2023-09-23T21:04:58.425437+0000","last_became_peered":"2023-09-23T21:04:58.425437+0000","last_unstale":"2023-09-23T21:09:29.139965+0000","last_undegraded":"2023-09-23T21:09:29.139965+0000","last_fullsized":"2023-09-23T21:09:29.139965+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:42.142780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"421'11308","reported_seq":22819,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.514520+0000","last_change":"2023-09-23T21:06:29.463527+0000","last_active":"2023-09-23T21:09:28.514520+0000","last_peered":"2023-09-23T21:09:28.514520+0000","last_clean":"2023-09-23T21:09:28.514520+0000","last_became_active":"2023-09-23T21:06:29.460421+0000","last_became_peered":"2023-09-23T21:06:29.460421+0000","last_unstale":"2023-09-23T21:09:28.514520+0000","last_undegraded":"2023-09-23T21:09:28.514520+0000","last_fullsized":"2023-09-23T21:09:28.514520+0000","mapping_epoch":400,"log_start":"408'7199","ondisk_log_start":"408'7199","created":343,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4109,"log_dups_size":0,"ondisk_log_size":4109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:34.142675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11308,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":80,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.635402+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:09:18.635402+0000","last_peered":"2023-09-23T21:09:18.635402+0000","last_clean":"2023-09-23T21:09:18.635402+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:09:18.635402+0000","last_undegraded":"2023-09-23T21:09:18.635402+0000","last_fullsized":"2023-09-23T21:09:18.635402+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":63,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.634180+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:09:18.634180+0000","last_peered":"2023-09-23T21:09:18.634180+0000","last_clean":"2023-09-23T21:09:18.634180+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:09:18.634180+0000","last_undegraded":"2023-09-23T21:09:18.634180+0000","last_fullsized":"2023-09-23T21:09:18.634180+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"421'10982","reported_seq":22107,"reported_epoch":421,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:28.515653+0000","last_change":"2023-09-23T21:06:45.516402+0000","last_active":"2023-09-23T21:09:28.515653+0000","last_peered":"2023-09-23T21:09:28.515653+0000","last_clean":"2023-09-23T21:03:37.387209+0000","last_became_active":"2023-09-23T21:06:45.506603+0000","last_became_peered":"2023-09-23T21:06:45.506603+0000","last_unstale":"2023-09-23T21:09:28.515653+0000","last_undegraded":"2023-09-23T21:09:28.515653+0000","last_fullsized":"2023-09-23T21:09:28.515653+0000","mapping_epoch":403,"log_start":"349'2088","ondisk_log_start":"349'2088","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8894,"log_dups_size":0,"ondisk_log_size":8894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:11.444898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10982,"num_write_kb":351424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5487,"num_bytes_recovered":359596032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0,1],"avail_no_missing":["2","0","1","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":348,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.643966+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:09:18.643966+0000","last_peered":"2023-09-23T21:09:18.643966+0000","last_clean":"2023-09-23T21:09:18.643966+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:09:18.643966+0000","last_undegraded":"2023-09-23T21:09:18.643966+0000","last_fullsized":"2023-09-23T21:09:18.643966+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":433,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.652655+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:09:18.652655+0000","last_peered":"2023-09-23T21:09:18.652655+0000","last_clean":"2023-09-23T21:09:18.652655+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:09:18.652655+0000","last_undegraded":"2023-09-23T21:09:18.652655+0000","last_fullsized":"2023-09-23T21:09:18.652655+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"421'10904","reported_seq":11012,"reported_epoch":421,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:28.542682+0000","last_change":"2023-09-23T21:04:58.574123+0000","last_active":"2023-09-23T21:09:28.542682+0000","last_peered":"2023-09-23T21:09:28.542682+0000","last_clean":"2023-09-23T21:04:31.477353+0000","last_became_active":"2023-09-23T21:04:58.570178+0000","last_became_peered":"2023-09-23T21:04:58.570178+0000","last_unstale":"2023-09-23T21:09:28.542682+0000","last_undegraded":"2023-09-23T21:09:28.542682+0000","last_fullsized":"2023-09-23T21:09:28.542682+0000","mapping_epoch":374,"log_start":"407'6794","ondisk_log_start":"407'6794","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4110,"log_dups_size":0,"ondisk_log_size":4110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:13.836102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10904,"num_write_kb":348928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,2,1],"avail_no_missing":["6","0","1","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":48,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.634858+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:09:07.634858+0000","last_peered":"2023-09-23T21:09:07.634858+0000","last_clean":"2023-09-23T21:09:07.634858+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:09:07.634858+0000","last_undegraded":"2023-09-23T21:09:07.634858+0000","last_fullsized":"2023-09-23T21:09:07.634858+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":461,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.644483+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:09:18.644483+0000","last_peered":"2023-09-23T21:09:18.644483+0000","last_clean":"2023-09-23T21:09:18.644483+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:09:18.644483+0000","last_undegraded":"2023-09-23T21:09:18.644483+0000","last_fullsized":"2023-09-23T21:09:18.644483+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"421'10880","reported_seq":13872,"reported_epoch":421,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:09:26.163021+0000","last_change":"2023-09-23T21:08:40.308232+0000","last_active":"2023-09-23T21:09:26.163021+0000","last_peered":"2023-09-23T21:09:26.163021+0000","last_clean":"2023-09-23T21:04:28.743706+0000","last_became_active":"2023-09-23T21:04:41.636090+0000","last_became_peered":"2023-09-23T21:04:41.636090+0000","last_unstale":"2023-09-23T21:09:26.163021+0000","last_undegraded":"2023-09-23T21:05:01.715888+0000","last_fullsized":"2023-09-23T21:04:41.398952+0000","mapping_epoch":371,"log_start":"404'6094","ondisk_log_start":"404'6094","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4786,"log_dups_size":0,"ondisk_log_size":4786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:01.429611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5373952,"num_objects":82,"num_object_clones":0,"num_object_copies":246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4020,"num_objects_misplaced":21,"num_objects_unfound":0,"num_objects_dirty":82,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10880,"num_write_kb":350784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1440,"num_bytes_recovered":94371840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,6],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6","objects":-3959},{"shards":"1,6,7","objects":21}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.637906+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:09:18.637906+0000","last_peered":"2023-09-23T21:09:18.637906+0000","last_clean":"2023-09-23T21:09:18.637906+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:09:18.637906+0000","last_undegraded":"2023-09-23T21:09:18.637906+0000","last_fullsized":"2023-09-23T21:09:18.637906+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":210,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:16.159050+0000","last_change":"2023-09-23T21:05:24.128028+0000","last_active":"2023-09-23T21:09:16.159050+0000","last_peered":"2023-09-23T21:09:16.159050+0000","last_clean":"2023-09-23T21:09:16.159050+0000","last_became_active":"2023-09-23T21:05:24.127746+0000","last_became_peered":"2023-09-23T21:05:24.127746+0000","last_unstale":"2023-09-23T21:09:16.159050+0000","last_undegraded":"2023-09-23T21:09:16.159050+0000","last_fullsized":"2023-09-23T21:09:16.159050+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:27.959573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"421'11030","reported_seq":16537,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:28.547188+0000","last_change":"2023-09-23T21:04:07.394290+0000","last_active":"2023-09-23T21:09:28.547188+0000","last_peered":"2023-09-23T21:09:28.547188+0000","last_clean":"2023-09-23T21:09:28.547188+0000","last_became_active":"2023-09-23T21:04:07.388701+0000","last_became_peered":"2023-09-23T21:04:07.388701+0000","last_unstale":"2023-09-23T21:09:28.547188+0000","last_undegraded":"2023-09-23T21:09:28.547188+0000","last_fullsized":"2023-09-23T21:09:28.547188+0000","mapping_epoch":364,"log_start":"349'2109","ondisk_log_start":"349'2109","created":343,"last_epoch_clean":365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8921,"log_dups_size":0,"ondisk_log_size":8921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:23.783164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11030,"num_write_kb":352897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2694,"num_bytes_recovered":176553984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":316,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.627942+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:09:07.627942+0000","last_peered":"2023-09-23T21:09:07.627942+0000","last_clean":"2023-09-23T21:09:07.627942+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:09:07.627942+0000","last_undegraded":"2023-09-23T21:09:07.627942+0000","last_fullsized":"2023-09-23T21:09:07.627942+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.628008+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:09:07.628008+0000","last_peered":"2023-09-23T21:09:07.628008+0000","last_clean":"2023-09-23T21:09:07.628008+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:09:07.628008+0000","last_undegraded":"2023-09-23T21:09:07.628008+0000","last_fullsized":"2023-09-23T21:09:07.628008+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":72,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.642341+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:09:18.642341+0000","last_peered":"2023-09-23T21:09:18.642341+0000","last_clean":"2023-09-23T21:09:18.642341+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:09:18.642341+0000","last_undegraded":"2023-09-23T21:09:18.642341+0000","last_fullsized":"2023-09-23T21:09:18.642341+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"421'11026","reported_seq":22204,"reported_epoch":421,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:28.528305+0000","last_change":"2023-09-23T21:08:42.553079+0000","last_active":"2023-09-23T21:09:28.528305+0000","last_peered":"2023-09-23T21:09:28.528305+0000","last_clean":"2023-09-23T21:03:37.385511+0000","last_became_active":"2023-09-23T21:08:42.521706+0000","last_became_peered":"2023-09-23T21:08:42.521706+0000","last_unstale":"2023-09-23T21:09:28.528305+0000","last_undegraded":"2023-09-23T21:09:28.528305+0000","last_fullsized":"2023-09-23T21:09:28.528305+0000","mapping_epoch":417,"log_start":"367'5457","ondisk_log_start":"367'5457","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":5569,"log_dups_size":0,"ondisk_log_size":5569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:45.707578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11026,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5513,"num_bytes_recovered":361299968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[2,1,6],"avail_no_missing":["2","0","1","3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":397,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.619133+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:09:07.619133+0000","last_peered":"2023-09-23T21:09:07.619133+0000","last_clean":"2023-09-23T21:09:07.619133+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:09:07.619133+0000","last_undegraded":"2023-09-23T21:09:07.619133+0000","last_fullsized":"2023-09-23T21:09:07.619133+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":372,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.644486+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:09:18.644486+0000","last_peered":"2023-09-23T21:09:18.644486+0000","last_clean":"2023-09-23T21:09:18.644486+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:09:18.644486+0000","last_undegraded":"2023-09-23T21:09:18.644486+0000","last_fullsized":"2023-09-23T21:09:18.644486+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":54,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.644331+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:09:18.644331+0000","last_peered":"2023-09-23T21:09:18.644331+0000","last_clean":"2023-09-23T21:09:18.644331+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:09:18.644331+0000","last_undegraded":"2023-09-23T21:09:18.644331+0000","last_fullsized":"2023-09-23T21:09:18.644331+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"421'11062","reported_seq":11263,"reported_epoch":421,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:28.542423+0000","last_change":"2023-09-23T21:08:40.306414+0000","last_active":"2023-09-23T21:09:28.542423+0000","last_peered":"2023-09-23T21:09:28.542423+0000","last_clean":"2023-09-23T21:04:31.455592+0000","last_became_active":"2023-09-23T21:05:34.233778+0000","last_became_peered":"2023-09-23T21:05:34.233778+0000","last_unstale":"2023-09-23T21:09:28.542423+0000","last_undegraded":"2023-09-23T21:09:28.542423+0000","last_fullsized":"2023-09-23T21:09:28.542423+0000","mapping_epoch":386,"log_start":"349'2173","ondisk_log_start":"349'2173","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":8889,"log_dups_size":0,"ondisk_log_size":8889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:54.990401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11062,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[2,6,4],"avail_no_missing":["2","0","3","4","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":400,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:24.427394+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:09:24.427394+0000","last_peered":"2023-09-23T21:09:24.427394+0000","last_clean":"2023-09-23T21:09:24.427394+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:09:24.427394+0000","last_undegraded":"2023-09-23T21:09:24.427394+0000","last_fullsized":"2023-09-23T21:09:24.427394+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":316,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.622295+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:09:07.622295+0000","last_peered":"2023-09-23T21:09:07.622295+0000","last_clean":"2023-09-23T21:09:07.622295+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:09:07.622295+0000","last_undegraded":"2023-09-23T21:09:07.622295+0000","last_fullsized":"2023-09-23T21:09:07.622295+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":53,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.625558+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:09:07.625558+0000","last_peered":"2023-09-23T21:09:07.625558+0000","last_clean":"2023-09-23T21:09:07.625558+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:09:07.625558+0000","last_undegraded":"2023-09-23T21:09:07.625558+0000","last_fullsized":"2023-09-23T21:09:07.625558+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"421'10874","reported_seq":17142,"reported_epoch":421,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:28.541459+0000","last_change":"2023-09-23T21:05:35.197637+0000","last_active":"2023-09-23T21:09:28.541459+0000","last_peered":"2023-09-23T21:09:28.541459+0000","last_clean":"2023-09-23T21:03:37.358495+0000","last_became_active":"2023-09-23T21:05:35.194590+0000","last_became_peered":"2023-09-23T21:05:35.194590+0000","last_unstale":"2023-09-23T21:09:28.541459+0000","last_undegraded":"2023-09-23T21:09:28.541459+0000","last_fullsized":"2023-09-23T21:09:28.541459+0000","mapping_epoch":387,"log_start":"407'6780","ondisk_log_start":"407'6780","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:17.908264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10874,"num_write_kb":347968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3071,"num_bytes_recovered":201261056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[6,5,2],"avail_no_missing":["6","1","2","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":238,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:24.427451+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:09:24.427451+0000","last_peered":"2023-09-23T21:09:24.427451+0000","last_clean":"2023-09-23T21:09:24.427451+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:09:24.427451+0000","last_undegraded":"2023-09-23T21:09:24.427451+0000","last_fullsized":"2023-09-23T21:09:24.427451+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":682,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.633074+0000","last_change":"2023-09-23T21:08:41.538586+0000","last_active":"2023-09-23T21:09:18.633074+0000","last_peered":"2023-09-23T21:09:18.633074+0000","last_clean":"2023-09-23T21:09:18.633074+0000","last_became_active":"2023-09-23T21:08:41.538107+0000","last_became_peered":"2023-09-23T21:08:41.538107+0000","last_unstale":"2023-09-23T21:09:18.633074+0000","last_undegraded":"2023-09-23T21:09:18.633074+0000","last_fullsized":"2023-09-23T21:09:18.633074+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:48.268707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":72,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.636414+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:09:18.636414+0000","last_peered":"2023-09-23T21:09:18.636414+0000","last_clean":"2023-09-23T21:09:18.636414+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:09:18.636414+0000","last_undegraded":"2023-09-23T21:09:18.636414+0000","last_fullsized":"2023-09-23T21:09:18.636414+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":310,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.644793+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:09:18.644793+0000","last_peered":"2023-09-23T21:09:18.644793+0000","last_clean":"2023-09-23T21:09:18.644793+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:09:18.644793+0000","last_undegraded":"2023-09-23T21:09:18.644793+0000","last_fullsized":"2023-09-23T21:09:18.644793+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"421'11122","reported_seq":11246,"reported_epoch":421,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:09:28.536614+0000","last_change":"2023-09-23T21:04:58.535597+0000","last_active":"2023-09-23T21:09:28.536614+0000","last_peered":"2023-09-23T21:09:28.536614+0000","last_clean":"2023-09-23T21:04:28.746613+0000","last_became_active":"2023-09-23T21:04:58.531676+0000","last_became_peered":"2023-09-23T21:04:58.531676+0000","last_unstale":"2023-09-23T21:09:28.536614+0000","last_undegraded":"2023-09-23T21:09:28.536614+0000","last_fullsized":"2023-09-23T21:09:28.536614+0000","mapping_epoch":374,"log_start":"408'7005","ondisk_log_start":"408'7005","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4117,"log_dups_size":0,"ondisk_log_size":4117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:06.891972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11122,"num_write_kb":355904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[6,1,7],"avail_no_missing":["6","0","1","3","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":54,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:16.158793+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:09:16.158793+0000","last_peered":"2023-09-23T21:09:16.158793+0000","last_clean":"2023-09-23T21:09:16.158793+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:09:16.158793+0000","last_undegraded":"2023-09-23T21:09:16.158793+0000","last_fullsized":"2023-09-23T21:09:16.158793+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.633418+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:09:18.633418+0000","last_peered":"2023-09-23T21:09:18.633418+0000","last_clean":"2023-09-23T21:09:18.633418+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:09:18.633418+0000","last_undegraded":"2023-09-23T21:09:18.633418+0000","last_fullsized":"2023-09-23T21:09:18.633418+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":60,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:19.249533+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:09:19.249533+0000","last_peered":"2023-09-23T21:09:19.249533+0000","last_clean":"2023-09-23T21:09:19.249533+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:09:19.249533+0000","last_undegraded":"2023-09-23T21:09:19.249533+0000","last_fullsized":"2023-09-23T21:09:19.249533+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"421'10934","reported_seq":21892,"reported_epoch":421,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:09:26.176786+0000","last_change":"2023-09-23T21:05:25.364916+0000","last_active":"2023-09-23T21:09:26.176786+0000","last_peered":"2023-09-23T21:09:26.176786+0000","last_clean":"2023-09-23T21:04:28.738580+0000","last_became_active":"2023-09-23T21:05:25.353916+0000","last_became_peered":"2023-09-23T21:05:25.353916+0000","last_unstale":"2023-09-23T21:09:26.176786+0000","last_undegraded":"2023-09-23T21:05:25.249729+0000","last_fullsized":"2023-09-23T21:09:26.176786+0000","mapping_epoch":383,"log_start":"405'6226","ondisk_log_start":"405'6226","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4708,"log_dups_size":0,"ondisk_log_size":4708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.083623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6029312,"num_objects":92,"num_object_clones":0,"num_object_copies":276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":87,"num_objects_misplaced":5,"num_objects_unfound":0,"num_objects_dirty":92,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10934,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5423,"num_bytes_recovered":355401728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,5],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,5","objects":5},{"shards":"1,2,7","objects":87}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":60,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.634761+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:09:18.634761+0000","last_peered":"2023-09-23T21:09:18.634761+0000","last_clean":"2023-09-23T21:09:18.634761+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:09:18.634761+0000","last_undegraded":"2023-09-23T21:09:18.634761+0000","last_fullsized":"2023-09-23T21:09:18.634761+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":318,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.629132+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:09:07.629132+0000","last_peered":"2023-09-23T21:09:07.629132+0000","last_clean":"2023-09-23T21:09:07.629132+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:09:07.629132+0000","last_undegraded":"2023-09-23T21:09:07.629132+0000","last_fullsized":"2023-09-23T21:09:07.629132+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":96,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.627779+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:09:07.627779+0000","last_peered":"2023-09-23T21:09:07.627779+0000","last_clean":"2023-09-23T21:09:07.627779+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:09:07.627779+0000","last_undegraded":"2023-09-23T21:09:07.627779+0000","last_fullsized":"2023-09-23T21:09:07.627779+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"421'10780","reported_seq":23681,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:26.131482+0000","last_change":"2023-09-23T21:06:24.681796+0000","last_active":"2023-09-23T21:09:26.131482+0000","last_peered":"2023-09-23T21:09:26.131482+0000","last_clean":"2023-09-23T21:09:26.131482+0000","last_became_active":"2023-09-23T21:06:24.677813+0000","last_became_peered":"2023-09-23T21:06:24.677813+0000","last_unstale":"2023-09-23T21:09:26.131482+0000","last_undegraded":"2023-09-23T21:09:26.131482+0000","last_fullsized":"2023-09-23T21:09:26.131482+0000","mapping_epoch":397,"log_start":"404'6045","ondisk_log_start":"404'6045","created":343,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4735,"log_dups_size":0,"ondisk_log_size":4735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:36.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5505024,"num_objects":84,"num_object_clones":0,"num_object_copies":252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10780,"num_write_kb":347648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6392,"num_bytes_recovered":418906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":80,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.635007+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:09:18.635007+0000","last_peered":"2023-09-23T21:09:18.635007+0000","last_clean":"2023-09-23T21:09:18.635007+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:09:18.635007+0000","last_undegraded":"2023-09-23T21:09:18.635007+0000","last_fullsized":"2023-09-23T21:09:18.635007+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":53,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.620903+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:09:07.620903+0000","last_peered":"2023-09-23T21:09:07.620903+0000","last_clean":"2023-09-23T21:09:07.620903+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:09:07.620903+0000","last_undegraded":"2023-09-23T21:09:07.620903+0000","last_fullsized":"2023-09-23T21:09:07.620903+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.637705+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:09:18.637705+0000","last_peered":"2023-09-23T21:09:18.637705+0000","last_clean":"2023-09-23T21:09:18.637705+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:09:18.637705+0000","last_undegraded":"2023-09-23T21:09:18.637705+0000","last_fullsized":"2023-09-23T21:09:18.637705+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"421'11129","reported_seq":22316,"reported_epoch":421,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:09:26.156549+0000","last_change":"2023-09-23T21:05:24.428394+0000","last_active":"2023-09-23T21:09:26.156549+0000","last_peered":"2023-09-23T21:09:26.156549+0000","last_clean":"2023-09-23T21:04:28.693694+0000","last_became_active":"2023-09-23T21:05:24.345127+0000","last_became_peered":"2023-09-23T21:05:24.345127+0000","last_unstale":"2023-09-23T21:09:26.156549+0000","last_undegraded":"2023-09-23T21:05:24.328308+0000","last_fullsized":"2023-09-23T21:09:26.156549+0000","mapping_epoch":382,"log_start":"405'6412","ondisk_log_start":"405'6412","created":343,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:02:59.493331+0000","last_clean_scrub_stamp":"2023-09-23T21:02:59.493331+0000","objects_scrubbed":0,"log_size":4717,"log_dups_size":0,"ondisk_log_size":4717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:25.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4521984,"num_objects":69,"num_object_clones":0,"num_object_copies":207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":66,"num_objects_misplaced":72,"num_objects_unfound":0,"num_objects_dirty":69,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11129,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5530,"num_bytes_recovered":362414080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[1,6,2],"avail_no_missing":["1","3","6","7"],"object_location_counts":[{"shards":"1,2,6","objects":3},{"shards":"1,6,7","objects":66}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":318,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.644707+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:09:18.644707+0000","last_peered":"2023-09-23T21:09:18.644707+0000","last_clean":"2023-09-23T21:09:18.644707+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:09:18.644707+0000","last_undegraded":"2023-09-23T21:09:18.644707+0000","last_fullsized":"2023-09-23T21:09:18.644707+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:19.249555+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:09:19.249555+0000","last_peered":"2023-09-23T21:09:19.249555+0000","last_clean":"2023-09-23T21:09:19.249555+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:09:19.249555+0000","last_undegraded":"2023-09-23T21:09:19.249555+0000","last_fullsized":"2023-09-23T21:09:19.249555+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":106,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.627566+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:09:07.627566+0000","last_peered":"2023-09-23T21:09:07.627566+0000","last_clean":"2023-09-23T21:09:07.627566+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:09:07.627566+0000","last_undegraded":"2023-09-23T21:09:07.627566+0000","last_fullsized":"2023-09-23T21:09:07.627566+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":68,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.634302+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:09:18.634302+0000","last_peered":"2023-09-23T21:09:18.634302+0000","last_clean":"2023-09-23T21:09:18.634302+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:09:18.634302+0000","last_undegraded":"2023-09-23T21:09:18.634302+0000","last_fullsized":"2023-09-23T21:09:18.634302+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":446,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:24.427468+0000","last_change":"2023-09-23T21:04:41.456569+0000","last_active":"2023-09-23T21:09:24.427468+0000","last_peered":"2023-09-23T21:09:24.427468+0000","last_clean":"2023-09-23T21:09:24.427468+0000","last_became_active":"2023-09-23T21:04:41.456258+0000","last_became_peered":"2023-09-23T21:04:41.456258+0000","last_unstale":"2023-09-23T21:09:24.427468+0000","last_undegraded":"2023-09-23T21:09:24.427468+0000","last_fullsized":"2023-09-23T21:09:24.427468+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:33.110294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":83,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.635727+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:09:07.635727+0000","last_peered":"2023-09-23T21:09:07.635727+0000","last_clean":"2023-09-23T21:09:07.635727+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:09:07.635727+0000","last_undegraded":"2023-09-23T21:09:07.635727+0000","last_fullsized":"2023-09-23T21:09:07.635727+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":88,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.635469+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:09:18.635469+0000","last_peered":"2023-09-23T21:09:18.635469+0000","last_clean":"2023-09-23T21:09:18.635469+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:09:18.635469+0000","last_undegraded":"2023-09-23T21:09:18.635469+0000","last_fullsized":"2023-09-23T21:09:18.635469+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":303,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.645283+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:09:18.645283+0000","last_peered":"2023-09-23T21:09:18.645283+0000","last_clean":"2023-09-23T21:09:18.645283+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:09:18.645283+0000","last_undegraded":"2023-09-23T21:09:18.645283+0000","last_fullsized":"2023-09-23T21:09:18.645283+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":80,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.636584+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:09:18.636584+0000","last_peered":"2023-09-23T21:09:18.636584+0000","last_clean":"2023-09-23T21:09:18.636584+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:09:18.636584+0000","last_undegraded":"2023-09-23T21:09:18.636584+0000","last_fullsized":"2023-09-23T21:09:18.636584+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":63,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.634762+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:09:18.634762+0000","last_peered":"2023-09-23T21:09:18.634762+0000","last_clean":"2023-09-23T21:09:18.634762+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:09:18.634762+0000","last_undegraded":"2023-09-23T21:09:18.634762+0000","last_fullsized":"2023-09-23T21:09:18.634762+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":71,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.634034+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:09:18.634034+0000","last_peered":"2023-09-23T21:09:18.634034+0000","last_clean":"2023-09-23T21:09:18.634034+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:09:18.634034+0000","last_undegraded":"2023-09-23T21:09:18.634034+0000","last_fullsized":"2023-09-23T21:09:18.634034+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":66,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T21:09:18.633672+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:09:18.633672+0000","last_peered":"2023-09-23T21:09:18.633672+0000","last_clean":"2023-09-23T21:09:18.633672+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:09:18.633672+0000","last_undegraded":"2023-09-23T21:09:18.633672+0000","last_fullsized":"2023-09-23T21:09:18.633672+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":48,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T21:09:07.635504+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:09:07.635504+0000","last_peered":"2023-09-23T21:09:07.635504+0000","last_clean":"2023-09-23T21:09:07.635504+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:09:07.635504+0000","last_undegraded":"2023-09-23T21:09:07.635504+0000","last_fullsized":"2023-09-23T21:09:07.635504+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":30932992,"num_objects":472,"num_object_clones":0,"num_object_copies":1416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12833,"num_objects_misplaced":177,"num_objects_unfound":0,"num_objects_dirty":472,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175902,"num_write_kb":5643905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45315,"num_bytes_recovered":2923626496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3637579776,"data_stored":3637575704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85301,"ondisk_log_size":85301,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201769,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463390,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":561700,"kb_used_data":5020,"kb_used_omap":0,"kb_used_meta":556672,"kb_avail":93183452,"statfs":{"total":95995035648,"available":95419854848,"internally_reserved":0,"allocated":5140480,"data_stored":3893864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":570032128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":2.181,"15min":1.958},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.3849999999999998,"5min":99.533000000000001,"15min":102.242},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.7809999999999999,"5min":2.3439999999999999,"15min":1.7769999999999999},"min":{"1min":0.34300000000000003,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":6.04,"5min":99.325000000000003,"15min":102.502},"last":0.316}]},{"osd":1,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7440000000000002,"5min":3.3039999999999998,"15min":2.5760000000000001},"min":{"1min":0.29599999999999999,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":7.915,"5min":119.077,"15min":119.077},"last":0.27800000000000002},{"interface":"front","average":{"1min":1.9319999999999999,"5min":3.1040000000000001,"15min":2.5939999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.0229999999999997,"5min":119.313,"15min":119.313},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":5.1950000000000003,"15min":3.7799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.5470000000000006,"5min":199.23400000000001,"15min":199.23400000000001},"last":0.443},{"interface":"front","average":{"1min":2.0870000000000002,"5min":5.1689999999999996,"15min":3.8319999999999999},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.21199999999999999},"max":{"1min":6.3019999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.339,"15min":2.7229999999999999},"min":{"1min":0.31,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":13.340999999999999,"5min":13.340999999999999,"15min":178.40000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.7509999999999999,"15min":2.7040000000000002},"min":{"1min":0.53100000000000003,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":7.7759999999999998,"5min":13.319000000000001,"15min":178.26599999999999},"last":0.47199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.423,"15min":0.90700000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.24299999999999999},"max":{"1min":6.4989999999999997,"5min":13.1,"15min":13.1},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.0269999999999999,"15min":0.871},"min":{"1min":0.31,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":5.9580000000000002,"5min":6.0250000000000004,"15min":8.6859999999999999},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:08:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.333,"15min":0.94799999999999995},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.1819999999999999,"5min":11.281000000000001,"15min":11.281000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.208,"15min":1.05},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9279999999999999,"5min":8.6229999999999993,"15min":9.5389999999999997},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.1080000000000001,"15min":0.97499999999999998},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":6.6299999999999999,"5min":7.8079999999999998,"15min":11.388999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.0669999999999999,"15min":1.0760000000000001},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.30099999999999999},"max":{"1min":6.4749999999999996,"5min":7.1849999999999996,"15min":8.8010000000000002},"last":0.57399999999999995}]}]},{"osd":1,"up_from":276,"seq":1185410973835,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":943052,"kb_used_data":479428,"kb_used_omap":5,"kb_used_meta":463610,"kb_avail":92802100,"statfs":{"total":95995035648,"available":95029350400,"internally_reserved":0,"allocated":490934272,"data_stored":488821214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":474737379},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.274,"5min":2.4860000000000002,"15min":1.1259999999999999},"min":{"1min":0.27600000000000002,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":23.486000000000001,"5min":23.486000000000001,"15min":23.486000000000001},"last":0.23100000000000001},{"interface":"front","average":{"1min":3.9540000000000002,"5min":1.996,"15min":1.2310000000000001},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":16.148,"5min":16.148,"15min":18.047999999999998},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4459999999999997,"5min":2.556,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":21.695,"5min":21.695,"15min":78.409000000000006},"last":0.36599999999999999},{"interface":"front","average":{"1min":4.8369999999999997,"5min":2.6619999999999999,"15min":1.6140000000000001},"min":{"1min":0.502,"5min":0.245,"15min":0.22},"max":{"1min":23.539000000000001,"5min":23.539000000000001,"15min":76.245000000000005},"last":0.26300000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":2.4750000000000001,"15min":3.0379999999999998},"min":{"1min":0.39500000000000002,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":23.407,"5min":23.407,"15min":172.44399999999999},"last":21.498000000000001},{"interface":"front","average":{"1min":3.9260000000000002,"5min":2.173,"15min":3.1059999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":17.033999999999999,"5min":17.033999999999999,"15min":172.58500000000001},"last":21.565000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.631,"15min":1.137},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.5389999999999997,"5min":12.238,"15min":17.006},"last":0.378},{"interface":"front","average":{"1min":3.3530000000000002,"5min":2.0640000000000001,"15min":1.1180000000000001},"min":{"1min":0.42199999999999999,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":23.332999999999998,"5min":23.332999999999998,"15min":23.332999999999998},"last":0.314}]},{"osd":5,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":16.196000000000002,"5min":16.196000000000002,"15min":17.155000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.5339999999999998,"5min":1.7969999999999999,"15min":1.284},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.9660000000000002,"5min":12.345000000000001,"15min":17.076000000000001},"last":0.30299999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":1.8089999999999999,"15min":1.73},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.222000000000001,"5min":16.222000000000001,"15min":127.995},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.6160000000000001,"5min":1.962,"15min":1.758},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8959999999999999,"5min":12.276999999999999,"15min":126.836},"last":0.34699999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":3.2639999999999998,"15min":1.756},"min":{"1min":0.38100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.859,"5min":151.791,"15min":151.791},"last":0.40999999999999998},{"interface":"front","average":{"1min":3.4039999999999999,"5min":3.552,"15min":1.6579999999999999},"min":{"1min":0.45000000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":23.376000000000001,"5min":151.809,"15min":151.809},"last":0.44}]}]},{"osd":0,"up_from":92,"seq":395136991576,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":813136,"kb_used_data":46408,"kb_used_omap":5,"kb_used_meta":766714,"kb_avail":92932016,"statfs":{"total":95995035648,"available":95162384384,"internally_reserved":0,"allocated":47521792,"data_stored":45526788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":785115770},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.3210000000000002,"15min":2.0219999999999998},"min":{"1min":0.26300000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":18.140999999999998,"5min":52.590000000000003,"15min":61.341999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":4.5570000000000004,"5min":3.754,"15min":2.1059999999999999},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.219999999999999,"5min":52.276000000000003,"15min":61.299999999999997},"last":0.23300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":3.5539999999999998,"5min":3.0030000000000001,"15min":2.2130000000000001},"min":{"1min":0.35699999999999998,"5min":0.217,"15min":0.161},"max":{"1min":12.916,"5min":45.912999999999997,"15min":183.77099999999999},"last":127.791},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8210000000000002,"15min":2.3719999999999999},"min":{"1min":0.33500000000000002,"5min":0.254,"15min":0.214},"max":{"1min":9.1370000000000005,"5min":46.234000000000002,"15min":183.78700000000001},"last":127.953}]},{"osd":3,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.613,"5min":1.655,"15min":2.4750000000000001},"min":{"1min":0.36099999999999999,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":10.755000000000001,"5min":12.113,"15min":186.143},"last":0.309},{"interface":"front","average":{"1min":2.7719999999999998,"5min":1.5620000000000001,"15min":2.5099999999999998},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":7.6260000000000003,"5min":12.348000000000001,"15min":186.97399999999999},"last":0.32300000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.401,"15min":1.3009999999999999},"min":{"1min":0.34200000000000003,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.9450000000000003,"5min":10.814,"15min":10.814},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.171,"15min":1.012},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.629,"5min":10.629,"15min":11.493},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.5329999999999999,"15min":1.879},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.984,"5min":10.763,"15min":154.30600000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.373,"15min":1.5640000000000001},"min":{"1min":0.42099999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":10.685,"5min":11.478,"15min":154.42400000000001},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":6.0860000000000003,"15min":3.2360000000000002},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.710000000000001,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.105,"5min":6.0289999999999999,"15min":3.2810000000000001},"min":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":8.0269999999999992,"5min":169.767,"15min":169.767},"last":0.39100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":1.605,"15min":2.3119999999999998},"min":{"1min":0.47399999999999998,"5min":0.35199999999999998,"15min":0.33600000000000002},"max":{"1min":10.661,"5min":10.74,"15min":164.26400000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.4830000000000001,"5min":1.3280000000000001,"15min":2.306},"min":{"1min":0.41199999999999998,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":164.23099999999999},"last":0.441}]}]},{"osd":2,"up_from":129,"seq":554050781467,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":934460,"kb_used_data":226228,"kb_used_omap":7,"kb_used_meta":708216,"kb_avail":92810692,"statfs":{"total":95995035648,"available":95038148608,"internally_reserved":0,"allocated":231657472,"data_stored":229615962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":725213265},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7930000000000001,"5min":1.5169999999999999,"15min":1.423},"min":{"1min":0.222,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":11.699999999999999,"5min":11.699999999999999,"15min":26.356000000000002},"last":0.374},{"interface":"front","average":{"1min":3.6800000000000002,"5min":2.444,"15min":1.8560000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":16.974,"5min":23.963999999999999,"15min":26.341000000000001},"last":0.98699999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":3.3039999999999998,"15min":2.802},"min":{"1min":0.24299999999999999,"5min":0.19,"15min":0.188},"max":{"1min":11.271000000000001,"5min":67.296999999999997,"15min":212.12},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.7069999999999999,"5min":2.5960000000000001,"15min":2.4860000000000002},"min":{"1min":0.34200000000000003,"5min":0.214,"15min":0.214},"max":{"1min":11.840999999999999,"5min":67.355000000000004,"15min":213.02799999999999},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:08:48 2023","interfaces":[{"interface":"back","average":{"1min":4.4729999999999999,"5min":3.5760000000000001,"15min":3.3439999999999999},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":15.327999999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":6.6970000000000001},{"interface":"front","average":{"1min":4.1790000000000003,"5min":3.4279999999999999,"15min":3.2949999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.215},"max":{"1min":33.889000000000003,"5min":138.87700000000001,"15min":138.87700000000001},"last":1.3180000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.169,"15min":1.139},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26200000000000001},"max":{"1min":11.744,"5min":11.744,"15min":12.586},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.5179999999999998,"5min":2.028,"15min":1.46},"min":{"1min":0.35699999999999998,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":16.725999999999999,"5min":16.725999999999999,"15min":16.725999999999999},"last":0.73699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":3.8119999999999998,"5min":2.222,"15min":2.2130000000000001},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":16.673999999999999,"5min":16.673999999999999,"15min":173.34},"last":0.76800000000000002},{"interface":"front","average":{"1min":2.2480000000000002,"5min":1.4770000000000001,"15min":1.8380000000000001},"min":{"1min":0.33800000000000002,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":12.368,"5min":13.701000000000001,"15min":173.15000000000001},"last":0.92300000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":3.6800000000000002,"15min":1.9590000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.323,"5min":175.72499999999999,"15min":175.72499999999999},"last":1.006},{"interface":"front","average":{"1min":2.0230000000000001,"5min":3.3780000000000001,"15min":1.915},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":11.782,"5min":176.20400000000001,"15min":176.20400000000001},"last":3.4609999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:08:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":1.665,"15min":2.0579999999999998},"min":{"1min":0.42399999999999999,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":11.811999999999999,"5min":11.811999999999999,"15min":177.30799999999999},"last":1.2869999999999999},{"interface":"front","average":{"1min":3.6339999999999999,"5min":2.109,"15min":2.2410000000000001},"min":{"1min":0.38200000000000001,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":16.875,"5min":16.875,"15min":177.40899999999999},"last":0.84099999999999997}]}]},{"osd":3,"up_from":148,"seq":635655160061,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":602728,"kb_used_data":5536,"kb_used_omap":2,"kb_used_meta":597181,"kb_avail":93142424,"statfs":{"total":95995035648,"available":95377842176,"internally_reserved":0,"allocated":5668864,"data_stored":3412610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":611513397},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":1.984,"15min":1.4970000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.184},"max":{"1min":17.425999999999998,"5min":44.963000000000001,"15min":44.963000000000001},"last":10.098000000000001},{"interface":"front","average":{"1min":1.629,"5min":1.1080000000000001,"15min":1.302},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.19500000000000001},"max":{"1min":7.9699999999999998,"5min":9.7370000000000001,"15min":37.634999999999998},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":19.158999999999999,"5min":7.0279999999999996,"15min":3.0649999999999999},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.20999999999999999},"max":{"1min":166.56999999999999,"5min":166.56999999999999,"15min":166.56999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":17.228999999999999,"5min":7.0049999999999999,"15min":3.202},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.182},"max":{"1min":166.291,"5min":166.291,"15min":166.291},"last":0.434}]},{"osd":2,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":9.6780000000000008,"5min":3.2469999999999999,"15min":1.9259999999999999},"min":{"1min":0.23599999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":88.825999999999993,"5min":88.825999999999993,"15min":88.825999999999993},"last":10.124000000000001},{"interface":"front","average":{"1min":7.3529999999999998,"5min":2.5790000000000002,"15min":1.732},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":85.626000000000005,"5min":85.626000000000005,"15min":85.626000000000005},"last":1.7849999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":2.8900000000000001,"15min":2.347},"min":{"1min":0.374,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":70.728999999999999,"15min":111.36},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.605,"5min":2.7679999999999998,"15min":2.1429999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":17.286999999999999,"5min":70.712000000000003,"15min":112.473},"last":10.021000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2029999999999998,"5min":1.7989999999999999,"15min":1.5920000000000001},"min":{"1min":0.39400000000000002,"5min":0.35199999999999998,"15min":0.28499999999999998},"max":{"1min":7.9050000000000002,"5min":36.488999999999997,"15min":64.528000000000006},"last":0.58699999999999997},{"interface":"front","average":{"1min":3.278,"5min":2.101,"15min":1.724},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":12.382999999999999,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.54600000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":3.3220000000000001,"5min":1.8080000000000001,"15min":1.7050000000000001},"min":{"1min":0.36099999999999999,"5min":0.318,"15min":0.312},"max":{"1min":12.317,"5min":12.317,"15min":19.568000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.3570000000000002,"5min":1.722,"15min":1.522},"min":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":17.324999999999999,"5min":17.324999999999999,"15min":17.324999999999999},"last":10.066000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:08:39 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.427,"15min":2.488},"min":{"1min":0.377,"5min":0.377,"15min":0.30499999999999999},"max":{"1min":7.9459999999999997,"5min":74.019000000000005,"15min":215.70099999999999},"last":0.62},{"interface":"front","average":{"1min":3.46,"5min":2.7970000000000002,"15min":2.8879999999999999},"min":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.32900000000000001},"max":{"1min":12.407,"5min":73.364999999999995,"15min":222.11699999999999},"last":0.57399999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986679,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590748,"kb_used_data":10900,"kb_used_omap":0,"kb_used_meta":579840,"kb_avail":93154404,"statfs":{"total":95995035648,"available":95390109696,"internally_reserved":0,"allocated":11161600,"data_stored":9914156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":593756160},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.3560000000000001,"15min":2.2829999999999999},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":10.273999999999999,"5min":12.869,"15min":102.358},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.883,"5min":1.4790000000000001,"15min":2.29},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":8.3190000000000008,"5min":12.848000000000001,"15min":102.58199999999999},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":7.0010000000000003,"5min":6.1950000000000003,"15min":3.9399999999999999},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":42.887,"5min":127.411,"15min":127.411},"last":1.286},{"interface":"front","average":{"1min":7.258,"5min":4.3179999999999996,"15min":3.2770000000000001},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.253},"max":{"1min":44.148000000000003,"5min":118.075,"15min":118.075},"last":1.599}]},{"osd":2,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0299999999999998,"5min":5.6589999999999998,"15min":3.6699999999999999},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":7.4480000000000004,"5min":164.517,"15min":164.517},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.4419999999999999,"5min":4.7889999999999997,"15min":3.4769999999999999},"min":{"1min":0.35499999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":5.9400000000000004,"5min":164.46799999999999,"15min":164.46799999999999},"last":4.3860000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.3380000000000001,"15min":1.4359999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27300000000000002},"max":{"1min":6.9390000000000001,"5min":6.9390000000000001,"15min":60.720999999999997},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.3859999999999999,"15min":1.4870000000000001},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.25600000000000001},"max":{"1min":6.4020000000000001,"5min":8.2899999999999991,"15min":60.631999999999998},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 21:09:22 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.87,"15min":0.76900000000000002},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":4.9080000000000004,"5min":6.0069999999999997,"15min":8.7089999999999996},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.85699999999999998,"15min":0.74099999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.21099999999999999},"max":{"1min":4.7140000000000004,"5min":4.9740000000000002,"15min":7.0179999999999998},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.2949999999999999,"15min":0.95299999999999996},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":6.4269999999999996,"5min":11.800000000000001,"15min":11.887},"last":0.312},{"interface":"front","average":{"1min":1.2070000000000001,"5min":1.341,"15min":0.92900000000000005},"min":{"1min":0.39300000000000002,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":4.5449999999999999,"5min":8.7739999999999991,"15min":8.7739999999999991},"last":0.35999999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:09:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.036,"15min":1.4950000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.7359999999999998,"5min":13.667999999999999,"15min":145.721},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.2949999999999999,"15min":1.667},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.28000000000000003},"max":{"1min":10.411,"5min":10.411,"15min":148.45699999999999},"last":0.35199999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725043,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":645704,"kb_used_data":8576,"kb_used_omap":0,"kb_used_meta":637120,"kb_avail":93099448,"statfs":{"total":95995035648,"available":95333834752,"internally_reserved":0,"allocated":8781824,"data_stored":7550580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":652410880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.0660000000000001,"15min":1.319},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.2640000000000002,"5min":10.488,"15min":80.105999999999995},"last":2.464},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.1799999999999999,"15min":1.4590000000000001},"min":{"1min":0.27000000000000002,"5min":0.222,"15min":0.222},"max":{"1min":6.6619999999999999,"5min":8.2759999999999998,"15min":80.152000000000001},"last":4.1319999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:09:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5309999999999997,"5min":6.2519999999999998,"15min":3.5259999999999998},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":32.777999999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":2.5529999999999999},{"interface":"front","average":{"1min":6.5460000000000003,"5min":6.2060000000000004,"15min":3.4700000000000002},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.247},"max":{"1min":32.761000000000003,"5min":233.75200000000001,"15min":233.75200000000001},"last":2.5369999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":4.9640000000000004,"15min":2.794},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.254},"max":{"1min":70.810000000000002,"5min":231.34700000000001,"15min":231.34700000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":8.1370000000000005,"5min":5.4619999999999997,"15min":3.0289999999999999},"min":{"1min":0.40200000000000002,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":70.507000000000005,"5min":231.381,"15min":231.381},"last":0.29199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.6519999999999999,"15min":2.786},"min":{"1min":0.33000000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":12.765000000000001,"5min":12.765000000000001,"15min":153.60300000000001},"last":0.80100000000000005},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.3959999999999999,"15min":2.7130000000000001},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":9.6120000000000001,"5min":9.6120000000000001,"15min":153.774},"last":1.464}]},{"osd":4,"last update":"Sat Sep 23 21:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.64900000000000002,"15min":0.63700000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.247},"max":{"1min":1.946,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.0580000000000001,"15min":0.76500000000000001},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":5.0369999999999999,"5min":8.3740000000000006,"15min":10.792},"last":0.90500000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":1.1990000000000001,"15min":1.1259999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.1130000000000004,"5min":9.8480000000000008,"15min":71.230000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.252,"15min":1.163},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":3.4359999999999999,"5min":12.673999999999999,"15min":70.266000000000005},"last":0.38}]},{"osd":7,"last update":"Sat Sep 23 21:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.262,"15min":1.0289999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.312},"max":{"1min":4.3399999999999999,"5min":9.6329999999999991,"15min":31.091999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.69899999999999995,"15min":0.80100000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":2.0179999999999998,"5min":4.29,"15min":31.780000000000001},"last":0.441}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":41615360,"data_stored":41615360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":468910080,"data_stored":468910080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":226295808,"data_stored":226295808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":65536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2166784,"data_stored":2162712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1441792,"data_stored":1441792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":65536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2897018880,"data_stored":2897018880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:09:35.270 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:09:35.361 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:35.389 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:35 smithi121 ceph-mon[211526]: pgmap v1363: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 76 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 12325/72 objects degraded (17118.056%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T21:09:35.389 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:35 smithi121 ceph-mon[219709]: pgmap v1363: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 76 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 12325/72 objects degraded (17118.056%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T21:09:35.391 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:35.426 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:35.616 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:35.617 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:35.652 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:35.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:35.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:35.658 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:35 smithi123 ceph-mon[193903]: pgmap v1363: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 76 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 12325/72 objects degraded (17118.056%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T21:09:35.757 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:35.798 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:35.965 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:35.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:36.045 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:09:36.068 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:09:36.071 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:36.103 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:09:36.113 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:36.131 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:09:36.154 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:09:36.176 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:09:36.203 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:09:36.223 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.225 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:09:36.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3843566996' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:09:36.239 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:36 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:09:36.240 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/418057937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:36.241 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3843566996' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:09:36.241 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:36 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:09:36.241 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/418057937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:36.339 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_6' removed 2023-09-23T21:09:36.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3843566996' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:09:36.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:36 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:09:36.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/418057937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:36.742 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:36.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:36.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:36.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:36.874 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:37.289 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:09:37.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[211526]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[211526]: failed to encode map e423 with expected crc 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[211526]: failed to encode map e423 with expected crc 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 12325/72 objects degraded (17118.056%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[211526]: pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[219709]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[219709]: failed to encode map e423 with expected crc 2023-09-23T21:09:37.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[219709]: failed to encode map e423 with expected crc 2023-09-23T21:09:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 12325/72 objects degraded (17118.056%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:37 smithi121 ceph-mon[219709]: pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:37 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:09:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:37 smithi123 ceph-mon[193903]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T21:09:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:37 smithi123 ceph-mon[193903]: failed to encode map e423 with expected crc 2023-09-23T21:09:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:37 smithi123 ceph-mon[193903]: failed to encode map e423 with expected crc 2023-09-23T21:09:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 12325/72 objects degraded (17118.056%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:09:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:37 smithi123 ceph-mon[193903]: pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:37.818 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:38 smithi123 ceph-mon[193903]: failed to encode map e423 with expected crc 2023-09-23T21:09:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:38 smithi123 ceph-mon[193903]: failed to encode map e423 with expected crc 2023-09-23T21:09:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:38 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12325/72 objects degraded (17118.056%), 3 pgs degraded, 4 pgs undersized) 2023-09-23T21:09:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:38 smithi121 ceph-mon[219709]: failed to encode map e423 with expected crc 2023-09-23T21:09:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:38 smithi121 ceph-mon[219709]: failed to encode map e423 with expected crc 2023-09-23T21:09:38.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:38 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12325/72 objects degraded (17118.056%), 3 pgs degraded, 4 pgs undersized) 2023-09-23T21:09:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:38 smithi121 ceph-mon[211526]: failed to encode map e423 with expected crc 2023-09-23T21:09:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:38 smithi121 ceph-mon[211526]: failed to encode map e423 with expected crc 2023-09-23T21:09:38.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:38 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12325/72 objects degraded (17118.056%), 3 pgs degraded, 4 pgs undersized) 2023-09-23T21:09:38.701 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:09:38.701 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:09:38.701 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-09-23T21:09:38.702 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_7 16 2023-09-23T21:09:38.739 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:38.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:38.747 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:38.845 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:38.879 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:39.131 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:39.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:39.237 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:39.270 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:39.321 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:39.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:39.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:39 smithi121 ceph-mon[211526]: pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:39.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:39 smithi121 ceph-mon[219709]: pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:39.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:39.454 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:39 smithi123 ceph-mon[193903]: pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:39.840 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:09:39.840 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:09:39.840 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50073 11.b 11:da39f2df:::benchmark_data_smithi121_477989_object50072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 362.431740252, 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2188515999999998, 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50073 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.842 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405039+0000", 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405103+0000", 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3901999999999999e-05 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405111+0000", 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4460000000000001e-06 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405116+0000", 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0629999999999996e-06 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416824+0000", 2023-09-23T21:09:39.844 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011707555 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416833+0000", 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1540000000000008e-06 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400621+0000", 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98378816000000002 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.845 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400630+0000", 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6950000000000006e-06 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.622813+0000", 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.222183198 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.846 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.623891+0000", 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001077929 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50086 11.b 11:d8a7cc4c:::benchmark_data_smithi121_477989_object50085:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 362.43165331300003, 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2482345829999999, 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.847 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50086 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.848 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:09:39.849 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.849 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.849 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.849 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.849 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:09:39.849 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.850 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.850 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.850 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.850 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405126+0000", 2023-09-23T21:09:39.850 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.850 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.851 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.851 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.851 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405188+0000", 2023-09-23T21:09:39.851 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2147e-05 2023-09-23T21:09:39.851 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.851 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.851 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405196+0000", 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1799999999999996e-06 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.405201+0000", 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9339999999999997e-06 2023-09-23T21:09:39.852 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416857+0000", 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011656025 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.853 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.416865+0000", 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8329999999999994e-06 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400651+0000", 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98378581600000004 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.854 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.855 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.855 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400658+0000", 2023-09-23T21:09:39.855 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.08e-06 2023-09-23T21:09:39.855 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.855 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.855 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.855 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.623919+0000", 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2232614909999999 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.624169+0000", 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024978700000000001 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.624334+0000", 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165315 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.856 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631814+0000", 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.007479482 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634635+0000", 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028213819999999999 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.650585+0000", 2023-09-23T21:09:39.857 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015949393999999999 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.650633+0000", 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8591999999999998e-05 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653361+0000", 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002727602 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.858 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.859 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.859 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.859 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.859 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50100 11.3 11:c9d587e5:::benchmark_data_smithi121_477989_object50099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:09:39.859 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:09:39.859 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 362.40817999500001, 2023-09-23T21:09:39.859 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2031901700000001, 2023-09-23T21:09:39.860 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.860 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.860 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.860 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.860 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.860 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50100 2023-09-23T21:09:39.861 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.861 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.861 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.861 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.861 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:09:39.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.862 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.862 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.862 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.862 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428600+0000", 2023-09-23T21:09:39.862 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.863 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.863 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.863 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.863 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428599+0000", 2023-09-23T21:09:39.863 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428631+0000", 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1956999999999999e-05 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428639+0000", 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8269999999999998e-06 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.864 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428644+0000", 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3550000000000004e-06 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428697+0000", 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2799000000000001e-05 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.865 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.428706+0000", 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0659999999999993e-06 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.399543+0000", 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97083674099999995 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.866 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.399551+0000", 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5429999999999994e-06 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607385+0000", 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2078334150000001 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607502+0000", 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117125 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.867 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.607664+0000", 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016177800000000001 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.614069+0000", 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064056510000000001 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.868 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629191+0000", 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015121252 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629222+0000", 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1405000000000002e-05 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.629265+0000", 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2824000000000002e-05 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.869 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631790+0000", 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025248559999999998 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50110 11.b 11:d197f0fb:::benchmark_data_smithi121_477989_object50109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 362.402242747, 2023-09-23T21:09:39.870 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2210893860000001, 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50110 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.871 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434537+0000", 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434536+0000", 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.872 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434604+0000", 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8382999999999999e-05 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434616+0000", 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2139e-05 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434623+0000", 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6510000000000004e-06 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.873 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434802+0000", 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017905300000000001 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.434811+0000", 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3820000000000004e-06 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.874 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400678+0000", 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96586726099999998 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400684+0000", 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2689999999999996e-06 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.625756+0000", 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2250717149999999 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.625892+0000", 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000135933 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.626051+0000", 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015894499999999999 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.631821+0000", 2023-09-23T21:09:39.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0057699200000000004 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634669+0000", 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028480839999999999 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653436+0000", 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018767817999999999 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.653479+0000", 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2070000000000002e-05 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655626+0000", 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002147599 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50121 11.b 11:d6d11928:::benchmark_data_smithi121_477989_object50120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 362.211790194, 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.033318296, 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50121 2023-09-23T21:09:39.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624989+0000", 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.624988+0000", 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625021+0000", 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3223999999999999e-05 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625034+0000", 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3191e-05 2023-09-23T21:09:39.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625041+0000", 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1130000000000003e-06 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625217+0000", 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000175921 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.882 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625227+0000", 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8390000000000002e-06 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400706+0000", 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77547898800000004 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.400713+0000", 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9789999999999996e-06 2023-09-23T21:09:39.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627606+0000", 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.226892662 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627745+0000", 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013900300000000001 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:09:39.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.627884+0000", 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000138932 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634608+0000", 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0067245739999999997 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.885 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.634714+0000", 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010603300000000001 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655900+0000", 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021185137999999999 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.655945+0000", 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5346000000000003e-05 2023-09-23T21:09:39.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.658308+0000", 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002362762 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50132 11.a 11:593fc135:::benchmark_data_smithi121_477989_object50131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 362.19404301100002, 2023-09-23T21:09:39.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9071047490000002, 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50132 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642737+0000", 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642735+0000", 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642763+0000", 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7504e-05 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642777+0000", 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3684e-05 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642781+0000", 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9110000000000001e-06 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642843+0000", 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1890999999999994e-05 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.642855+0000", 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1999e-05 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.396214+0000", 2023-09-23T21:09:39.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75335825499999998 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.396226+0000", 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2177e-05 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540693+0000", 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.144467481 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540826+0000", 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013311 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.540929+0000", 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010215799999999999 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547150+0000", 2023-09-23T21:09:39.893 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062215100000000004 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547194+0000", 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3538999999999998e-05 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547204+0000", 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0873000000000001e-05 2023-09-23T21:09:39.894 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.547271+0000", 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6261999999999997e-05 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.549841+0000", 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025705569999999998 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88131 11.9 11:9f73b961:::benchmark_data_smithi121_477989_object88130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.745423+0000", 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 311.09135674100003, 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1797363559999998, 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88131 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.896 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745423+0000", 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745423+0000", 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745422+0000", 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745500+0000", 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8769000000000004e-05 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745526+0000", 2023-09-23T21:09:39.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5327e-05 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.745544+0000", 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8533999999999999e-05 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746670+0000", 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011258279999999999 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746810+0000", 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013983100000000001 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746968+0000", 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015768100000000001 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768727+0000", 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021759737000000001 2023-09-23T21:09:39.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768736+0000", 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3599999999999996e-06 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772896+0000", 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0041604820000001 2023-09-23T21:09:39.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772908+0000", 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1946e-05 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.906549+0000", 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.133640728 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.909440+0000", 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028911649999999998 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.924853+0000", 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015413377000000001 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.925159+0000", 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00030577099999999998 2023-09-23T21:09:39.903 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88138 11.9 11:97b0517a:::benchmark_data_smithi121_477989_object88137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 311.09012347800001, 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2300521670000002, 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.904 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88138 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:09:39.905 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746656+0000", 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746706+0000", 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0890000000000002e-05 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746713+0000", 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2639999999999997e-06 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.746717+0000", 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0899999999999998e-06 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.753050+0000", 2023-09-23T21:09:39.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0063330749999999996 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.753173+0000", 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012362899999999999 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,7", 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.753316+0000", 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000142149 2023-09-23T21:09:39.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768761+0000", 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015445296000000001 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768768+0000", 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.905e-06 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772942+0000", 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.004173762 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.772951+0000", 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4150000000000005e-06 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.909629+0000", 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1366782120000001 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.910655+0000", 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001025518 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.976619+0000", 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.065964751000000002 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:30.976708+0000", 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8727000000000006e-05 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88123 11.4 11:2a8ebd38:::benchmark_data_smithi121_477989_object88122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.759676+0000", 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 311.07710304699998, 2023-09-23T21:09:39.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6256372400000001, 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88123 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759676+0000", 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759676+0000", 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759675+0000", 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759724+0000", 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8857000000000003e-05 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759737+0000", 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3185e-05 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.759743+0000", 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4309999999999999e-06 2023-09-23T21:09:39.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768032+0000", 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0082888340000000001 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768043+0000", 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.039e-05 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767443+0000", 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99940036899999996 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767455+0000", 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2167000000000001e-05 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.383270+0000", 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6158152189999999 2023-09-23T21:09:39.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385314+0000", 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00204339 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88130 11.4 11:2d9f9be9:::benchmark_data_smithi121_477989_object88129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.767371+0000", 2023-09-23T21:09:39.918 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 311.06940832599997, 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.618074526, 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88130 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.919 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767371+0000", 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767371+0000", 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767370+0000", 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:09:39.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767417+0000", 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6795000000000002e-05 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767430+0000", 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3681e-05 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767437+0000", 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8730000000000001e-06 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768067+0000", 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00062932299999999999 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768074+0000", 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7039999999999994e-06 2023-09-23T21:09:39.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767486+0000", 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99941159099999999 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767495+0000", 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7420000000000003e-06 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385352+0000", 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6178574299999999 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385446+0000", 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3614000000000002e-05 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88143 11.4 11:20aacae1:::benchmark_data_smithi121_477989_object88142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:09:39.924 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 311.06933513299998, 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6459621049999997, 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88143 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.925 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767444+0000", 2023-09-23T21:09:39.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767481+0000", 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7512000000000003e-05 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767489+0000", 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5429999999999996e-06 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.767493+0000", 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.279e-06 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768835+0000", 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001341255 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.768843+0000", 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9899999999999997e-06 2023-09-23T21:09:39.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767519+0000", 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99867673400000001 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.767528+0000", 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3119999999999994e-06 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.385470+0000", 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.617942703 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.386142+0000", 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00067205699999999999 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.386575+0000", 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00043233799999999999 2023-09-23T21:09:39.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.402719+0000", 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016144479999999999 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.406453+0000", 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037342669999999999 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.409931+0000", 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034772359999999999 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.409993+0000", 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2419e-05 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.413406+0000", 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034134259999999998 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.932 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88145 11.4 11:238c6080:::benchmark_data_smithi121_477989_object88144:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.769668+0000", 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 311.06711167399999, 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6476349180000001, 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.933 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88145 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769668+0000", 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769668+0000", 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769667+0000", 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769698+0000", 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0843999999999998e-05 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769706+0000", 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6850000000000001e-06 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769711+0000", 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9289999999999997e-06 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769775+0000", 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4603000000000005e-05 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.769785+0000", 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1949999999999992e-06 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.770501+0000", 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0007162979999999 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:09:39.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.770513+0000", 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2418999999999999e-05 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.399922+0000", 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6294089870000001 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.400078+0000", 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015531400000000001 2023-09-23T21:09:39.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.400281+0000", 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000202995 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.406464+0000", 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061830230000000002 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.409889+0000", 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034254329999999999 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.413472+0000", 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035832640000000001 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.413534+0000", 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2169999999999996e-05 2023-09-23T21:09:39.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.417303+0000", 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003768291 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88155 11.d e370/364 11:b7e40311:::benchmark_data_smithi121_477989_object88154:head v 370'5510, mlcod=367'5508)", 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.067298+0000", 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 308.76948197399997, 2023-09-23T21:09:39.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.751661201000001, 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067298+0000", 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067298+0000", 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067297+0000", 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067345+0000", 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7917000000000002e-05 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067358+0000", 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2945000000000001e-05 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.067365+0000", 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7700000000000001e-06 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.811897+0000", 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.744531416000001 2023-09-23T21:09:39.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.811910+0000", 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2858e-05 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.818932+0000", 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070229439999999997 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.945 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.818959+0000", 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6258000000000002e-05 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88162 11.d e370/364 11:b0e3ae3a:::benchmark_data_smithi121_477989_object88161:head v 370'5511, mlcod=367'5508)", 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.408324+0000", 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 308.428455391, 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.412904649000001, 2023-09-23T21:09:39.946 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408324+0000", 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408324+0000", 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408323+0000", 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408368+0000", 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5136000000000003e-05 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408384+0000", 2023-09-23T21:09:39.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6011999999999998e-05 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.408391+0000", 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5039999999999999e-06 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.816146+0000", 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.407755393999999 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.816161+0000", 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4467999999999999e-05 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821197+0000", 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050365870000000004 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821229+0000", 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1568e-05 2023-09-23T21:09:39.950 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88168 11.d e370/364 11:b40d873b:::benchmark_data_smithi121_477989_object88167:head v 370'5512, mlcod=367'5508)", 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.427202+0000", 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 308.40957740699997, 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.394081789000001, 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.951 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427202+0000", 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427202+0000", 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427201+0000", 2023-09-23T21:09:39.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427246+0000", 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5252e-05 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427266+0000", 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9473000000000001e-05 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.953 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.427273+0000", 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.515e-06 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.817746+0000", 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.390472430999999 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.817758+0000", 2023-09-23T21:09:39.954 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2257e-05 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821251+0000", 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034928160000000001 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821284+0000", 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3104e-05 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.955 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88182 11.d e370/364 11:b50fd016:::benchmark_data_smithi121_477989_object88181:head v 370'5513, mlcod=367'5508)", 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 308.371756097, 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.356282992000001, 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:09:39.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465023+0000", 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465088+0000", 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5238000000000001e-05 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465102+0000", 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.396e-05 2023-09-23T21:09:39.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.465108+0000", 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.733e-06 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.818992+0000", 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.35388459 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.819016+0000", 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3479e-05 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821286+0000", 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022702270000000001 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821306+0000", 2023-09-23T21:09:39.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0452999999999999e-05 2023-09-23T21:09:39.961 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.961 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.961 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.962 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:88186 11.d e370/364 11:b93f9f29:::benchmark_data_smithi121_477989_object88185:head v 370'5514, mlcod=367'5508)", 2023-09-23T21:09:39.962 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:31.490693+0000", 2023-09-23T21:09:39.962 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 308.346086378, 2023-09-23T21:09:39.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.330656862000001, 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490693+0000", 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490693+0000", 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490692+0000", 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490727+0000", 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5203999999999997e-05 2023-09-23T21:09:39.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490743+0000", 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5498e-05 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:31.490749+0000", 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5200000000000003e-06 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.820123+0000", 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 18.3293739 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.820138+0000", 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4338999999999999e-05 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.966 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821337+0000", 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011991079999999999 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:49.821350+0000", 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3208e-05 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.967 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:155566 11.4 11:2734e66b:::benchmark_data_smithi121_477989_object67377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:08:47.141465+0000", 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 52.695314263999997, 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.27124920499999999, 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 155566 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141465+0000", 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141465+0000", 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141464+0000", 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141469+0000", 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8269999999999997e-06 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141482+0000", 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2619000000000001e-05 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141487+0000", 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9579999999999998e-06 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141520+0000", 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3424999999999999e-05 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141633+0000", 2023-09-23T21:09:39.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113132 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.141739+0000", 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000106088 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.143894+0000", 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021542670000000001 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.170336+0000", 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.026442164000000001 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.410571+0000", 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.240235426 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.410628+0000", 2023-09-23T21:09:39.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6894000000000001e-05 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.412714+0000", 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020863689999999998 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:155571 11.1 11:89ae14fc:::benchmark_data_smithi121_477989_object67382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T21:09:39.974 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:08:47.149649+0000", 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 52.687130420000003, 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.26640120299999998, 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 155571 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149649+0000", 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149649+0000", 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149648+0000", 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149654+0000", 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3709999999999999e-06 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149666+0000", 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2846e-05 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.977 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149672+0000", 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0349999999999998e-06 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149717+0000", 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4138000000000003e-05 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.149855+0000", 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0001381 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 0,1,2,3", 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.150014+0000", 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000158954 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.152215+0000", 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022013929999999998 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.153341+0000", 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011263309999999999 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.153595+0000", 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00025399500000000002 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.154399+0000", 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00080339699999999997 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.415857+0000", 2023-09-23T21:09:39.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.26145795199999999 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.415954+0000", 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7523999999999997e-05 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:08:47.416050+0000", 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6036e-05 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.982 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:169611 11.a 11:54ebd492:::benchmark_data_smithi121_477989_object81422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:15.935235+0000", 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 23.901544701999999, 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.27588458300000002, 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:09:39.983 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 169611 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935235+0000", 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935235+0000", 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935234+0000", 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935237+0000", 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6900000000000001e-06 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:09:39.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935247+0000", 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0268000000000001e-05 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935257+0000", 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0097e-05 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:09:39.986 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935307+0000", 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9125999999999997e-05 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935420+0000", 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113674 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.935511+0000", 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0914000000000004e-05 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.938330+0000", 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028188390000000001 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.938338+0000", 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6249999999999998e-06 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:15.938367+0000", 2023-09-23T21:09:39.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8929e-05 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:16.210923+0000", 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.272555989 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:16.210980+0000", 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7515000000000001e-05 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:09:39.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:16.211119+0000", 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013931999999999999 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:09:39.990 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:09:39.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:39.991 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:39.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:39.992 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:40.093 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:40.123 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:40.273 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:09:40.431 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:40.433 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1938271974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:40.433 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:40.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:40.513 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:09:40.534 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:40.554 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1938271974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:40.554 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:40.554 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1938271974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:40.554 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:09:40.581 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:40.630 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:40.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:40.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:40.733 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:40.759 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:40.768 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:09:40.853 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:40.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:40.971 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:40.973 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:40.999 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:41.100 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:41.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:41.204 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:41.232 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:41.386 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:41.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:09:41.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:41.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T21:09:41.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: failed to encode map e424 with expected crc 2023-09-23T21:09:41.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:41.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:41.387 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:41 smithi123 ceph-mon[193903]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:41.485 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:41.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:41.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:09:41.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:41.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T21:09:41.488 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: failed to encode map e424 with expected crc 2023-09-23T21:09:41.488 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:41.488 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:41.488 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[219709]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:41.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:41.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:41.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:09:41.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:09:41.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:09:41.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T21:09:41.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: failed to encode map e424 with expected crc 2023-09-23T21:09:41.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:41.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:41.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:41 smithi121 ceph-mon[211526]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:41.579 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:09:41.579 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1838,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":424,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695503081,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":595872,"bytes_used":5132013568,"bytes_avail":666833235968,"bytes_total":671965249536},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":29,"modified":"2023-09-23T21:08:15.075906+0000","services":{}},"progress_events":{"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:09:41.590 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:41.620 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:41.929 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:41.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:42.032 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:42.060 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:42.463 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:09:42.464 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:09:42.464 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:09:42.464 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:09:42.638 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T21:09:42.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:09:42.680 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:42.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:42.783 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:42.808 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: failed to encode map e424 with expected crc 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: failed to encode map e424 with expected crc 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2062419229' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2023-09-23T21:09:42.809 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: failed to encode map e424 with expected crc 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: failed to encode map e424 with expected crc 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2062419229' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:09:42.810 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.812 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:42.812 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:42.848 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:42.865 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:42.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:42.969 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:42.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: failed to encode map e424 with expected crc 2023-09-23T21:09:42.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: failed to encode map e424 with expected crc 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2062419229' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T21:09:42.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2023-09-23T21:09:42.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-09-23T21:09:42.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:42.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:43.002 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:43.053 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:43.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:43.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:43.157 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:43.167 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:09:43.186 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:43.284 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:43.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:43.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:43.324 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:43.388 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:43.415 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:43.503 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:09:43.613 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:43.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:43.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:43.718 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:43.745 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:43.787 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:43.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:43.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:43.890 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:43.920 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:44.078 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]': finished 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: pgmap v1370: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:44.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2223824806' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T21:09:44.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:43 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T21:09:44.159 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_7' created 2023-09-23T21:09:44.190 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]': finished 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T21:09:44.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: pgmap v1370: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2223824806' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]': finished 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: failed to encode map e425 with expected crc 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: pgmap v1370: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2223824806' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T21:09:44.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:43 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-09-23T21:09:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:09:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:09:44] "GET /metrics HTTP/1.1" 200 34261 "" "Prometheus/2.43.0" 2023-09-23T21:09:45.120 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:45.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:45.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:45.227 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:45.258 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:45.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.296 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.297 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.297 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T21:09:45.297 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T21:09:45.297 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.297 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.297 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:45.297 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:45 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:45.345 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-09-23T21:09:45.353 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: failed to encode map e426 with expected crc 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:45.355 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:45 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:45.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:45.457 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:45.485 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:45.643 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:45.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:45.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:45.747 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:45.778 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:45.986 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:09:45.986 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:09:45.987 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:09:45.987 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:09:45.987 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:09:46.059 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:46.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:46.089 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:46.164 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:46.192 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:46.213 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:46.294 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[211526]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[211526]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[211526]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[211526]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[211526]: pgmap v1373: 89 pgs: 3 creating+activating, 5 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[219709]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[219709]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[219709]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[219709]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.295 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:46 smithi121 ceph-mon[219709]: pgmap v1373: 89 pgs: 3 creating+activating, 5 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:46.455 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:46.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:46 smithi123 ceph-mon[193903]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:46 smithi123 ceph-mon[193903]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T21:09:46.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:46 smithi123 ceph-mon[193903]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:46 smithi123 ceph-mon[193903]: failed to encode map e427 with expected crc 2023-09-23T21:09:46.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:46 smithi123 ceph-mon[193903]: pgmap v1373: 89 pgs: 3 creating+activating, 5 creating+peering, 8 unknown, 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:46.560 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:46.589 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:09:46.592 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:46.616 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:09:46.638 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:09:46.658 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:09:46.678 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:09:46.698 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:09:46.718 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:09:46.736 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:09:46.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:09:47.052 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:09:47.318 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:09:47.319 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[211526]: failed to encode map e428 with expected crc 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[211526]: failed to encode map e428 with expected crc 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[211526]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1747554944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[219709]: failed to encode map e428 with expected crc 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[219709]: failed to encode map e428 with expected crc 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[219709]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T21:09:47.341 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1747554944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:47.410 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:09:47.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:47 smithi123 ceph-mon[193903]: failed to encode map e428 with expected crc 2023-09-23T21:09:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:47 smithi123 ceph-mon[193903]: failed to encode map e428 with expected crc 2023-09-23T21:09:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:47 smithi123 ceph-mon[193903]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T21:09:47.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1747554944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:48 smithi123 ceph-mon[193903]: pgmap v1375: 89 pgs: 3 creating+activating, 5 creating+peering, 81 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:48 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:09:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:48 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:09:48.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[211526]: pgmap v1375: 89 pgs: 3 creating+activating, 5 creating+peering, 81 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:09:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:09:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[219709]: pgmap v1375: 89 pgs: 3 creating+activating, 5 creating+peering, 81 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:09:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:09:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:48.441 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:48.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:48.547 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:48.576 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:48.850 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:48.851 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:49.334 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:49 smithi121 ceph-mon[219709]: from='client.36660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:49.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:49 smithi121 ceph-mon[211526]: from='client.36660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:49.351 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1375,"stamp":"2023-09-23T21:09:47.143143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":119,"num_bytes_recovered":3579692,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":230,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4979968,"kb_used_data":624904,"kb_used_omap":21,"kb_used_meta":4354986,"kb_avail":651236096,"statfs":{"total":671965249536,"available":666865762304,"internally_reserved":0,"allocated":639901696,"data_stored":627679381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4459506147},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.126052"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":198,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.161056+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:09:44.161056+0000","last_peered":"2023-09-23T21:09:44.161056+0000","last_clean":"2023-09-23T21:09:44.161056+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:09:44.161056+0000","last_undegraded":"2023-09-23T21:09:44.161056+0000","last_fullsized":"2023-09-23T21:09:44.161056+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":68,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.025997+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:09:45.025997+0000","last_peered":"2023-09-23T21:09:45.025997+0000","last_clean":"2023-09-23T21:09:45.025997+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:09:45.025997+0000","last_undegraded":"2023-09-23T21:09:45.025997+0000","last_fullsized":"2023-09-23T21:09:45.025997+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":340,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.029103+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:09:44.029103+0000","last_peered":"2023-09-23T21:09:44.029103+0000","last_clean":"2023-09-23T21:09:44.029103+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:09:44.029103+0000","last_undegraded":"2023-09-23T21:09:44.029103+0000","last_fullsized":"2023-09-23T21:09:44.029103+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":64,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.022120+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:09:43.022120+0000","last_peered":"2023-09-23T21:09:43.022120+0000","last_clean":"2023-09-23T21:09:43.022120+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:09:43.022120+0000","last_undegraded":"2023-09-23T21:09:43.022120+0000","last_fullsized":"2023-09-23T21:09:43.022120+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":86,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022408+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:09:44.022408+0000","last_peered":"2023-09-23T21:09:44.022408+0000","last_clean":"2023-09-23T21:09:44.022408+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:09:44.022408+0000","last_undegraded":"2023-09-23T21:09:44.022408+0000","last_fullsized":"2023-09-23T21:09:44.022408+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":303,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.162309+0000","last_change":"2023-09-23T21:09:44.161988+0000","last_active":"2023-09-23T21:09:44.162309+0000","last_peered":"2023-09-23T21:09:44.162309+0000","last_clean":"2023-09-23T21:09:44.162309+0000","last_became_active":"2023-09-23T21:09:44.161643+0000","last_became_peered":"2023-09-23T21:09:44.161643+0000","last_unstale":"2023-09-23T21:09:44.162309+0000","last_undegraded":"2023-09-23T21:09:44.162309+0000","last_fullsized":"2023-09-23T21:09:44.162309+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:17.083615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":63,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.022766+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:09:43.022766+0000","last_peered":"2023-09-23T21:09:43.022766+0000","last_clean":"2023-09-23T21:09:43.022766+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:09:43.022766+0000","last_undegraded":"2023-09-23T21:09:43.022766+0000","last_fullsized":"2023-09-23T21:09:43.022766+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":68,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022348+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:09:44.022348+0000","last_peered":"2023-09-23T21:09:44.022348+0000","last_clean":"2023-09-23T21:09:44.022348+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:09:44.022348+0000","last_undegraded":"2023-09-23T21:09:44.022348+0000","last_fullsized":"2023-09-23T21:09:44.022348+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":48,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.270932+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:09:40.270932+0000","last_peered":"2023-09-23T21:09:40.270932+0000","last_clean":"2023-09-23T21:09:40.270932+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:09:40.270932+0000","last_undegraded":"2023-09-23T21:09:40.270932+0000","last_fullsized":"2023-09-23T21:09:40.270932+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":68,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022724+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:09:44.022724+0000","last_peered":"2023-09-23T21:09:44.022724+0000","last_clean":"2023-09-23T21:09:44.022724+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:09:44.022724+0000","last_undegraded":"2023-09-23T21:09:44.022724+0000","last_fullsized":"2023-09-23T21:09:44.022724+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":84,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.161214+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:09:44.161214+0000","last_peered":"2023-09-23T21:09:44.161214+0000","last_clean":"2023-09-23T21:09:44.161214+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:09:44.161214+0000","last_undegraded":"2023-09-23T21:09:44.161214+0000","last_fullsized":"2023-09-23T21:09:44.161214+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.029101+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:09:44.029101+0000","last_peered":"2023-09-23T21:09:44.029101+0000","last_clean":"2023-09-23T21:09:44.029101+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:09:44.029101+0000","last_undegraded":"2023-09-23T21:09:44.029101+0000","last_fullsized":"2023-09-23T21:09:44.029101+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":373,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028329+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:09:44.028329+0000","last_peered":"2023-09-23T21:09:44.028329+0000","last_clean":"2023-09-23T21:09:44.028329+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:09:44.028329+0000","last_undegraded":"2023-09-23T21:09:44.028329+0000","last_fullsized":"2023-09-23T21:09:44.028329+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":343,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.867835+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:09:40.867835+0000","last_peered":"2023-09-23T21:09:40.867835+0000","last_clean":"2023-09-23T21:09:40.867835+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:09:40.867835+0000","last_undegraded":"2023-09-23T21:09:40.867835+0000","last_fullsized":"2023-09-23T21:09:40.867835+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":117,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.019469+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:09:43.019469+0000","last_peered":"2023-09-23T21:09:43.019469+0000","last_clean":"2023-09-23T21:09:43.019469+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:09:43.019469+0000","last_undegraded":"2023-09-23T21:09:43.019469+0000","last_fullsized":"2023-09-23T21:09:43.019469+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.160936+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:09:44.160936+0000","last_peered":"2023-09-23T21:09:44.160936+0000","last_clean":"2023-09-23T21:09:44.160936+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:09:44.160936+0000","last_undegraded":"2023-09-23T21:09:44.160936+0000","last_fullsized":"2023-09-23T21:09:44.160936+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":86,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022697+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:09:44.022697+0000","last_peered":"2023-09-23T21:09:44.022697+0000","last_clean":"2023-09-23T21:09:44.022697+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:09:44.022697+0000","last_undegraded":"2023-09-23T21:09:44.022697+0000","last_fullsized":"2023-09-23T21:09:44.022697+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":86,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.161306+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:09:44.161306+0000","last_peered":"2023-09-23T21:09:44.161306+0000","last_clean":"2023-09-23T21:09:44.161306+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:09:44.161306+0000","last_undegraded":"2023-09-23T21:09:44.161306+0000","last_fullsized":"2023-09-23T21:09:44.161306+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.867729+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:09:40.867729+0000","last_peered":"2023-09-23T21:09:40.867729+0000","last_clean":"2023-09-23T21:09:40.867729+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:09:40.867729+0000","last_undegraded":"2023-09-23T21:09:40.867729+0000","last_fullsized":"2023-09-23T21:09:40.867729+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.022193+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:09:43.022193+0000","last_peered":"2023-09-23T21:09:43.022193+0000","last_clean":"2023-09-23T21:09:43.022193+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:09:43.022193+0000","last_undegraded":"2023-09-23T21:09:43.022193+0000","last_fullsized":"2023-09-23T21:09:43.022193+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":423,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.027725+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:09:44.027725+0000","last_peered":"2023-09-23T21:09:44.027725+0000","last_clean":"2023-09-23T21:09:44.027725+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:09:44.027725+0000","last_undegraded":"2023-09-23T21:09:44.027725+0000","last_fullsized":"2023-09-23T21:09:44.027725+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.164721+0000","last_change":"2023-09-23T21:09:44.164036+0000","last_active":"2023-09-23T21:09:44.164721+0000","last_peered":"2023-09-23T21:09:44.164721+0000","last_clean":"2023-09-23T21:09:44.164721+0000","last_became_active":"2023-09-23T21:09:44.163672+0000","last_became_peered":"2023-09-23T21:09:44.163672+0000","last_unstale":"2023-09-23T21:09:44.164721+0000","last_undegraded":"2023-09-23T21:09:44.164721+0000","last_fullsized":"2023-09-23T21:09:44.164721+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:10.564576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":9,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.077841+0000","last_change":"2023-09-23T21:09:45.077841+0000","last_active":"2023-09-23T21:09:45.077841+0000","last_peered":"2023-09-23T21:09:45.077841+0000","last_clean":"2023-09-23T21:09:45.077841+0000","last_became_active":"2023-09-23T21:09:45.077550+0000","last_became_peered":"2023-09-23T21:09:45.077550+0000","last_unstale":"2023-09-23T21:09:45.077841+0000","last_undegraded":"2023-09-23T21:09:45.077841+0000","last_fullsized":"2023-09-23T21:09:45.077841+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:19.178954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":129,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.867807+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:09:40.867807+0000","last_peered":"2023-09-23T21:09:40.867807+0000","last_clean":"2023-09-23T21:09:40.867807+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:09:40.867807+0000","last_undegraded":"2023-09-23T21:09:40.867807+0000","last_fullsized":"2023-09-23T21:09:40.867807+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":48,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.270645+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:09:40.270645+0000","last_peered":"2023-09-23T21:09:40.270645+0000","last_clean":"2023-09-23T21:09:40.270645+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:09:40.270645+0000","last_undegraded":"2023-09-23T21:09:40.270645+0000","last_fullsized":"2023-09-23T21:09:40.270645+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":9,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.040484+0000","last_change":"2023-09-23T21:09:45.040484+0000","last_active":"2023-09-23T21:09:45.040484+0000","last_peered":"2023-09-23T21:09:45.040484+0000","last_clean":"2023-09-23T21:09:45.040484+0000","last_became_active":"2023-09-23T21:09:45.040324+0000","last_became_peered":"2023-09-23T21:09:45.040324+0000","last_unstale":"2023-09-23T21:09:45.040484+0000","last_undegraded":"2023-09-23T21:09:45.040484+0000","last_fullsized":"2023-09-23T21:09:45.040484+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:54.268283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":126,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.159055+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:09:44.159055+0000","last_peered":"2023-09-23T21:09:44.159055+0000","last_clean":"2023-09-23T21:09:44.159055+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:09:44.159055+0000","last_undegraded":"2023-09-23T21:09:44.159055+0000","last_fullsized":"2023-09-23T21:09:44.159055+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":388,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.029072+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:09:44.029072+0000","last_peered":"2023-09-23T21:09:44.029072+0000","last_clean":"2023-09-23T21:09:44.029072+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:09:44.029072+0000","last_undegraded":"2023-09-23T21:09:44.029072+0000","last_fullsized":"2023-09-23T21:09:44.029072+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":3,"reported_epoch":427,"state":"creating+activating","last_fresh":"2023-09-23T21:09:45.025333+0000","last_change":"2023-09-23T21:09:45.025333+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:45.025333+0000","last_undegraded":"2023-09-23T21:09:45.025333+0000","last_fullsized":"2023-09-23T21:09:45.025333+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:15.852527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":["3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":52,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.270791+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:09:40.270791+0000","last_peered":"2023-09-23T21:09:40.270791+0000","last_clean":"2023-09-23T21:09:40.270791+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:09:40.270791+0000","last_undegraded":"2023-09-23T21:09:40.270791+0000","last_fullsized":"2023-09-23T21:09:40.270791+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":315,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028101+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:09:44.028101+0000","last_peered":"2023-09-23T21:09:44.028101+0000","last_clean":"2023-09-23T21:09:44.028101+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:09:44.028101+0000","last_undegraded":"2023-09-23T21:09:44.028101+0000","last_fullsized":"2023-09-23T21:09:44.028101+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":1,"reported_epoch":426,"state":"creating+peering","last_fresh":"2023-09-23T21:09:44.024320+0000","last_change":"2023-09-23T21:09:44.024320+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:44.024320+0000","last_undegraded":"2023-09-23T21:09:44.024320+0000","last_fullsized":"2023-09-23T21:09:44.024320+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:34.212928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.161015+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:09:44.161015+0000","last_peered":"2023-09-23T21:09:44.161015+0000","last_clean":"2023-09-23T21:09:44.161015+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:09:44.161015+0000","last_undegraded":"2023-09-23T21:09:44.161015+0000","last_fullsized":"2023-09-23T21:09:44.161015+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.163930+0000","last_change":"2023-09-23T21:09:44.163930+0000","last_active":"2023-09-23T21:09:44.163930+0000","last_peered":"2023-09-23T21:09:44.163930+0000","last_clean":"2023-09-23T21:09:44.163930+0000","last_became_active":"2023-09-23T21:09:44.163614+0000","last_became_peered":"2023-09-23T21:09:44.163614+0000","last_unstale":"2023-09-23T21:09:44.163930+0000","last_undegraded":"2023-09-23T21:09:44.163930+0000","last_fullsized":"2023-09-23T21:09:44.163930+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:26.549908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":9,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.040470+0000","last_change":"2023-09-23T21:09:45.040470+0000","last_active":"2023-09-23T21:09:45.040470+0000","last_peered":"2023-09-23T21:09:45.040470+0000","last_clean":"2023-09-23T21:09:45.040470+0000","last_became_active":"2023-09-23T21:09:45.040298+0000","last_became_peered":"2023-09-23T21:09:45.040298+0000","last_unstale":"2023-09-23T21:09:45.040470+0000","last_undegraded":"2023-09-23T21:09:45.040470+0000","last_fullsized":"2023-09-23T21:09:45.040470+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:06.840153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":85,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022851+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:09:44.022851+0000","last_peered":"2023-09-23T21:09:44.022851+0000","last_clean":"2023-09-23T21:09:44.022851+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:09:44.022851+0000","last_undegraded":"2023-09-23T21:09:44.022851+0000","last_fullsized":"2023-09-23T21:09:44.022851+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.160858+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:09:44.160858+0000","last_peered":"2023-09-23T21:09:44.160858+0000","last_clean":"2023-09-23T21:09:44.160858+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:09:44.160858+0000","last_undegraded":"2023-09-23T21:09:44.160858+0000","last_fullsized":"2023-09-23T21:09:44.160858+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":9,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.041603+0000","last_change":"2023-09-23T21:09:45.041603+0000","last_active":"2023-09-23T21:09:45.041603+0000","last_peered":"2023-09-23T21:09:45.041603+0000","last_clean":"2023-09-23T21:09:45.041603+0000","last_became_active":"2023-09-23T21:09:45.041377+0000","last_became_peered":"2023-09-23T21:09:45.041377+0000","last_unstale":"2023-09-23T21:09:45.041603+0000","last_undegraded":"2023-09-23T21:09:45.041603+0000","last_fullsized":"2023-09-23T21:09:45.041603+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:46.148257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":353,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028143+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:09:44.028143+0000","last_peered":"2023-09-23T21:09:44.028143+0000","last_clean":"2023-09-23T21:09:44.028143+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:09:44.028143+0000","last_undegraded":"2023-09-23T21:09:44.028143+0000","last_fullsized":"2023-09-23T21:09:44.028143+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":437,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.020449+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:09:43.020449+0000","last_peered":"2023-09-23T21:09:43.020449+0000","last_clean":"2023-09-23T21:09:43.020449+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:09:43.020449+0000","last_undegraded":"2023-09-23T21:09:43.020449+0000","last_fullsized":"2023-09-23T21:09:43.020449+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":10,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.041052+0000","last_change":"2023-09-23T21:09:45.040594+0000","last_active":"2023-09-23T21:09:45.041052+0000","last_peered":"2023-09-23T21:09:45.041052+0000","last_clean":"2023-09-23T21:09:45.041052+0000","last_became_active":"2023-09-23T21:09:45.040285+0000","last_became_peered":"2023-09-23T21:09:45.040285+0000","last_unstale":"2023-09-23T21:09:45.041052+0000","last_undegraded":"2023-09-23T21:09:45.041052+0000","last_fullsized":"2023-09-23T21:09:45.041052+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:36.237549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":52,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.867623+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:09:40.867623+0000","last_peered":"2023-09-23T21:09:40.867623+0000","last_clean":"2023-09-23T21:09:40.867623+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:09:40.867623+0000","last_undegraded":"2023-09-23T21:09:40.867623+0000","last_fullsized":"2023-09-23T21:09:40.867623+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":466,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028252+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:09:44.028252+0000","last_peered":"2023-09-23T21:09:44.028252+0000","last_clean":"2023-09-23T21:09:44.028252+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:09:44.028252+0000","last_undegraded":"2023-09-23T21:09:44.028252+0000","last_fullsized":"2023-09-23T21:09:44.028252+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":10,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.080323+0000","last_change":"2023-09-23T21:09:45.080008+0000","last_active":"2023-09-23T21:09:45.080323+0000","last_peered":"2023-09-23T21:09:45.080323+0000","last_clean":"2023-09-23T21:09:45.080323+0000","last_became_active":"2023-09-23T21:09:45.079685+0000","last_became_peered":"2023-09-23T21:09:45.079685+0000","last_unstale":"2023-09-23T21:09:45.080323+0000","last_undegraded":"2023-09-23T21:09:45.080323+0000","last_fullsized":"2023-09-23T21:09:45.080323+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:24.022184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":77,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022432+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:09:44.022432+0000","last_peered":"2023-09-23T21:09:44.022432+0000","last_clean":"2023-09-23T21:09:44.022432+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:09:44.022432+0000","last_undegraded":"2023-09-23T21:09:44.022432+0000","last_fullsized":"2023-09-23T21:09:44.022432+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.025833+0000","last_change":"2023-09-23T21:09:44.164862+0000","last_active":"2023-09-23T21:09:45.025833+0000","last_peered":"2023-09-23T21:09:45.025833+0000","last_clean":"2023-09-23T21:09:45.025833+0000","last_became_active":"2023-09-23T21:09:44.164337+0000","last_became_peered":"2023-09-23T21:09:44.164337+0000","last_unstale":"2023-09-23T21:09:45.025833+0000","last_undegraded":"2023-09-23T21:09:45.025833+0000","last_fullsized":"2023-09-23T21:09:45.025833+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:57.424412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":1,"reported_epoch":426,"state":"creating+peering","last_fresh":"2023-09-23T21:09:44.023461+0000","last_change":"2023-09-23T21:09:44.023461+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:44.023461+0000","last_undegraded":"2023-09-23T21:09:44.023461+0000","last_fullsized":"2023-09-23T21:09:44.023461+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:15.193017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":322,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.159541+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:09:44.159541+0000","last_peered":"2023-09-23T21:09:44.159541+0000","last_clean":"2023-09-23T21:09:44.159541+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:09:44.159541+0000","last_undegraded":"2023-09-23T21:09:44.159541+0000","last_fullsized":"2023-09-23T21:09:44.159541+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":90,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.270907+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:09:40.270907+0000","last_peered":"2023-09-23T21:09:40.270907+0000","last_clean":"2023-09-23T21:09:40.270907+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:09:40.270907+0000","last_undegraded":"2023-09-23T21:09:40.270907+0000","last_fullsized":"2023-09-23T21:09:40.270907+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.029135+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:09:44.029135+0000","last_peered":"2023-09-23T21:09:44.029135+0000","last_clean":"2023-09-23T21:09:44.029135+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:09:44.029135+0000","last_undegraded":"2023-09-23T21:09:44.029135+0000","last_fullsized":"2023-09-23T21:09:44.029135+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":1,"reported_epoch":426,"state":"creating+peering","last_fresh":"2023-09-23T21:09:44.024529+0000","last_change":"2023-09-23T21:09:44.024529+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:44.024529+0000","last_undegraded":"2023-09-23T21:09:44.024529+0000","last_fullsized":"2023-09-23T21:09:44.024529+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:17.052514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":403,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.161150+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:09:44.161150+0000","last_peered":"2023-09-23T21:09:44.161150+0000","last_clean":"2023-09-23T21:09:44.161150+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:09:44.161150+0000","last_undegraded":"2023-09-23T21:09:44.161150+0000","last_fullsized":"2023-09-23T21:09:44.161150+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":377,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028173+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:09:44.028173+0000","last_peered":"2023-09-23T21:09:44.028173+0000","last_clean":"2023-09-23T21:09:44.028173+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:09:44.028173+0000","last_undegraded":"2023-09-23T21:09:44.028173+0000","last_fullsized":"2023-09-23T21:09:44.028173+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":60,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.159468+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:09:44.159468+0000","last_peered":"2023-09-23T21:09:44.159468+0000","last_clean":"2023-09-23T21:09:44.159468+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:09:44.159468+0000","last_undegraded":"2023-09-23T21:09:44.159468+0000","last_fullsized":"2023-09-23T21:09:44.159468+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":9,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.843960+0000","last_change":"2023-09-23T21:09:45.843960+0000","last_active":"2023-09-23T21:09:45.843960+0000","last_peered":"2023-09-23T21:09:45.843960+0000","last_clean":"2023-09-23T21:09:45.843960+0000","last_became_active":"2023-09-23T21:09:45.843562+0000","last_became_peered":"2023-09-23T21:09:45.843562+0000","last_unstale":"2023-09-23T21:09:45.843960+0000","last_undegraded":"2023-09-23T21:09:45.843960+0000","last_fullsized":"2023-09-23T21:09:45.843960+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:10.329012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":404,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.019539+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:09:43.019539+0000","last_peered":"2023-09-23T21:09:43.019539+0000","last_clean":"2023-09-23T21:09:43.019539+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:09:43.019539+0000","last_undegraded":"2023-09-23T21:09:43.019539+0000","last_fullsized":"2023-09-23T21:09:43.019539+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":322,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.029163+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:09:44.029163+0000","last_peered":"2023-09-23T21:09:44.029163+0000","last_clean":"2023-09-23T21:09:44.029163+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:09:44.029163+0000","last_undegraded":"2023-09-23T21:09:44.029163+0000","last_fullsized":"2023-09-23T21:09:44.029163+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":60,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.159206+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:09:44.159206+0000","last_peered":"2023-09-23T21:09:44.159206+0000","last_clean":"2023-09-23T21:09:44.159206+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:09:44.159206+0000","last_undegraded":"2023-09-23T21:09:44.159206+0000","last_fullsized":"2023-09-23T21:09:44.159206+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":10,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.036823+0000","last_change":"2023-09-23T21:09:45.036017+0000","last_active":"2023-09-23T21:09:45.036823+0000","last_peered":"2023-09-23T21:09:45.036823+0000","last_clean":"2023-09-23T21:09:45.036823+0000","last_became_active":"2023-09-23T21:09:45.035744+0000","last_became_peered":"2023-09-23T21:09:45.035744+0000","last_unstale":"2023-09-23T21:09:45.036823+0000","last_undegraded":"2023-09-23T21:09:45.036823+0000","last_fullsized":"2023-09-23T21:09:45.036823+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:55.125986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":242,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.020288+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:09:43.020288+0000","last_peered":"2023-09-23T21:09:43.020288+0000","last_clean":"2023-09-23T21:09:43.020288+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:09:43.020288+0000","last_undegraded":"2023-09-23T21:09:43.020288+0000","last_fullsized":"2023-09-23T21:09:43.020288+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":686,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.021243+0000","last_change":"2023-09-23T21:08:41.538586+0000","last_active":"2023-09-23T21:09:43.021243+0000","last_peered":"2023-09-23T21:09:43.021243+0000","last_clean":"2023-09-23T21:09:43.021243+0000","last_became_active":"2023-09-23T21:08:41.538107+0000","last_became_peered":"2023-09-23T21:08:41.538107+0000","last_unstale":"2023-09-23T21:09:43.021243+0000","last_undegraded":"2023-09-23T21:09:43.021243+0000","last_fullsized":"2023-09-23T21:09:43.021243+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:48.268707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022587+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:09:44.022587+0000","last_peered":"2023-09-23T21:09:44.022587+0000","last_clean":"2023-09-23T21:09:44.022587+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:09:44.022587+0000","last_undegraded":"2023-09-23T21:09:44.022587+0000","last_fullsized":"2023-09-23T21:09:44.022587+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":315,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028264+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:09:44.028264+0000","last_peered":"2023-09-23T21:09:44.028264+0000","last_clean":"2023-09-23T21:09:44.028264+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:09:44.028264+0000","last_undegraded":"2023-09-23T21:09:44.028264+0000","last_fullsized":"2023-09-23T21:09:44.028264+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":1,"reported_epoch":426,"state":"creating+peering","last_fresh":"2023-09-23T21:09:44.024643+0000","last_change":"2023-09-23T21:09:44.024643+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:44.024643+0000","last_undegraded":"2023-09-23T21:09:44.024643+0000","last_fullsized":"2023-09-23T21:09:44.024643+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:01.305138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":60,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028840+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:09:44.028840+0000","last_peered":"2023-09-23T21:09:44.028840+0000","last_clean":"2023-09-23T21:09:44.028840+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:09:44.028840+0000","last_undegraded":"2023-09-23T21:09:44.028840+0000","last_fullsized":"2023-09-23T21:09:44.028840+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.021739+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:09:43.021739+0000","last_peered":"2023-09-23T21:09:43.021739+0000","last_clean":"2023-09-23T21:09:43.021739+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:09:43.021739+0000","last_undegraded":"2023-09-23T21:09:43.021739+0000","last_fullsized":"2023-09-23T21:09:43.021739+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":64,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.021903+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:09:43.021903+0000","last_peered":"2023-09-23T21:09:43.021903+0000","last_clean":"2023-09-23T21:09:43.021903+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:09:43.021903+0000","last_undegraded":"2023-09-23T21:09:43.021903+0000","last_fullsized":"2023-09-23T21:09:43.021903+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":4,"reported_epoch":427,"state":"creating+activating","last_fresh":"2023-09-23T21:09:45.030382+0000","last_change":"2023-09-23T21:09:45.025020+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:45.030382+0000","last_undegraded":"2023-09-23T21:09:45.030382+0000","last_fullsized":"2023-09-23T21:09:45.030382+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:37.785460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":64,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.023079+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:09:43.023079+0000","last_peered":"2023-09-23T21:09:43.023079+0000","last_clean":"2023-09-23T21:09:43.023079+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:09:43.023079+0000","last_undegraded":"2023-09-23T21:09:43.023079+0000","last_fullsized":"2023-09-23T21:09:43.023079+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":324,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.159683+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:09:44.159683+0000","last_peered":"2023-09-23T21:09:44.159683+0000","last_clean":"2023-09-23T21:09:44.159683+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:09:44.159683+0000","last_undegraded":"2023-09-23T21:09:44.159683+0000","last_fullsized":"2023-09-23T21:09:44.159683+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":100,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.271041+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:09:40.271041+0000","last_peered":"2023-09-23T21:09:40.271041+0000","last_clean":"2023-09-23T21:09:40.271041+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:09:40.271041+0000","last_undegraded":"2023-09-23T21:09:40.271041+0000","last_fullsized":"2023-09-23T21:09:40.271041+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":4,"reported_epoch":427,"state":"creating+activating","last_fresh":"2023-09-23T21:09:45.030724+0000","last_change":"2023-09-23T21:09:45.025345+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:45.030724+0000","last_undegraded":"2023-09-23T21:09:45.030724+0000","last_fullsized":"2023-09-23T21:09:45.030724+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:26.841578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":85,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.160654+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:09:44.160654+0000","last_peered":"2023-09-23T21:09:44.160654+0000","last_clean":"2023-09-23T21:09:44.160654+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:09:44.160654+0000","last_undegraded":"2023-09-23T21:09:44.160654+0000","last_fullsized":"2023-09-23T21:09:44.160654+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":60,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.158785+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:09:44.158785+0000","last_peered":"2023-09-23T21:09:44.158785+0000","last_clean":"2023-09-23T21:09:44.158785+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:09:44.158785+0000","last_undegraded":"2023-09-23T21:09:44.158785+0000","last_fullsized":"2023-09-23T21:09:44.158785+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022325+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:09:44.022325+0000","last_peered":"2023-09-23T21:09:44.022325+0000","last_clean":"2023-09-23T21:09:44.022325+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:09:44.022325+0000","last_undegraded":"2023-09-23T21:09:44.022325+0000","last_fullsized":"2023-09-23T21:09:44.022325+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":1,"reported_epoch":426,"state":"creating+peering","last_fresh":"2023-09-23T21:09:44.023365+0000","last_change":"2023-09-23T21:09:44.023365+0000","last_active":"2023-09-23T21:09:44.015845+0000","last_peered":"2023-09-23T21:09:44.015845+0000","last_clean":"2023-09-23T21:09:44.015845+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T21:09:44.023365+0000","last_undegraded":"2023-09-23T21:09:44.023365+0000","last_fullsized":"2023-09-23T21:09:44.023365+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:44.361487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":323,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.029311+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:09:44.029311+0000","last_peered":"2023-09-23T21:09:44.029311+0000","last_clean":"2023-09-23T21:09:44.029311+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:09:44.029311+0000","last_undegraded":"2023-09-23T21:09:44.029311+0000","last_fullsized":"2023-09-23T21:09:44.029311+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":68,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.021984+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:09:43.021984+0000","last_peered":"2023-09-23T21:09:43.021984+0000","last_clean":"2023-09-23T21:09:43.021984+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:09:43.021984+0000","last_undegraded":"2023-09-23T21:09:43.021984+0000","last_fullsized":"2023-09-23T21:09:43.021984+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":110,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.270664+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:09:40.270664+0000","last_peered":"2023-09-23T21:09:40.270664+0000","last_clean":"2023-09-23T21:09:40.270664+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:09:40.270664+0000","last_undegraded":"2023-09-23T21:09:40.270664+0000","last_fullsized":"2023-09-23T21:09:40.270664+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":72,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.022000+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:09:43.022000+0000","last_peered":"2023-09-23T21:09:43.022000+0000","last_clean":"2023-09-23T21:09:43.022000+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:09:43.022000+0000","last_undegraded":"2023-09-23T21:09:43.022000+0000","last_fullsized":"2023-09-23T21:09:43.022000+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":459,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.163759+0000","last_change":"2023-09-23T21:09:44.163759+0000","last_active":"2023-09-23T21:09:44.163759+0000","last_peered":"2023-09-23T21:09:44.163759+0000","last_clean":"2023-09-23T21:09:44.163759+0000","last_became_active":"2023-09-23T21:09:44.163405+0000","last_became_peered":"2023-09-23T21:09:44.163405+0000","last_unstale":"2023-09-23T21:09:44.163759+0000","last_undegraded":"2023-09-23T21:09:44.163759+0000","last_fullsized":"2023-09-23T21:09:44.163759+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:52.346296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":87,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.868048+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:09:40.868048+0000","last_peered":"2023-09-23T21:09:40.868048+0000","last_clean":"2023-09-23T21:09:40.868048+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:09:40.868048+0000","last_undegraded":"2023-09-23T21:09:40.868048+0000","last_fullsized":"2023-09-23T21:09:40.868048+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":94,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.025971+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:09:45.025971+0000","last_peered":"2023-09-23T21:09:45.025971+0000","last_clean":"2023-09-23T21:09:45.025971+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:09:45.025971+0000","last_undegraded":"2023-09-23T21:09:45.025971+0000","last_fullsized":"2023-09-23T21:09:45.025971+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":308,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.028365+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:09:44.028365+0000","last_peered":"2023-09-23T21:09:44.028365+0000","last_clean":"2023-09-23T21:09:44.028365+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:09:44.028365+0000","last_undegraded":"2023-09-23T21:09:44.028365+0000","last_fullsized":"2023-09-23T21:09:44.028365+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":427,"state":"active+clean","last_fresh":"2023-09-23T21:09:45.026298+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:09:45.026298+0000","last_peered":"2023-09-23T21:09:45.026298+0000","last_clean":"2023-09-23T21:09:45.026298+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:09:45.026298+0000","last_undegraded":"2023-09-23T21:09:45.026298+0000","last_fullsized":"2023-09-23T21:09:45.026298+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":68,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.160010+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:09:44.160010+0000","last_peered":"2023-09-23T21:09:44.160010+0000","last_clean":"2023-09-23T21:09:44.160010+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:09:44.160010+0000","last_undegraded":"2023-09-23T21:09:44.160010+0000","last_fullsized":"2023-09-23T21:09:44.160010+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":77,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T21:09:44.022575+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:09:44.022575+0000","last_peered":"2023-09-23T21:09:44.022575+0000","last_clean":"2023-09-23T21:09:44.022575+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:09:44.022575+0000","last_undegraded":"2023-09-23T21:09:44.022575+0000","last_fullsized":"2023-09-23T21:09:44.022575+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":70,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T21:09:43.022636+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:09:43.022636+0000","last_peered":"2023-09-23T21:09:43.022636+0000","last_clean":"2023-09-23T21:09:43.022636+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:09:43.022636+0000","last_undegraded":"2023-09-23T21:09:43.022636+0000","last_fullsized":"2023-09-23T21:09:43.022636+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":52,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T21:09:40.867426+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:09:40.867426+0000","last_peered":"2023-09-23T21:09:40.867426+0000","last_clean":"2023-09-23T21:09:40.867426+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:09:40.867426+0000","last_undegraded":"2023-09-23T21:09:40.867426+0000","last_fullsized":"2023-09-23T21:09:40.867426+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201772,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463393,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":569160,"kb_used_data":4992,"kb_used_omap":0,"kb_used_meta":564160,"kb_avail":93175992,"statfs":{"total":95995035648,"available":95412215808,"internally_reserved":0,"allocated":5111808,"data_stored":3867458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":577699840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":2.085,"15min":1.9430000000000001},"min":{"1min":0.245,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":5.5780000000000003,"5min":99.533000000000001,"15min":102.242},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.4399999999999999,"5min":2.177,"15min":1.8160000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":11.157999999999999,"5min":99.325000000000003,"15min":102.502},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7440000000000002,"5min":3.3039999999999998,"15min":2.5760000000000001},"min":{"1min":0.29599999999999999,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":7.915,"5min":119.077,"15min":119.077},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.9319999999999999,"5min":3.1040000000000001,"15min":2.5939999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.0229999999999997,"5min":119.313,"15min":119.313},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":6.4370000000000003,"5min":4.766,"15min":4.1180000000000003},"min":{"1min":0.29799999999999999,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":81.801000000000002,"5min":199.23400000000001,"15min":199.23400000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":6.6970000000000001,"5min":4.5650000000000004,"15min":4.1740000000000004},"min":{"1min":0.33800000000000002,"5min":0.28499999999999998,"15min":0.21199999999999999},"max":{"1min":81.510999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.71599999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.3380000000000001,"15min":2.6429999999999998},"min":{"1min":0.315,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":7.2130000000000001,"5min":13.340999999999999,"15min":178.40000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.466,"5min":1.6000000000000001,"15min":2.6440000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":5.415,"5min":13.319000000000001,"15min":178.26599999999999},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":0.93200000000000005,"15min":0.94299999999999995},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.0890000000000004,"5min":9.0890000000000004,"15min":13.1},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.78400000000000003,"15min":0.82999999999999996},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":8.6859999999999999},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.94599999999999995,"15min":0.94399999999999995},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.0519999999999996,"5min":9.0519999999999996,"15min":11.281000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.86599999999999999,"15min":0.99399999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.0190000000000001,"5min":6.6890000000000001,"15min":9.5389999999999997},"last":0.622}]},{"osd":7,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.1080000000000001,"15min":0.97499999999999998},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":6.6299999999999999,"5min":7.8079999999999998,"15min":11.388999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.0669999999999999,"15min":1.0760000000000001},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.30099999999999999},"max":{"1min":6.4749999999999996,"5min":7.1849999999999996,"15min":8.8010000000000002},"last":0.72899999999999998}]}]},{"osd":1,"up_from":276,"seq":1185410973839,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":843220,"kb_used_data":361548,"kb_used_omap":5,"kb_used_meta":481658,"kb_avail":92901932,"statfs":{"total":95995035648,"available":95131578368,"internally_reserved":0,"allocated":370225152,"data_stored":368088155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":493218531},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.274,"5min":2.4860000000000002,"15min":1.1259999999999999},"min":{"1min":0.27600000000000002,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":23.486000000000001,"5min":23.486000000000001,"15min":23.486000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.9540000000000002,"5min":1.996,"15min":1.2310000000000001},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":16.148,"5min":16.148,"15min":18.047999999999998},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4459999999999997,"5min":2.556,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":21.695,"5min":21.695,"15min":78.409000000000006},"last":0.30499999999999999},{"interface":"front","average":{"1min":4.8369999999999997,"5min":2.6619999999999999,"15min":1.6140000000000001},"min":{"1min":0.502,"5min":0.245,"15min":0.22},"max":{"1min":23.539000000000001,"5min":23.539000000000001,"15min":76.245000000000005},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":2.4750000000000001,"15min":3.0379999999999998},"min":{"1min":0.39500000000000002,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":23.407,"5min":23.407,"15min":172.44399999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":3.9260000000000002,"5min":2.173,"15min":3.1059999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":17.033999999999999,"5min":17.033999999999999,"15min":172.58500000000001},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.631,"15min":1.137},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.5389999999999997,"5min":12.238,"15min":17.006},"last":0.46999999999999997},{"interface":"front","average":{"1min":3.3530000000000002,"5min":2.0640000000000001,"15min":1.1180000000000001},"min":{"1min":0.42199999999999999,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":23.332999999999998,"5min":23.332999999999998,"15min":23.332999999999998},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":16.196000000000002,"5min":16.196000000000002,"15min":17.155000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.5339999999999998,"5min":1.7969999999999999,"15min":1.284},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.9660000000000002,"5min":12.345000000000001,"15min":17.076000000000001},"last":0.629}]},{"osd":6,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":1.8089999999999999,"15min":1.73},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.222000000000001,"5min":16.222000000000001,"15min":127.995},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.6160000000000001,"5min":1.962,"15min":1.758},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8959999999999999,"5min":12.276999999999999,"15min":126.836},"last":0.435}]},{"osd":7,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":3.2639999999999998,"15min":1.756},"min":{"1min":0.38100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.859,"5min":151.791,"15min":151.791},"last":0.48999999999999999},{"interface":"front","average":{"1min":3.4039999999999999,"5min":3.552,"15min":1.6579999999999999},"min":{"1min":0.45000000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":23.376000000000001,"5min":151.809,"15min":151.809},"last":0.54700000000000004}]}]},{"osd":0,"up_from":92,"seq":395136991579,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":799796,"kb_used_data":33068,"kb_used_omap":5,"kb_used_meta":766714,"kb_avail":92945356,"statfs":{"total":95995035648,"available":95176044544,"internally_reserved":0,"allocated":33861632,"data_stored":31854138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":785115770},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.3210000000000002,"15min":2.0219999999999998},"min":{"1min":0.26300000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":18.140999999999998,"5min":52.590000000000003,"15min":61.341999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":4.5570000000000004,"5min":3.754,"15min":2.1059999999999999},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.219999999999999,"5min":52.276000000000003,"15min":61.299999999999997},"last":0.62}]},{"osd":2,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":3.5539999999999998,"5min":3.0030000000000001,"15min":2.2130000000000001},"min":{"1min":0.35699999999999998,"5min":0.217,"15min":0.161},"max":{"1min":12.916,"5min":45.912999999999997,"15min":183.77099999999999},"last":0.38},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8210000000000002,"15min":2.3719999999999999},"min":{"1min":0.33500000000000002,"5min":0.254,"15min":0.214},"max":{"1min":9.1370000000000005,"5min":46.234000000000002,"15min":183.78700000000001},"last":0.65000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.613,"5min":1.655,"15min":2.4750000000000001},"min":{"1min":0.36099999999999999,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":10.755000000000001,"5min":12.113,"15min":186.143},"last":0.60099999999999998},{"interface":"front","average":{"1min":2.7719999999999998,"5min":1.5620000000000001,"15min":2.5099999999999998},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":7.6260000000000003,"5min":12.348000000000001,"15min":186.97399999999999},"last":0.66600000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.401,"15min":1.3009999999999999},"min":{"1min":0.34200000000000003,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":5.9450000000000003,"5min":10.814,"15min":10.814},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.7450000000000001,"5min":1.171,"15min":1.012},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":10.629,"5min":10.629,"15min":11.493},"last":0.434}]},{"osd":5,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.5329999999999999,"15min":1.879},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":5.984,"5min":10.763,"15min":154.30600000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.373,"15min":1.5640000000000001},"min":{"1min":0.42099999999999999,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":10.685,"5min":11.478,"15min":154.42400000000001},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":6.0860000000000003,"15min":3.2360000000000002},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.710000000000001,"5min":170.12700000000001,"15min":170.12700000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.105,"5min":6.0289999999999999,"15min":3.2810000000000001},"min":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.28299999999999997},"max":{"1min":8.0269999999999992,"5min":169.767,"15min":169.767},"last":0.84699999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:08:55 2023","interfaces":[{"interface":"back","average":{"1min":2.391,"5min":1.605,"15min":2.3119999999999998},"min":{"1min":0.47399999999999998,"5min":0.35199999999999998,"15min":0.33600000000000002},"max":{"1min":10.661,"5min":10.74,"15min":164.26400000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.4830000000000001,"5min":1.3280000000000001,"15min":2.306},"min":{"1min":0.41199999999999998,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":164.23099999999999},"last":0.87}]}]},{"osd":2,"up_from":129,"seq":554050781470,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919220,"kb_used_data":210988,"kb_used_omap":7,"kb_used_meta":708216,"kb_avail":92825932,"statfs":{"total":95995035648,"available":95053754368,"internally_reserved":0,"allocated":216051712,"data_stored":213986148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":725213265},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.6259999999999999,"15min":1.381},"min":{"1min":0.217,"5min":0.19400000000000001,"15min":0.17699999999999999},"max":{"1min":10.077,"5min":11.699999999999999,"15min":26.356000000000002},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.879,"5min":2.7610000000000001,"15min":1.9299999999999999},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.2360000000000007,"5min":23.963999999999999,"15min":26.341000000000001},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":3.3039999999999998,"15min":2.802},"min":{"1min":0.24299999999999999,"5min":0.19,"15min":0.188},"max":{"1min":11.271000000000001,"5min":67.296999999999997,"15min":212.12},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.7069999999999999,"5min":2.5960000000000001,"15min":2.4860000000000002},"min":{"1min":0.34200000000000003,"5min":0.214,"15min":0.214},"max":{"1min":11.840999999999999,"5min":67.355000000000004,"15min":213.02799999999999},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:08:48 2023","interfaces":[{"interface":"back","average":{"1min":4.4729999999999999,"5min":3.5760000000000001,"15min":3.3439999999999999},"min":{"1min":0.26900000000000002,"5min":0.255,"15min":0.21199999999999999},"max":{"1min":15.327999999999999,"5min":138.90299999999999,"15min":138.90299999999999},"last":0.31},{"interface":"front","average":{"1min":4.1790000000000003,"5min":3.4279999999999999,"15min":3.2949999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.215},"max":{"1min":33.889000000000003,"5min":138.87700000000001,"15min":138.87700000000001},"last":0.46999999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.1910000000000001,"15min":1.022},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.26200000000000001},"max":{"1min":6.3470000000000004,"5min":11.744,"15min":12.586},"last":0.443},{"interface":"front","average":{"1min":2.3370000000000002,"5min":2.2109999999999999,"15min":1.444},"min":{"1min":0.35399999999999998,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":9.3480000000000008,"5min":16.725999999999999,"15min":16.725999999999999},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.3119999999999998,"15min":1.5569999999999999},"min":{"1min":0.37,"5min":0.31900000000000001,"15min":0.29999999999999999},"max":{"1min":9.3170000000000002,"5min":16.673999999999999,"15min":16.673999999999999},"last":0.377},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.353,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":4.0369999999999999,"5min":12.368,"15min":13.701000000000001},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.351,"15min":1.835},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.069,"5min":12.323,"15min":175.72499999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.456,"5min":1.286,"15min":1.8160000000000001},"min":{"1min":0.38,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.415,"5min":11.782,"15min":176.20400000000001},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.5089999999999999,"15min":1.3140000000000001},"min":{"1min":0.41899999999999998,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":9.9710000000000001,"5min":11.811999999999999,"15min":12.609},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.8330000000000002,"5min":2.359,"15min":1.589},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":11.41,"5min":16.875,"15min":16.875},"last":0.45500000000000002}]}]},{"osd":3,"up_from":148,"seq":635655160064,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617948,"kb_used_data":5524,"kb_used_omap":2,"kb_used_meta":612413,"kb_avail":93127204,"statfs":{"total":95995035648,"available":95362256896,"internally_reserved":0,"allocated":5656576,"data_stored":3380632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":627110965},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.3239999999999998,"15min":1.546},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.184},"max":{"1min":10.098000000000001,"5min":44.963000000000001,"15min":44.963000000000001},"last":0.378},{"interface":"front","average":{"1min":3.2349999999999999,"5min":1.597,"15min":1.325},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":16.920999999999999,"5min":16.920999999999999,"15min":37.634999999999998},"last":0.34000000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":19.158999999999999,"5min":7.0279999999999996,"15min":3.0649999999999999},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.20999999999999999},"max":{"1min":166.56999999999999,"5min":166.56999999999999,"15min":166.56999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":17.228999999999999,"5min":7.0049999999999999,"15min":3.202},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.182},"max":{"1min":166.291,"5min":166.291,"15min":166.291},"last":0.32000000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":3.649,"15min":2.0089999999999999},"min":{"1min":0.24199999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":17.373999999999999,"5min":88.825999999999993,"15min":88.825999999999993},"last":0.28999999999999998},{"interface":"front","average":{"1min":4.734,"5min":3.29,"15min":1.861},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":17.145,"5min":85.626000000000005,"15min":85.626000000000005},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.986,"15min":2.2850000000000001},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.976,"5min":38.600000000000001,"15min":111.36},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.915,"15min":2.1389999999999998},"min":{"1min":0.34399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":10.021000000000001,"5min":38.524000000000001,"15min":112.473},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9609999999999999,"5min":2.109,"15min":1.4259999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":16.855,"5min":36.488999999999997,"15min":36.488999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.0920000000000001,"15min":1.6180000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29299999999999998},"max":{"1min":6.0179999999999998,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.50700000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":19.748999999999999,"5min":5.399,"15min":2.7919999999999998},"min":{"1min":0.376,"5min":0.318,"15min":0.312},"max":{"1min":304.22399999999999,"5min":304.22399999999999,"15min":304.22399999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":19.741,"5min":5.3079999999999998,"15min":2.6560000000000001},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":304.15800000000002,"5min":304.15800000000002,"15min":304.15800000000002},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":3.4020000000000001,"5min":1.7669999999999999,"15min":1.722},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.30499999999999999},"max":{"1min":16.899000000000001,"5min":16.899000000000001,"15min":74.019000000000005},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.97,"5min":1.964,"15min":1.9870000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.32900000000000001},"max":{"1min":6.0490000000000004,"5min":12.407,"15min":73.364999999999995},"last":0.58399999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986682,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":586808,"kb_used_data":4400,"kb_used_omap":0,"kb_used_meta":582400,"kb_avail":93158344,"statfs":{"total":95995035648,"available":95394144256,"internally_reserved":0,"allocated":4505600,"data_stored":3253553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":596377600},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.272,"15min":1.9079999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":6.5469999999999997,"5min":12.869,"15min":102.358},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.097,"5min":1.4610000000000001,"15min":1.9339999999999999},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":10.439,"5min":12.848000000000001,"15min":102.58199999999999},"last":0.54400000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":7.0010000000000003,"5min":6.1950000000000003,"15min":3.9399999999999999},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":42.887,"5min":127.411,"15min":127.411},"last":0.60199999999999998},{"interface":"front","average":{"1min":7.258,"5min":4.3179999999999996,"15min":3.2770000000000001},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.253},"max":{"1min":44.148000000000003,"5min":118.075,"15min":118.075},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:09:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4260000000000002,"5min":5.2359999999999998,"15min":4.0259999999999998},"min":{"1min":0.28999999999999998,"5min":0.27300000000000002,"15min":0.246},"max":{"1min":137.89500000000001,"5min":137.89500000000001,"15min":164.517},"last":0.68200000000000005},{"interface":"front","average":{"1min":9.3119999999999994,"5min":4.7770000000000001,"15min":3.8620000000000001},"min":{"1min":0.33100000000000002,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":138.20599999999999,"5min":138.20599999999999,"15min":164.46799999999999},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.3380000000000001,"15min":1.4359999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27300000000000002},"max":{"1min":6.9390000000000001,"5min":6.9390000000000001,"15min":60.720999999999997},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.3859999999999999,"15min":1.4870000000000001},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.25600000000000001},"max":{"1min":6.4020000000000001,"5min":8.2899999999999991,"15min":60.631999999999998},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:09:22 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.87,"15min":0.76900000000000002},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":4.9080000000000004,"5min":6.0069999999999997,"15min":8.7089999999999996},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.85699999999999998,"15min":0.74099999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.21099999999999999},"max":{"1min":4.7140000000000004,"5min":4.9740000000000002,"15min":7.0179999999999998},"last":0.65200000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:09:36 2023","interfaces":[{"interface":"back","average":{"1min":3.1389999999999998,"5min":1.3700000000000001,"15min":1.03},"min":{"1min":0.312,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":36.557000000000002,"5min":36.557000000000002,"15min":36.557000000000002},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.423,"5min":1.2350000000000001,"15min":0.995},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.26200000000000001},"max":{"1min":38.468000000000004,"5min":38.468000000000004,"15min":38.468000000000004},"last":0.83499999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:09:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.036,"15min":1.4950000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.7359999999999998,"5min":13.667999999999999,"15min":145.721},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.2949999999999999,"15min":1.667},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.28000000000000003},"max":{"1min":10.411,"5min":10.411,"15min":148.45699999999999},"last":0.63300000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725046,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643816,"kb_used_data":4384,"kb_used_omap":0,"kb_used_meta":639424,"kb_avail":93101336,"statfs":{"total":95995035648,"available":95335768064,"internally_reserved":0,"allocated":4489216,"data_stored":3249297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":654770176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.0660000000000001,"15min":1.319},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.2640000000000002,"5min":10.488,"15min":80.105999999999995},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.1799999999999999,"15min":1.4590000000000001},"min":{"1min":0.27000000000000002,"5min":0.222,"15min":0.222},"max":{"1min":6.6619999999999999,"5min":8.2759999999999998,"15min":80.152000000000001},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:09:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5309999999999997,"5min":6.2519999999999998,"15min":3.5259999999999998},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":32.777999999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":6.5460000000000003,"5min":6.2060000000000004,"15min":3.4700000000000002},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.247},"max":{"1min":32.761000000000003,"5min":233.75200000000001,"15min":233.75200000000001},"last":0.437}]},{"osd":2,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":6.8789999999999996,"5min":4.9640000000000004,"15min":2.794},"min":{"1min":0.33500000000000002,"5min":0.28799999999999998,"15min":0.254},"max":{"1min":70.810000000000002,"5min":231.34700000000001,"15min":231.34700000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":8.1370000000000005,"5min":5.4619999999999997,"15min":3.0289999999999999},"min":{"1min":0.40200000000000002,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":70.507000000000005,"5min":231.381,"15min":231.381},"last":0.28499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.6519999999999999,"15min":2.786},"min":{"1min":0.33000000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":12.765000000000001,"5min":12.765000000000001,"15min":153.60300000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.3959999999999999,"15min":2.7130000000000001},"min":{"1min":0.34499999999999997,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":9.6120000000000001,"5min":9.6120000000000001,"15min":153.774},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.64900000000000002,"15min":0.63700000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.247},"max":{"1min":1.946,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.0580000000000001,"15min":0.76500000000000001},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":5.0369999999999999,"5min":8.3740000000000006,"15min":10.792},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":1.1990000000000001,"15min":1.1259999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.1130000000000004,"5min":9.8480000000000008,"15min":71.230000000000004},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.252,"15min":1.163},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":3.4359999999999999,"5min":12.673999999999999,"15min":70.266000000000005},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.262,"15min":1.0289999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.312},"max":{"1min":4.3399999999999999,"5min":9.6329999999999991,"15min":31.091999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.69899999999999995,"15min":0.80100000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":2.0179999999999998,"5min":4.29,"15min":31.780000000000001},"last":0.38200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:09:49.473 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:49.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:49 smithi123 ceph-mon[193903]: from='client.36660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:09:49.545 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:09:49.577 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:49.619 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:49.669 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:49.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:49.772 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:49.803 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:49.847 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:49.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:49.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:49.952 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:49.980 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:50.022 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:50.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:50.033 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:50.060 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:09:50.126 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:50.152 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:50.193 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:50.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:50.300 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:50.310 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:50 smithi121 ceph-mon[211526]: pgmap v1376: 89 pgs: 3 creating+activating, 5 creating+peering, 81 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:50.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3447903370' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:50.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:50 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:50.311 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:50 smithi121 ceph-mon[219709]: pgmap v1376: 89 pgs: 3 creating+activating, 5 creating+peering, 81 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:50.311 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3447903370' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:50.311 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:50 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:50.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:50 smithi123 ceph-mon[193903]: pgmap v1376: 89 pgs: 3 creating+activating, 5 creating+peering, 81 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:50.323 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3447903370' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:50.324 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:50 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:09:50.326 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:50.367 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:50.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:50.467 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:09:50.475 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:50.500 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:50.542 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:50.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:50.646 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:50.672 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:50.713 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:50.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:50.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:50.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:50.844 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:50.886 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:50.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:50.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:50.993 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:51.019 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:51.064 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-09-23T21:09:51.243 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:51.243 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.243 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:51.243 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2867669061' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[211526]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T21:09:51.244 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2867669061' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:09:51.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:09:51.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:51 smithi121 ceph-mon[219709]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:09:51.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:09:51.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T21:09:51.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2867669061' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:09:51.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:09:51.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: failed to encode map e429 with expected crc 2023-09-23T21:09:51.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:51 smithi123 ceph-mon[193903]: failed to encode map e429 with expected crc 2023-09-23T21:09:52.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[211526]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.066 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[211526]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.212 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:52.225 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:52.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:52.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:52.342 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:52.358 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:09:52.371 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[211526]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[211526]: pgmap v1379: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[211526]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[219709]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[219709]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[219709]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[219709]: pgmap v1379: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:52.372 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:52 smithi121 ceph-mon[219709]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.374 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:52 smithi123 ceph-mon[193903]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:52 smithi123 ceph-mon[193903]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:52 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:09:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:52 smithi123 ceph-mon[193903]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T21:09:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:52 smithi123 ceph-mon[193903]: pgmap v1379: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:52.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:52 smithi123 ceph-mon[193903]: failed to encode map e430 with expected crc 2023-09-23T21:09:52.521 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:52.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:52.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:52.625 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:09:52.657 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:52.705 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:09:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:52.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:09:52.811 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:52.840 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:52.990 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:09:53.133 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-09-23T21:09:53.137 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:09:53.137 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:09:53.184 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T21:09:53.185 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:09:53.185 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_563784 2023-09-23T21:09:53.185 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:09:53.185 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:09:53.293 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:53.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:53.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:53.303 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:53 smithi121 ceph-mon[211526]: pgmap v1380: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:53.303 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:53 smithi121 ceph-mon[219709]: pgmap v1380: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:53.404 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:53.437 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:53.492 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:53.494 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:53 smithi123 ceph-mon[193903]: pgmap v1380: 89 pgs: 89 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:09:53.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:53.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:53.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:53.659 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:53.855 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:09:53.856 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:09:54.185 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 2814 2798 174.827 174.875 0.00375162 0.00570166 2023-09-23T21:09:54.353 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:54.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:54.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:54.458 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:54.492 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:54.553 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:54.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:54.581 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:09:54.620 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:09:54.660 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:09:54.661 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:09:54.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4268214357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4268214357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:54.690 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:54.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:09:54.741 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:09:54.748 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:09:54.753 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4268214357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:09:54.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:54.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:09:54.782 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:09:54.823 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:09:54.854 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:09:54.862 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:09:54.870 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.870 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.870 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.870 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.871 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.871 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.871 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.871 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:09:54.924 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:09:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:09:54] "GET /metrics HTTP/1.1" 200 34261 "" "Prometheus/2.43.0" 2023-09-23T21:09:55.185 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 5647 5631 175.93 177.062 0.0186152 0.00563972 2023-09-23T21:09:55.762 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:55.890 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:09:55.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:55.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:09:55.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:55 smithi121 ceph-mon[211526]: pgmap v1381: 89 pgs: 89 active+clean; 127 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 16 MiB/s wr, 251 op/s 2023-09-23T21:09:55.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:55 smithi121 ceph-mon[219709]: pgmap v1381: 89 pgs: 89 active+clean; 127 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 16 MiB/s wr, 251 op/s 2023-09-23T21:09:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:55 smithi123 ceph-mon[193903]: pgmap v1381: 89 pgs: 89 active+clean; 127 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 16 MiB/s wr, 251 op/s 2023-09-23T21:09:55.998 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:09:56.066 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:56.185 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 7785 7769 161.824 133.625 0.00504788 0.00617329 2023-09-23T21:09:57.086 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:09:57.186 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 10539 10523 164.353 172.125 0.00500407 0.00607558 2023-09-23T21:09:57.264 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:09:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:57.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:09:57.367 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:57.398 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:57.423 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:09:57.454 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:57.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:57.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:57.558 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:57.589 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:57.643 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:57.646 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:57 smithi123 ceph-mon[193903]: pgmap v1382: 89 pgs: 89 active+clean; 356 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T21:09:57.647 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:57.647 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:57.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:57.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:57 smithi121 ceph-mon[219709]: pgmap v1382: 89 pgs: 89 active+clean; 356 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T21:09:57.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:57 smithi121 ceph-mon[211526]: pgmap v1382: 89 pgs: 89 active+clean; 356 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s 2023-09-23T21:09:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:09:57.748 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:57.810 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:58.187 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 12727 12711 158.795 136.75 0.00463521 0.00628989 2023-09-23T21:09:58.526 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:09:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:58.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:09:58.635 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:09:58.667 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:58.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:09:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:58.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:09:58.836 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:09:58.868 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:58.926 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:09:58.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:58.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:09:59.033 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:09:59.093 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:09:59.188 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 14657 14641 152.433 120.625 0.00188951 0.00648787 2023-09-23T21:09:59.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:09:59.424 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:09:59.680 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:09:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:09:59.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:09:59.785 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:09:59.849 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:00.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:09:59 smithi121 ceph-mon[219709]: pgmap v1383: 89 pgs: 89 active+clean; 356 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s 2023-09-23T21:10:00.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:09:59 smithi121 ceph-mon[211526]: pgmap v1383: 89 pgs: 89 active+clean; 356 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s 2023-09-23T21:10:00.187 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 16695 16679 148.854 127.375 0.0045313 0.00670979 2023-09-23T21:10:00.303 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:00.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:00.408 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:00.435 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:09:59 smithi123 ceph-mon[193903]: pgmap v1383: 89 pgs: 89 active+clean; 356 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s 2023-09-23T21:10:01.038 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:01.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:01.147 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:01.179 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.188 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 18492 18476 144.286 112.312 0.0203901 0.00691695 2023-09-23T21:10:01.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:01.324 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:01.335 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:01.364 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:01 smithi123 ceph-mon[193903]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T21:10:01.364 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:01 smithi123 ceph-mon[193903]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T21:10:01.367 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:01 smithi121 ceph-mon[211526]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T21:10:01.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:01 smithi121 ceph-mon[211526]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T21:10:01.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:01 smithi121 ceph-mon[219709]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-09-23T21:10:01.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:01 smithi121 ceph-mon[219709]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T21:10:01.426 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:01.532 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:01.564 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.619 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:01.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:01.724 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:01.755 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.811 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:01.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:01.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:01.864 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:01.919 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:01.949 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:01.999 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:02.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:02.104 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:02.188 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 20222 20206 140.268 108.125 0.0151096 0.00712277 2023-09-23T21:10:02.203 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:02.210 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:02 smithi121 ceph-mon[219709]: pgmap v1384: 89 pgs: 89 active+clean; 927 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:10:02.210 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:02 smithi121 ceph-mon[211526]: pgmap v1384: 89 pgs: 89 active+clean; 927 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:10:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:02 smithi123 ceph-mon[193903]: pgmap v1384: 89 pgs: 89 active+clean; 927 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:10:02.359 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:10:02.373 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:10:02.656 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:02.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:02.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:02.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:02.823 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:03.036 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:10:03.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1366625829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:03.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:03.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1366625829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:03.064 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:03.188 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 21722 21706 135.617 93.75 0.00421864 0.00736729 2023-09-23T21:10:03.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1366625829' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:03.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:04.188 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 23484 23468 133.299 110.125 0.00960803 0.00739545 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: from='client.56767 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: pgmap v1386: 89 pgs: 89 active+clean; 1.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[219709]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: from='client.56767 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:10:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T21:10:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: pgmap v1386: 89 pgs: 89 active+clean; 1.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T21:10:04.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:04 smithi121 ceph-mon[211526]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: from='client.56767 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:10:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T21:10:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: failed to encode map e431 with expected crc 2023-09-23T21:10:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: pgmap v1386: 89 pgs: 89 active+clean; 1.0 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T21:10:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:04 smithi123 ceph-mon[193903]: failed to encode map e431 with expected crc 2023-09-23T21:10:05.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:10:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:10:04] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.43.0" 2023-09-23T21:10:05.188 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 24495 24479 127.456 63.1875 0.00714756 0.00780832 2023-09-23T21:10:05.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:05 smithi123 ceph-mon[193903]: pgmap v1387: 89 pgs: 89 active+clean; 1.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T21:10:05.487 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:05.488 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:05 smithi121 ceph-mon[211526]: pgmap v1387: 89 pgs: 89 active+clean; 1.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T21:10:05.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:05 smithi121 ceph-mon[219709]: pgmap v1387: 89 pgs: 89 active+clean; 1.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T21:10:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:05.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:05.590 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:05.623 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:05.683 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:05.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:05.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:05.790 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:05.817 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:06.188 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 15 25771 25756 123.792 79.8125 0.0062115 0.0080723 2023-09-23T21:10:06.592 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:06.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:06.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:06.699 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:06.700 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1384,"stamp":"2023-09-23T21:10:01.151226+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":971773856,"num_objects":14843,"num_object_clones":0,"num_object_copies":44529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14843,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":15119,"num_write_kb":950861,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":119,"num_bytes_recovered":3579692,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15035,"ondisk_log_size":15035,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7468080,"kb_used_data":3072120,"kb_used_omap":21,"kb_used_meta":4395882,"kb_avail":648747984,"statfs":{"total":671965249536,"available":664317935616,"internally_reserved":0,"allocated":3145850880,"data_stored":3133584003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":4501383651},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,3,7,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":971177984,"num_objects":14819,"num_object_clones":0,"num_object_copies":44457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14819,"num_write_kb":948416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.092224"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":201,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386562+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:09:50.386562+0000","last_peered":"2023-09-23T21:09:50.386562+0000","last_clean":"2023-09-23T21:09:50.386562+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:09:50.386562+0000","last_undegraded":"2023-09-23T21:09:50.386562+0000","last_fullsized":"2023-09-23T21:09:50.386562+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.055044+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:09:50.055044+0000","last_peered":"2023-09-23T21:09:50.055044+0000","last_clean":"2023-09-23T21:09:50.055044+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:09:50.055044+0000","last_undegraded":"2023-09-23T21:09:50.055044+0000","last_fullsized":"2023-09-23T21:09:50.055044+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":344,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:51.189587+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:09:51.189587+0000","last_peered":"2023-09-23T21:09:51.189587+0000","last_clean":"2023-09-23T21:09:51.189587+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:09:51.189587+0000","last_undegraded":"2023-09-23T21:09:51.189587+0000","last_fullsized":"2023-09-23T21:09:51.189587+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":68,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411072+0000","last_change":"2023-09-23T21:04:58.672869+0000","last_active":"2023-09-23T21:09:50.411072+0000","last_peered":"2023-09-23T21:09:50.411072+0000","last_clean":"2023-09-23T21:09:50.411072+0000","last_became_active":"2023-09-23T21:04:58.672603+0000","last_became_peered":"2023-09-23T21:04:58.672603+0000","last_unstale":"2023-09-23T21:09:50.411072+0000","last_undegraded":"2023-09-23T21:09:50.411072+0000","last_fullsized":"2023-09-23T21:09:50.411072+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:51.431910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054056+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:09:50.054056+0000","last_peered":"2023-09-23T21:09:50.054056+0000","last_clean":"2023-09-23T21:09:50.054056+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:09:50.054056+0000","last_undegraded":"2023-09-23T21:09:50.054056+0000","last_fullsized":"2023-09-23T21:09:50.054056+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":306,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388416+0000","last_change":"2023-09-23T21:09:44.161988+0000","last_active":"2023-09-23T21:09:50.388416+0000","last_peered":"2023-09-23T21:09:50.388416+0000","last_clean":"2023-09-23T21:09:50.388416+0000","last_became_active":"2023-09-23T21:09:44.161643+0000","last_became_peered":"2023-09-23T21:09:44.161643+0000","last_unstale":"2023-09-23T21:09:50.388416+0000","last_undegraded":"2023-09-23T21:09:50.388416+0000","last_fullsized":"2023-09-23T21:09:50.388416+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:17.083615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":68,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411298+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:09:50.411298+0000","last_peered":"2023-09-23T21:09:50.411298+0000","last_clean":"2023-09-23T21:09:50.411298+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:09:50.411298+0000","last_undegraded":"2023-09-23T21:09:50.411298+0000","last_fullsized":"2023-09-23T21:09:50.411298+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.053973+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:09:50.053973+0000","last_peered":"2023-09-23T21:09:50.053973+0000","last_clean":"2023-09-23T21:09:50.053973+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:09:50.053973+0000","last_undegraded":"2023-09-23T21:09:50.053973+0000","last_fullsized":"2023-09-23T21:09:50.053973+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":53,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057742+0000","last_change":"2023-09-23T21:05:34.227277+0000","last_active":"2023-09-23T21:09:50.057742+0000","last_peered":"2023-09-23T21:09:50.057742+0000","last_clean":"2023-09-23T21:09:50.057742+0000","last_became_active":"2023-09-23T21:05:34.227018+0000","last_became_peered":"2023-09-23T21:05:34.227018+0000","last_unstale":"2023-09-23T21:09:50.057742+0000","last_undegraded":"2023-09-23T21:09:50.057742+0000","last_fullsized":"2023-09-23T21:09:50.057742+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:18.900660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":71,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054882+0000","last_change":"2023-09-23T21:04:41.542337+0000","last_active":"2023-09-23T21:09:50.054882+0000","last_peered":"2023-09-23T21:09:50.054882+0000","last_clean":"2023-09-23T21:09:50.054882+0000","last_became_active":"2023-09-23T21:04:41.542087+0000","last_became_peered":"2023-09-23T21:04:41.542087+0000","last_unstale":"2023-09-23T21:09:50.054882+0000","last_undegraded":"2023-09-23T21:09:50.054882+0000","last_fullsized":"2023-09-23T21:09:50.054882+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:03.303056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":87,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.055134+0000","last_change":"2023-09-23T21:05:01.732930+0000","last_active":"2023-09-23T21:09:50.055134+0000","last_peered":"2023-09-23T21:09:50.055134+0000","last_clean":"2023-09-23T21:09:50.055134+0000","last_became_active":"2023-09-23T21:05:01.692217+0000","last_became_peered":"2023-09-23T21:05:01.692217+0000","last_unstale":"2023-09-23T21:09:50.055134+0000","last_undegraded":"2023-09-23T21:09:50.055134+0000","last_fullsized":"2023-09-23T21:09:50.055134+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:17.616575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:51.524530+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:09:51.524530+0000","last_peered":"2023-09-23T21:09:51.524530+0000","last_clean":"2023-09-23T21:09:51.524530+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:09:51.524530+0000","last_undegraded":"2023-09-23T21:09:51.524530+0000","last_fullsized":"2023-09-23T21:09:51.524530+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":376,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388503+0000","last_change":"2023-09-23T21:05:24.157835+0000","last_active":"2023-09-23T21:09:50.388503+0000","last_peered":"2023-09-23T21:09:50.388503+0000","last_clean":"2023-09-23T21:09:50.388503+0000","last_became_active":"2023-09-23T21:05:24.136644+0000","last_became_peered":"2023-09-23T21:05:24.136644+0000","last_unstale":"2023-09-23T21:09:50.388503+0000","last_undegraded":"2023-09-23T21:09:50.388503+0000","last_fullsized":"2023-09-23T21:09:50.388503+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:17.317402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":348,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388503+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:09:50.388503+0000","last_peered":"2023-09-23T21:09:50.388503+0000","last_clean":"2023-09-23T21:09:50.388503+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:09:50.388503+0000","last_undegraded":"2023-09-23T21:09:50.388503+0000","last_fullsized":"2023-09-23T21:09:50.388503+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":121,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.055904+0000","last_change":"2023-09-23T21:05:24.137430+0000","last_active":"2023-09-23T21:09:50.055904+0000","last_peered":"2023-09-23T21:09:50.055904+0000","last_clean":"2023-09-23T21:09:50.055904+0000","last_became_active":"2023-09-23T21:05:24.128608+0000","last_became_peered":"2023-09-23T21:05:24.128608+0000","last_unstale":"2023-09-23T21:09:50.055904+0000","last_undegraded":"2023-09-23T21:09:50.055904+0000","last_fullsized":"2023-09-23T21:09:50.055904+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:01.023903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":80,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054327+0000","last_change":"2023-09-23T21:05:11.421345+0000","last_active":"2023-09-23T21:09:50.054327+0000","last_peered":"2023-09-23T21:09:50.054327+0000","last_clean":"2023-09-23T21:09:50.054327+0000","last_became_active":"2023-09-23T21:05:11.421042+0000","last_became_peered":"2023-09-23T21:05:11.421042+0000","last_unstale":"2023-09-23T21:09:50.054327+0000","last_undegraded":"2023-09-23T21:09:50.054327+0000","last_fullsized":"2023-09-23T21:09:50.054327+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:18.027428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":89,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054724+0000","last_change":"2023-09-23T21:05:11.421865+0000","last_active":"2023-09-23T21:09:50.054724+0000","last_peered":"2023-09-23T21:09:50.054724+0000","last_clean":"2023-09-23T21:09:50.054724+0000","last_became_active":"2023-09-23T21:05:11.421576+0000","last_became_peered":"2023-09-23T21:05:11.421576+0000","last_unstale":"2023-09-23T21:09:50.054724+0000","last_undegraded":"2023-09-23T21:09:50.054724+0000","last_fullsized":"2023-09-23T21:09:50.054724+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:55.613451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":89,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.055212+0000","last_change":"2023-09-23T21:05:40.111425+0000","last_active":"2023-09-23T21:09:50.055212+0000","last_peered":"2023-09-23T21:09:50.055212+0000","last_clean":"2023-09-23T21:09:50.055212+0000","last_became_active":"2023-09-23T21:05:40.111039+0000","last_became_peered":"2023-09-23T21:05:40.111039+0000","last_unstale":"2023-09-23T21:09:50.055212+0000","last_undegraded":"2023-09-23T21:09:50.055212+0000","last_fullsized":"2023-09-23T21:09:50.055212+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:14.874281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":230,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388032+0000","last_change":"2023-09-23T21:04:58.436345+0000","last_active":"2023-09-23T21:09:50.388032+0000","last_peered":"2023-09-23T21:09:50.388032+0000","last_clean":"2023-09-23T21:09:50.388032+0000","last_became_active":"2023-09-23T21:04:58.436050+0000","last_became_peered":"2023-09-23T21:04:58.436050+0000","last_unstale":"2023-09-23T21:09:50.388032+0000","last_undegraded":"2023-09-23T21:09:50.388032+0000","last_fullsized":"2023-09-23T21:09:50.388032+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:14.456266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":68,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411146+0000","last_change":"2023-09-23T21:04:58.433512+0000","last_active":"2023-09-23T21:09:50.411146+0000","last_peered":"2023-09-23T21:09:50.411146+0000","last_clean":"2023-09-23T21:09:50.411146+0000","last_became_active":"2023-09-23T21:04:58.433210+0000","last_became_peered":"2023-09-23T21:04:58.433210+0000","last_unstale":"2023-09-23T21:09:50.411146+0000","last_undegraded":"2023-09-23T21:09:50.411146+0000","last_fullsized":"2023-09-23T21:09:50.411146+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:18.930513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":426,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.387905+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:09:50.387905+0000","last_peered":"2023-09-23T21:09:50.387905+0000","last_clean":"2023-09-23T21:09:50.387905+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:09:50.387905+0000","last_undegraded":"2023-09-23T21:09:50.387905+0000","last_fullsized":"2023-09-23T21:09:50.387905+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054622+0000","last_change":"2023-09-23T21:09:44.164036+0000","last_active":"2023-09-23T21:09:50.054622+0000","last_peered":"2023-09-23T21:09:50.054622+0000","last_clean":"2023-09-23T21:09:50.054622+0000","last_became_active":"2023-09-23T21:09:44.163672+0000","last_became_peered":"2023-09-23T21:09:44.163672+0000","last_unstale":"2023-09-23T21:09:50.054622+0000","last_undegraded":"2023-09-23T21:09:50.054622+0000","last_fullsized":"2023-09-23T21:09:50.054622+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:10.564576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"430'1125","reported_seq":1137,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.944953+0000","last_change":"2023-09-23T21:09:45.077841+0000","last_active":"2023-09-23T21:10:00.944953+0000","last_peered":"2023-09-23T21:10:00.944953+0000","last_clean":"2023-09-23T21:10:00.944953+0000","last_became_active":"2023-09-23T21:09:45.077550+0000","last_became_peered":"2023-09-23T21:09:45.077550+0000","last_unstale":"2023-09-23T21:10:00.944953+0000","last_undegraded":"2023-09-23T21:10:00.944953+0000","last_fullsized":"2023-09-23T21:10:00.944953+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":1125,"log_dups_size":0,"ondisk_log_size":1125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:19.178954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73728000,"num_objects":1125,"num_object_clones":0,"num_object_copies":3375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1125,"num_write_kb":72000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":134,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388200+0000","last_change":"2023-09-23T21:05:11.446105+0000","last_active":"2023-09-23T21:09:50.388200+0000","last_peered":"2023-09-23T21:09:50.388200+0000","last_clean":"2023-09-23T21:09:50.388200+0000","last_became_active":"2023-09-23T21:05:11.424599+0000","last_became_peered":"2023-09-23T21:05:11.424599+0000","last_unstale":"2023-09-23T21:09:50.388200+0000","last_undegraded":"2023-09-23T21:09:50.388200+0000","last_fullsized":"2023-09-23T21:09:50.388200+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:29.472000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057503+0000","last_change":"2023-09-23T21:05:34.180005+0000","last_active":"2023-09-23T21:09:50.057503+0000","last_peered":"2023-09-23T21:09:50.057503+0000","last_clean":"2023-09-23T21:09:50.057503+0000","last_became_active":"2023-09-23T21:05:34.179694+0000","last_became_peered":"2023-09-23T21:05:34.179694+0000","last_unstale":"2023-09-23T21:09:50.057503+0000","last_undegraded":"2023-09-23T21:09:50.057503+0000","last_fullsized":"2023-09-23T21:09:50.057503+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:03.840198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"430'1153","reported_seq":1165,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.937402+0000","last_change":"2023-09-23T21:09:45.040484+0000","last_active":"2023-09-23T21:10:00.937402+0000","last_peered":"2023-09-23T21:10:00.937402+0000","last_clean":"2023-09-23T21:10:00.937402+0000","last_became_active":"2023-09-23T21:09:45.040324+0000","last_became_peered":"2023-09-23T21:09:45.040324+0000","last_unstale":"2023-09-23T21:10:00.937402+0000","last_undegraded":"2023-09-23T21:10:00.937402+0000","last_fullsized":"2023-09-23T21:10:00.937402+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":1153,"log_dups_size":0,"ondisk_log_size":1153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:54.268283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75563008,"num_objects":1153,"num_object_clones":0,"num_object_copies":3459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1153,"num_write_kb":73792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":129,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411219+0000","last_change":"2023-09-23T21:05:24.431109+0000","last_active":"2023-09-23T21:09:50.411219+0000","last_peered":"2023-09-23T21:09:50.411219+0000","last_clean":"2023-09-23T21:09:50.411219+0000","last_became_active":"2023-09-23T21:05:24.339953+0000","last_became_peered":"2023-09-23T21:05:24.339953+0000","last_unstale":"2023-09-23T21:09:50.411219+0000","last_undegraded":"2023-09-23T21:09:50.411219+0000","last_fullsized":"2023-09-23T21:09:50.411219+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:20.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":391,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386375+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:09:50.386375+0000","last_peered":"2023-09-23T21:09:50.386375+0000","last_clean":"2023-09-23T21:09:50.386375+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:09:50.386375+0000","last_undegraded":"2023-09-23T21:09:50.386375+0000","last_fullsized":"2023-09-23T21:09:50.386375+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"430'998","reported_seq":1010,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.026845+0000","last_change":"2023-09-23T21:09:45.078034+0000","last_active":"2023-09-23T21:10:00.026845+0000","last_peered":"2023-09-23T21:10:00.026845+0000","last_clean":"2023-09-23T21:10:00.026845+0000","last_became_active":"2023-09-23T21:09:45.077754+0000","last_became_peered":"2023-09-23T21:09:45.077754+0000","last_unstale":"2023-09-23T21:10:00.026845+0000","last_undegraded":"2023-09-23T21:10:00.026845+0000","last_fullsized":"2023-09-23T21:10:00.026845+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":998,"log_dups_size":0,"ondisk_log_size":998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:15.852527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65404928,"num_objects":998,"num_object_clones":0,"num_object_copies":2994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":998,"num_write_kb":63872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":57,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057734+0000","last_change":"2023-09-23T21:05:40.115342+0000","last_active":"2023-09-23T21:09:50.057734+0000","last_peered":"2023-09-23T21:09:50.057734+0000","last_clean":"2023-09-23T21:09:50.057734+0000","last_became_active":"2023-09-23T21:05:40.111077+0000","last_became_peered":"2023-09-23T21:05:40.111077+0000","last_unstale":"2023-09-23T21:09:50.057734+0000","last_undegraded":"2023-09-23T21:09:50.057734+0000","last_fullsized":"2023-09-23T21:09:50.057734+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:46.666670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":318,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388795+0000","last_change":"2023-09-23T21:03:38.414984+0000","last_active":"2023-09-23T21:09:50.388795+0000","last_peered":"2023-09-23T21:09:50.388795+0000","last_clean":"2023-09-23T21:09:50.388795+0000","last_became_active":"2023-09-23T21:03:38.414682+0000","last_became_peered":"2023-09-23T21:03:38.414682+0000","last_unstale":"2023-09-23T21:09:50.388795+0000","last_undegraded":"2023-09-23T21:09:50.388795+0000","last_fullsized":"2023-09-23T21:09:50.388795+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:21.525336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"430'896","reported_seq":909,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:59.119275+0000","last_change":"2023-09-23T21:09:45.078224+0000","last_active":"2023-09-23T21:09:59.119275+0000","last_peered":"2023-09-23T21:09:59.119275+0000","last_clean":"2023-09-23T21:09:59.119275+0000","last_became_active":"2023-09-23T21:09:45.077946+0000","last_became_peered":"2023-09-23T21:09:45.077946+0000","last_unstale":"2023-09-23T21:09:59.119275+0000","last_undegraded":"2023-09-23T21:09:59.119275+0000","last_fullsized":"2023-09-23T21:09:59.119275+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":896,"log_dups_size":0,"ondisk_log_size":896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:34.212928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58720256,"num_objects":896,"num_object_clones":0,"num_object_copies":2688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":896,"num_write_kb":57344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054414+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:09:50.054414+0000","last_peered":"2023-09-23T21:09:50.054414+0000","last_clean":"2023-09-23T21:09:50.054414+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:09:50.054414+0000","last_undegraded":"2023-09-23T21:09:50.054414+0000","last_fullsized":"2023-09-23T21:09:50.054414+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":77,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386814+0000","last_change":"2023-09-23T21:09:44.163930+0000","last_active":"2023-09-23T21:09:50.386814+0000","last_peered":"2023-09-23T21:09:50.386814+0000","last_clean":"2023-09-23T21:09:50.386814+0000","last_became_active":"2023-09-23T21:09:44.163614+0000","last_became_peered":"2023-09-23T21:09:44.163614+0000","last_unstale":"2023-09-23T21:09:50.386814+0000","last_undegraded":"2023-09-23T21:09:50.386814+0000","last_fullsized":"2023-09-23T21:09:50.386814+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:26.549908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"430'1150","reported_seq":1162,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.931328+0000","last_change":"2023-09-23T21:09:45.040470+0000","last_active":"2023-09-23T21:10:00.931328+0000","last_peered":"2023-09-23T21:10:00.931328+0000","last_clean":"2023-09-23T21:10:00.931328+0000","last_became_active":"2023-09-23T21:09:45.040298+0000","last_became_peered":"2023-09-23T21:09:45.040298+0000","last_unstale":"2023-09-23T21:10:00.931328+0000","last_undegraded":"2023-09-23T21:10:00.931328+0000","last_fullsized":"2023-09-23T21:10:00.931328+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":1150,"log_dups_size":0,"ondisk_log_size":1150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:06.840153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75366400,"num_objects":1150,"num_object_clones":0,"num_object_copies":3450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1150,"num_write_kb":73600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":88,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054286+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:09:50.054286+0000","last_peered":"2023-09-23T21:09:50.054286+0000","last_clean":"2023-09-23T21:09:50.054286+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:09:50.054286+0000","last_undegraded":"2023-09-23T21:09:50.054286+0000","last_fullsized":"2023-09-23T21:09:50.054286+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054240+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:09:50.054240+0000","last_peered":"2023-09-23T21:09:50.054240+0000","last_clean":"2023-09-23T21:09:50.054240+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:09:50.054240+0000","last_undegraded":"2023-09-23T21:09:50.054240+0000","last_fullsized":"2023-09-23T21:09:50.054240+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"430'1100","reported_seq":1112,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.937333+0000","last_change":"2023-09-23T21:09:45.041603+0000","last_active":"2023-09-23T21:10:00.937333+0000","last_peered":"2023-09-23T21:10:00.937333+0000","last_clean":"2023-09-23T21:10:00.937333+0000","last_became_active":"2023-09-23T21:09:45.041377+0000","last_became_peered":"2023-09-23T21:09:45.041377+0000","last_unstale":"2023-09-23T21:10:00.937333+0000","last_undegraded":"2023-09-23T21:10:00.937333+0000","last_fullsized":"2023-09-23T21:10:00.937333+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":1100,"log_dups_size":0,"ondisk_log_size":1100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:46.148257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72089600,"num_objects":1100,"num_object_clones":0,"num_object_copies":3300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1100,"num_write_kb":70400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":356,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388848+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:09:50.388848+0000","last_peered":"2023-09-23T21:09:50.388848+0000","last_clean":"2023-09-23T21:09:50.388848+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:09:50.388848+0000","last_undegraded":"2023-09-23T21:09:50.388848+0000","last_fullsized":"2023-09-23T21:09:50.388848+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":441,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057785+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:09:50.057785+0000","last_peered":"2023-09-23T21:09:50.057785+0000","last_clean":"2023-09-23T21:09:50.057785+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:09:50.057785+0000","last_undegraded":"2023-09-23T21:09:50.057785+0000","last_fullsized":"2023-09-23T21:09:50.057785+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"430'510","reported_seq":523,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:56.190791+0000","last_change":"2023-09-23T21:09:45.040594+0000","last_active":"2023-09-23T21:09:56.190791+0000","last_peered":"2023-09-23T21:09:56.190791+0000","last_clean":"2023-09-23T21:09:56.190791+0000","last_became_active":"2023-09-23T21:09:45.040285+0000","last_became_peered":"2023-09-23T21:09:45.040285+0000","last_unstale":"2023-09-23T21:09:56.190791+0000","last_undegraded":"2023-09-23T21:09:56.190791+0000","last_fullsized":"2023-09-23T21:09:56.190791+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":510,"log_dups_size":0,"ondisk_log_size":510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:36.237549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":510,"num_write_kb":32640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.387906+0000","last_change":"2023-09-23T21:05:24.132025+0000","last_active":"2023-09-23T21:09:50.387906+0000","last_peered":"2023-09-23T21:09:50.387906+0000","last_clean":"2023-09-23T21:09:50.387906+0000","last_became_active":"2023-09-23T21:05:24.131797+0000","last_became_peered":"2023-09-23T21:05:24.131797+0000","last_unstale":"2023-09-23T21:09:50.387906+0000","last_undegraded":"2023-09-23T21:09:50.387906+0000","last_fullsized":"2023-09-23T21:09:50.387906+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:32.069339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":469,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388468+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:09:50.388468+0000","last_peered":"2023-09-23T21:09:50.388468+0000","last_clean":"2023-09-23T21:09:50.388468+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:09:50.388468+0000","last_undegraded":"2023-09-23T21:09:50.388468+0000","last_fullsized":"2023-09-23T21:09:50.388468+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"430'468","reported_seq":481,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:56.181352+0000","last_change":"2023-09-23T21:09:45.080008+0000","last_active":"2023-09-23T21:09:56.181352+0000","last_peered":"2023-09-23T21:09:56.181352+0000","last_clean":"2023-09-23T21:09:56.181352+0000","last_became_active":"2023-09-23T21:09:45.079685+0000","last_became_peered":"2023-09-23T21:09:45.079685+0000","last_unstale":"2023-09-23T21:09:56.181352+0000","last_undegraded":"2023-09-23T21:09:56.181352+0000","last_fullsized":"2023-09-23T21:09:56.181352+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":468,"log_dups_size":0,"ondisk_log_size":468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:24.022184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30670848,"num_objects":468,"num_object_clones":0,"num_object_copies":1404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":468,"num_write_kb":29952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054134+0000","last_change":"2023-09-23T21:05:24.123350+0000","last_active":"2023-09-23T21:09:50.054134+0000","last_peered":"2023-09-23T21:09:50.054134+0000","last_clean":"2023-09-23T21:09:50.054134+0000","last_became_active":"2023-09-23T21:05:24.122996+0000","last_became_peered":"2023-09-23T21:05:24.122996+0000","last_unstale":"2023-09-23T21:09:50.054134+0000","last_undegraded":"2023-09-23T21:09:50.054134+0000","last_fullsized":"2023-09-23T21:09:50.054134+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:13.864877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":177,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054477+0000","last_change":"2023-09-23T21:09:44.164862+0000","last_active":"2023-09-23T21:09:50.054477+0000","last_peered":"2023-09-23T21:09:50.054477+0000","last_clean":"2023-09-23T21:09:50.054477+0000","last_became_active":"2023-09-23T21:09:44.164337+0000","last_became_peered":"2023-09-23T21:09:44.164337+0000","last_unstale":"2023-09-23T21:09:50.054477+0000","last_undegraded":"2023-09-23T21:09:50.054477+0000","last_fullsized":"2023-09-23T21:09:50.054477+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:57.424412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"430'994","reported_seq":1007,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:59.433423+0000","last_change":"2023-09-23T21:09:45.033715+0000","last_active":"2023-09-23T21:09:59.433423+0000","last_peered":"2023-09-23T21:09:59.433423+0000","last_clean":"2023-09-23T21:09:59.433423+0000","last_became_active":"2023-09-23T21:09:45.033437+0000","last_became_peered":"2023-09-23T21:09:45.033437+0000","last_unstale":"2023-09-23T21:09:59.433423+0000","last_undegraded":"2023-09-23T21:09:59.433423+0000","last_fullsized":"2023-09-23T21:09:59.433423+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":994,"log_dups_size":0,"ondisk_log_size":994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:15.193017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65142784,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":994,"num_write_kb":63616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":325,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386803+0000","last_change":"2023-09-23T21:04:29.792666+0000","last_active":"2023-09-23T21:09:50.386803+0000","last_peered":"2023-09-23T21:09:50.386803+0000","last_clean":"2023-09-23T21:09:50.386803+0000","last_became_active":"2023-09-23T21:04:29.792360+0000","last_became_peered":"2023-09-23T21:04:29.792360+0000","last_unstale":"2023-09-23T21:09:50.386803+0000","last_undegraded":"2023-09-23T21:09:50.386803+0000","last_fullsized":"2023-09-23T21:09:50.386803+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:31.208596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":95,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057825+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:09:50.057825+0000","last_peered":"2023-09-23T21:09:50.057825+0000","last_clean":"2023-09-23T21:09:50.057825+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:09:50.057825+0000","last_undegraded":"2023-09-23T21:09:50.057825+0000","last_fullsized":"2023-09-23T21:09:50.057825+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386180+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:09:50.386180+0000","last_peered":"2023-09-23T21:09:50.386180+0000","last_clean":"2023-09-23T21:09:50.386180+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:09:50.386180+0000","last_undegraded":"2023-09-23T21:09:50.386180+0000","last_fullsized":"2023-09-23T21:09:50.386180+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"430'918","reported_seq":931,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:59.129142+0000","last_change":"2023-09-23T21:09:45.843731+0000","last_active":"2023-09-23T21:09:59.129142+0000","last_peered":"2023-09-23T21:09:59.129142+0000","last_clean":"2023-09-23T21:09:59.129142+0000","last_became_active":"2023-09-23T21:09:45.843440+0000","last_became_peered":"2023-09-23T21:09:45.843440+0000","last_unstale":"2023-09-23T21:09:59.129142+0000","last_undegraded":"2023-09-23T21:09:59.129142+0000","last_fullsized":"2023-09-23T21:09:59.129142+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":918,"log_dups_size":0,"ondisk_log_size":918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:17.052514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60162048,"num_objects":918,"num_object_clones":0,"num_object_copies":2754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":918,"num_write_kb":58752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":406,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386727+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:09:50.386727+0000","last_peered":"2023-09-23T21:09:50.386727+0000","last_clean":"2023-09-23T21:09:50.386727+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:09:50.386727+0000","last_undegraded":"2023-09-23T21:09:50.386727+0000","last_fullsized":"2023-09-23T21:09:50.386727+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":380,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388748+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:09:50.388748+0000","last_peered":"2023-09-23T21:09:50.388748+0000","last_clean":"2023-09-23T21:09:50.388748+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:09:50.388748+0000","last_undegraded":"2023-09-23T21:09:50.388748+0000","last_fullsized":"2023-09-23T21:09:50.388748+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386731+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:09:50.386731+0000","last_peered":"2023-09-23T21:09:50.386731+0000","last_clean":"2023-09-23T21:09:50.386731+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:09:50.386731+0000","last_undegraded":"2023-09-23T21:09:50.386731+0000","last_fullsized":"2023-09-23T21:09:50.386731+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"430'1126","reported_seq":1138,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.919388+0000","last_change":"2023-09-23T21:09:45.843960+0000","last_active":"2023-09-23T21:10:00.919388+0000","last_peered":"2023-09-23T21:10:00.919388+0000","last_clean":"2023-09-23T21:10:00.919388+0000","last_became_active":"2023-09-23T21:09:45.843562+0000","last_became_peered":"2023-09-23T21:09:45.843562+0000","last_unstale":"2023-09-23T21:10:00.919388+0000","last_undegraded":"2023-09-23T21:10:00.919388+0000","last_fullsized":"2023-09-23T21:10:00.919388+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":1126,"log_dups_size":0,"ondisk_log_size":1126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:10.329012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73793536,"num_objects":1126,"num_object_clones":0,"num_object_copies":3378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1126,"num_write_kb":72064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":408,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.055724+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:09:50.055724+0000","last_peered":"2023-09-23T21:09:50.055724+0000","last_clean":"2023-09-23T21:09:50.055724+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:09:50.055724+0000","last_undegraded":"2023-09-23T21:09:50.055724+0000","last_fullsized":"2023-09-23T21:09:50.055724+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":326,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:51.189559+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:09:51.189559+0000","last_peered":"2023-09-23T21:09:51.189559+0000","last_clean":"2023-09-23T21:09:51.189559+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:09:51.189559+0000","last_undegraded":"2023-09-23T21:09:51.189559+0000","last_fullsized":"2023-09-23T21:09:51.189559+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":63,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411229+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:09:50.411229+0000","last_peered":"2023-09-23T21:09:50.411229+0000","last_clean":"2023-09-23T21:09:50.411229+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:09:50.411229+0000","last_undegraded":"2023-09-23T21:09:50.411229+0000","last_fullsized":"2023-09-23T21:09:50.411229+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"430'476","reported_seq":489,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:56.183764+0000","last_change":"2023-09-23T21:09:45.036017+0000","last_active":"2023-09-23T21:09:56.183764+0000","last_peered":"2023-09-23T21:09:56.183764+0000","last_clean":"2023-09-23T21:09:56.183764+0000","last_became_active":"2023-09-23T21:09:45.035744+0000","last_became_peered":"2023-09-23T21:09:45.035744+0000","last_unstale":"2023-09-23T21:09:56.183764+0000","last_undegraded":"2023-09-23T21:09:56.183764+0000","last_fullsized":"2023-09-23T21:09:56.183764+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":476,"log_dups_size":0,"ondisk_log_size":476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:55.125986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31195136,"num_objects":476,"num_object_clones":0,"num_object_copies":1428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":476,"num_write_kb":30464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":246,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.055942+0000","last_change":"2023-09-23T21:04:41.434048+0000","last_active":"2023-09-23T21:09:50.055942+0000","last_peered":"2023-09-23T21:09:50.055942+0000","last_clean":"2023-09-23T21:09:50.055942+0000","last_became_active":"2023-09-23T21:04:41.430664+0000","last_became_peered":"2023-09-23T21:04:41.430664+0000","last_unstale":"2023-09-23T21:09:50.055942+0000","last_undegraded":"2023-09-23T21:09:50.055942+0000","last_fullsized":"2023-09-23T21:09:50.055942+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:11.559268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":690,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386170+0000","last_change":"2023-09-23T21:08:41.538586+0000","last_active":"2023-09-23T21:09:50.386170+0000","last_peered":"2023-09-23T21:09:50.386170+0000","last_clean":"2023-09-23T21:09:50.386170+0000","last_became_active":"2023-09-23T21:08:41.538107+0000","last_became_peered":"2023-09-23T21:08:41.538107+0000","last_unstale":"2023-09-23T21:09:50.386170+0000","last_undegraded":"2023-09-23T21:09:50.386170+0000","last_fullsized":"2023-09-23T21:09:50.386170+0000","mapping_epoch":416,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:48.268707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054642+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:09:50.054642+0000","last_peered":"2023-09-23T21:09:50.054642+0000","last_clean":"2023-09-23T21:09:50.054642+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:09:50.054642+0000","last_undegraded":"2023-09-23T21:09:50.054642+0000","last_fullsized":"2023-09-23T21:09:50.054642+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":318,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388600+0000","last_change":"2023-09-23T21:05:40.113981+0000","last_active":"2023-09-23T21:09:50.388600+0000","last_peered":"2023-09-23T21:09:50.388600+0000","last_clean":"2023-09-23T21:09:50.388600+0000","last_became_active":"2023-09-23T21:05:40.113619+0000","last_became_peered":"2023-09-23T21:05:40.113619+0000","last_unstale":"2023-09-23T21:09:50.388600+0000","last_undegraded":"2023-09-23T21:09:50.388600+0000","last_fullsized":"2023-09-23T21:09:50.388600+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:49.774784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"430'987","reported_seq":1000,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:59.116354+0000","last_change":"2023-09-23T21:09:45.078090+0000","last_active":"2023-09-23T21:09:59.116354+0000","last_peered":"2023-09-23T21:09:59.116354+0000","last_clean":"2023-09-23T21:09:59.116354+0000","last_became_active":"2023-09-23T21:09:45.077774+0000","last_became_peered":"2023-09-23T21:09:45.077774+0000","last_unstale":"2023-09-23T21:09:59.116354+0000","last_undegraded":"2023-09-23T21:09:59.116354+0000","last_fullsized":"2023-09-23T21:09:59.116354+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":987,"log_dups_size":0,"ondisk_log_size":987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:01.305138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64684032,"num_objects":987,"num_object_clones":0,"num_object_copies":2961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":987,"num_write_kb":63168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":63,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386262+0000","last_change":"2023-09-23T21:05:11.430320+0000","last_active":"2023-09-23T21:09:50.386262+0000","last_peered":"2023-09-23T21:09:50.386262+0000","last_clean":"2023-09-23T21:09:50.386262+0000","last_became_active":"2023-09-23T21:05:11.427530+0000","last_became_peered":"2023-09-23T21:05:11.427530+0000","last_unstale":"2023-09-23T21:09:50.386262+0000","last_undegraded":"2023-09-23T21:09:50.386262+0000","last_fullsized":"2023-09-23T21:09:50.386262+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:08.238606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":77,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.386464+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:09:50.386464+0000","last_peered":"2023-09-23T21:09:50.386464+0000","last_clean":"2023-09-23T21:09:50.386464+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:09:50.386464+0000","last_undegraded":"2023-09-23T21:09:50.386464+0000","last_fullsized":"2023-09-23T21:09:50.386464+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.387092+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:09:50.387092+0000","last_peered":"2023-09-23T21:09:50.387092+0000","last_clean":"2023-09-23T21:09:50.387092+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:09:50.387092+0000","last_undegraded":"2023-09-23T21:09:50.387092+0000","last_fullsized":"2023-09-23T21:09:50.387092+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"430'973","reported_seq":985,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.030873+0000","last_change":"2023-09-23T21:09:45.038161+0000","last_active":"2023-09-23T21:10:00.030873+0000","last_peered":"2023-09-23T21:10:00.030873+0000","last_clean":"2023-09-23T21:10:00.030873+0000","last_became_active":"2023-09-23T21:09:45.037907+0000","last_became_peered":"2023-09-23T21:09:45.037907+0000","last_unstale":"2023-09-23T21:10:00.030873+0000","last_undegraded":"2023-09-23T21:10:00.030873+0000","last_fullsized":"2023-09-23T21:10:00.030873+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":973,"log_dups_size":0,"ondisk_log_size":973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:37.785460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63766528,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":973,"num_write_kb":62272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":68,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.387031+0000","last_change":"2023-09-23T21:05:01.646895+0000","last_active":"2023-09-23T21:09:50.387031+0000","last_peered":"2023-09-23T21:09:50.387031+0000","last_clean":"2023-09-23T21:09:50.387031+0000","last_became_active":"2023-09-23T21:05:01.646594+0000","last_became_peered":"2023-09-23T21:05:01.646594+0000","last_unstale":"2023-09-23T21:09:50.387031+0000","last_undegraded":"2023-09-23T21:09:50.387031+0000","last_fullsized":"2023-09-23T21:09:50.387031+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:02.174001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":327,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411165+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:09:50.411165+0000","last_peered":"2023-09-23T21:09:50.411165+0000","last_clean":"2023-09-23T21:09:50.411165+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:09:50.411165+0000","last_undegraded":"2023-09-23T21:09:50.411165+0000","last_fullsized":"2023-09-23T21:09:50.411165+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":105,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057627+0000","last_change":"2023-09-23T21:04:58.432098+0000","last_active":"2023-09-23T21:09:50.057627+0000","last_peered":"2023-09-23T21:09:50.057627+0000","last_clean":"2023-09-23T21:09:50.057627+0000","last_became_active":"2023-09-23T21:04:58.431828+0000","last_became_peered":"2023-09-23T21:04:58.431828+0000","last_unstale":"2023-09-23T21:09:50.057627+0000","last_undegraded":"2023-09-23T21:09:50.057627+0000","last_fullsized":"2023-09-23T21:09:50.057627+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:56.163017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"430'1040","reported_seq":1053,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:10:00.026012+0000","last_change":"2023-09-23T21:09:45.033672+0000","last_active":"2023-09-23T21:10:00.026012+0000","last_peered":"2023-09-23T21:10:00.026012+0000","last_clean":"2023-09-23T21:10:00.026012+0000","last_became_active":"2023-09-23T21:09:45.033296+0000","last_became_peered":"2023-09-23T21:09:45.033296+0000","last_unstale":"2023-09-23T21:10:00.026012+0000","last_undegraded":"2023-09-23T21:10:00.026012+0000","last_fullsized":"2023-09-23T21:10:00.026012+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":1040,"log_dups_size":0,"ondisk_log_size":1040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:26.841578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68157440,"num_objects":1040,"num_object_clones":0,"num_object_copies":3120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1040,"num_write_kb":66560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":88,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054717+0000","last_change":"2023-09-23T21:05:01.713048+0000","last_active":"2023-09-23T21:09:50.054717+0000","last_peered":"2023-09-23T21:09:50.054717+0000","last_clean":"2023-09-23T21:09:50.054717+0000","last_became_active":"2023-09-23T21:04:58.428974+0000","last_became_peered":"2023-09-23T21:04:58.428974+0000","last_unstale":"2023-09-23T21:09:50.054717+0000","last_undegraded":"2023-09-23T21:09:50.054717+0000","last_fullsized":"2023-09-23T21:09:50.054717+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:37.239048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":63,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411096+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:09:50.411096+0000","last_peered":"2023-09-23T21:09:50.411096+0000","last_clean":"2023-09-23T21:09:50.411096+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:09:50.411096+0000","last_undegraded":"2023-09-23T21:09:50.411096+0000","last_fullsized":"2023-09-23T21:09:50.411096+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.053898+0000","last_change":"2023-09-23T21:04:58.424071+0000","last_active":"2023-09-23T21:09:50.053898+0000","last_peered":"2023-09-23T21:09:50.053898+0000","last_clean":"2023-09-23T21:09:50.053898+0000","last_became_active":"2023-09-23T21:04:58.422459+0000","last_became_peered":"2023-09-23T21:04:58.422459+0000","last_unstale":"2023-09-23T21:09:50.053898+0000","last_undegraded":"2023-09-23T21:09:50.053898+0000","last_fullsized":"2023-09-23T21:09:50.053898+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:19.299423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"430'905","reported_seq":918,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:59.433181+0000","last_change":"2023-09-23T21:09:45.038290+0000","last_active":"2023-09-23T21:09:59.433181+0000","last_peered":"2023-09-23T21:09:59.433181+0000","last_clean":"2023-09-23T21:09:59.433181+0000","last_became_active":"2023-09-23T21:09:45.037976+0000","last_became_peered":"2023-09-23T21:09:45.037976+0000","last_unstale":"2023-09-23T21:09:59.433181+0000","last_undegraded":"2023-09-23T21:09:59.433181+0000","last_fullsized":"2023-09-23T21:09:59.433181+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":905,"log_dups_size":0,"ondisk_log_size":905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:44.361487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59310080,"num_objects":905,"num_object_clones":0,"num_object_copies":2715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":905,"num_write_kb":57920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":326,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388491+0000","last_change":"2023-09-23T21:04:41.423899+0000","last_active":"2023-09-23T21:09:50.388491+0000","last_peered":"2023-09-23T21:09:50.388491+0000","last_clean":"2023-09-23T21:09:50.388491+0000","last_became_active":"2023-09-23T21:04:41.423582+0000","last_became_peered":"2023-09-23T21:04:41.423582+0000","last_unstale":"2023-09-23T21:09:50.388491+0000","last_undegraded":"2023-09-23T21:09:50.388491+0000","last_fullsized":"2023-09-23T21:09:50.388491+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:08.226163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":72,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.410980+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:09:50.410980+0000","last_peered":"2023-09-23T21:09:50.410980+0000","last_clean":"2023-09-23T21:09:50.410980+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:09:50.410980+0000","last_undegraded":"2023-09-23T21:09:50.410980+0000","last_fullsized":"2023-09-23T21:09:50.410980+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":115,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057527+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:09:50.057527+0000","last_peered":"2023-09-23T21:09:50.057527+0000","last_clean":"2023-09-23T21:09:50.057527+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:09:50.057527+0000","last_undegraded":"2023-09-23T21:09:50.057527+0000","last_fullsized":"2023-09-23T21:09:50.057527+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":77,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-09-23T21:09:51.525041+0000","last_change":"2023-09-23T21:05:11.411263+0000","last_active":"2023-09-23T21:09:51.525041+0000","last_peered":"2023-09-23T21:09:51.525041+0000","last_clean":"2023-09-23T21:09:51.525041+0000","last_became_active":"2023-09-23T21:05:11.410881+0000","last_became_peered":"2023-09-23T21:05:11.410881+0000","last_unstale":"2023-09-23T21:09:51.525041+0000","last_undegraded":"2023-09-23T21:09:51.525041+0000","last_fullsized":"2023-09-23T21:09:51.525041+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:20.245864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":463,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.057460+0000","last_change":"2023-09-23T21:09:44.163759+0000","last_active":"2023-09-23T21:09:50.057460+0000","last_peered":"2023-09-23T21:09:50.057460+0000","last_clean":"2023-09-23T21:09:50.057460+0000","last_became_active":"2023-09-23T21:09:44.163405+0000","last_became_peered":"2023-09-23T21:09:44.163405+0000","last_unstale":"2023-09-23T21:09:50.057460+0000","last_undegraded":"2023-09-23T21:09:50.057460+0000","last_fullsized":"2023-09-23T21:09:50.057460+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:52.346296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":92,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388109+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:09:50.388109+0000","last_peered":"2023-09-23T21:09:50.388109+0000","last_clean":"2023-09-23T21:09:50.388109+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:09:50.388109+0000","last_undegraded":"2023-09-23T21:09:50.388109+0000","last_fullsized":"2023-09-23T21:09:50.388109+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":97,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054376+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:09:50.054376+0000","last_peered":"2023-09-23T21:09:50.054376+0000","last_clean":"2023-09-23T21:09:50.054376+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:09:50.054376+0000","last_undegraded":"2023-09-23T21:09:50.054376+0000","last_fullsized":"2023-09-23T21:09:50.054376+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":311,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.388698+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:09:50.388698+0000","last_peered":"2023-09-23T21:09:50.388698+0000","last_clean":"2023-09-23T21:09:50.388698+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:09:50.388698+0000","last_undegraded":"2023-09-23T21:09:50.388698+0000","last_fullsized":"2023-09-23T21:09:50.388698+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":89,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054795+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:09:50.054795+0000","last_peered":"2023-09-23T21:09:50.054795+0000","last_clean":"2023-09-23T21:09:50.054795+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:09:50.054795+0000","last_undegraded":"2023-09-23T21:09:50.054795+0000","last_fullsized":"2023-09-23T21:09:50.054795+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":71,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054518+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:09:50.054518+0000","last_peered":"2023-09-23T21:09:50.054518+0000","last_clean":"2023-09-23T21:09:50.054518+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:09:50.054518+0000","last_undegraded":"2023-09-23T21:09:50.054518+0000","last_fullsized":"2023-09-23T21:09:50.054518+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":80,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.054768+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:09:50.054768+0000","last_peered":"2023-09-23T21:09:50.054768+0000","last_clean":"2023-09-23T21:09:50.054768+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:09:50.054768+0000","last_undegraded":"2023-09-23T21:09:50.054768+0000","last_fullsized":"2023-09-23T21:09:50.054768+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":74,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.411281+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:09:50.411281+0000","last_peered":"2023-09-23T21:09:50.411281+0000","last_clean":"2023-09-23T21:09:50.411281+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:09:50.411281+0000","last_undegraded":"2023-09-23T21:09:50.411281+0000","last_fullsized":"2023-09-23T21:09:50.411281+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":57,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-09-23T21:09:50.387753+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:09:50.387753+0000","last_peered":"2023-09-23T21:09:50.387753+0000","last_clean":"2023-09-23T21:09:50.387753+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:09:50.387753+0000","last_undegraded":"2023-09-23T21:09:50.387753+0000","last_fullsized":"2023-09-23T21:09:50.387753+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":971177984,"num_objects":14819,"num_object_clones":0,"num_object_copies":44457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14819,"num_write_kb":948416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2875588608,"data_stored":2875588608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":14819,"ondisk_log_size":14819,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":37484,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201775,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463396,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1187680,"kb_used_data":614168,"kb_used_omap":0,"kb_used_meta":573504,"kb_avail":92557472,"statfs":{"total":95995035648,"available":94778851328,"internally_reserved":0,"allocated":628908032,"data_stored":627663560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":587268096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":2.085,"15min":1.9430000000000001},"min":{"1min":0.245,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":5.5780000000000003,"5min":99.533000000000001,"15min":102.242},"last":4.298},{"interface":"front","average":{"1min":1.4399999999999999,"5min":2.177,"15min":1.8160000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":11.157999999999999,"5min":99.325000000000003,"15min":102.502},"last":4.0709999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7440000000000002,"5min":3.3039999999999998,"15min":2.5760000000000001},"min":{"1min":0.29599999999999999,"5min":0.251,"15min":0.24299999999999999},"max":{"1min":7.915,"5min":119.077,"15min":119.077},"last":1.319},{"interface":"front","average":{"1min":1.9319999999999999,"5min":3.1040000000000001,"15min":2.5939999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.0229999999999997,"5min":119.313,"15min":119.313},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":6.4370000000000003,"5min":4.766,"15min":4.1180000000000003},"min":{"1min":0.29799999999999999,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":81.801000000000002,"5min":199.23400000000001,"15min":199.23400000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":6.6970000000000001,"5min":4.5650000000000004,"15min":4.1740000000000004},"min":{"1min":0.33800000000000002,"5min":0.28499999999999998,"15min":0.21199999999999999},"max":{"1min":81.510999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.72099999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.3380000000000001,"15min":2.6429999999999998},"min":{"1min":0.315,"5min":0.29399999999999998,"15min":0.26900000000000002},"max":{"1min":7.2130000000000001,"5min":13.340999999999999,"15min":178.40000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.466,"5min":1.6000000000000001,"15min":2.6440000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":5.415,"5min":13.319000000000001,"15min":178.26599999999999},"last":1.0649999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":0.93200000000000005,"15min":0.94299999999999995},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.0890000000000004,"5min":9.0890000000000004,"15min":13.1},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.78400000000000003,"15min":0.82999999999999996},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.24299999999999999},"max":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":8.6859999999999999},"last":0.23999999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.94599999999999995,"15min":0.94399999999999995},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.0519999999999996,"5min":9.0519999999999996,"15min":11.281000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.86599999999999999,"15min":0.99399999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.0190000000000001,"5min":6.6890000000000001,"15min":9.5389999999999997},"last":0.35299999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.1080000000000001,"15min":0.97499999999999998},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":6.6299999999999999,"5min":7.8079999999999998,"15min":11.388999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.0669999999999999,"15min":1.0760000000000001},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.30099999999999999},"max":{"1min":6.4749999999999996,"5min":7.1849999999999996,"15min":8.8010000000000002},"last":0.42199999999999999}]}]},{"osd":1,"up_from":276,"seq":1185410973841,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":953776,"kb_used_data":463912,"kb_used_omap":5,"kb_used_meta":489850,"kb_avail":92791376,"statfs":{"total":95995035648,"available":95018369024,"internally_reserved":0,"allocated":475045888,"data_stored":472905062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":501607139},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.274,"5min":2.4860000000000002,"15min":1.1259999999999999},"min":{"1min":0.27600000000000002,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":23.486000000000001,"5min":23.486000000000001,"15min":23.486000000000001},"last":0.95299999999999996},{"interface":"front","average":{"1min":3.9540000000000002,"5min":1.996,"15min":1.2310000000000001},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":16.148,"5min":16.148,"15min":18.047999999999998},"last":0.29899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":5.4459999999999997,"5min":2.556,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":21.695,"5min":21.695,"15min":78.409000000000006},"last":1.2490000000000001},{"interface":"front","average":{"1min":4.8369999999999997,"5min":2.6619999999999999,"15min":1.6140000000000001},"min":{"1min":0.502,"5min":0.245,"15min":0.22},"max":{"1min":23.539000000000001,"5min":23.539000000000001,"15min":76.245000000000005},"last":0.997}]},{"osd":3,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":2.4750000000000001,"15min":3.0379999999999998},"min":{"1min":0.39500000000000002,"5min":0.25,"15min":0.23899999999999999},"max":{"1min":23.407,"5min":23.407,"15min":172.44399999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":3.9260000000000002,"5min":2.173,"15min":3.1059999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":17.033999999999999,"5min":17.033999999999999,"15min":172.58500000000001},"last":0.46999999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.631,"15min":1.137},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.5389999999999997,"5min":12.238,"15min":17.006},"last":1.5349999999999999},{"interface":"front","average":{"1min":3.3530000000000002,"5min":2.0640000000000001,"15min":1.1180000000000001},"min":{"1min":0.42199999999999999,"5min":0.33300000000000002,"15min":0.29399999999999998},"max":{"1min":23.332999999999998,"5min":23.332999999999998,"15min":23.332999999999998},"last":1.0289999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.7909999999999999,"15min":1.2170000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":16.196000000000002,"5min":16.196000000000002,"15min":17.155000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.5339999999999998,"5min":1.7969999999999999,"15min":1.284},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.9660000000000002,"5min":12.345000000000001,"15min":17.076000000000001},"last":1.573}]},{"osd":6,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":1.8089999999999999,"15min":1.73},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":16.222000000000001,"5min":16.222000000000001,"15min":127.995},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.6160000000000001,"5min":1.962,"15min":1.758},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8959999999999999,"5min":12.276999999999999,"15min":126.836},"last":1.6060000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:09:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":3.2639999999999998,"15min":1.756},"min":{"1min":0.38100000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.859,"5min":151.791,"15min":151.791},"last":1.6339999999999999},{"interface":"front","average":{"1min":3.4039999999999999,"5min":3.552,"15min":1.6579999999999999},"min":{"1min":0.45000000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":23.376000000000001,"5min":151.809,"15min":151.809},"last":1.0609999999999999}]}]},{"osd":0,"up_from":92,"seq":395136991582,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1092328,"kb_used_data":325600,"kb_used_omap":5,"kb_used_meta":766714,"kb_avail":92652824,"statfs":{"total":95995035648,"available":94876491776,"internally_reserved":0,"allocated":333414400,"data_stored":331396017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":785115770},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:09:01 2023","interfaces":[{"interface":"back","average":{"1min":3.218,"5min":3.3210000000000002,"15min":2.0219999999999998},"min":{"1min":0.26300000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":18.140999999999998,"5min":52.590000000000003,"15min":61.341999999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":4.5570000000000004,"5min":3.754,"15min":2.1059999999999999},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":18.219999999999999,"5min":52.276000000000003,"15min":61.299999999999997},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:09:56 2023","interfaces":[{"interface":"back","average":{"1min":9.6219999999999999,"5min":4.6950000000000003,"15min":2.7639999999999998},"min":{"1min":0.22900000000000001,"5min":0.217,"15min":0.161},"max":{"1min":127.791,"5min":127.791,"15min":183.77099999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":10.083,"5min":4.617,"15min":2.8940000000000001},"min":{"1min":0.29299999999999998,"5min":0.254,"15min":0.214},"max":{"1min":127.953,"5min":127.953,"15min":183.78700000000001},"last":0.29499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:09:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3730000000000002,"5min":2.323,"15min":2.3330000000000002},"min":{"1min":0.23999999999999999,"5min":0.22900000000000001,"15min":0.183},"max":{"1min":46.692,"5min":46.692,"15min":186.143},"last":0.45100000000000001},{"interface":"front","average":{"1min":4.3070000000000004,"5min":2.1779999999999999,"15min":2.371},"min":{"1min":0.26800000000000002,"5min":0.24299999999999999,"15min":0.23000000000000001},"max":{"1min":46.762,"5min":46.762,"15min":186.97399999999999},"last":0.432}]},{"osd":4,"last update":"Sat Sep 23 21:09:56 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.248,"15min":1.2350000000000001},"min":{"1min":0.35599999999999998,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":3.286,"5min":10.814,"15min":10.814},"last":1.968},{"interface":"front","average":{"1min":1.171,"5min":1.2090000000000001,"15min":0.97499999999999998},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":7.343,"5min":10.629,"15min":11.493},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:09:56 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.276,"15min":1.3},"min":{"1min":0.42099999999999999,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":3.3220000000000001,"5min":10.763,"15min":10.763},"last":2.0129999999999999},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.345,"15min":1.0029999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":7.3920000000000003,"5min":11.478,"15min":11.478},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:09:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":2.9279999999999999,"15min":3.2370000000000001},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":17.773,"5min":136.404,"15min":170.12700000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.331,"5min":2.9009999999999998,"15min":3.3170000000000002},"min":{"1min":0.374,"5min":0.36599999999999999,"15min":0.28299999999999997},"max":{"1min":17.684999999999999,"5min":136.44999999999999,"15min":169.767},"last":1.7230000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:09:56 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.534,"15min":1.1870000000000001},"min":{"1min":0.38200000000000001,"5min":0.35199999999999998,"15min":0.33600000000000002},"max":{"1min":7.4160000000000004,"5min":10.74,"15min":11.42},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.6850000000000001,"5min":1.3720000000000001,"15min":1.2},"min":{"1min":0.39200000000000002,"5min":0.38700000000000001,"15min":0.34599999999999997},"max":{"1min":10.237,"5min":10.237,"15min":10.237},"last":1.762}]}]},{"osd":2,"up_from":129,"seq":554050781473,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428652,"kb_used_data":714660,"kb_used_omap":7,"kb_used_meta":713976,"kb_avail":92316500,"statfs":{"total":95995035648,"available":94532096000,"internally_reserved":0,"allocated":731811840,"data_stored":729732020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":731111505},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.6259999999999999,"15min":1.381},"min":{"1min":0.217,"5min":0.19400000000000001,"15min":0.17699999999999999},"max":{"1min":10.077,"5min":11.699999999999999,"15min":26.356000000000002},"last":0.95699999999999996},{"interface":"front","average":{"1min":2.879,"5min":2.7610000000000001,"15min":1.9299999999999999},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.2360000000000007,"5min":23.963999999999999,"15min":26.341000000000001},"last":0.68500000000000005}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":3.3039999999999998,"15min":2.802},"min":{"1min":0.24299999999999999,"5min":0.19,"15min":0.188},"max":{"1min":11.271000000000001,"5min":67.296999999999997,"15min":212.12},"last":0.90400000000000003},{"interface":"front","average":{"1min":2.7069999999999999,"5min":2.5960000000000001,"15min":2.4860000000000002},"min":{"1min":0.34200000000000003,"5min":0.214,"15min":0.214},"max":{"1min":11.840999999999999,"5min":67.355000000000004,"15min":213.02799999999999},"last":1.0700000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:09:51 2023","interfaces":[{"interface":"back","average":{"1min":7.0279999999999996,"5min":4.6260000000000003,"15min":3.2269999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.21199999999999999},"max":{"1min":108.389,"5min":138.90299999999999,"15min":138.90299999999999},"last":1.4830000000000001},{"interface":"front","average":{"1min":6.6790000000000003,"5min":4.4960000000000004,"15min":3.081},"min":{"1min":0.29199999999999998,"5min":0.27600000000000002,"15min":0.215},"max":{"1min":108.245,"5min":138.87700000000001,"15min":138.87700000000001},"last":0.92300000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.1910000000000001,"15min":1.022},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.26200000000000001},"max":{"1min":6.3470000000000004,"5min":11.744,"15min":12.586},"last":1.0369999999999999},{"interface":"front","average":{"1min":2.3370000000000002,"5min":2.2109999999999999,"15min":1.444},"min":{"1min":0.35399999999999998,"5min":0.32200000000000001,"15min":0.28299999999999997},"max":{"1min":9.3480000000000008,"5min":16.725999999999999,"15min":16.725999999999999},"last":0.73399999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":2.3119999999999998,"15min":1.5569999999999999},"min":{"1min":0.37,"5min":0.31900000000000001,"15min":0.29999999999999999},"max":{"1min":9.3170000000000002,"5min":16.673999999999999,"15min":16.673999999999999},"last":0.76300000000000001},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.353,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":4.0369999999999999,"5min":12.368,"15min":13.701000000000001},"last":0.31}]},{"osd":6,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.351,"15min":1.835},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.069,"5min":12.323,"15min":175.72499999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.456,"5min":1.286,"15min":1.8160000000000001},"min":{"1min":0.38,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.415,"5min":11.782,"15min":176.20400000000001},"last":1.204}]},{"osd":7,"last update":"Sat Sep 23 21:09:42 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.5089999999999999,"15min":1.3140000000000001},"min":{"1min":0.41899999999999998,"5min":0.39600000000000002,"15min":0.32700000000000001},"max":{"1min":9.9710000000000001,"5min":11.811999999999999,"15min":12.609},"last":1.236},{"interface":"front","average":{"1min":2.8330000000000002,"5min":2.359,"15min":1.589},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":11.41,"5min":16.875,"15min":16.875},"last":0.79900000000000004}]}]},{"osd":3,"up_from":148,"seq":635655160067,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1228748,"kb_used_data":608132,"kb_used_omap":2,"kb_used_meta":620605,"kb_avail":92516404,"statfs":{"total":95995035648,"available":94736797696,"internally_reserved":0,"allocated":622727168,"data_stored":620436936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":635499573},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.3239999999999998,"15min":1.546},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.184},"max":{"1min":10.098000000000001,"5min":44.963000000000001,"15min":44.963000000000001},"last":1.0649999999999999},{"interface":"front","average":{"1min":3.2349999999999999,"5min":1.597,"15min":1.325},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":16.920999999999999,"5min":16.920999999999999,"15min":37.634999999999998},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:09:03 2023","interfaces":[{"interface":"back","average":{"1min":19.158999999999999,"5min":7.0279999999999996,"15min":3.0649999999999999},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.20999999999999999},"max":{"1min":166.56999999999999,"5min":166.56999999999999,"15min":166.56999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":17.228999999999999,"5min":7.0049999999999999,"15min":3.202},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.182},"max":{"1min":166.291,"5min":166.291,"15min":166.291},"last":0.64000000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":3.649,"15min":2.0089999999999999},"min":{"1min":0.24199999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":17.373999999999999,"5min":88.825999999999993,"15min":88.825999999999993},"last":0.70399999999999996},{"interface":"front","average":{"1min":4.734,"5min":3.29,"15min":1.861},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":17.145,"5min":85.626000000000005,"15min":85.626000000000005},"last":0.80500000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.986,"15min":2.2850000000000001},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.976,"5min":38.600000000000001,"15min":111.36},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.915,"15min":2.1389999999999998},"min":{"1min":0.34399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":10.021000000000001,"5min":38.524000000000001,"15min":112.473},"last":0.60799999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":2.9609999999999999,"5min":2.109,"15min":1.4259999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":16.855,"5min":36.488999999999997,"15min":36.488999999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.0920000000000001,"15min":1.6180000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29299999999999998},"max":{"1min":6.0179999999999998,"5min":36.295000000000002,"15min":36.295000000000002},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":19.748999999999999,"5min":5.399,"15min":2.7919999999999998},"min":{"1min":0.376,"5min":0.318,"15min":0.312},"max":{"1min":304.22399999999999,"5min":304.22399999999999,"15min":304.22399999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":19.741,"5min":5.3079999999999998,"15min":2.6560000000000001},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.30299999999999999},"max":{"1min":304.15800000000002,"5min":304.15800000000002,"15min":304.15800000000002},"last":0.63500000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:09:43 2023","interfaces":[{"interface":"back","average":{"1min":3.4020000000000001,"5min":1.7669999999999999,"15min":1.722},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.30499999999999999},"max":{"1min":16.899000000000001,"5min":16.899000000000001,"15min":74.019000000000005},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.97,"5min":1.964,"15min":1.9870000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.32900000000000001},"max":{"1min":6.0490000000000004,"5min":12.407,"15min":73.364999999999995},"last":0.73599999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986685,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":832936,"kb_used_data":246752,"kb_used_omap":0,"kb_used_meta":586176,"kb_avail":92912216,"statfs":{"total":95995035648,"available":95142109184,"internally_reserved":0,"allocated":252674048,"data_stored":251421412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":600244224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.272,"15min":1.9079999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.20399999999999999},"max":{"1min":6.5469999999999997,"5min":12.869,"15min":102.358},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.097,"5min":1.4610000000000001,"15min":1.9339999999999999},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":10.439,"5min":12.848000000000001,"15min":102.58199999999999},"last":0.63}]},{"osd":1,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":7.0010000000000003,"5min":6.1950000000000003,"15min":3.9399999999999999},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":42.887,"5min":127.411,"15min":127.411},"last":0.98399999999999999},{"interface":"front","average":{"1min":7.258,"5min":4.3179999999999996,"15min":3.2770000000000001},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.253},"max":{"1min":44.148000000000003,"5min":118.075,"15min":118.075},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:09:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4260000000000002,"5min":5.2359999999999998,"15min":4.0259999999999998},"min":{"1min":0.28999999999999998,"5min":0.27300000000000002,"15min":0.246},"max":{"1min":137.89500000000001,"5min":137.89500000000001,"15min":164.517},"last":0.78000000000000003},{"interface":"front","average":{"1min":9.3119999999999994,"5min":4.7770000000000001,"15min":3.8620000000000001},"min":{"1min":0.33100000000000002,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":138.20599999999999,"5min":138.20599999999999,"15min":164.46799999999999},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:09:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.3380000000000001,"15min":1.4359999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27300000000000002},"max":{"1min":6.9390000000000001,"5min":6.9390000000000001,"15min":60.720999999999997},"last":1.5940000000000001},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.3859999999999999,"15min":1.4870000000000001},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.25600000000000001},"max":{"1min":6.4020000000000001,"5min":8.2899999999999991,"15min":60.631999999999998},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:09:22 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.87,"15min":0.76900000000000002},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":4.9080000000000004,"5min":6.0069999999999997,"15min":8.7089999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.85699999999999998,"15min":0.74099999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.21099999999999999},"max":{"1min":4.7140000000000004,"5min":4.9740000000000002,"15min":7.0179999999999998},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:09:36 2023","interfaces":[{"interface":"back","average":{"1min":3.1389999999999998,"5min":1.3700000000000001,"15min":1.03},"min":{"1min":0.312,"5min":0.28299999999999997,"15min":0.26100000000000001},"max":{"1min":36.557000000000002,"5min":36.557000000000002,"15min":36.557000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.423,"5min":1.2350000000000001,"15min":0.995},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.26200000000000001},"max":{"1min":38.468000000000004,"5min":38.468000000000004,"15min":38.468000000000004},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:09:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.036,"15min":1.4950000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.7359999999999998,"5min":13.667999999999999,"15min":145.721},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.5069999999999999,"5min":1.2949999999999999,"15min":1.667},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.28000000000000003},"max":{"1min":10.411,"5min":10.411,"15min":148.45699999999999},"last":0.53800000000000003}]}]},{"osd":5,"up_from":39,"seq":167503725049,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":743960,"kb_used_data":98896,"kb_used_omap":0,"kb_used_meta":645056,"kb_avail":93001192,"statfs":{"total":95995035648,"available":95233220608,"internally_reserved":0,"allocated":101269504,"data_stored":100028996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":660537344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8759999999999999,"5min":1.3400000000000001,"15min":1.369},"min":{"1min":0.254,"5min":0.22,"15min":0.22},"max":{"1min":12.381,"5min":12.381,"15min":80.105999999999995},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.903,"5min":1.3069999999999999,"15min":1.468},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.4030000000000005,"5min":9.4030000000000005,"15min":80.152000000000001},"last":0.85599999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:09:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5309999999999997,"5min":6.2519999999999998,"15min":3.5259999999999998},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.254},"max":{"1min":32.777999999999999,"5min":233.77099999999999,"15min":233.77099999999999},"last":2.1640000000000001},{"interface":"front","average":{"1min":6.5460000000000003,"5min":6.2060000000000004,"15min":3.4700000000000002},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.247},"max":{"1min":32.761000000000003,"5min":233.75200000000001,"15min":233.75200000000001},"last":2.1829999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":5.1909999999999998,"15min":2.8180000000000001},"min":{"1min":0.36199999999999999,"5min":0.28799999999999998,"15min":0.254},"max":{"1min":11.538,"5min":231.34700000000001,"15min":231.34700000000001},"last":0.377},{"interface":"front","average":{"1min":1.492,"5min":5.4219999999999997,"15min":2.9569999999999999},"min":{"1min":0.28499999999999998,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":6.375,"5min":231.381,"15min":231.381},"last":1.3540000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.482,"15min":2.2040000000000002},"min":{"1min":0.27800000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":8.4469999999999992,"5min":12.765000000000001,"15min":98.369},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.2149999999999999,"5min":1.599,"15min":2.2210000000000001},"min":{"1min":0.33600000000000002,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":7.8129999999999997,"5min":9.6120000000000001,"15min":98.683000000000007},"last":3.9449999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.64900000000000002,"15min":0.63700000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.247},"max":{"1min":1.946,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.0580000000000001,"15min":0.76500000000000001},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":5.0369999999999999,"5min":8.3740000000000006,"15min":10.792},"last":0.36399999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:09:44 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.81999999999999995,"15min":1.048},"min":{"1min":0.32200000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.6499999999999999,"5min":7.1130000000000004,"15min":71.230000000000004},"last":0.31},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.76700000000000002,"15min":1.0720000000000001},"min":{"1min":0.30299999999999999,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":2.734,"5min":4.0880000000000001,"15min":70.266000000000005},"last":0.34000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:09:17 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.262,"15min":1.0289999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.312},"max":{"1min":4.3399999999999999,"5min":9.6329999999999991,"15min":31.091999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.69899999999999995,"15min":0.80100000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":2.0179999999999998,"5min":4.29,"15min":31.780000000000001},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":368705536,"data_stored":368705536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":191299584,"data_stored":191299584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":611581952,"data_stored":611581952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":672333824,"data_stored":672333824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":268304384,"data_stored":268304384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":104726528,"data_stored":104726528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":658636800,"data_stored":658636800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:10:06.706 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:10:06.707 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:06.707 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T21:10:06.707 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T21:10:06.735 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:06.788 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:06.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:06.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:06.896 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:06.927 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:06.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:06.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:06.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:07.084 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:07.196 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 27348 27332 121.916 98.5 0.011825 0.00819382 2023-09-23T21:10:07.227 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:10:07.227 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:10:07.227 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:10:07.227 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:10:07.227 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:10:07.252 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.306 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:07.412 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:07.475 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.574 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:10:07.590 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:10:07.630 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:07.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:07.642 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:10:07.687 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:10:07.720 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:10:07.736 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:07.759 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:10:07.770 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:07.798 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:10:07.827 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:07.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:07.835 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:07 smithi123 ceph-mon[193903]: pgmap v1388: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T21:10:07.835 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/906259676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:07.835 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:10:07.853 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.853 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.853 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.853 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.854 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.854 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.854 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.854 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:10:07.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:07 smithi121 ceph-mon[211526]: pgmap v1388: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T21:10:07.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/906259676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:07.882 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:07 smithi121 ceph-mon[219709]: pgmap v1388: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T21:10:07.882 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/906259676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:07.935 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:07.968 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.030 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:08.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:08.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:08.136 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:08.167 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.196 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 29020 29004 120.755 104.5 0.00762679 0.00827695 2023-09-23T21:10:08.224 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:08.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:08.330 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:08.357 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:08.584 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:10:09.196 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 30329 30313 118.323 81.8125 0.00231029 0.00844317 2023-09-23T21:10:09.629 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:09.632 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:09 smithi121 ceph-mon[211526]: pgmap v1389: 89 pgs: 89 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T21:10:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:09.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:09.634 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:09 smithi121 ceph-mon[219709]: pgmap v1389: 89 pgs: 89 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T21:10:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:09 smithi123 ceph-mon[193903]: pgmap v1389: 89 pgs: 89 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-09-23T21:10:09.734 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:09.762 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:10:09.807 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:09.905 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:09.959 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:09.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:10.012 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:10.075 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:10.162 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:10.199 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 31544 31528 115.809 75.9375 0.00629256 0.00862923 2023-09-23T21:10:11.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 33112 33096 114.816 98 0.00502388 0.00870407 2023-09-23T21:10:11.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:11 smithi121 ceph-mon[211526]: pgmap v1390: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T21:10:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:11 smithi121 ceph-mon[219709]: pgmap v1390: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T21:10:11.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:11 smithi123 ceph-mon[193903]: pgmap v1390: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-09-23T21:10:11.755 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:11.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:11.810 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:10:11.866 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:11.892 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:11.934 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:11.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:11.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:12.040 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:12.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:12.200 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 34390 34374 112.978 79.875 0.00239167 0.00882995 2023-09-23T21:10:12.543 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:12.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:12.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:12.649 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:12.679 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:12.739 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:12.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:12.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:12.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:12.861 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:12.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:12.909 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:12.921 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:12.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:13.038 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:13.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:13.146 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:13.178 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:13.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:10:13.201866+0000 min lat: 0.00130643 max lat: 0.263773 avg lat: 0.00902735 2023-09-23T21:10:13.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:10:13.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 35468 35452 110.698 67.375 0.0083629 0.00902735 2023-09-23T21:10:13.226 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:13.683 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:13.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:13.785 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:13.810 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:13 smithi123 ceph-mon[193903]: pgmap v1391: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:10:13.813 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:13.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:13 smithi121 ceph-mon[219709]: pgmap v1391: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:10:13.851 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:13 smithi121 ceph-mon[211526]: pgmap v1391: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T21:10:13.871 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:13.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:13.977 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:14.063 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:14.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 36735 36719 109.198 79.1875 0.0113329 0.00915009 2023-09-23T21:10:14.574 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4025050492' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:14.574 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:14 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:14.574 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/493745956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:10:14.576 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.7. 2023-09-23T21:10:14.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:14 smithi121 conmon[211495]: 2023-09-23T21:10:14.247+0000 7fdc50c66700 -1 mon.a@0(leader).osd e431 definitely_dead 0 2023-09-23T21:10:14.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4025050492' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:14.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:14 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:14.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/493745956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:10:14.596 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:10:14.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:10:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:10:14] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.43.0" 2023-09-23T21:10:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4025050492' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:14 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T21:10:14.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/493745956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:10:15.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 37277 37261 105.776 33.875 0.00231348 0.0091901 2023-09-23T21:10:15.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:15.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:15.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:15.705 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:15.731 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:15.773 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:15.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:15.878 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:15.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:15.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:10:15.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/493745956' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:10:15.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T21:10:15.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: pgmap v1393: 89 pgs: 4 peering, 85 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[219709]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:10:15.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/493745956' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:10:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T21:10:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: pgmap v1393: 89 pgs: 4 peering, 85 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T21:10:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:15 smithi121 ceph-mon[211526]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T21:10:15.906 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:15.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/493745956' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: failed to encode map e432 with expected crc 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: pgmap v1393: 89 pgs: 4 peering, 85 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T21:10:15.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:15 smithi123 ceph-mon[193903]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T21:10:16.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 37277 37261 101.18 0 - 0.0091901 2023-09-23T21:10:16.656 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:16.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:16.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:16.790 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[211526]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:16 smithi121 ceph-mon[219709]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:16 smithi123 ceph-mon[193903]: failed to encode map e433 with expected crc 2023-09-23T21:10:17.135 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:17.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:17.201 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 37592 37576 97.786 9.84375 0.012446 0.0102211 2023-09-23T21:10:17.250 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:17.262 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T21:10:17.289 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.349 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:17.456 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:17.489 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.553 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:17.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:17.665 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:17.695 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.747 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:17.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:17.794 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:10:17.823 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:17.823 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:17.834 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:10:17.852 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[211526]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[211526]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[211526]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[211526]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[211526]: pgmap v1396: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[219709]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[219709]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[219709]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[219709]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.854 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:17 smithi121 ceph-mon[219709]: pgmap v1396: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T21:10:17.855 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:17.877 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:10:17.914 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:10:17.932 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:17.956 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:10:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:17 smithi123 ceph-mon[193903]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:17 smithi123 ceph-mon[193903]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T21:10:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:17 smithi123 ceph-mon[193903]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:17 smithi123 ceph-mon[193903]: failed to encode map e434 with expected crc 2023-09-23T21:10:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:17 smithi123 ceph-mon[193903]: pgmap v1396: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T21:10:18.006 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:10:18.040 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:10:18.076 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:10:18.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:10:18.105 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:18.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:18.202 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 38716 38700 96.685 70.25 0.0100206 0.0103363 2023-09-23T21:10:18.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:18.279 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:18.382 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:18.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:18.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:18.495 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:18.563 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:18.663 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:10:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:18 smithi121 ceph-mon[211526]: failed to encode map e434 with expected crc 2023-09-23T21:10:18.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2898341429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:18 smithi121 ceph-mon[219709]: failed to encode map e434 with expected crc 2023-09-23T21:10:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2898341429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:18.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:18 smithi123 ceph-mon[193903]: failed to encode map e434 with expected crc 2023-09-23T21:10:18.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2898341429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:19.202 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 39691 39675 95.3104 60.9375 0.00355242 0.010455 2023-09-23T21:10:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:19 smithi121 ceph-mon[219709]: pgmap v1397: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:10:19.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:19 smithi121 ceph-mon[211526]: pgmap v1397: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:10:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:19 smithi123 ceph-mon[193903]: pgmap v1397: 89 pgs: 1 remapped+peering, 4 peering, 84 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T21:10:20.163 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:20.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:20.202 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 40619 40603 93.9291 58 0.00381625 0.0106184 2023-09-23T21:10:20.268 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:20.296 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:20.463 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:10:20.620 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:20.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:20.727 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:20.850 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:20.892 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:20.962 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:20.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:21.081 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:21.116 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:21.169 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:21.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:21.202 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 41593 41577 92.749 60.875 0.00687759 0.0107766 2023-09-23T21:10:21.275 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:21.354 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:21 smithi123 ceph-mon[193903]: pgmap v1398: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 85 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 4716/119751 objects degraded (3.938%); 2086/119751 objects misplaced (1.742%); 1.6 MiB/s, 1 keys/s, 24 objects/s recovering 2023-09-23T21:10:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:21 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 4716/119751 objects degraded (3.938%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:21.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:21 smithi121 ceph-mon[211526]: pgmap v1398: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 85 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 4716/119751 objects degraded (3.938%); 2086/119751 objects misplaced (1.742%); 1.6 MiB/s, 1 keys/s, 24 objects/s recovering 2023-09-23T21:10:21.527 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:21 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 4716/119751 objects degraded (3.938%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:21.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:21 smithi121 ceph-mon[219709]: pgmap v1398: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 85 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 4716/119751 objects degraded (3.938%); 2086/119751 objects misplaced (1.742%); 1.6 MiB/s, 1 keys/s, 24 objects/s recovering 2023-09-23T21:10:21.527 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:21 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 4716/119751 objects degraded (3.938%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:21.570 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:21.586 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:21.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:21.692 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:21.770 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:22.202 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 42775 42759 92.0982 73.875 0.00574822 0.0108539 2023-09-23T21:10:22.263 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 3, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:10:22.263 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:22.264 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T21:10:22.264 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:10:22.264 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:10:22.264 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T21:10:22.650 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:10:22.728 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:10:22.995 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:23.104 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:23.140 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:23.200 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:23.205 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 43517 43501 90.5668 46.375 0.0156475 0.0110346 2023-09-23T21:10:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:23.309 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:23.344 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:23.408 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:23.410 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:23 smithi123 ceph-mon[193903]: pgmap v1399: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 7025/122238 objects degraded (5.747%); 2086/122238 objects misplaced (1.707%); 1.3 MiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T21:10:23.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:23.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:23 smithi121 ceph-mon[211526]: pgmap v1399: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 7025/122238 objects degraded (5.747%); 2086/122238 objects misplaced (1.707%); 1.3 MiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T21:10:23.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:23 smithi121 ceph-mon[219709]: pgmap v1399: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 7025/122238 objects degraded (5.747%); 2086/122238 objects misplaced (1.707%); 1.3 MiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T21:10:23.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:23.547 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:23.603 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:23.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:23.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:23.707 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:23.791 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:24.205 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 44667 44651 89.964 71.875 0.00502047 0.0111103 2023-09-23T21:10:24.908 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:24.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:24.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:25.013 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:25.040 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:10:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:10:24] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.43.0" 2023-09-23T21:10:25.044 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:25.103 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:25.122 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:25.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:25.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 45670 45654 89.1053 62.6875 0.0193383 0.0112168 2023-09-23T21:10:25.227 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:25.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:25 smithi121 ceph-mon[211526]: pgmap v1400: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 7025/129870 objects degraded (5.409%); 1644/129870 objects misplaced (1.266%); 4.0 MiB/s, 1 keys/s, 63 objects/s recovering 2023-09-23T21:10:25.315 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:25 smithi121 ceph-mon[219709]: pgmap v1400: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 7025/129870 objects degraded (5.409%); 1644/129870 objects misplaced (1.266%); 4.0 MiB/s, 1 keys/s, 63 objects/s recovering 2023-09-23T21:10:25.340 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:25.575 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:25.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:25.626 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:25 smithi123 ceph-mon[193903]: pgmap v1400: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 7025/129870 objects degraded (5.409%); 1644/129870 objects misplaced (1.266%); 4.0 MiB/s, 1 keys/s, 63 objects/s recovering 2023-09-23T21:10:25.682 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:46137 12.3 12:c60b72d8:::benchmark_data_smithi121_563784_object46136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:25.712666+0000", 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0079574569999999994, 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079913199999999997, 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:10:25.722 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 46137 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.712666+0000", 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.712666+0000", 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.712665+0000", 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.712713+0000", 2023-09-23T21:10:25.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8191e-05 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.712724+0000", 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.134e-05 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.712731+0000", 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1099999999999997e-06 2023-09-23T21:10:25.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.712985+0000", 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00025374799999999998 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.713144+0000", 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015893400000000001 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.713323+0000", 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017904999999999999 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.718236+0000", 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049126159999999999 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.727 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:46139 12.e 12:702b1359:::benchmark_data_smithi121_563784_object46138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:25.715675+0000", 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0049492260000000001, 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050760349999999996, 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 46139 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:25.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715675+0000", 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715675+0000", 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:25.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715673+0000", 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715724+0000", 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0429000000000001e-05 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715737+0000", 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2830000000000001e-05 2023-09-23T21:10:25.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715745+0000", 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4589999999999998e-06 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.719065+0000", 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003319476 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:25.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.719212+0000", 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014777 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.719382+0000", 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016913799999999999 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.732 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:46126 12.a e434/426 12:509b9797:::benchmark_data_smithi121_563784_object46125:head v 434'2990, mlcod=434'2988)", 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:25.715767+0000", 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0048564389999999997, 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050586260000000001, 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715767+0000", 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715767+0000", 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715767+0000", 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:25.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715788+0000", 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1481e-05 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715796+0000", 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6750000000000002e-06 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715801+0000", 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0690000000000001e-06 2023-09-23T21:10:25.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715910+0000", 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010888899999999999 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.715930+0000", 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0224000000000002e-05 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:46138 12.a e434/426)", 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:25.718546+0000", 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0020781599999999999, 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023387009999999999, 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.718546+0000", 2023-09-23T21:10:25.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.718546+0000", 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.718545+0000", 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.718568+0000", 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3099999999999999e-05 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.718579+0000", 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1429e-05 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:25.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.718586+0000", 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4659999999999996e-06 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:46145 12.8 e434/426)", 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:25.720821+0000", 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 4294967295.9998021, 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112796, 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:25.740 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720821+0000", 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720821+0000", 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:25.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720821+0000", 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720845+0000", 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4015e-05 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720861+0000", 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6393999999999999e-05 2023-09-23T21:10:25.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720868+0000", 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3130000000000001e-06 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720913+0000", 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4431000000000002e-05 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:25.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.720937+0000", 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4396000000000002e-05 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 5 2023-09-23T21:10:25.744 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:10:25.769 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:26.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 46691 46675 88.3391 63.8125 0.0107724 0.0113149 2023-09-23T21:10:26.258 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3711012360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:26.258 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:26.259 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3711012360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:26.259 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:26.290 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:10:26.334 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:10:26.335 INFO:teuthology.orchestra.run.smithi123.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T21:10:26.335 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:10:26.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3711012360' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:26.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:27.207 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 47535 47519 87.2928 52.75 0.0028051 0.0114228 2023-09-23T21:10:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:27 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:10:27.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:27 smithi123 ceph-mon[193903]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T21:10:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:27 smithi123 ceph-mon[193903]: failed to encode map e435 with expected crc 2023-09-23T21:10:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:27 smithi123 ceph-mon[193903]: failed to encode map e435 with expected crc 2023-09-23T21:10:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:27 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7025/129870 objects degraded (5.409%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:27 smithi123 ceph-mon[193903]: pgmap v1402: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 7025/137505 objects degraded (5.109%); 1644/137505 objects misplaced (1.196%); 3.8 MiB/s, 0 keys/s, 60 objects/s recovering 2023-09-23T21:10:27.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[211526]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[211526]: failed to encode map e435 with expected crc 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[211526]: failed to encode map e435 with expected crc 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7025/129870 objects degraded (5.409%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[211526]: pgmap v1402: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 7025/137505 objects degraded (5.109%); 1644/137505 objects misplaced (1.196%); 3.8 MiB/s, 0 keys/s, 60 objects/s recovering 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[219709]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T21:10:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[219709]: failed to encode map e435 with expected crc 2023-09-23T21:10:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[219709]: failed to encode map e435 with expected crc 2023-09-23T21:10:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7025/129870 objects degraded (5.409%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:27 smithi121 ceph-mon[219709]: pgmap v1402: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 7025/137505 objects degraded (5.109%); 1644/137505 objects misplaced (1.196%); 3.8 MiB/s, 0 keys/s, 60 objects/s recovering 2023-09-23T21:10:27.827 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:27.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:27.938 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:27.967 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:28.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 48272 48256 86.1152 46.0625 0.016894 0.0116066 2023-09-23T21:10:28.700 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:28.700 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:28.701 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:28.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:28.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:28.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:28.726 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:28.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:28.837 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:28.868 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:29.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 49140 49124 85.2275 54.25 0.0146788 0.0117281 2023-09-23T21:10:29.468 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:29.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:29.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:29.580 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:29.618 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:29.645 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:10:29.683 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:29.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:29.719 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:29 smithi121 ceph-mon[211526]: pgmap v1403: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 7025/137505 objects degraded (5.109%); 1644/137505 objects misplaced (1.196%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:10:29.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:29 smithi121 ceph-mon[219709]: pgmap v1403: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 7025/137505 objects degraded (5.109%); 1644/137505 objects misplaced (1.196%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:10:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:29 smithi123 ceph-mon[193903]: pgmap v1403: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s; 7025/137505 objects degraded (5.109%); 1644/137505 objects misplaced (1.196%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T21:10:29.789 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:29.823 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:29.878 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:29.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:29.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:30.071 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:30.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 50166 50150 84.6573 64.125 0.0153781 0.011806 2023-09-23T21:10:30.264 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:30.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:30.370 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:30.402 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:30.409 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:10:30.438 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:30.456 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:30.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:10:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:30.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:30.498 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:10:30.537 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:10:30.561 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:30.575 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:10:30.594 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:30.616 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:10:30.648 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:10:30.652 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:30.656 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1839573125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:30.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:30.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:30.691 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:10:30.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:10:30.737 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1839573125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:30.737 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1839573125' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:30.757 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:30.840 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:31.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 51217 51201 84.1582 65.6875 0.0140822 0.0118773 2023-09-23T21:10:31.221 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:31.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:31.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:31.326 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:31.395 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:31.485 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:31 smithi121 ceph-mon[211526]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 7025/148449 objects degraded (4.732%); 1100/148449 objects misplaced (0.741%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:10:31.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:31 smithi121 ceph-mon[219709]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 7025/148449 objects degraded (4.732%); 1100/148449 objects misplaced (0.741%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:10:31.522 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:10:31.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:31 smithi123 ceph-mon[193903]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 7025/148449 objects degraded (4.732%); 1100/148449 objects misplaced (0.741%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:10:32.154 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 3, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:10:32.154 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:32.155 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T21:10:32.155 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:10:32.155 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:10:32.155 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T21:10:32.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 52188 52172 83.5566 60.6875 0.00750935 0.0119618 2023-09-23T21:10:32.530 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:32.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:32.640 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:32.681 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:32.749 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:32.755 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7025/148449 objects degraded (4.732%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:32.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:32.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:32.857 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:32.884 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7025/148449 objects degraded (4.732%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7025/148449 objects degraded (4.732%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:32.944 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:32.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:33.053 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:33.132 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:33.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:10:33.210149+0000 min lat: 0.00130643 max lat: 2.40445 avg lat: 0.0120554 2023-09-23T21:10:33.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:10:33.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 53112 53096 82.9117 57.75 0.0158841 0.0120554 2023-09-23T21:10:33.507 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:33.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:33.614 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:33.648 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:33.700 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:33.724 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:33.725 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:33 smithi121 ceph-mon[211526]: pgmap v1405: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 7025/151041 objects degraded (4.651%); 1100/151041 objects misplaced (0.728%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:10:33.725 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:33 smithi121 ceph-mon[219709]: pgmap v1405: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 7025/151041 objects degraded (4.651%); 1100/151041 objects misplaced (0.728%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:10:33.897 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:33.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:33.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:33 smithi123 ceph-mon[193903]: pgmap v1405: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 7025/151041 objects degraded (4.651%); 1100/151041 objects misplaced (0.728%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T21:10:34.002 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:34.036 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:34.085 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:34.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:34.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:34.197 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:34.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 54095 54079 82.388 61.4375 0.00502427 0.0121309 2023-09-23T21:10:34.286 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:34.392 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:10:34.392 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:10:35.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:10:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:10:34] "GET /metrics HTTP/1.1" 200 36703 "" "Prometheus/2.43.0" 2023-09-23T21:10:35.209 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 55165 55149 82.0187 66.875 0.014618 0.0121876 2023-09-23T21:10:35.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:10:35.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:35 smithi121 ceph-mon[211526]: pgmap v1406: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 7025/155529 objects degraded (4.517%); 604/155529 objects misplaced (0.388%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:10:35.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:35 smithi121 ceph-mon[219709]: pgmap v1406: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 7025/155529 objects degraded (4.517%); 604/155529 objects misplaced (0.388%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:10:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:35 smithi123 ceph-mon[193903]: pgmap v1406: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 7025/155529 objects degraded (4.517%); 604/155529 objects misplaced (0.388%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:10:36.210 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 55761 55745 80.978 37.25 0.131984 0.0123438 2023-09-23T21:10:36.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7025/155529 objects degraded (4.517%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7025/155529 objects degraded (4.517%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:36.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7025/155529 objects degraded (4.517%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:37.126 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:10:37.210 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 56773 56757 80.5738 63.25 0.0159404 0.0124056 2023-09-23T21:10:37.258 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:37.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:37.363 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:37.392 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:37.456 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:37.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:37.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:37.562 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:37.660 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:37 smithi123 ceph-mon[193903]: pgmap v1407: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 7025/165408 objects degraded (4.247%); 604/165408 objects misplaced (0.365%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:10:37.753 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:37.853 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:37.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:37.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:37 smithi121 ceph-mon[211526]: pgmap v1407: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 7025/165408 objects degraded (4.247%); 604/165408 objects misplaced (0.365%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:10:37.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:37 smithi121 ceph-mon[219709]: pgmap v1407: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 7025/165408 objects degraded (4.247%); 604/165408 objects misplaced (0.365%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T21:10:37.964 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:38.038 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:38.210 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 57659 57643 80.0139 55.375 0.0101932 0.0124919 2023-09-23T21:10:38.359 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:38.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:38.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:38.433 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:10:38.433 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:10:38.433 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:10:38.474 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:38.546 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:38.596 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:38.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:38.703 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:38.776 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:39.008 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:39.029 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:39.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:39.135 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:39.160 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:39.203 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:39.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:39.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:39.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 58255 58239 79.0847 37.25 0.00221358 0.0125352 2023-09-23T21:10:39.308 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:39.338 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:39.651 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:10:39.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[211526]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T21:10:39.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[211526]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[211526]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[211526]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[211526]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4139244532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:39.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[211526]: pgmap v1409: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 7025/165408 objects degraded (4.247%); 604/165408 objects misplaced (0.365%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:10:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[219709]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T21:10:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[219709]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[219709]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[219709]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[219709]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4139244532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:39.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:39 smithi121 ceph-mon[219709]: pgmap v1409: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 7025/165408 objects degraded (4.247%); 604/165408 objects misplaced (0.365%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:10:39.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:39 smithi123 ceph-mon[193903]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T21:10:39.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:39 smithi123 ceph-mon[193903]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:39 smithi123 ceph-mon[193903]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:39 smithi123 ceph-mon[193903]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:39 smithi123 ceph-mon[193903]: failed to encode map e436 with expected crc 2023-09-23T21:10:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4139244532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:10:39.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:39 smithi123 ceph-mon[193903]: pgmap v1409: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 7025/165408 objects degraded (4.247%); 604/165408 objects misplaced (0.365%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T21:10:40.210 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 58561 58545 77.8095 19.125 0.00257912 0.0128299 2023-09-23T21:10:40.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[211526]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4139244532' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[211526]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[211526]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[211526]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[211526]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:40.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[219709]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4139244532' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[219709]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T21:10:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[219709]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[219709]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:40 smithi121 ceph-mon[219709]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:40 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:40 smithi123 ceph-mon[193903]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4139244532' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:10:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:40 smithi123 ceph-mon[193903]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T21:10:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:40 smithi123 ceph-mon[193903]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:40 smithi123 ceph-mon[193903]: failed to encode map e437 with expected crc 2023-09-23T21:10:40.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:40 smithi123 ceph-mon[193903]: failed to encode map e437 with expected crc 2023-09-23T21:10:41.135 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:41.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:41.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 59590 59574 77.5283 64.3125 0.00250915 0.0128898 2023-09-23T21:10:41.242 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:41.330 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:41 smithi121 ceph-mon[211526]: pgmap v1411: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 4706/174474 objects degraded (2.697%); 2136/174474 objects misplaced (1.224%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:10:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:41 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7025/165408 objects degraded (4.247%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:41 smithi121 ceph-mon[219709]: pgmap v1411: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 4706/174474 objects degraded (2.697%); 2136/174474 objects misplaced (1.224%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:10:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:41 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7025/165408 objects degraded (4.247%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:41 smithi123 ceph-mon[193903]: pgmap v1411: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 4706/174474 objects degraded (2.697%); 2136/174474 objects misplaced (1.224%); 10 MiB/s, 160 objects/s recovering 2023-09-23T21:10:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:41 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7025/165408 objects degraded (4.247%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:42.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 60450 60434 77.0432 53.75 0.0175563 0.0129742 2023-09-23T21:10:42.242 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:42.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:42.357 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:42.428 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:43.022 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:10:43.022 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:10:43.022 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:10:43.022 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:10:43.022 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:10:43.022 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:10:43.041 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:10:43.053 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:43.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:43.080 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:10:43.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:10:43.159 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:43.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 61390 61374 76.6774 58.75 0.014393 0.0130352 2023-09-23T21:10:43.226 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:43.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:43.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:43.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3678842033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:43.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:43 smithi123 ceph-mon[193903]: pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 4706/176664 objects degraded (2.664%); 2136/176664 objects misplaced (1.209%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:10:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3678842033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[219709]: pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 4706/176664 objects degraded (2.664%); 2136/176664 objects misplaced (1.209%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:10:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3678842033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:43 smithi121 ceph-mon[211526]: pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 4706/176664 objects degraded (2.664%); 2136/176664 objects misplaced (1.209%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:10:44.161 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:44.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:44.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:44.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 62213 62197 76.1826 51.4375 0.0238211 0.0131202 2023-09-23T21:10:44.221 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:10:44.268 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:44.352 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:44.904 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:10:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:10:44] "GET /metrics HTTP/1.1" 200 36708 "" "Prometheus/2.43.0" 2023-09-23T21:10:44.925 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:44.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:45.032 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:45.110 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:45.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 63096 63080 75.7789 55.1875 0.0208394 0.0131898 2023-09-23T21:10:45.219 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:45.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:45.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:45.325 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:45.385 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:45.416 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:45.534 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:45.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:45.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:45.540 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:45 smithi121 ceph-mon[211526]: pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s; 4706/180573 objects degraded (2.606%); 2136/180573 objects misplaced (1.183%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:10:45.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:45 smithi121 ceph-mon[219709]: pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s; 4706/180573 objects degraded (2.606%); 2136/180573 objects misplaced (1.183%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:10:45.542 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:10:45.542 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:10:45.542 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:10:45.542 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:10:45.542 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151000 10.16 10:6fb6daa8:::benchmark_data_smithi121_358375_object66255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 526.12951811100004, 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.019180497, 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151000 2023-09-23T21:10:45.543 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409671+0000", 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.544 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409669+0000", 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409680+0000", 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1647e-05 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.545 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409699+0000", 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8763e-05 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409711+0000", 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1793e-05 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.409930+0000", 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00021934400000000001 2023-09-23T21:10:45.546 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410095+0000", 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016493700000000001 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 1,4", 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.410262+0000", 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016674600000000001 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.547 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.415289+0000", 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050267660000000002 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428299+0000", 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013010763 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.428313+0000", 2023-09-23T21:10:45.548 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4064e-05 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428836+0000", 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0005229959999999 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428851+0000", 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4960000000000001e-05 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.549 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151024 10.6 10.8ded8286 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 526.04025738999997, 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.930075294, 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.550 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151024 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498932+0000", 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.551 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498930+0000", 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498938+0000", 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3890000000000005e-06 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.552 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498955+0000", 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6996000000000001e-05 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.498965+0000", 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0629999999999995e-06 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499015+0000", 2023-09-23T21:10:45.553 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0868000000000002e-05 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.499026+0000", 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0883999999999999e-05 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428989+0000", 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92996320600000004 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.554 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429007+0000", 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7419e-05 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151064 10.16 10.14ca33f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 525.90349162699999, 2023-09-23T21:10:45.555 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79318550200000004, 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151064 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:10:45.556 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635697+0000", 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635696+0000", 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.557 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635703+0000", 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8220000000000001e-06 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635720+0000", 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7116999999999999e-05 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635729+0000", 2023-09-23T21:10:45.558 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1929999999999999e-06 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635796+0000", 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7163999999999996e-05 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.635810+0000", 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4469e-05 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.559 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428874+0000", 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79306313900000003 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428883+0000", 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2390000000000008e-06 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.560 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151070 10.6 10.e7155d86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 525.85280592499998, 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74265465799999997, 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151070 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.561 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686383+0000", 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.562 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686381+0000", 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686389+0000", 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.204e-06 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686407+0000", 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8130000000000001e-05 2023-09-23T21:10:45.563 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686416+0000", 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2369999999999998e-06 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686477+0000", 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1432000000000004e-05 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.564 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.686488+0000", 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1026000000000001e-05 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429028+0000", 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.74253917700000005 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429038+0000", 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.007e-05 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.565 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151071 10.16 10.6fc596f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 525.84556998400001, 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73529290700000005, 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.566 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151071 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693619+0000", 2023-09-23T21:10:45.567 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693617+0000", 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693627+0000", 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4120000000000007e-06 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.568 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693644+0000", 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7669999999999999e-05 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693654+0000", 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1980000000000007e-06 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693712+0000", 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8579999999999998e-05 2023-09-23T21:10:45.569 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.693726+0000", 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3847e-05 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428903+0000", 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73517710400000003 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.570 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428912+0000", 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7469999999999994e-06 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151082 10.16 10.dd3bf356 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 525.82555189100003, 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71530539400000004, 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.571 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151082 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.572 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713637+0000", 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713635+0000", 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713645+0000", 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0113e-05 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.573 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713670+0000", 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4507000000000001e-05 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713679+0000", 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1710000000000008e-06 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713749+0000", 2023-09-23T21:10:45.574 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0607999999999999e-05 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.713761+0000", 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1841999999999999e-05 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428934+0000", 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.71517288000000001 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.575 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428942+0000", 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3040000000000005e-06 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151084 10.6 10.dfef7e86 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:10:45.576 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 525.81880395600001, 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70867917800000002, 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151084 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.577 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720385+0000", 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.578 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720383+0000", 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720392+0000", 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1699999999999997e-06 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720411+0000", 2023-09-23T21:10:45.579 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9681e-05 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720422+0000", 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0587999999999999e-05 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720488+0000", 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.648e-05 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.580 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.720504+0000", 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6172999999999999e-05 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429056+0000", 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70855175100000001 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429064+0000", 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0150000000000003e-06 2023-09-23T21:10:45.581 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151096 10.16 10.c63710d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 525.74873377799997, 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63851427199999999, 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.582 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151096 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790455+0000", 2023-09-23T21:10:45.583 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790453+0000", 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790471+0000", 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7876999999999998e-05 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.584 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790488+0000", 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6642000000000001e-05 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790498+0000", 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6569999999999992e-06 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790553+0000", 2023-09-23T21:10:45.585 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5707e-05 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:01:59.790563+0000", 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6520000000000001e-06 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428961+0000", 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.63839818400000004 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.586 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.428969+0000", 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3799999999999994e-06 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.45470.0:151153 10.6 10.b41dbb46 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 525.49766143600004, 2023-09-23T21:10:45.587 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.387565296, 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.45470", 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/1941162221", 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151153 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:10:45.588 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041528+0000", 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041526+0000", 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.589 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041533+0000", 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1250000000000004e-06 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041546+0000", 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2289e-05 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.590 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041552+0000", 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6980000000000001e-06 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041613+0000", 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0640000000000002e-05 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.041623+0000", 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0309000000000001e-05 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.591 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429084+0000", 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.38746082799999998 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:02:00.429093+0000", 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6910000000000003e-06 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.592 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5656 12.e 12:763500b5:::benchmark_data_smithi121_563784_object5655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.187588+0000", 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 50.351600955000002, 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26195850999999998, 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.593 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5656 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187588+0000", 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.594 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187588+0000", 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187587+0000", 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187608+0000", 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0772e-05 2023-09-23T21:10:45.595 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187618+0000", 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8719999999999997e-06 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187625+0000", 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8809999999999998e-06 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.596 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.188379+0000", 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00075411100000000002 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.188487+0000", 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000108107 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.188629+0000", 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014239799999999999 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.597 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.189770+0000", 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001140624 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190075+0000", 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00030567000000000002 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.598 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449465+0000", 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25938914899999999 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449508+0000", 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2911999999999998e-05 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449547+0000", 2023-09-23T21:10:45.599 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9008000000000003e-05 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5662 12.3 12:c07a2b72:::benchmark_data_smithi121_563784_object5661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.189983+0000", 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 50.349206451999997, 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.258899518, 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.600 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5662 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.189983+0000", 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.601 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.189983+0000", 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.189981+0000", 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190006+0000", 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4655e-05 2023-09-23T21:10:45.602 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190017+0000", 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1547e-05 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190025+0000", 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6629999999999993e-06 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.603 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190145+0000", 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119834 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190297+0000", 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015222900000000001 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:10:45.604 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190437+0000", 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139818 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191017+0000", 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057978499999999998 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191791+0000", 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00077446100000000005 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.605 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191827+0000", 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5568999999999998e-05 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448836+0000", 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25700960099999998 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.606 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448882+0000", 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5640999999999999e-05 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5664 12.e 12:7a1707df:::benchmark_data_smithi121_563784_object5663:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.190425+0000", 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 50.348763998000003, 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26194191, 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.607 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5664 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190425+0000", 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.608 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190425+0000", 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190424+0000", 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.609 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190445+0000", 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0849e-05 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190456+0000", 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1362999999999999e-05 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190464+0000", 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.763e-06 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.610 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190557+0000", 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2649e-05 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190687+0000", 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000129698 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:10:45.611 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190812+0000", 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012547599999999999 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449587+0000", 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25877486 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449621+0000", 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4032999999999998e-05 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.612 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452262+0000", 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026407990000000001 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452330+0000", 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8670000000000005e-05 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.613 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452367+0000", 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6483000000000003e-05 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5668 12.e 12:73426eaa:::benchmark_data_smithi121_563784_object5667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.191572+0000", 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 50.347617462999999, 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26293898500000001, 2023-09-23T21:10:45.614 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5668 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191572+0000", 2023-09-23T21:10:45.615 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191572+0000", 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191571+0000", 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.616 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191597+0000", 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.652e-05 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191608+0000", 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0976e-05 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.617 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191615+0000", 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3259999999999998e-06 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191655+0000", 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9363000000000002e-05 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191781+0000", 2023-09-23T21:10:45.618 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012657599999999999 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448896+0000", 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25711451899999999 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452403+0000", 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003506852 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.619 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452411+0000", 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1950000000000003e-06 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.454435+0000", 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00202381 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.620 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.454475+0000", 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0309000000000003e-05 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.454511+0000", 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.54e-05 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.621 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:5665 12.8 e430/426 12:15d27e1f:::benchmark_data_smithi121_563784_object5664:head v 430'349, mlcod=430'347)", 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.191790+0000", 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 50.347399445000001, 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.258457398, 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191790+0000", 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191790+0000", 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191789+0000", 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:45.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191811+0000", 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1956000000000001e-05 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191821+0000", 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5680000000000005e-06 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191827+0000", 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6900000000000003e-06 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448796+0000", 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25696847699999997 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448823+0000", 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6993000000000001e-05 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450211+0000", 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001388465 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450247+0000", 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5800000000000003e-05 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5676 12.3 12:cd92a09f:::benchmark_data_smithi121_563784_object5675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.192638+0000", 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 50.346550765000003, 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26064975299999998, 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5676 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.627 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.192638+0000", 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.192638+0000", 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.192637+0000", 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.192668+0000", 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0812000000000002e-05 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448819+0000", 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25615120499999999 2023-09-23T21:10:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448829+0000", 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0135e-05 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448916+0000", 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7004999999999998e-05 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449071+0000", 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015446000000000001 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449235+0000", 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016359600000000001 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450551+0000", 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001316247 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452019+0000", 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001468331 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.453191+0000", 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001171598 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.453244+0000", 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3724999999999998e-05 2023-09-23T21:10:45.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.453288+0000", 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3569000000000002e-05 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39674 12.3 12:c3c1b119:::benchmark_data_smithi121_563784_object39673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.119423+0000", 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 26.419766393, 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27609113200000002, 2023-09-23T21:10:45.633 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39674 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119423+0000", 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119423+0000", 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119421+0000", 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119448+0000", 2023-09-23T21:10:45.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7127e-05 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119462+0000", 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3419e-05 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119471+0000", 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3700000000000001e-06 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.126557+0000", 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070855839999999998 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.126680+0000", 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000123134 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.126873+0000", 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019315900000000001 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.130909+0000", 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004035772 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.134583+0000", 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036738930000000001 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393864+0000", 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25928128299999997 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393920+0000", 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5995000000000002e-05 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.395514+0000", 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001593726 2023-09-23T21:10:45.639 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39678 12.3 12:c3b9d452:::benchmark_data_smithi121_563784_object39677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 26.419706846, 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27732642000000002, 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.640 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39678 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119502+0000", 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0346000000000001e-05 2023-09-23T21:10:45.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119519+0000", 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6858999999999999e-05 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119526+0000", 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7719999999999997e-06 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.128199+0000", 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086730639999999994 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.128325+0000", 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012607699999999999 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.128436+0000", 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000111576 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.130914+0000", 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024774319999999999 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.134631+0000", 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003717321 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.395566+0000", 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26093502200000002 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.395613+0000", 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6745000000000001e-05 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.396809+0000", 2023-09-23T21:10:45.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0011957440000000001 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39679 12.3 12:c2c45921:::benchmark_data_smithi121_563784_object39678:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.120298+0000", 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 26.418891279, 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27795734500000002, 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.647 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.648 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.649 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.649 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39679 2023-09-23T21:10:45.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.649 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120298+0000", 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120298+0000", 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120297+0000", 2023-09-23T21:10:45.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120316+0000", 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8694000000000001e-05 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120327+0000", 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0954999999999999e-05 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120338+0000", 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1939e-05 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.129660+0000", 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0093211189999999992 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.129825+0000", 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016582500000000001 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.130013+0000", 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000187243 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.131019+0000", 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001006297 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.134666+0000", 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036467190000000001 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.396869+0000", 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26220349100000001 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.396924+0000", 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5009000000000001e-05 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.398255+0000", 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0013308720000000001 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:42856 12.c 12:39b21385:::benchmark_data_smithi121_563784_object42855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:22.268596+0000", 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 23.270593094999999, 2023-09-23T21:10:45.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28378214000000002, 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 42856 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268596+0000", 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268596+0000", 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268594+0000", 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268618+0000", 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3674999999999999e-05 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268628+0000", 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6770000000000007e-06 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268634+0000", 2023-09-23T21:10:45.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3589999999999996e-06 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268771+0000", 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013668799999999999 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268910+0000", 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139715 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:10:45.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.269058+0000", 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148126 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.270664+0000", 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016054019999999999 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.273197+0000", 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025333159999999999 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.552284+0000", 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27908699199999998 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.552337+0000", 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3019999999999997e-05 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.552378+0000", 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0812999999999997e-05 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:45565 12.e 12:76acf3e4:::benchmark_data_smithi121_563784_object45564:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:24.926474+0000", 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 20.612714917000002, 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25843736099999998, 2023-09-23T21:10:45.662 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 45565 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926474+0000", 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926474+0000", 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926474+0000", 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:45.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926494+0000", 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9983000000000002e-05 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926500+0000", 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6309999999999997e-06 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926505+0000", 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0019999999999997e-06 2023-09-23T21:10:45.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.930072+0000", 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035662440000000001 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.930192+0000", 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000120729 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.930308+0000", 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116129 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.934316+0000", 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040077189999999999 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.941684+0000", 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0073674220000000002 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.183506+0000", 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24182295300000001 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.183563+0000", 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6267000000000002e-05 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.184911+0000", 2023-09-23T21:10:45.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0013487060000000001 2023-09-23T21:10:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:45.669 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:45.669 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:10:45.669 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:45.699 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:45 smithi123 ceph-mon[193903]: pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s; 4706/180573 objects degraded (2.606%); 2136/180573 objects misplaced (1.183%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T21:10:45.701 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:45.769 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:45.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:45.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:45.875 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:45.952 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:46.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 64047 64031 75.4706 59.4375 0.0150732 0.0132454 2023-09-23T21:10:46.837 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 3, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:10:46.837 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:10:46.837 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:10:46.838 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T21:10:46.879 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:46.890 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4706/180573 objects degraded (2.606%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:46.890 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4706/180573 objects degraded (2.606%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:46.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:46.947 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:10:46.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4706/180573 objects degraded (2.606%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:46.992 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:47.082 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:10:47.193 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:47.211 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 64624 64608 74.7411 36.0625 0.0211853 0.0133714 2023-09-23T21:10:47.494 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:47.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:47.611 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:47.710 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:47.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:47 smithi121 ceph-mon[211526]: pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 4706/189804 objects degraded (2.479%); 1838/189804 objects misplaced (0.968%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:10:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:47 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:10:47.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:47 smithi121 ceph-mon[219709]: pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 4706/189804 objects degraded (2.479%); 1838/189804 objects misplaced (0.968%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:10:47.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:47 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:10:47.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:47 smithi123 ceph-mon[193903]: pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 4706/189804 objects degraded (2.479%); 1838/189804 objects misplaced (0.968%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:10:47.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:47 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:10:48.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 65245 65229 74.0873 38.8125 0.0101678 0.0134917 2023-09-23T21:10:48.518 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:48.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:48.629 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:48.667 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:48.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:48.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:48.835 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:48.835 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:48.865 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:48.871 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:48.894 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:48.929 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:48.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:49.033 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:10:49.093 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:49.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 66058 66042 73.6718 50.8125 0.020242 0.0135677 2023-09-23T21:10:49.329 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:10:49.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:49.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:10:49.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:49 smithi121 ceph-mon[219709]: pgmap v1415: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 4706/189804 objects degraded (2.479%); 1838/189804 objects misplaced (0.968%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T21:10:49.340 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:49 smithi121 ceph-mon[211526]: pgmap v1415: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 4706/189804 objects degraded (2.479%); 1838/189804 objects misplaced (0.968%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T21:10:49.440 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:49.476 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:49.542 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:49.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:49.548 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:49 smithi123 ceph-mon[193903]: pgmap v1415: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 4706/189804 objects degraded (2.479%); 1838/189804 objects misplaced (0.968%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T21:10:49.648 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:49.680 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:49.733 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:49.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:49.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:49.837 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:49.869 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:49.982 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:49.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:49.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:50.087 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:50.124 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:50.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 66865 66849 73.2643 50.4375 0.0171926 0.0136437 2023-09-23T21:10:50.291 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:50.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:50.351 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:10:50.395 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:50.425 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4181207561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:50.428 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:50.485 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:50.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:50.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:50.589 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:50.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4181207561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:50.654 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4181207561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:10:50.670 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:51.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 67758 67742 72.9634 55.8125 0.00960771 0.0136998 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4181207561' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 4706/198927 objects degraded (2.366%); 1497/198927 objects misplaced (0.753%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4181207561' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 4706/198927 objects degraded (2.366%); 1497/198927 objects misplaced (0.753%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:51.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:51.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:51 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:51.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:10:51.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4181207561' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:10:51.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: failed to encode map e438 with expected crc 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 4706/198927 objects degraded (2.366%); 1497/198927 objects misplaced (0.753%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:51 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:10:52.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 68400 68384 72.407 40.125 0.0122199 0.0138054 2023-09-23T21:10:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4706/198927 objects degraded (2.366%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:52.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4706/198927 objects degraded (2.366%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:52.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4706/198927 objects degraded (2.366%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:52.749 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:52.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:52.857 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:52.974 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:53.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:10:53.213179+0000 min lat: 0.00130643 max lat: 2.40445 avg lat: 0.0138728 2023-09-23T21:10:53.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:10:53.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 69224 69208 72.0586 51.5 0.00953828 0.0138728 2023-09-23T21:10:53.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:53 smithi121 ceph-mon[211526]: pgmap v1418: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 4706/201141 objects degraded (2.340%); 1497/201141 objects misplaced (0.744%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:53.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:53 smithi121 ceph-mon[219709]: pgmap v1418: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 4706/201141 objects degraded (2.340%); 1497/201141 objects misplaced (0.744%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:53.666 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:53.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:53.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:53 smithi123 ceph-mon[193903]: pgmap v1418: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 4706/201141 objects degraded (2.340%); 1497/201141 objects misplaced (0.744%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:53.776 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:10:53.805 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:54.212 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 69958 69942 71.6294 45.875 0.0153138 0.0139565 2023-09-23T21:10:54.520 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:10:54.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:54.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:10:54.629 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:54.663 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:54.722 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:54.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:54.782 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:10:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:10:54] "GET /metrics HTTP/1.1" 200 36708 "" "Prometheus/2.43.0" 2023-09-23T21:10:54.830 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:54.861 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:54.921 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:54.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:55.025 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:10:55.058 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:55.115 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:10:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:55.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:10:55.143 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:10:55.214 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 70784 70768 71.3046 51.625 0.0339071 0.0140181 2023-09-23T21:10:55.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:55.261 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:55.320 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:10:55.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:55.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:10:55.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:55 smithi121 ceph-mon[211526]: pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 4706/204780 objects degraded (2.298%); 1497/204780 objects misplaced (0.731%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:55.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:55 smithi121 ceph-mon[219709]: pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 4706/204780 objects degraded (2.298%); 1497/204780 objects misplaced (0.731%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:55.429 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:55.630 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:55 smithi123 ceph-mon[193903]: pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 4706/204780 objects degraded (2.298%); 1497/204780 objects misplaced (0.731%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:10:55.766 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:10:55.766 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:10:55.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.766 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:71159 12.a 12:5cf454a0:::benchmark_data_smithi121_563784_object71158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e438)", 2023-09-23T21:10:55.766 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:55.735240+0000", 2023-09-23T21:10:55.766 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.018590802999999999, 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018630923000000001, 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 71159 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735240+0000", 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735240+0000", 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:55.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735239+0000", 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735278+0000", 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9031e-05 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:55.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735291+0000", 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3667e-05 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735300+0000", 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3620000000000006e-06 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735351+0000", 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1782e-05 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735493+0000", 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000141967 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T21:10:55.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.735639+0000", 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014556000000000001 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.738400+0000", 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027613070000000002 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:71157 12.f e438/426 12:f12a4264:::benchmark_data_smithi121_563784_object71156:head v 438'4415, mlcod=438'4413)", 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:55.745696+0000", 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0081354540000000003, 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0082636370000000008, 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745696+0000", 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745696+0000", 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745694+0000", 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745742+0000", 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7623000000000001e-05 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745764+0000", 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1486000000000001e-05 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745774+0000", 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0168e-05 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746353+0000", 2023-09-23T21:10:55.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057927300000000005 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746376+0000", 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2575e-05 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:71155 12.8 e438/426 12:18417222:::benchmark_data_smithi121_563784_object71154:head v 438'4407, mlcod=438'4404)", 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:55.745806+0000", 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0080249339999999992, 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0082190260000000008, 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:55.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745806+0000", 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745806+0000", 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745806+0000", 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:55.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745833+0000", 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7166999999999999e-05 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745841+0000", 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5290000000000008e-06 2023-09-23T21:10:55.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.745847+0000", 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7729999999999996e-06 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746170+0000", 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00032334699999999998 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746194+0000", 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3578999999999999e-05 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:71163 12.5 e438/426 12:aa096e6a:::benchmark_data_smithi121_563784_object71162:head v 438'4430, mlcod=438'4428)", 2023-09-23T21:10:55.782 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:55.746050+0000", 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0077810249999999996, 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.008036629, 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746050+0000", 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746050+0000", 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746049+0000", 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746164+0000", 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115118 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746175+0000", 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0282000000000001e-05 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.746182+0000", 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0960000000000003e-06 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.749014+0000", 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028317529999999998 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.749037+0000", 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2997e-05 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:71160 12.7 e438/426 12:e2de2475:::benchmark_data_smithi121_563784_object71159:head v 438'4492, mlcod=438'4490)", 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:55.751421+0000", 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0024097419999999999, 2023-09-23T21:10:55.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027288189999999999, 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751421+0000", 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751421+0000", 2023-09-23T21:10:55.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751420+0000", 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751446+0000", 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6072e-05 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751460+0000", 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3976e-05 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751469+0000", 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0720000000000006e-06 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751511+0000", 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1418000000000002e-05 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751530+0000", 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9449999999999998e-05 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:71156 12.d e438/426 12:b311e27e:::benchmark_data_smithi121_563784_object71155:head v 438'4480, mlcod=438'4478)", 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:55.751484+0000", 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0023468759999999999, 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027377899999999999, 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:55.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751484+0000", 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751484+0000", 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751484+0000", 2023-09-23T21:10:55.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751505+0000", 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1330000000000001e-05 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:55.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751511+0000", 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2890000000000003e-06 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751516+0000", 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8049999999999997e-06 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:55.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751591+0000", 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4773999999999999e-05 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751611+0000", 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0208e-05 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:71165 12.7 e438/426 12:e79cf9ef:::benchmark_data_smithi121_563784_object71164:head v 438'4493, mlcod=438'4490)", 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:55.751521+0000", 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0023103450000000001, 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027709879999999998, 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751521+0000", 2023-09-23T21:10:55.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751521+0000", 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751520+0000", 2023-09-23T21:10:55.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751540+0000", 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9504000000000002e-05 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751545+0000", 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4110000000000004e-06 2023-09-23T21:10:55.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.751549+0000", 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0450000000000002e-06 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.754072+0000", 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025225460000000001 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:10:55.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:55.754093+0000", 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0999e-05 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 7 2023-09-23T21:10:55.801 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:10:55.802 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:55.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:55.906 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:10:55.977 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:56.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 71538 71522 70.9197 47.125 0.0103169 0.0140959 2023-09-23T21:10:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4706/204780 objects degraded (2.298%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4706/204780 objects degraded (2.298%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4706/204780 objects degraded (2.298%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:10:57.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 72256 72240 70.512 44.875 0.0105325 0.0141767 2023-09-23T21:10:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:10:57.377 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:10:57.399 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:10:57.452 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:10:57.497 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:10:57.500 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:10:57.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:57.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:10:57.538 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:10:57.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:10:57.612 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:57.629 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:10:57.641 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:57.645 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:10:57.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.658 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.658 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.658 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.658 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.658 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:10:57.701 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:10:57.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:57.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:57.807 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:10:57.837 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 267 (0%) 2023-09-23T21:10:57.837 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T21:10:57.837 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.4019 -> 6.4019 (expected baseline 5.40399) 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.7 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0276 2023-09-23T21:10:57.838 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.2762 2023-09-23T21:10:57.839 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:57.876 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:10:57.895 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:10:57.902 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:57 smithi123 ceph-mon[193903]: pgmap v1420: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 4706/212478 objects degraded (2.215%); 1113/212478 objects misplaced (0.524%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:10:57.902 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3828613964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:57.902 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:57.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:10:57.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:57 smithi121 ceph-mon[211526]: pgmap v1420: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 4706/212478 objects degraded (2.215%); 1113/212478 objects misplaced (0.524%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:10:57.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3828613964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:57.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:57.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:57 smithi121 ceph-mon[219709]: pgmap v1420: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 4706/212478 objects degraded (2.215%); 1113/212478 objects misplaced (0.524%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:10:57.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3828613964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:10:57.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:10:58.003 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:58.038 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:58.093 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:58.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:58.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:10:58.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 72912 72896 70.0581 41 0.00289915 0.0142579 2023-09-23T21:10:58.226 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:58.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:58 smithi121 ceph-mon[211526]: from='client.46721 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:10:58.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:58 smithi121 ceph-mon[219709]: from='client.46721 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:10:58.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:58 smithi123 ceph-mon[193903]: from='client.46721 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:10:59.216 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 73555 73539 69.6056 40.1875 0.00366177 0.0143461 2023-09-23T21:10:59.586 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:10:59.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:59.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:10:59.696 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:10:59.729 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:59.772 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:10:59 smithi121 ceph-mon[211526]: pgmap v1421: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 4706/214554 objects degraded (2.193%); 1113/214554 objects misplaced (0.519%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:10:59.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:10:59 smithi121 ceph-mon[219709]: pgmap v1421: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 4706/214554 objects degraded (2.193%); 1113/214554 objects misplaced (0.519%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:10:59.788 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:10:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:10:59.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:10:59.896 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:10:59.929 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:10:59 smithi123 ceph-mon[193903]: pgmap v1421: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 4706/214554 objects degraded (2.193%); 1113/214554 objects misplaced (0.519%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:10:59.931 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:10:59.997 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:00.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:00.102 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:00.132 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:00.142 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:11:00.142 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T21:11:00.186 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:00.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:00.217 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 74388 74372 69.3438 52.0625 0.019137 0.0144144 2023-09-23T21:11:00.294 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:00.405 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:01.217 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 75004 74988 68.8903 38.5 0.0101229 0.0145109 2023-09-23T21:11:01.283 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:01.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:01.391 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:01.412 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:01.423 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:01.476 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:01.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:01.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:01.509 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:11:01.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:01 smithi121 ceph-mon[211526]: pgmap v1422: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 4706/221025 objects degraded (2.129%); 720/221025 objects misplaced (0.326%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:11:01.579 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:01 smithi121 ceph-mon[219709]: pgmap v1422: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 4706/221025 objects degraded (2.129%); 720/221025 objects misplaced (0.326%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:11:01.588 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:01 smithi123 ceph-mon[193903]: pgmap v1422: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 4706/221025 objects degraded (2.129%); 720/221025 objects misplaced (0.326%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T21:11:01.755 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:01.813 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:02.146 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:02.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:02.217 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 15 75751 75736 68.5695 46.75 0.0125071 0.0145773 2023-09-23T21:11:02.268 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:02.299 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:02.354 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:02.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4706/221025 objects degraded (2.129%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:02.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4706/221025 objects degraded (2.129%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:02.460 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:02.492 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:02.541 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:02.546 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4706/221025 objects degraded (2.129%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:02.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:02.647 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:02.762 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:03.217 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 76523 76507 68.2784 48.1875 0.0242834 0.0146401 2023-09-23T21:11:03.320 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:11:03.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1247284221' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:03.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:03 smithi121 ceph-mon[211526]: pgmap v1423: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 4706/223038 objects degraded (2.110%); 720/223038 objects misplaced (0.323%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:11:03.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1247284221' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:03.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:03.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:03 smithi121 ceph-mon[219709]: pgmap v1423: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 4706/223038 objects degraded (2.110%); 720/223038 objects misplaced (0.323%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:11:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1247284221' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:03 smithi123 ceph-mon[193903]: pgmap v1423: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 4706/223038 objects degraded (2.110%); 720/223038 objects misplaced (0.323%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:11:04.221 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 77406 77390 68.0903 55.1875 0.0127054 0.0146815 2023-09-23T21:11:04.281 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:04.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:04.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:04.465 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:04.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:04.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[219709]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:04 smithi121 ceph-mon[211526]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.677 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:04.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:04.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:04 smithi123 ceph-mon[193903]: failed to encode map e439 with expected crc 2023-09-23T21:11:04.781 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:04.850 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:05.111 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:11:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:11:04] "GET /metrics HTTP/1.1" 200 36702 "" "Prometheus/2.43.0" 2023-09-23T21:11:05.223 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 77991 77975 67.6508 36.5625 0.018406 0.0147762 2023-09-23T21:11:05.341 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:05.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:05.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:05 smithi121 ceph-mon[219709]: pgmap v1425: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 4706/226707 objects degraded (2.076%); 720/226707 objects misplaced (0.318%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T21:11:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:05 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:05 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:05 smithi121 ceph-mon[211526]: pgmap v1425: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 4706/226707 objects degraded (2.076%); 720/226707 objects misplaced (0.318%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T21:11:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:05 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:05 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:05.448 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:05.478 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:05.536 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:05.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:05.641 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:05.670 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:05 smithi123 ceph-mon[193903]: pgmap v1425: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s; 4706/226707 objects degraded (2.076%); 720/226707 objects misplaced (0.318%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T21:11:05.670 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:05 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:05.670 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:05 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:05.674 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:05.724 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:05.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:05.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:05.830 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:05.904 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:06.223 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 78720 78704 67.3483 45.5625 0.00665824 0.014844 2023-09-23T21:11:06.270 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:06.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:06.384 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:06.453 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:06.499 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:06.602 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:06 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:11:06.602 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:06 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:11:06.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:06 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:06.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:06 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:11:06.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:06 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:11:06.603 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:06 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:06.791 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:06.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:06.905 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:06.908 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:06.908 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:06.935 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:06 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:11:06.935 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:06 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:11:06.935 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:06 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:06.937 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:06.994 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:07.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:07.102 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:07.130 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:07.183 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:07.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:07.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:07.224 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 79453 79437 67.0566 45.8125 0.0180602 0.0149081 2023-09-23T21:11:07.297 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:07.325 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:07.351 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 267 (0%) 2023-09-23T21:11:07.351 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.4019 -> 6.4019 (expected baseline 5.40399) 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.7 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0293 2023-09-23T21:11:07.352 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.2935 2023-09-23T21:11:07.362 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:11:07.379 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:07.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:07.487 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:07.580 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:07.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:07 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4706/226707 objects degraded (2.076%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:07.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:07 smithi121 ceph-mon[211526]: pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 4706/234243 objects degraded (2.009%); 185/234243 objects misplaced (0.079%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:11:07.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:07 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4706/226707 objects degraded (2.076%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:07.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:07 smithi121 ceph-mon[219709]: pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 4706/234243 objects degraded (2.009%); 185/234243 objects misplaced (0.079%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:11:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:07 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4706/226707 objects degraded (2.076%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:07 smithi123 ceph-mon[193903]: pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 4706/234243 objects degraded (2.009%); 185/234243 objects misplaced (0.079%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T21:11:08.224 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 80126 80110 66.7235 42.0625 0.00329259 0.0149375 2023-09-23T21:11:08.297 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:08.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:08.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:08.402 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:08.433 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:08.490 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:11:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[211526]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[211526]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T21:11:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[211526]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[211526]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[211526]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[211526]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[211526]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[219709]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[219709]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T21:11:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[219709]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[219709]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[219709]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[219709]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:08 smithi121 ceph-mon[219709]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:08 smithi123 ceph-mon[193903]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:08 smithi123 ceph-mon[193903]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T21:11:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:08 smithi123 ceph-mon[193903]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:08 smithi123 ceph-mon[193903]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:08 smithi123 ceph-mon[193903]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:08 smithi123 ceph-mon[193903]: failed to encode map e440 with expected crc 2023-09-23T21:11:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:08 smithi123 ceph-mon[193903]: failed to encode map e440 with expected crc 2023-09-23T21:11:09.110 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:11:09.229 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 80571 80555 66.2076 27.8125 0.00858555 0.0150981 2023-09-23T21:11:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[219709]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[219709]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[219709]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T21:11:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[219709]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[219709]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[219709]: pgmap v1429: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 4706/236037 objects degraded (1.994%); 185/236037 objects misplaced (0.078%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:11:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[211526]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[211526]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[211526]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T21:11:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[211526]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[211526]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:09 smithi121 ceph-mon[211526]: pgmap v1429: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 4706/236037 objects degraded (1.994%); 185/236037 objects misplaced (0.078%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:11:09.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:09 smithi123 ceph-mon[193903]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:09 smithi123 ceph-mon[193903]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:09 smithi123 ceph-mon[193903]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T21:11:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:09 smithi123 ceph-mon[193903]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:09 smithi123 ceph-mon[193903]: failed to encode map e441 with expected crc 2023-09-23T21:11:09.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:09 smithi123 ceph-mon[193903]: pgmap v1429: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 4706/236037 objects degraded (1.994%); 185/236037 objects misplaced (0.078%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:11:10.163 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:11:10.164 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T21:11:10.231 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 81230 81214 65.8812 41.1875 0.0282364 0.0151739 2023-09-23T21:11:10.288 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:10.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:10.397 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:10.428 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:10.479 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:10.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:10.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:10.588 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:10.714 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:10.898 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:10.937 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:10.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:10.962 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:11.047 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:11.118 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:11:11.169 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:11.169 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:11:11.227 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:11:11.233 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 81914 81898 65.5823 42.75 0.0304483 0.0152414 2023-09-23T21:11:11.241 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1094647748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:11.242 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1094647748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:11.281 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:11:11.325 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:11:11.394 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:11.394 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:11:11.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:11.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:11.443 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:11:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1094647748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:11.488 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:11:11.507 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:11.510 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.510 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.510 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.510 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.511 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.511 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.511 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.511 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:11:11.556 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:11.645 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:12.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 82697 82681 65.3703 48.9375 0.0168018 0.0152919 2023-09-23T21:11:12.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:12 smithi121 ceph-mon[211526]: pgmap v1430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 4706/242166 objects degraded (1.943%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:11:12.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4706/236037 objects degraded (1.994%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:12.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:12 smithi121 ceph-mon[219709]: pgmap v1430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 4706/242166 objects degraded (1.943%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:11:12.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4706/236037 objects degraded (1.994%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:12 smithi123 ceph-mon[193903]: pgmap v1430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 4706/242166 objects degraded (1.943%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T21:11:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4706/236037 objects degraded (1.994%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:11:13.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:11:13.236339+0000 min lat: 0.00130643 max lat: 2.40445 avg lat: 0.0153566 2023-09-23T21:11:13.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:11:13.236 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 83392 83376 65.0962 43.4375 0.0189084 0.0153566 2023-09-23T21:11:13.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:13.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:13.833 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:13.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:13.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:13.937 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:14.125 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:14.235 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 84023 84007 64.7796 39.4375 0.0341872 0.0154317 2023-09-23T21:11:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:14 smithi121 ceph-mon[219709]: pgmap v1431: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 2397/243930 objects degraded (0.983%); 2001/243930 objects misplaced (0.820%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:11:14.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:14 smithi121 ceph-mon[211526]: pgmap v1431: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 2397/243930 objects degraded (0.983%); 2001/243930 objects misplaced (0.820%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:11:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:14 smithi123 ceph-mon[193903]: pgmap v1431: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 2397/243930 objects degraded (0.983%); 2001/243930 objects misplaced (0.820%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:11:14.914 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:11:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:11:14] "GET /metrics HTTP/1.1" 200 36703 "" "Prometheus/2.43.0" 2023-09-23T21:11:14.945 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:14.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:14.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:15.055 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:15.084 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:15.143 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:15.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:15.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:15.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 84846 84830 64.6155 51.4375 0.0310408 0.0154709 2023-09-23T21:11:15.250 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:15.337 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:15 smithi123 ceph-mon[193903]: pgmap v1432: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 34 MiB/s wr, 540 op/s; 2397/247224 objects degraded (0.970%); 2001/247224 objects misplaced (0.809%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:11:15.612 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:15.614 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:15 smithi121 ceph-mon[211526]: pgmap v1432: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 34 MiB/s wr, 540 op/s; 2397/247224 objects degraded (0.970%); 2001/247224 objects misplaced (0.809%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:11:15.615 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:15 smithi121 ceph-mon[219709]: pgmap v1432: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 34 MiB/s wr, 540 op/s; 2397/247224 objects degraded (0.970%); 2001/247224 objects misplaced (0.809%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T21:11:15.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:15.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:15.717 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:15.796 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:15.916 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:15.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:15.972 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50084 11.2 11:41af7975:::benchmark_data_smithi121_477989_object50083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.400207+0000", 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 458.56724269799997, 2023-09-23T21:11:15.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1853260130000001, 2023-09-23T21:11:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:15.974 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50084 2023-09-23T21:11:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400207+0000", 2023-09-23T21:11:15.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400207+0000", 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400206+0000", 2023-09-23T21:11:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.400237+0000", 2023-09-23T21:11:15.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1195999999999997e-05 2023-09-23T21:11:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.401315+0000", 2023-09-23T21:11:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0010784480000000001 2023-09-23T21:11:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.401329+0000", 2023-09-23T21:11:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3125000000000001e-05 2023-09-23T21:11:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.405972+0000", 2023-09-23T21:11:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046429330000000001 2023-09-23T21:11:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.405982+0000", 2023-09-23T21:11:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0834e-05 2023-09-23T21:11:15.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393675+0000", 2023-09-23T21:11:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98769265399999995 2023-09-23T21:11:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393689+0000", 2023-09-23T21:11:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4436e-05 2023-09-23T21:11:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.573878+0000", 2023-09-23T21:11:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.180188276 2023-09-23T21:11:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.574056+0000", 2023-09-23T21:11:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000178381 2023-09-23T21:11:15.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:11:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.574262+0000", 2023-09-23T21:11:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020565500000000001 2023-09-23T21:11:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580224+0000", 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005962686 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580239+0000", 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4504e-05 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.583594+0000", 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033555109999999998 2023-09-23T21:11:15.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.583631+0000", 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6523000000000003e-05 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585533+0000", 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001902321 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:15.986 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50093 11.2 11:4ca61590:::benchmark_data_smithi121_477989_object50092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.417151+0000", 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 458.55029934300001, 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.174274069, 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:15.987 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50093 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417151+0000", 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417151+0000", 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:15.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417149+0000", 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417171+0000", 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1328999999999999e-05 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417182+0000", 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.134e-05 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417190+0000", 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9400000000000002e-06 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417234+0000", 2023-09-23T21:11:15.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4304000000000001e-05 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.417244+0000", 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5000000000000005e-06 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393853+0000", 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.97660954799999999 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.991 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.393862+0000", 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0089999999999996e-06 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.578019+0000", 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.184156268 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.578132+0000", 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011379000000000001 2023-09-23T21:11:15.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.578284+0000", 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000151621 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.583554+0000", 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052696110000000004 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585553+0000", 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001998948 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.589246+0000", 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003693174 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.589295+0000", 2023-09-23T21:11:15.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9301000000000001e-05 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.591425+0000", 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021296150000000001 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50094 11.2 11:49734ba9:::benchmark_data_smithi121_477989_object50093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:11:15.995 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.422439+0000", 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 458.54501140299999, 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1746549339999999, 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50094 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:15.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422439+0000", 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422439+0000", 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422437+0000", 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422463+0000", 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6129e-05 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422479+0000", 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6099e-05 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422488+0000", 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6589999999999996e-06 2023-09-23T21:11:15.998 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422554+0000", 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6344e-05 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.422567+0000", 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2478e-05 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394015+0000", 2023-09-23T21:11:15.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.97144863199999998 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394025+0000", 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7990000000000005e-06 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580262+0000", 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1862366040000001 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580377+0000", 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115528 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.580486+0000", 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000108835 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585561+0000", 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050745069999999998 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585596+0000", 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5036000000000001e-05 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.595075+0000", 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0094792490000000004 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.595175+0000", 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000100534 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.597094+0000", 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001918169 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.003 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50111 11.2 11:45a36be7:::benchmark_data_smithi121_477989_object50110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.437062+0000", 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 458.53038800500002, 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1613549390000002, 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50111 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.004 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437062+0000", 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437062+0000", 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437061+0000", 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437081+0000", 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0035999999999999e-05 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437091+0000", 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8660000000000001e-06 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437096+0000", 2023-09-23T21:11:16.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7989999999999999e-06 2023-09-23T21:11:16.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.007 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.007 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.007 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437159+0000", 2023-09-23T21:11:16.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2572999999999999e-05 2023-09-23T21:11:16.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.008 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.437172+0000", 2023-09-23T21:11:16.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3294e-05 2023-09-23T21:11:16.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394055+0000", 2023-09-23T21:11:16.009 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.956883068 2023-09-23T21:11:16.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394061+0000", 2023-09-23T21:11:16.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0079999999999999e-06 2023-09-23T21:11:16.010 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.010 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.010 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.010 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.581905+0000", 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1878437550000001 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.582005+0000", 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9733000000000002e-05 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.582122+0000", 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116845 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585626+0000", 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035039440000000002 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585635+0000", 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3479999999999993e-06 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.597148+0000", 2023-09-23T21:11:16.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011512992 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.597185+0000", 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7446e-05 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.598417+0000", 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012315150000000001 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.013 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:50125 11.2 11:4350bada:::benchmark_data_smithi121_477989_object50124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:03:37.628739+0000", 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 458.33871049999999, 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9714031510000001, 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.014 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.015 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.015 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.015 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 50125 2023-09-23T21:11:16.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.015 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.015 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.015 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628739+0000", 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628739+0000", 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628738+0000", 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628757+0000", 2023-09-23T21:11:16.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8709999999999999e-05 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628774+0000", 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7088e-05 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628782+0000", 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4679999999999992e-06 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628843+0000", 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0961999999999997e-05 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:37.628853+0000", 2023-09-23T21:11:16.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6399999999999992e-06 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394173+0000", 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.765319466 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:38.394181+0000", 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6770000000000001e-06 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585661+0000", 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1914793539999999 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585773+0000", 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112244 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.585937+0000", 2023-09-23T21:11:16.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016426900000000001 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.589209+0000", 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0032715309999999998 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.591450+0000", 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022412119999999998 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.598481+0000", 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070309209999999999 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.598543+0000", 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1839000000000002e-05 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:03:39.600143+0000", 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016000369999999999 2023-09-23T21:11:16.023 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94323 11.2 11:4f2271be:::benchmark_data_smithi121_477989_object6134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.497029+0000", 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.470421096, 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0223154210000001, 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.024 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94323 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497029+0000", 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497029+0000", 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497027+0000", 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497032+0000", 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7949999999999998e-06 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497040+0000", 2023-09-23T21:11:16.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.875e-06 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497045+0000", 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3820000000000003e-06 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497093+0000", 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7811e-05 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.497105+0000", 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2347000000000001e-05 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.513818+0000", 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.016712711 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.513924+0000", 2023-09-23T21:11:16.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010610799999999999 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.514008+0000", 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3659999999999995e-05 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516509+0000", 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025014540000000002 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516558+0000", 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8269999999999997e-05 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519004+0000", 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024466309999999999 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519173+0000", 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000168226 2023-09-23T21:11:16.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519218+0000", 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5862000000000002e-05 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519344+0000", 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012590999999999999 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.031 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94333 11.2 11:4e5950f9:::benchmark_data_smithi121_477989_object6144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.516562+0000", 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.45088845100003, 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0030305859999999, 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94333 2023-09-23T21:11:16.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516562+0000", 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516562+0000", 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.033 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516560+0000", 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516567+0000", 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1690000000000003e-06 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516579+0000", 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2296000000000001e-05 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516587+0000", 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7689999999999996e-06 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516624+0000", 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7214e-05 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.516634+0000", 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3919999999999992e-06 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516590+0000", 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99995588499999999 2023-09-23T21:11:16.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516719+0000", 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000129018 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.516798+0000", 2023-09-23T21:11:16.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9468000000000006e-05 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519416+0000", 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026183119999999998 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519446+0000", 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9918000000000001e-05 2023-09-23T21:11:16.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519454+0000", 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.413e-06 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519491+0000", 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6854999999999999e-05 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519526+0000", 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5567e-05 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519592+0000", 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6006000000000005e-05 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94386 11.2 11:4212ad5d:::benchmark_data_smithi121_477989_object6197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.644675+0000", 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.322775175, 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.88700020899999998, 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94386 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.041 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644675+0000", 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644675+0000", 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644673+0000", 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644683+0000", 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0200999999999999e-05 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644702+0000", 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9069e-05 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644713+0000", 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0939e-05 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644765+0000", 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1374e-05 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.644776+0000", 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1126e-05 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519633+0000", 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.87485688800000005 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519721+0000", 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.886e-05 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.519799+0000", 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7966999999999994e-05 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528429+0000", 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086300349999999994 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531481+0000", 2023-09-23T21:11:16.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003051144 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531505+0000", 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4156e-05 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531538+0000", 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3365999999999999e-05 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531597+0000", 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9021000000000003e-05 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531675+0000", 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7866000000000003e-05 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94417 11.2 11:44af34fd:::benchmark_data_smithi121_477989_object6228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.699432+0000", 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.26801842100002, 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.83896963999999996, 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94417 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699432+0000", 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699432+0000", 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699431+0000", 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699435+0000", 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.49e-06 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699453+0000", 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8845e-05 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.699460+0000", 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4030000000000004e-06 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.701531+0000", 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020708800000000002 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.052 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.701540+0000", 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3440000000000007e-06 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.522132+0000", 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.82059189200000004 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.053 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.522247+0000", 2023-09-23T21:11:16.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115277 2023-09-23T21:11:16.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.054 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.054 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.054 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.522377+0000", 2023-09-23T21:11:16.054 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012988 2023-09-23T21:11:16.054 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531714+0000", 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0093365370000000007 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531733+0000", 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9267999999999999e-05 2023-09-23T21:11:16.055 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531751+0000", 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7833e-05 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538195+0000", 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0064444460000000004 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.056 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538248+0000", 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2831999999999998e-05 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538401+0000", 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015315399999999999 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94430 11.2 11:4c4574b1:::benchmark_data_smithi121_477989_object6241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.057 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.727165+0000", 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.240285293, 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.81163672499999995, 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94430 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.058 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727165+0000", 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727165+0000", 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.059 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727163+0000", 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727171+0000", 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4919999999999996e-06 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727187+0000", 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6348e-05 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.060 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727196+0000", 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9809999999999992e-06 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727246+0000", 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9808999999999997e-05 2023-09-23T21:11:16.061 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.727257+0000", 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.147e-05 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.062 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.525318+0000", 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79806034599999998 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.525430+0000", 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112805 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.063 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.525546+0000", 2023-09-23T21:11:16.064 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011565000000000001 2023-09-23T21:11:16.064 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.064 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.064 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.064 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531773+0000", 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062273459999999999 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538463+0000", 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0066899999999999998 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.065 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538530+0000", 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6587999999999993e-05 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538622+0000", 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1810000000000004e-05 2023-09-23T21:11:16.066 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538674+0000", 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2438999999999997e-05 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538801+0000", 2023-09-23T21:11:16.067 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000127193 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94445 11.2 11:4d352ece:::benchmark_data_smithi121_477989_object6256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:11:16.068 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.21909971299999, 2023-09-23T21:11:16.069 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.79675490599999998, 2023-09-23T21:11:16.069 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.069 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.069 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.069 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.069 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.069 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94445 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.070 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748350+0000", 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:16.071 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748353+0000", 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7010000000000001e-06 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748357+0000", 2023-09-23T21:11:16.072 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8940000000000001e-06 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748361+0000", 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2219999999999998e-06 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.073 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748402+0000", 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1729e-05 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.748413+0000", 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0552e-05 2023-09-23T21:11:16.074 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528479+0000", 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.78006554900000002 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528635+0000", 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000156188 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.075 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.528773+0000", 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013783900000000001 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531810+0000", 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030374569999999999 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538539+0000", 2023-09-23T21:11:16.076 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067294110000000002 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538578+0000", 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8931999999999999e-05 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.544961+0000", 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063821549999999996 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.077 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545014+0000", 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3165999999999999e-05 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545105+0000", 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1481e-05 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.078 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94484 11.2 11:43d1d258:::benchmark_data_smithi121_477989_object6295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.841431+0000", 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.126018923, 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.70393903800000002, 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94484 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.079 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841431+0000", 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841431+0000", 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841430+0000", 2023-09-23T21:11:16.080 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841436+0000", 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2360000000000001e-06 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841448+0000", 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1894e-05 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.081 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841456+0000", 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0299999999999994e-06 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841516+0000", 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0090000000000002e-05 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.841529+0000", 2023-09-23T21:11:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3399e-05 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531837+0000", 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.69030767299999996 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.531992+0000", 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015467200000000001 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.083 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.532149+0000", 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015686199999999999 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538518+0000", 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063691370000000004 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538548+0000", 2023-09-23T21:11:16.084 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9923000000000001e-05 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538889+0000", 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00034130500000000003 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545165+0000", 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0062759620000000004 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.085 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545224+0000", 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8542000000000003e-05 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545370+0000", 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014655500000000001 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:94488 11.2 11:46589f20:::benchmark_data_smithi121_477989_object6299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:06:44.846779+0000", 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 271.12067091199998, 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.69881891100000004, 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:11:16.087 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 94488 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846779+0000", 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.088 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846779+0000", 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846778+0000", 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.089 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846785+0000", 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1930000000000004e-06 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846797+0000", 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.254e-05 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.846805+0000", 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1310000000000006e-06 2023-09-23T21:11:16.090 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.847172+0000", 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00036666999999999998 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:44.847182+0000", 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8889999999999997e-06 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.091 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.534969+0000", 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.68778722800000003 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.535105+0000", 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0001362 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.535236+0000", 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013034900000000001 2023-09-23T21:11:16.092 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538556+0000", 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033200830000000001 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538849+0000", 2023-09-23T21:11:16.093 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00029309900000000001 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.538929+0000", 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9872999999999999e-05 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.094 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545451+0000", 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065226119999999997 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545484+0000", 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2274999999999998e-05 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.095 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:06:45.545598+0000", 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114301 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64046 12.a 12:573871b3:::benchmark_data_smithi121_563784_object64045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.211038+0000", 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.756412035, 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.370912299, 2023-09-23T21:11:16.096 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64046 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211038+0000", 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.097 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211038+0000", 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211036+0000", 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.098 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211059+0000", 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2813e-05 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211070+0000", 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.079e-05 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.099 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211090+0000", 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9534e-05 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211151+0000", 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1367999999999999e-05 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211306+0000", 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015519700000000001 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.100 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.211395+0000", 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8341e-05 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.216208+0000", 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048134450000000004 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217401+0000", 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001193482 2023-09-23T21:11:16.101 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581867+0000", 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36446511599999998 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581915+0000", 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8087999999999998e-05 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.102 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581950+0000", 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5589000000000001e-05 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64051 12.a 12:53619f52:::benchmark_data_smithi121_563784_object64050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.217299+0000", 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.750150605000002, 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36474712999999997, 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.103 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64051 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217299+0000", 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.104 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217299+0000", 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217298+0000", 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217319+0000", 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1532e-05 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.105 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217330+0000", 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0577000000000001e-05 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217336+0000", 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9900000000000002e-06 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.106 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217423+0000", 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6472999999999997e-05 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217525+0000", 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000102495 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217623+0000", 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7948999999999993e-05 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.107 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222711+0000", 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050875649999999996 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222716+0000", 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3179999999999997e-06 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.108 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581985+0000", 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35926943 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582019+0000", 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3358999999999997e-05 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582046+0000", 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7844e-05 2023-09-23T21:11:16.109 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.110 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.110 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.110 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.110 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.110 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:16.111 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64053 12.a 12:51e134cc:::benchmark_data_smithi121_563784_object64052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:16.111 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.217362+0000", 2023-09-23T21:11:16.112 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.750088325, 2023-09-23T21:11:16.112 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36478179599999999, 2023-09-23T21:11:16.112 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.112 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:16.112 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64053 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217362+0000", 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.113 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217362+0000", 2023-09-23T21:11:16.114 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.114 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.114 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.114 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.114 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217361+0000", 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217383+0000", 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2631000000000001e-05 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217393+0000", 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1749999999999994e-06 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.115 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217398+0000", 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3720000000000004e-06 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.219583+0000", 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021848480000000001 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.219681+0000", 2023-09-23T21:11:16.116 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7987999999999998e-05 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.219779+0000", 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8375000000000006e-05 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222721+0000", 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029418589999999998 2023-09-23T21:11:16.117 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.223504+0000", 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00078336300000000003 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582079+0000", 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.358574485 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582116+0000", 2023-09-23T21:11:16.118 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6939e-05 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582143+0000", 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7542000000000002e-05 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64054 12.a 12:55e294f6:::benchmark_data_smithi121_563784_object64053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.217402+0000", 2023-09-23T21:11:16.119 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.750047670000001, 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.36501321599999997, 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64054 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217402+0000", 2023-09-23T21:11:16.120 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217402+0000", 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217402+0000", 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.121 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217419+0000", 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7258e-05 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217423+0000", 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.284e-06 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.217426+0000", 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9299999999999999e-06 2023-09-23T21:11:16.122 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.221143+0000", 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037162330000000002 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.221229+0000", 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6159e-05 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.123 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,3", 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.221331+0000", 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000101803 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222874+0000", 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0015432390000000001 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.223616+0000", 2023-09-23T21:11:16.124 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00074220600000000003 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582338+0000", 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35872155300000003 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582383+0000", 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5451999999999998e-05 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.125 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.126 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.126 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582416+0000", 2023-09-23T21:11:16.126 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2495e-05 2023-09-23T21:11:16.126 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.126 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.126 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.126 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64058 12.9 12:96e1be5b:::benchmark_data_smithi121_563784_object64057:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.222525+0000", 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.744925390999999, 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35918054300000002, 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64058 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.127 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222525+0000", 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222525+0000", 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.128 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222523+0000", 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222544+0000", 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0568999999999999e-05 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222554+0000", 2023-09-23T21:11:16.129 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7990000000000005e-06 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222560+0000", 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1e-06 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222609+0000", 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9200999999999999e-05 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.130 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222734+0000", 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000125158 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.222877+0000", 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014274000000000001 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.224795+0000", 2023-09-23T21:11:16.131 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019176779999999999 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.225002+0000", 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000207698 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581613+0000", 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35661079699999998 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.132 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581665+0000", 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1950999999999998e-05 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581705+0000", 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0145000000000003e-05 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.133 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64060 12.9 12:9e6900ac:::benchmark_data_smithi121_563784_object64059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.226519+0000", 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.740930542000001, 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35529443900000002, 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64060 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.134 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226519+0000", 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226519+0000", 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.135 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226518+0000", 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226540+0000", 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1506e-05 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226551+0000", 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1416999999999999e-05 2023-09-23T21:11:16.136 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226558+0000", 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3170000000000004e-06 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226601+0000", 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2224000000000001e-05 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.137 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226688+0000", 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7295999999999999e-05 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.226777+0000", 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9591000000000003e-05 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.138 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.228013+0000", 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012360050000000001 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.228632+0000", 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00061863 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581746+0000", 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35311409300000002 2023-09-23T21:11:16.139 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581783+0000", 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6426e-05 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581814+0000", 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1265e-05 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.140 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64063 12.9 12:9a84f663:::benchmark_data_smithi121_563784_object64062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.231592+0000", 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.735858010000001, 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.35098681900000001, 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64063 2023-09-23T21:11:16.141 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231592+0000", 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231592+0000", 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.142 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231591+0000", 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231611+0000", 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0650999999999999e-05 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.143 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231620+0000", 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5120000000000008e-06 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231626+0000", 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9370000000000001e-06 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.144 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:16.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231660+0000", 2023-09-23T21:11:16.145 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3803000000000001e-05 2023-09-23T21:11:16.145 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.145 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.145 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:16.145 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231752+0000", 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2059999999999996e-05 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,6", 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.231840+0000", 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8265999999999998e-05 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.146 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.581576+0000", 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.34973652500000002 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582175+0000", 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00059885500000000003 2023-09-23T21:11:16.147 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582503+0000", 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00032773299999999997 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:16.148 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582547+0000", 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3940000000000003e-05 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.582579+0000", 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1785000000000001e-05 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.149 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.150 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.150 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:16.150 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:16.150 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:11:16.151 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:16.205 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:16.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:16.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 85548 85532 64.3657 43.875 0.0205939 0.0155308 2023-09-23T21:11:16.300 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3382031840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:16.312 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:16.325 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:11:16.415 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3382031840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:16.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3382031840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:16.692 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:11:17.237 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 86349 86333 64.1954 50.0625 0.00872314 0.015572 2023-09-23T21:11:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3382031840' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T21:11:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2397/247224 objects degraded (0.970%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:11:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[219709]: pgmap v1434: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 2397/254658 objects degraded (0.941%); 1656/254658 objects misplaced (0.650%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T21:11:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3382031840' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T21:11:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2397/247224 objects degraded (0.970%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:11:17.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:17 smithi121 ceph-mon[211526]: pgmap v1434: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 2397/254658 objects degraded (0.941%); 1656/254658 objects misplaced (0.650%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T21:11:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:11:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3382031840' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:17.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T21:11:17.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: failed to encode map e442 with expected crc 2023-09-23T21:11:17.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2397/247224 objects degraded (0.970%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:11:17.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:17 smithi123 ceph-mon[193903]: pgmap v1434: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 2397/254658 objects degraded (0.941%); 1656/254658 objects misplaced (0.650%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T21:11:17.801 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:17.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:17.911 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:17.949 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:18.002 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:18.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:18.109 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:18.143 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:18.202 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:18.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:18.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 87059 87043 63.9588 44.375 0.00729515 0.0156292 2023-09-23T21:11:18.310 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:18.383 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:18.793 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:11:18.924 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:18.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:19.042 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:19.199 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:19.242 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 87739 87723 63.7094 42.5 0.021598 0.0156898 2023-09-23T21:11:19.435 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:19.468 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:19.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:19.568 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:19.603 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:19.605 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:19.662 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:19.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:19.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:19.767 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:19.877 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:19.935 INFO:teuthology.orchestra.run.smithi121.stdout:moved 13 / 267 (4.86891%) 2023-09-23T21:11:19.935 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.4019 -> 6.28366 (expected baseline 5.40399) 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.7 with 22 -> 23 pgs (0.659176 -> 0.689139 * mean) 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 41 -> 42 pgs (1.22846 -> 1.25843 * mean) 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0311 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0342 2023-09-23T21:11:19.936 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 weight 1.0000 -> 0.6939 2023-09-23T21:11:19.937 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 weight 1.0000 -> 0.7563 2023-09-23T21:11:19.937 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 1.0000 -> 0.8931 2023-09-23T21:11:19.963 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:11:20.089 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:11:19 smithi121 conmon[198376]: 2023-09-23T21:11:19.933+0000 7f479dcb1700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T21:11:20.245 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 88410 88394 63.4572 41.9375 0.0129713 0.0157522 2023-09-23T21:11:20.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:20 smithi121 ceph-mon[211526]: pgmap v1435: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 2397/256749 objects degraded (0.934%); 1656/256749 objects misplaced (0.645%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T21:11:20.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:20 smithi121 ceph-mon[219709]: pgmap v1435: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 2397/256749 objects degraded (0.934%); 1656/256749 objects misplaced (0.645%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T21:11:20.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:20 smithi123 ceph-mon[193903]: pgmap v1435: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s; 2397/256749 objects degraded (0.934%); 1656/256749 objects misplaced (0.645%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T21:11:20.991 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:21.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:21.075 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:11:21.101 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:21.132 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:21.175 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:21.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:21.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:21.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 89083 89067 63.2142 42.0625 0.00563213 0.0157937 2023-09-23T21:11:21.281 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:21.306 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:21 smithi121 ceph-mon[211526]: from='client.36750 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:21.306 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:21 smithi121 ceph-mon[211526]: pgmap v1436: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 2397/263376 objects degraded (0.910%); 1656/263376 objects misplaced (0.629%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:11:21.306 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:21 smithi121 ceph-mon[219709]: from='client.36750 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:21.306 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:21 smithi121 ceph-mon[219709]: pgmap v1436: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 2397/263376 objects degraded (0.910%); 1656/263376 objects misplaced (0.629%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:11:21.309 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:21 smithi123 ceph-mon[193903]: from='client.36750 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:21 smithi123 ceph-mon[193903]: pgmap v1436: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s; 2397/263376 objects degraded (0.910%); 1656/263376 objects misplaced (0.629%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:11:22.001 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:22.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:22.107 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:22.144 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:22.203 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:22.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:22.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 89677 89661 62.921 37.125 0.0142426 0.015887 2023-09-23T21:11:22.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2397/263376 objects degraded (0.910%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:22.310 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:22.367 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2397/263376 objects degraded (0.910%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:22.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2397/263376 objects degraded (0.910%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:23.065 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:23.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:23.175 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:23.202 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:23.203 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T21:11:23.203 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T21:11:23.244 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:23.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.0066 2023-09-23T21:11:23.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 90400 2023-09-23T21:11:23.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T21:11:23.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T21:11:23.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 62.7732 2023-09-23T21:11:23.246 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 32.7641 2023-09-23T21:11:23.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 177.062 2023-09-23T21:11:23.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T21:11:23.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 1004 2023-09-23T21:11:23.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 524.236 2023-09-23T21:11:23.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 2833 2023-09-23T21:11:23.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T21:11:23.247 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0159265 2023-09-23T21:11:23.248 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.0399715 2023-09-23T21:11:23.248 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 2.40445 2023-09-23T21:11:23.248 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00130643 2023-09-23T21:11:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:23.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:23.256 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:11:23.349 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:23.378 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:23.404 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:23.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:23 smithi121 ceph-mon[219709]: pgmap v1437: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 2397/265449 objects degraded (0.903%); 1314/265449 objects misplaced (0.495%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T21:11:23.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:23 smithi121 ceph-mon[211526]: pgmap v1437: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 2397/265449 objects degraded (0.903%); 1314/265449 objects misplaced (0.495%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T21:11:23.427 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:23.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:23.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:23 smithi123 ceph-mon[193903]: pgmap v1437: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 2397/265449 objects degraded (0.903%); 1314/265449 objects misplaced (0.495%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T21:11:23.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:23.623 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:23.977 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:23.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:23.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:24.100 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:24.129 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:24.176 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:11:24.187 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:24.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:24.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:24.212 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:11:24.213 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:11:24.213 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:11:24.214 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.214 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:91044 12.8 12:1c3d5d64:::benchmark_data_smithi121_563784_object641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:11:24.214 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:24.190956+0000", 2023-09-23T21:11:24.214 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.022916279000000001, 2023-09-23T21:11:24.214 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022947683, 2023-09-23T21:11:24.214 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:24.214 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 91044 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.190956+0000", 2023-09-23T21:11:24.215 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.190956+0000", 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.190955+0000", 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:24.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.190963+0000", 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8180000000000004e-06 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.190977+0000", 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3708e-05 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.190985+0000", 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8750000000000006e-06 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.191042+0000", 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6876999999999998e-05 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.191167+0000", 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012464 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.191279+0000", 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011255399999999999 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.197020+0000", 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0057407889999999996 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:24.197089+0000", 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8974999999999996e-05 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:24.220 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:24.221 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:24.221 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:11:24.221 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 1 2023-09-23T21:11:24.221 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:11:24.271 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:11:24.296 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:24.351 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:11:24.387 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:11:24.404 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:24.419 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:11:24.451 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:11:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1438826180' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:24.490 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.500 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:11:24.544 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1438826180' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:24.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1438826180' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:24.815 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:11:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:11:24] "GET /metrics HTTP/1.1" 200 36703 "" "Prometheus/2.43.0" 2023-09-23T21:11:25.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:25 smithi121 ceph-mon[219709]: pgmap v1438: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 708 op/s; 2397/267516 objects degraded (0.896%); 1314/267516 objects misplaced (0.491%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T21:11:25.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:25 smithi121 ceph-mon[211526]: pgmap v1438: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 708 op/s; 2397/267516 objects degraded (0.896%); 1314/267516 objects misplaced (0.491%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T21:11:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:25 smithi123 ceph-mon[193903]: pgmap v1438: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 708 op/s; 2397/267516 objects degraded (0.896%); 1314/267516 objects misplaced (0.491%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T21:11:25.927 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:25.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:26.039 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:26.121 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:26.364 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:26.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:26.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:26.472 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:26.534 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:26.603 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:11:26.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2397/267516 objects degraded (0.896%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:26.787 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:26.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:26.901 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:26.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2397/267516 objects degraded (0.896%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:26.928 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2397/267516 objects degraded (0.896%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:26.930 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:26.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:26.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:27.082 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:27.115 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:27.173 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:27.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:27.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:27.281 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:27.311 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:27.369 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:27.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:27.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:27.479 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:27.510 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:27.563 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:27.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:27.619 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:27.674 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:27.711 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:27.762 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:27.767 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:27 smithi123 ceph-mon[193903]: pgmap v1439: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 94 B/s rd, 32 MiB/s wr, 643 op/s; 2397/266913 objects degraded (0.898%); 889/266913 objects misplaced (0.333%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T21:11:27.767 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:27.767 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:27.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:27.869 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:27.898 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:27.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:27 smithi121 ceph-mon[211526]: pgmap v1439: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 94 B/s rd, 32 MiB/s wr, 643 op/s; 2397/266913 objects degraded (0.898%); 889/266913 objects misplaced (0.333%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T21:11:27.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:27.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:27.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:27 smithi121 ceph-mon[219709]: pgmap v1439: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 94 B/s rd, 32 MiB/s wr, 643 op/s; 2397/266913 objects degraded (0.898%); 889/266913 objects misplaced (0.333%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T21:11:27.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:27.909 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:27.962 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:27.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:28.069 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:28.090 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:28.192 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:28.723 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:11:28.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1096774270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:29.070 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1096774270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:29.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1096774270' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:11:29.661 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:11:29.730 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1096774270' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[211526]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[211526]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[211526]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[211526]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[211526]: pgmap v1441: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 700 op/s; 2397/264783 objects degraded (0.905%); 889/264783 objects misplaced (0.336%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:29.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1096774270' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:29.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[219709]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T21:11:29.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[219709]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[219709]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[219709]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:29 smithi121 ceph-mon[219709]: pgmap v1441: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 700 op/s; 2397/264783 objects degraded (0.905%); 889/264783 objects misplaced (0.336%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:11:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:29 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1096774270' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:11:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:29 smithi123 ceph-mon[193903]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T21:11:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:29 smithi123 ceph-mon[193903]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:29 smithi123 ceph-mon[193903]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:29 smithi123 ceph-mon[193903]: failed to encode map e443 with expected crc 2023-09-23T21:11:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:29 smithi123 ceph-mon[193903]: pgmap v1441: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 700 op/s; 2397/264783 objects degraded (0.905%); 889/264783 objects misplaced (0.336%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:11:30.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:30 smithi123 ceph-mon[193903]: failed to encode map e443 with expected crc 2023-09-23T21:11:31.019 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:30 smithi121 ceph-mon[211526]: failed to encode map e443 with expected crc 2023-09-23T21:11:31.019 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:30 smithi121 ceph-mon[219709]: failed to encode map e443 with expected crc 2023-09-23T21:11:31.064 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:31.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:31.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:31.173 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:31.200 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:31.561 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:31.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:31.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:31.672 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:31.707 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:31.763 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:31.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:31.847 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:31 smithi121 ceph-mon[211526]: pgmap v1442: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 705 op/s; 2397/257994 objects degraded (0.929%); 889/257994 objects misplaced (0.345%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:11:31.847 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:31 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2397/264783 objects degraded (0.905%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:31.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:31 smithi121 ceph-mon[219709]: pgmap v1442: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 705 op/s; 2397/257994 objects degraded (0.929%); 889/257994 objects misplaced (0.345%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:11:31.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:31 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2397/264783 objects degraded (0.905%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:31.872 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:31.969 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:31 smithi123 ceph-mon[193903]: pgmap v1442: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 705 op/s; 2397/257994 objects degraded (0.929%); 889/257994 objects misplaced (0.345%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:11:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:31 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2397/264783 objects degraded (0.905%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:32.140 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:32.255 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:32.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:32.363 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:32.414 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:32.621 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:32.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:32.636 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 267 (0%) 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.4019 -> 6.4019 (expected baseline 5.40399) 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.7 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0312 2023-09-23T21:11:32.637 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.3122 2023-09-23T21:11:32.646 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:11:32.674 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:32.674 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:32.728 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:32.820 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:33.021 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:11:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:34 smithi121 ceph-mon[211526]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:34 smithi121 ceph-mon[211526]: pgmap v1443: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 701 op/s; 2397/256068 objects degraded (0.936%); 460/256068 objects misplaced (0.180%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:11:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:34 smithi121 ceph-mon[219709]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:34 smithi121 ceph-mon[219709]: pgmap v1443: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 701 op/s; 2397/256068 objects degraded (0.936%); 460/256068 objects misplaced (0.180%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:11:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:34 smithi123 ceph-mon[193903]: from='client.36765 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:34 smithi123 ceph-mon[193903]: pgmap v1443: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 701 op/s; 2397/256068 objects degraded (0.936%); 460/256068 objects misplaced (0.180%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:11:34.899 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:34.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:34.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:34.980 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:11:35.012 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:35.055 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:11:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:11:34] "GET /metrics HTTP/1.1" 200 36705 "" "Prometheus/2.43.0" 2023-09-23T21:11:35.107 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:35.309 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T21:11:35.309 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T21:11:35.618 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:35.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:35.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:35.727 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:35.759 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:35.822 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:35.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:35.833 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:35.928 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:35.957 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.010 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:36.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:36.037 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:36.118 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:36.147 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.205 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:36.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:36.316 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:36.419 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:36 smithi123 ceph-mon[193903]: pgmap v1444: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 711 op/s; 2397/252735 objects degraded (0.948%); 460/252735 objects misplaced (0.182%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:11:36.552 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:36 smithi121 ceph-mon[219709]: pgmap v1444: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 711 op/s; 2397/252735 objects degraded (0.948%); 460/252735 objects misplaced (0.182%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:11:36.553 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:36 smithi121 ceph-mon[211526]: pgmap v1444: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 6.8 MiB/s wr, 711 op/s; 2397/252735 objects degraded (0.948%); 460/252735 objects misplaced (0.182%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:11:36.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:11:36.607 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:36.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:36.642 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:11:36.675 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:11:36.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:11:36.714 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:36.731 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:11:36.755 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:36.760 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:11:36.789 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:11:36.815 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:11:36.816 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:11:36.816 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:11:36.817 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:11:36.817 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:11:36.817 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:11:36.817 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:11:36.817 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:11:36.817 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:11:37.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/520599530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2397/252735 objects degraded (0.948%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[219709]: pgmap v1446: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 2397/245514 objects degraded (0.976%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T21:11:37.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T21:11:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/520599530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2397/252735 objects degraded (0.948%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:37 smithi121 ceph-mon[211526]: pgmap v1446: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 2397/245514 objects degraded (0.976%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T21:11:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T21:11:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: failed to encode map e444 with expected crc 2023-09-23T21:11:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/520599530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2397/252735 objects degraded (0.948%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:37 smithi123 ceph-mon[193903]: pgmap v1446: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 2397/245514 objects degraded (0.976%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T21:11:38.180 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:38.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:38.287 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:38.323 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:38.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[219709]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[219709]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[219709]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T21:11:38.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[219709]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[219709]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[211526]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[211526]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[211526]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T21:11:38.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[211526]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.466 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:38 smithi121 ceph-mon[211526]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.475 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:38 smithi123 ceph-mon[193903]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:38 smithi123 ceph-mon[193903]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:38 smithi123 ceph-mon[193903]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T21:11:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:38 smithi123 ceph-mon[193903]: failed to encode map e445 with expected crc 2023-09-23T21:11:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:38 smithi123 ceph-mon[193903]: failed to encode map e445 with expected crc 2023-09-23T21:11:39.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:39.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:39.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:39.289 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:39.358 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:39.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:39 smithi121 ceph-mon[211526]: pgmap v1448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 87 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 603/243705 objects degraded (0.247%); 1541/243705 objects misplaced (0.632%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:11:39.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:39 smithi121 ceph-mon[219709]: pgmap v1448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 87 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 603/243705 objects degraded (0.247%); 1541/243705 objects misplaced (0.632%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:11:39.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:39 smithi123 ceph-mon[193903]: pgmap v1448: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 87 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 603/243705 objects degraded (0.247%); 1541/243705 objects misplaced (0.632%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T21:11:39.896 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:39.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:39.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:40.008 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:40.045 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:40.099 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:40.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:40.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:40.204 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:40.232 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:40.284 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:40.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:40.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:40.389 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:40.482 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5665 12.8 12:15d27e1f:::benchmark_data_smithi121_563784_object5664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.191214+0000", 2023-09-23T21:11:40.688 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 105.49347591599999, 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25962825, 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5665 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191214+0000", 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191214+0000", 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191213+0000", 2023-09-23T21:11:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191235+0000", 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1696e-05 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191247+0000", 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2706e-05 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191255+0000", 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9689999999999994e-06 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191315+0000", 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9877999999999998e-05 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191447+0000", 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013184199999999999 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191579+0000", 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000131815 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.192399+0000", 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00081998399999999995 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.193882+0000", 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001483266 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450745+0000", 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25686271500000002 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450799+0000", 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4178999999999997e-05 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450842+0000", 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3460000000000001e-05 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5678 12.8 12:1b339262:::benchmark_data_smithi121_563784_object5677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.194613+0000", 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 105.49007729500001, 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25634367000000002, 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5678 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194613+0000", 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194613+0000", 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194611+0000", 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194633+0000", 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1742e-05 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194644+0000", 2023-09-23T21:11:40.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1097999999999999e-05 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194652+0000", 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1380000000000007e-06 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194705+0000", 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2296000000000002e-05 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194826+0000", 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121285 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194992+0000", 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016586699999999999 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.195572+0000", 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057995500000000005 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196822+0000", 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012501229999999999 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450881+0000", 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25405888999999998 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450922+0000", 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1588000000000002e-05 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450956+0000", 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4133e-05 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.702 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5685 12.8 12:15babe76:::benchmark_data_smithi121_563784_object5684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.196121+0000", 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 105.488569576, 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25495290599999998, 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5685 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196121+0000", 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196121+0000", 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196120+0000", 2023-09-23T21:11:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196139+0000", 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9334000000000002e-05 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196148+0000", 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5060000000000001e-06 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196155+0000", 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8850000000000001e-06 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196198+0000", 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2894e-05 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196353+0000", 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000154618 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196471+0000", 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011869 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.197034+0000", 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00056226799999999999 2023-09-23T21:11:40.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.197221+0000", 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018697700000000001 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450995+0000", 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25377428099999999 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.451040+0000", 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.545e-05 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.451073+0000", 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2987000000000001e-05 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.709 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39654 12.8 12:16533dd6:::benchmark_data_smithi121_563784_object39653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.105135+0000", 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.579554873000006, 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28269233500000002, 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39654 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105135+0000", 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105135+0000", 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105133+0000", 2023-09-23T21:11:40.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105156+0000", 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2793999999999999e-05 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105168+0000", 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1663e-05 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105177+0000", 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7560000000000005e-06 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105230+0000", 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3582999999999998e-05 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.107274+0000", 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020435539999999999 2023-09-23T21:11:40.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.107403+0000", 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012961799999999999 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112112+0000", 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047086740000000004 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.117495+0000", 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053830939999999997 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118386+0000", 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089051600000000005 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387793+0000", 2023-09-23T21:11:40.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26940751400000001 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387828+0000", 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4320999999999999e-05 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39656 12.8 12:13f662b9:::benchmark_data_smithi121_563784_object39655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:11:40.716 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.105196+0000", 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.579494314000002, 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28281587899999999, 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39656 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105196+0000", 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105196+0000", 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105195+0000", 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:40.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105216+0000", 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0993999999999999e-05 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105226+0000", 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3770000000000002e-06 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105232+0000", 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.993e-06 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.110437+0000", 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005205749 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.110581+0000", 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014404700000000001 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.110712+0000", 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000130331 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112286+0000", 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0015745100000000001 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.117524+0000", 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052376530000000001 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387925+0000", 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27040097099999999 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387968+0000", 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3333e-05 2023-09-23T21:11:40.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388012+0000", 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3569999999999998e-05 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39666 12.8 12:13f06d34:::benchmark_data_smithi121_563784_object39665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.112486+0000", 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.572204604999996, 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27566012400000001, 2023-09-23T21:11:40.723 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39666 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112486+0000", 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112486+0000", 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112485+0000", 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112513+0000", 2023-09-23T21:11:40.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8552e-05 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112524+0000", 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1253999999999999e-05 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112532+0000", 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8839999999999994e-06 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112589+0000", 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6700999999999998e-05 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112728+0000", 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139264 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112919+0000", 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000190896 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.117569+0000", 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046497190000000001 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388030+0000", 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27046082700000001 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388068+0000", 2023-09-23T21:11:40.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7994000000000002e-05 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388111+0000", 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3056999999999998e-05 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388146+0000", 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4963999999999999e-05 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.729 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:39667 12.d e434/426 12:bc28a1ff:::benchmark_data_smithi121_563784_object39666:head v 434'2460, mlcod=434'2458)", 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.113990+0000", 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.570699860999994, 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27380560799999998, 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.113990+0000", 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.113990+0000", 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.113989+0000", 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114015+0000", 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5075e-05 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114027+0000", 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2434e-05 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114035+0000", 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5549999999999997e-06 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114098+0000", 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3459000000000003e-05 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114117+0000", 2023-09-23T21:11:40.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9517e-05 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387746+0000", 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27362806299999998 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387796+0000", 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0308e-05 2023-09-23T21:11:40.734 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:39668 12.1 e434/426 12:80006d8a:::benchmark_data_smithi121_563784_object39667:head v 434'2477, mlcod=434'2475)", 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.114602+0000", 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.570088534000007, 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27316755700000001, 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114602+0000", 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114602+0000", 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114601+0000", 2023-09-23T21:11:40.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114641+0000", 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0445000000000003e-05 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114653+0000", 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1449e-05 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114660+0000", 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1459999999999999e-06 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114907+0000", 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024668299999999998 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114926+0000", 2023-09-23T21:11:40.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9363000000000001e-05 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118975+0000", 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040494900000000002 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387769+0000", 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26879371800000001 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.739 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39675 12.8 12:1749fe44:::benchmark_data_smithi121_563784_object39674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.115745+0000", 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.568945130000003, 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27250877699999998, 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.740 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39675 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115745+0000", 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115745+0000", 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115743+0000", 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115770+0000", 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6319999999999999e-05 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115783+0000", 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3441000000000001e-05 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115792+0000", 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5739999999999996e-06 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115843+0000", 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1226e-05 2023-09-23T21:11:40.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115991+0000", 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148052 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.116123+0000", 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013253399999999999 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387870+0000", 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27174642599999999 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388035+0000", 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016523899999999999 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388186+0000", 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000151216 2023-09-23T21:11:40.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388224+0000", 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7710999999999999e-05 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388254+0000", 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9691000000000001e-05 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.746 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:39654 12.8 e434/426 ondisk, result = 0)", 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.118308+0000", 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.566382485999995, 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26952806499999998, 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118308+0000", 2023-09-23T21:11:40.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118308+0000", 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118307+0000", 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118313+0000", 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1890000000000004e-06 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118325+0000", 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1575e-05 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118331+0000", 2023-09-23T21:11:40.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.63e-06 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118373+0000", 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2197000000000003e-05 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118381+0000", 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1359999999999999e-06 2023-09-23T21:11:40.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387836+0000", 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26945519200000001 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:39675 12.8 e434/426 ondisk, result = 0)", 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.119601+0000", 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.565089353999994, 2023-09-23T21:11:40.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26827622200000001, 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119601+0000", 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119601+0000", 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119600+0000", 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119606+0000", 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9150000000000001e-06 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387737+0000", 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26813114799999999 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387751+0000", 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3926e-05 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387855+0000", 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010456899999999999 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387864+0000", 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3629999999999994e-06 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387877+0000", 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3247999999999999e-05 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:39656 12.8 e434/426 ondisk, result = 0)", 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.564633768999997, 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26797944299999998, 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:11:40.756 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120060+0000", 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.357e-06 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387767+0000", 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.267706369 2023-09-23T21:11:40.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387774+0000", 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9870000000000002e-06 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387910+0000", 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013650700000000001 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387917+0000", 2023-09-23T21:11:40.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3440000000000003e-06 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388036+0000", 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000118429 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:39680 12.d e434/426 12:b632df5a:::benchmark_data_smithi121_563784_object39679:head v 434'2461, mlcod=434'2458)", 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:11:40.760 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 81.564120845999994, 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27253574400000002, 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120592+0000", 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3876999999999999e-05 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388029+0000", 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26743616399999998 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388035+0000", 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.144e-06 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388065+0000", 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0766000000000003e-05 2023-09-23T21:11:40.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388090+0000", 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4352999999999999e-05 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393065+0000", 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049749590000000002 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393105+0000", 2023-09-23T21:11:40.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0209000000000001e-05 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:45566 12.2 12:441f91eb:::benchmark_data_smithi121_563784_object45565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:24.926407+0000", 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 75.758283069000001, 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25631767, 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.765 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 45566 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926407+0000", 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926407+0000", 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926406+0000", 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926434+0000", 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8530999999999998e-05 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926450+0000", 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6041000000000001e-05 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926458+0000", 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6760000000000007e-06 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.927760+0000", 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0013016869999999999 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.927887+0000", 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000126747 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:11:40.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.928045+0000", 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015825400000000001 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.932292+0000", 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042472120000000002 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.934877+0000", 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025848360000000001 2023-09-23T21:11:40.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.180998+0000", 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24612136300000001 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.181043+0000", 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5117999999999997e-05 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.182725+0000", 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016813500000000001 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64065 12.b 12:dac96c8a:::benchmark_data_smithi121_563784_object64064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.247452+0000", 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 54.437238256999997, 2023-09-23T21:11:40.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33990143499999997, 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64065 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247452+0000", 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247452+0000", 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247451+0000", 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247472+0000", 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0517999999999999e-05 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247481+0000", 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8910000000000007e-06 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247488+0000", 2023-09-23T21:11:40.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3620000000000002e-06 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247529+0000", 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1146000000000002e-05 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247660+0000", 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000130681 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247796+0000", 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000136675 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.250174+0000", 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023775049999999998 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.252502+0000", 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002328526 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587275+0000", 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33477227999999998 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587320+0000", 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5782999999999999e-05 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587353+0000", 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2848999999999997e-05 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64078 12.8 12:185b8dbf:::benchmark_data_smithi121_563784_object64077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.270175+0000", 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 54.414514664000002, 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.31567373100000001, 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.779 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64078 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270175+0000", 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270175+0000", 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270174+0000", 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270196+0000", 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2175e-05 2023-09-23T21:11:40.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270208+0000", 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2510000000000001e-05 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270217+0000", 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3990000000000004e-06 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270271+0000", 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4182999999999999e-05 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270377+0000", 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010607099999999999 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270477+0000", 2023-09-23T21:11:40.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9956000000000003e-05 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.272458+0000", 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001980739 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.272707+0000", 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024946199999999999 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.585765+0000", 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.313057431 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.585811+0000", 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6387999999999997e-05 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.585849+0000", 2023-09-23T21:11:40.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8064e-05 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:65124 12.8 12:160dd34e:::benchmark_data_smithi121_563784_object65123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:47.875050+0000", 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.809640053999999, 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25459828800000001, 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.786 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 65124 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875050+0000", 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875050+0000", 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875048+0000", 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875075+0000", 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7010999999999999e-05 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875090+0000", 2023-09-23T21:11:40.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4419999999999999e-05 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875099+0000", 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5510000000000005e-06 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875160+0000", 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0829000000000003e-05 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875322+0000", 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000161729 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.875479+0000", 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000157116 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.883232+0000", 2023-09-23T21:11:40.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0077530819999999997 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.126846+0000", 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243613782 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.129569+0000", 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027233589999999998 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.129614+0000", 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4648999999999999e-05 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.129648+0000", 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4413999999999999e-05 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:65136 12.3 e437/436 12:c700e000:::benchmark_data_smithi121_563784_object65135:head v 437'3967, mlcod=437'3965)", 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:47.885343+0000", 2023-09-23T21:11:40.792 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 52.799347597999997, 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25090902799999998, 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.885343+0000", 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.885343+0000", 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.885342+0000", 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:47.885364+0000", 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1767999999999999e-05 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.794 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.127963+0000", 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24259919399999999 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.127972+0000", 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9870000000000005e-06 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.129693+0000", 2023-09-23T21:11:40.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0017215990000000001 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.129711+0000", 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7201e-05 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.136217+0000", 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065063430000000004 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:48.136252+0000", 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4607000000000003e-05 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89072 12.8 12:1c946405:::benchmark_data_smithi121_563784_object89071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.116493+0000", 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 19.568196886999999, 2023-09-23T21:11:40.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.29639990999999999, 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89072 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116493+0000", 2023-09-23T21:11:40.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116493+0000", 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116492+0000", 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116516+0000", 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3909000000000001e-05 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116527+0000", 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1314e-05 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116533+0000", 2023-09-23T21:11:40.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1700000000000002e-06 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116582+0000", 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9094000000000001e-05 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116715+0000", 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000133008 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:11:40.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116864+0000", 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148355 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.128352+0000", 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011488717000000001 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.365879+0000", 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.237526404 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412801+0000", 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.046921870999999997 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412854+0000", 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3232000000000001e-05 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412893+0000", 2023-09-23T21:11:40.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9164e-05 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89077 12.b 12:d83995cf:::benchmark_data_smithi121_563784_object89076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.124562+0000", 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 19.560128453000001, 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28833729600000002, 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:11:40.804 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89077 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124562+0000", 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124562+0000", 2023-09-23T21:11:40.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124560+0000", 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124587+0000", 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7463000000000001e-05 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124601+0000", 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3940999999999999e-05 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124610+0000", 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8389999999999996e-06 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124680+0000", 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0059000000000001e-05 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124804+0000", 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012447900000000001 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124899+0000", 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4778000000000006e-05 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.128535+0000", 2023-09-23T21:11:40.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036354719999999998 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.129942+0000", 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0014074910000000001 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412822+0000", 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28287962100000003 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:11:40.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412864+0000", 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2592999999999997e-05 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412899+0000", 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4516e-05 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:11:40.810 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:11:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:40.812 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:40.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:40.912 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:40.940 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:40.990 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:40.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:41.095 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:41.262 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:41.461 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:41.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:41.567 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:41.597 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:41.642 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:41.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:41.745 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:41.777 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:41.781 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:11:41.832 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:41.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:41.938 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:42.052 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:42.282 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:11:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[219709]: pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 87 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 603/238305 objects degraded (0.253%); 1541/238305 objects misplaced (0.647%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:11:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3808387002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:42.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 603/238305 objects degraded (0.253%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:42.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[211526]: pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 87 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 603/238305 objects degraded (0.253%); 1541/238305 objects misplaced (0.647%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:11:42.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3808387002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:42.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:42.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 603/238305 objects degraded (0.253%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:42.696 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:42.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:42.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:42 smithi123 ceph-mon[193903]: pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 87 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 603/238305 objects degraded (0.253%); 1541/238305 objects misplaced (0.647%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:11:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3808387002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:42 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:11:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 603/238305 objects degraded (0.253%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:11:42.758 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:11:42.801 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:42.867 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:43.198 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:11:43.518 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:43.519 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:43.582 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:43.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:43.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:43.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:43.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T21:11:43.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.591 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: failed to encode map e446 with expected crc 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:11:43.592 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:43.691 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:43.727 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:43.889 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:43.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:43.994 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:43.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:44.025 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:44.081 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:44.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:44.187 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:44.218 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:44.272 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:44.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:44.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:44.377 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:44.463 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:44.489 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 267 (0%) 2023-09-23T21:11:44.489 INFO:teuthology.orchestra.run.smithi121.stdout:avg 33.375 2023-09-23T21:11:44.489 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 6.4019 -> 6.4019 (expected baseline 5.40399) 2023-09-23T21:11:44.489 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.7 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-09-23T21:11:44.489 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-09-23T21:11:44.489 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:11:44.489 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:11:44.490 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:11:44.490 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:11:44.490 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0293 2023-09-23T21:11:44.490 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.2934 2023-09-23T21:11:44.543 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:11:44.569 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:44 smithi121 ceph-mon[211526]: pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 603/236760 objects degraded (0.255%); 1541/236760 objects misplaced (0.651%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T21:11:44.570 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:44 smithi121 ceph-mon[219709]: pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 603/236760 objects degraded (0.255%); 1541/236760 objects misplaced (0.651%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T21:11:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:44 smithi123 ceph-mon[193903]: pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 603/236760 objects degraded (0.255%); 1541/236760 objects misplaced (0.651%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T21:11:44.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:11:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:11:44] "GET /metrics HTTP/1.1" 200 36708 "" "Prometheus/2.43.0" 2023-09-23T21:11:45.219 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:45.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:45.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:45.442 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:45.517 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:45 smithi121 ceph-mon[211526]: from='client.36777 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:45.517 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:45 smithi121 ceph-mon[211526]: pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 603/233598 objects degraded (0.258%); 1250/233598 objects misplaced (0.535%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:11:45.517 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:45 smithi121 ceph-mon[219709]: from='client.36777 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:45.517 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:45 smithi121 ceph-mon[219709]: pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 603/233598 objects degraded (0.258%); 1250/233598 objects misplaced (0.535%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:11:45.522 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:45.565 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:45 smithi123 ceph-mon[193903]: from='client.36777 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:11:45.565 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:45 smithi123 ceph-mon[193903]: pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 603/233598 objects degraded (0.258%); 1250/233598 objects misplaced (0.535%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T21:11:45.640 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:11:46.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:46.122 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:46.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:46.223 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:11:46.280 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:11:46.281 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:11:46.281 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:11:46.281 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:11:46.366 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:46.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 603/233598 objects degraded (0.258%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:46.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 603/233598 objects degraded (0.258%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:46.884 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:11:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:46.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:11:46.957 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 603/233598 objects degraded (0.258%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:46.975 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:11:47.005 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:47.103 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:47.192 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:47.276 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:47.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:47.387 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:47.424 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:11:47.429 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:47.511 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:47.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:47.615 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:47.649 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:47.679 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:47 smithi123 ceph-mon[193903]: pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 603/226734 objects degraded (0.266%); 1250/226734 objects misplaced (0.551%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:11:47.717 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:47.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:47.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:47.744 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:11:47.790 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:11:47.821 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:47.821 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:11:47.876 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:11:47.887 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:47 smithi121 ceph-mon[211526]: pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 603/226734 objects degraded (0.266%); 1250/226734 objects misplaced (0.551%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:11:47.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:47 smithi121 ceph-mon[219709]: pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 603/226734 objects degraded (0.266%); 1250/226734 objects misplaced (0.551%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T21:11:47.894 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:47.936 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:11:47.976 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:11:48.024 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:11:48.055 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.066 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:11:48.132 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:48.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:48.245 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:48.273 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:48.333 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:48.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:48.440 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:48.472 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:48.611 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:48.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:48.723 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:48.759 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:48.871 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1674325931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:48.919 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:48.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:48.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:49.022 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:49.051 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:49.101 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:49.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:49.111 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:49.111 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:49.166 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1674325931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:49.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1674325931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:49.205 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:49.246 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:49.342 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:49.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:49.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:49.446 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:49.474 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:49.517 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:49.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:49.622 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:49.649 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:49 smithi123 ceph-mon[193903]: pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 603/224919 objects degraded (0.268%); 959/224919 objects misplaced (0.426%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:11:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:49 smithi121 ceph-mon[211526]: pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 603/224919 objects degraded (0.268%); 959/224919 objects misplaced (0.426%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:11:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:49 smithi121 ceph-mon[219709]: pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 603/224919 objects degraded (0.268%); 959/224919 objects misplaced (0.426%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:11:49.913 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:49.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:49.939 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:11:50.027 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:50.061 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:50.131 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:50.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:50.241 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:50.314 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:50.390 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:11:50.508 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:50.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:50.621 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:50.656 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:50.717 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:50.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:50.822 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:50.868 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:50.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:50.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:51.087 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:11:51.118 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1347473367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:51.118 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:51.158 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:11:51.169 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:51.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1347473367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:51.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:51.376 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1347473367' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:51.376 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:11:51.395 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:11:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:51.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:11:51.506 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:51.594 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:51.681 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:51.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:51.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:51.786 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:51.814 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:51.910 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 3, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:11:51.910 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:11:51.943 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:11:51.957 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:51.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:52.061 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:52.169 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:52.169 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:52.176 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:52.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:52.189 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T21:11:52.190 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.190 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.190 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.190 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.190 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.190 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 603/219111 objects degraded (0.275%); 959/219111 objects misplaced (0.438%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:11:52.190 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 603/224919 objects degraded (0.268%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T21:11:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.416 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 603/219111 objects degraded (0.275%); 959/219111 objects misplaced (0.438%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 603/224919 objects degraded (0.268%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.417 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: failed to encode map e447 with expected crc 2023-09-23T21:11:52.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 603/219111 objects degraded (0.275%); 959/219111 objects misplaced (0.438%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T21:11:52.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 603/224919 objects degraded (0.268%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:52.419 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:52.536 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:52.569 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.584 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:52.595 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:52.690 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:52.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:52.798 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:11:52.843 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:52.970 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:11:52.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:11:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:52.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:11:53.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:53.126 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:53.187 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:53.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:53.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:53.297 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:53.335 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:53.400 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:53.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:53.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:53.438 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:11:53.508 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:53.542 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:53.664 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:53.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:53.774 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:53.861 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:54.462 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:11:54.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:54 smithi123 ceph-mon[193903]: pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 603/217344 objects degraded (0.277%); 959/217344 objects misplaced (0.441%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:11:54.534 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:54 smithi121 ceph-mon[211526]: pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 603/217344 objects degraded (0.277%); 959/217344 objects misplaced (0.441%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:11:54.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:54 smithi121 ceph-mon[219709]: pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 603/217344 objects degraded (0.277%); 959/217344 objects misplaced (0.441%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T21:11:54.919 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:11:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:11:54] "GET /metrics HTTP/1.1" 200 36708 "" "Prometheus/2.43.0" 2023-09-23T21:11:55.201 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:11:55.201 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:11:55.335 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:55.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:55.443 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:55.478 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:55.534 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:55.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:55.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:55.647 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:55.722 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:55.991 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:11:56.044 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:56.150 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:56.193 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.252 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:11:56.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:11:56.285 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:11:56.307 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:56 smithi121 ceph-mon[211526]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 603/214476 objects degraded (0.281%); 665/214476 objects misplaced (0.310%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T21:11:56.307 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:56 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4059116189' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:56.307 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:56 smithi121 ceph-mon[219709]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 603/214476 objects degraded (0.281%); 665/214476 objects misplaced (0.310%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T21:11:56.308 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:56 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4059116189' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:56.343 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:11:56.359 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:11:56.391 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.393 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:11:56.440 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:11:56.457 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:11:56.467 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:11:56.495 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:11:56.495 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:11:56.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:11:56.538 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:11:56.586 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:11:56.603 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:56.621 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.621 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.622 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.622 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.622 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.622 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.622 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.622 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:11:56.630 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:56 smithi123 ceph-mon[193903]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 603/214476 objects degraded (0.281%); 665/214476 objects misplaced (0.310%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T21:11:56.631 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:56 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4059116189' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:11:56.633 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:56.707 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:56.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:56.811 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:56.979 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:57.262 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:57 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 603/214476 objects degraded (0.281%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:57.262 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2937592988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:11:57.626 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:57 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 603/214476 objects degraded (0.281%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:57.626 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2937592988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:11:57.626 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:57 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:11:57.658 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:11:57.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:57 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 603/214476 objects degraded (0.281%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:11:57.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2937592988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:11:57.670 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:11:57.670 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:57 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:11:58.191 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:11:58.192 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:11:58.192 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:11:58.192 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:11:58.192 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:11:58.323 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:58.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:58.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:58.428 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:58.461 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 603/208371 objects degraded (0.289%); 665/208371 objects misplaced (0.319%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T21:11:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:11:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T21:11:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:58 smithi123 ceph-mon[193903]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.509 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:58.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:58.613 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:11:58.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 603/208371 objects degraded (0.289%); 665/208371 objects misplaced (0.319%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T21:11:58.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:58.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:11:58.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T21:11:58.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[219709]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 603/208371 objects degraded (0.289%); 665/208371 objects misplaced (0.319%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:58 smithi121 ceph-mon[211526]: failed to encode map e448 with expected crc 2023-09-23T21:11:58.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:58.957 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:11:59.086 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:11:59.104 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:11:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:59.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:11:59.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:11:59.369 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:59.406 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:11:59.604 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:11:59 smithi123 ceph-mon[193903]: failed to encode map e448 with expected crc 2023-09-23T21:11:59.620 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:11:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:59.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:11:59.630 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:11:59 smithi121 ceph-mon[211526]: failed to encode map e448 with expected crc 2023-09-23T21:11:59.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:11:59 smithi121 ceph-mon[219709]: failed to encode map e448 with expected crc 2023-09-23T21:11:59.731 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:11:59.764 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:11:59.816 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:11:59.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:11:59.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:11:59.900 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:11:59.927 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:11:59.963 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:00.067 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:00.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:00.171 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:00.196 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:00.237 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:00.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:00.341 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:00.387 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:12:00.446 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:00.474 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:00 smithi121 ceph-mon[211526]: pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 603/206811 objects degraded (0.292%); 313/206811 objects misplaced (0.151%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:12:00.474 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/715289728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:00.474 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:00.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:00 smithi121 ceph-mon[219709]: pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 603/206811 objects degraded (0.292%); 313/206811 objects misplaced (0.151%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:12:00.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/715289728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:00.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:00.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:00 smithi123 ceph-mon[193903]: pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 603/206811 objects degraded (0.292%); 313/206811 objects misplaced (0.151%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:12:00.661 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/715289728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:00.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:01.048 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:12:01.234 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:12:01.234 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:12:01.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:12:01.234 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:12:01.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:100855 11.a e405/387 11:57aa010d:::benchmark_data_smithi121_477989_object12666:head v 405'6177, mlcod=405'6174)", 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:58.083849+0000", 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 303.14619805199999, 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.089286727999999996, 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083849+0000", 2023-09-23T21:12:01.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083849+0000", 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083849+0000", 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083856+0000", 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4619999999999999e-06 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.237 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083867+0000", 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0609e-05 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083873+0000", 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9340000000000003e-06 2023-09-23T21:12:01.238 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083908+0000", 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5049000000000003e-05 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.083928+0000", 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0531999999999999e-05 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.239 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.173104+0000", 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.089175885999999996 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.173136+0000", 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2038000000000001e-05 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:100854 11.3 e405/400 11:c1cb70a5:::benchmark_data_smithi121_477989_object12665:head v 405'6454, mlcod=405'6452)", 2023-09-23T21:12:01.240 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:58.085349+0000", 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 303.14469832899999, 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.086642550999999998, 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085349+0000", 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.241 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085349+0000", 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085348+0000", 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.242 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085355+0000", 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9110000000000004e-06 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085366+0000", 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0582e-05 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085372+0000", 2023-09-23T21:12:01.243 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6099999999999997e-06 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085410+0000", 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8022000000000003e-05 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.085427+0000", 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7632999999999999e-05 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.244 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.171960+0000", 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.086532991000000004 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.171992+0000", 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1494e-05 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.245 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:100853 11.7 e405/386 11:e99b384d:::benchmark_data_smithi121_477989_object12664:head v 405'6482, mlcod=0'0)", 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:58.086548+0000", 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 303.14349930499998, 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10248246, 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.086548+0000", 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.246 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.086548+0000", 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.086548+0000", 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.086556+0000", 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7360000000000007e-06 2023-09-23T21:12:01.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.166182+0000", 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.079625878999999997 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.166196+0000", 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3929e-05 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.166267+0000", 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1033999999999998e-05 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.166296+0000", 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9094e-05 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.189000+0000", 2023-09-23T21:12:01.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022703813999999999 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.189031+0000", 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0630000000000003e-05 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:100825 11.5 e405/389 11:a220c0a4:::benchmark_data_smithi121_477989_object12636:head v 405'6268, mlcod=0'0)", 2023-09-23T21:12:01.250 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:58.089413+0000", 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 303.140634014, 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.087034785000000003, 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.089413+0000", 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.251 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.089413+0000", 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.089413+0000", 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.089421+0000", 2023-09-23T21:12:01.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4030000000000007e-06 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.166195+0000", 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.076773501999999993 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.166204+0000", 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9649999999999997e-06 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.169226+0000", 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0030225040000000001 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.169252+0000", 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5704e-05 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.176412+0000", 2023-09-23T21:12:01.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0071598950000000003 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.176448+0000", 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6514999999999999e-05 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102725 11.7 e406/386 11:eb662857:::benchmark_data_smithi121_477989_object14536:head v 406'6593, mlcod=0'0)", 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:01.989096+0000", 2023-09-23T21:12:01.255 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.24095190999998, 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10644952100000001, 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989096+0000", 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989096+0000", 2023-09-23T21:12:01.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989095+0000", 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989105+0000", 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5980000000000003e-06 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989114+0000", 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.533e-06 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989120+0000", 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6509999999999998e-06 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989177+0000", 2023-09-23T21:12:01.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7119e-05 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.989201+0000", 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4465e-05 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095498+0000", 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.106296558 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.259 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095545+0000", 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7196e-05 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102727 11.c e406/383 11:3d5510ba:::benchmark_data_smithi121_477989_object14538:head v 406'6402, mlcod=349'1759)", 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:01.994294+0000", 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.23575366400001, 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.109017789, 2023-09-23T21:12:01.260 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994294+0000", 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994294+0000", 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994293+0000", 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994305+0000", 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2931999999999999e-05 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994318+0000", 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2479e-05 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994325+0000", 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5320000000000003e-06 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994374+0000", 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9976999999999999e-05 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994399+0000", 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4637000000000001e-05 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.103238+0000", 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.1088387 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.103312+0000", 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3789999999999997e-05 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102730 11.7 e406/386 11:e9dd503f:::benchmark_data_smithi121_477989_object14541:head v 406'6594, mlcod=0'0)", 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:01.994339+0000", 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.23570863100002, 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.102173844, 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994339+0000", 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994339+0000", 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994339+0000", 2023-09-23T21:12:01.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994344+0000", 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8740000000000003e-06 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994349+0000", 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4659999999999998e-06 2023-09-23T21:12:01.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.994352+0000", 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4869999999999998e-06 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.996276+0000", 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001923742 2023-09-23T21:12:01.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.996305+0000", 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8527999999999999e-05 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.096477+0000", 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.100172316 2023-09-23T21:12:01.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.096513+0000", 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5642999999999997e-05 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.271 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(11.5 406/389 [PushOp(11:aef35a7c:::benchmark_data_smithi121_477989_object14531:head, version: 347'900, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:aef35a7c:::benchmark_data_smithi121_477989_object14531:head@347'900, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:01.997395+0000", 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.232652492, 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.096510265999999997, 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997395+0000", 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997395+0000", 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997394+0000", 2023-09-23T21:12:01.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997437+0000", 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2953e-05 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997456+0000", 2023-09-23T21:12:01.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8807000000000001e-05 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997467+0000", 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0981e-05 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997530+0000", 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2912999999999999e-05 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997537+0000", 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5660000000000001e-06 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.093905+0000", 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.096368055999999994 2023-09-23T21:12:01.276 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102729 11.a e406/387 11:5db3acc5:::benchmark_data_smithi121_477989_object14540:head v 406'6299, mlcod=406'6297)", 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:01.997491+0000", 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.232556117, 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.098895809000000001, 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997491+0000", 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997491+0000", 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997491+0000", 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997496+0000", 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5110000000000003e-06 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997503+0000", 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9410000000000001e-06 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997507+0000", 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0380000000000001e-06 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997547+0000", 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9740000000000002e-05 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997572+0000", 2023-09-23T21:12:01.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4601999999999999e-05 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.096347+0000", 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.098775692999999998 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.096387+0000", 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9697999999999998e-05 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(11.5 406/389 [PushOp(11:ad4ca379:::benchmark_data_smithi121_477989_object14534:head, version: 347'901, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:ad4ca379:::benchmark_data_smithi121_477989_object14534:head@347'901, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:01.997518+0000", 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.23252987299998, 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.098533275000000003, 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997518+0000", 2023-09-23T21:12:01.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997518+0000", 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997517+0000", 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997545+0000", 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7849000000000001e-05 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997556+0000", 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1063999999999999e-05 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:01.997564+0000", 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5490000000000001e-06 2023-09-23T21:12:01.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.093970+0000", 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.096405759999999993 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.093979+0000", 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5889999999999992e-06 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.096051+0000", 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020716950000000001 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102735 11.8 e406/386 11:11d66875:::benchmark_data_smithi121_477989_object14546:head v 406'6485, mlcod=406'6483)", 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:02.001180+0000", 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.228867794, 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.099023847999999998, 2023-09-23T21:12:01.286 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.001180+0000", 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.001180+0000", 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.001179+0000", 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.001191+0000", 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1979e-05 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.094000+0000", 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.092809412999999993 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.094009+0000", 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2609999999999999e-06 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095439+0000", 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014293509999999999 2023-09-23T21:12:01.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095460+0000", 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1699e-05 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.100164+0000", 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0047040850000000002 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.100204+0000", 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9042999999999999e-05 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102750 11.c e406/383 11:355c3041:::benchmark_data_smithi121_477989_object14561:head v 406'6403, mlcod=349'1759)", 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:02.028400+0000", 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 299.20164755500002, 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.074943070000000001, 2023-09-23T21:12:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.028400+0000", 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.028400+0000", 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.028399+0000", 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.028410+0000", 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0957e-05 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.091822+0000", 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.063412131999999996 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.091838+0000", 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5721000000000001e-05 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.091906+0000", 2023-09-23T21:12:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7837999999999997e-05 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.091934+0000", 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8045e-05 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.103317+0000", 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01138309 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.103343+0000", 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6318000000000001e-05 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:167856 11.7 e420/386 11:e503f1e9:::benchmark_data_smithi121_477989_object79667:head v 420'10704, mlcod=0'0)", 2023-09-23T21:12:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:12.167639+0000", 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.062408213, 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.11021307599999999, 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167639+0000", 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167639+0000", 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167639+0000", 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167644+0000", 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3879999999999999e-06 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167650+0000", 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9429999999999997e-06 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167654+0000", 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6100000000000002e-06 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167697+0000", 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3207999999999997e-05 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167722+0000", 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4669999999999999e-05 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.277813+0000", 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.110091304 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.277852+0000", 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9307999999999997e-05 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:167859 11.7 e420/386 11:eee5f220:::benchmark_data_smithi121_477989_object79670:head v 420'10705, mlcod=0'0)", 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:12.167943+0000", 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.062104635, 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.109929633, 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167943+0000", 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167943+0000", 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167943+0000", 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167948+0000", 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0950000000000003e-06 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167955+0000", 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2049999999999996e-06 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.167959+0000", 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4719999999999997e-06 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.172987+0000", 2023-09-23T21:12:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050278349999999996 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.173005+0000", 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7657999999999999e-05 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.277855+0000", 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10485014300000001 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.277872+0000", 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7549000000000001e-05 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(11.5 420/389 [PushOp(11:a644326a:::benchmark_data_smithi121_477989_object79677:head, version: 366'4950, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:a644326a:::benchmark_data_smithi121_477989_object79677:head@366'4950, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:12.174664+0000", 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.05538304000001, 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.101266575, 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174664+0000", 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174664+0000", 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174664+0000", 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174697+0000", 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2944999999999997e-05 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174713+0000", 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6872e-05 2023-09-23T21:12:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174723+0000", 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6909999999999992e-06 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174797+0000", 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4085000000000006e-05 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174805+0000", 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.622e-06 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.275931+0000", 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10112623900000001 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(11.c 420/383 [PushOp(11:3b3912b1:::benchmark_data_smithi121_477989_object79675:head, version: 366'4975, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:3b3912b1:::benchmark_data_smithi121_477989_object79675:head@366'4975, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:12.174889+0000", 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.055158358, 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.101459859, 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174889+0000", 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174889+0000", 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174888+0000", 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.174954+0000", 2023-09-23T21:12:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5247999999999996e-05 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273728+0000", 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.098774760000000003 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273748+0000", 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9945999999999998e-05 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273877+0000", 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128985 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273887+0000", 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0008e-05 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.276349+0000", 2023-09-23T21:12:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024616999999999998 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:167871 11.3 e420/400 11:cd45ea9b:::benchmark_data_smithi121_477989_object79682:head v 420'10770, mlcod=420'10768)", 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:12.181906+0000", 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 169.04814184899999, 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.098377310999999995, 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.181906+0000", 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.181906+0000", 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.181904+0000", 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.181915+0000", 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0651e-05 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273724+0000", 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.091809368000000002 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273736+0000", 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.15e-05 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273776+0000", 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0358999999999998e-05 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.273797+0000", 2023-09-23T21:12:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0978000000000001e-05 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.280246+0000", 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0064486580000000003 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:12.280283+0000", 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7178000000000002e-05 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:173606 11.c e421/383 11:38764841:::benchmark_data_smithi121_477989_object85417:head v 421'10850, mlcod=349'1759)", 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:09:23.474274+0000", 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 157.75577371399999, 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.064501649999999994, 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.474274+0000", 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.474274+0000", 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.474273+0000", 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.474278+0000", 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1549999999999998e-06 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.474286+0000", 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9189999999999999e-06 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.530701+0000", 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.056414525 2023-09-23T21:12:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.530756+0000", 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5501000000000003e-05 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.530783+0000", 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6363000000000002e-05 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.538737+0000", 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079541330000000004 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:09:23.538775+0000", 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8652000000000002e-05 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:45584 12.c e434/426 12:33467756:::benchmark_data_smithi121_563784_object45583:head v 434'2811, mlcod=434'2809)", 2023-09-23T21:12:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:10:24.941958+0000", 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 96.288089077999999, 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.11121592299999999, 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.941958+0000", 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.941958+0000", 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.941957+0000", 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.941997+0000", 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9743999999999998e-05 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.942014+0000", 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6461999999999999e-05 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.942020+0000", 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9160000000000004e-06 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.050641+0000", 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10862032100000001 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.050734+0000", 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3080999999999999e-05 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.053138+0000", 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024044100000000001 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.053174+0000", 2023-09-23T21:12:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6069000000000003e-05 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:45591 12.f e434/426 12:f64a9d2e:::benchmark_data_smithi121_563784_object45590:head v 434'2831, mlcod=434'2829)", 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:10:24.946957+0000", 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 96.283090068999996, 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10534331800000001, 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.946957+0000", 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.946957+0000", 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.946956+0000", 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.946978+0000", 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1291999999999999e-05 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.050672+0000", 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.103694233 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.050697+0000", 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4660000000000001e-05 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.050764+0000", 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7572999999999998e-05 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.050779+0000", 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4819e-05 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.052271+0000", 2023-09-23T21:12:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014921489999999999 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.052301+0000", 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9594000000000002e-05 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:01.336 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:12:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[211526]: failed to encode map e449 with expected crc 2023-09-23T21:12:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[211526]: failed to encode map e449 with expected crc 2023-09-23T21:12:01.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[211526]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T21:12:01.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[211526]: pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 603/201342 objects degraded (0.299%); 313/201342 objects misplaced (0.155%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:12:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[219709]: failed to encode map e449 with expected crc 2023-09-23T21:12:01.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[219709]: failed to encode map e449 with expected crc 2023-09-23T21:12:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[219709]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T21:12:01.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:01 smithi121 ceph-mon[219709]: pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 603/201342 objects degraded (0.299%); 313/201342 objects misplaced (0.155%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:12:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:01 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:01.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:01 smithi123 ceph-mon[193903]: failed to encode map e449 with expected crc 2023-09-23T21:12:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:01 smithi123 ceph-mon[193903]: failed to encode map e449 with expected crc 2023-09-23T21:12:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:01 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:01 smithi123 ceph-mon[193903]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T21:12:01.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:01 smithi123 ceph-mon[193903]: pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 603/201342 objects degraded (0.299%); 313/201342 objects misplaced (0.155%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:12:02.024 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:12:02.468 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:02.565 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:02.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:02.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 603/201342 objects degraded (0.299%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:02.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 603/201342 objects degraded (0.299%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:02.677 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:02.687 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 603/201342 objects degraded (0.299%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:02.760 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:02.891 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:02.976 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:02.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:02.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:03.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:03.122 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:03.175 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:03.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:03.289 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:03.314 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:03.358 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:03 smithi123 ceph-mon[193903]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 603/199629 objects degraded (0.302%); 313/199629 objects misplaced (0.157%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:12:03.379 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:03.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:03.484 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:03.519 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:03 smithi121 ceph-mon[211526]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 603/199629 objects degraded (0.302%); 313/199629 objects misplaced (0.157%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:12:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:03 smithi121 ceph-mon[219709]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 603/199629 objects degraded (0.302%); 313/199629 objects misplaced (0.157%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:12:03.696 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:03.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:03.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:03.803 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:03.841 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:03.996 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:04.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:04.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:04.101 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:04.172 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:04.253 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:12:04.253 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:12:04.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:116501 12.f 12:fc4290b9:::benchmark_data_smithi121_563784_object26098:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:04.237166+0000", 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.015623836, 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015655189, 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:04.254 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 116501 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237166+0000", 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237166+0000", 2023-09-23T21:12:04.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237165+0000", 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237168+0000", 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3100000000000001e-06 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237176+0000", 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7759999999999997e-06 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237182+0000", 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3839999999999996e-06 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:04.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237213+0000", 2023-09-23T21:12:04.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1142000000000001e-05 2023-09-23T21:12:04.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:04.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237309+0000", 2023-09-23T21:12:04.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5884000000000006e-05 2023-09-23T21:12:04.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:12:04.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.237384+0000", 2023-09-23T21:12:04.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5405000000000001e-05 2023-09-23T21:12:04.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.241469+0000", 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040845259999999998 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.241593+0000", 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012472500000000001 2023-09-23T21:12:04.264 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:116506 12.4 12:25437a64:::benchmark_data_smithi121_563784_object26103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:04.246119+0000", 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0066702599999999999, 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0067956270000000003, 2023-09-23T21:12:04.265 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 116506 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:04.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246119+0000", 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246119+0000", 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:04.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246118+0000", 2023-09-23T21:12:04.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:04.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:04.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246125+0000", 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0469999999999996e-06 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246141+0000", 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6483000000000001e-05 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:04.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246148+0000", 2023-09-23T21:12:04.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5729999999999996e-06 2023-09-23T21:12:04.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:04.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246198+0000", 2023-09-23T21:12:04.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0488999999999997e-05 2023-09-23T21:12:04.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:04.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246315+0000", 2023-09-23T21:12:04.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000116297 2023-09-23T21:12:04.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3", 2023-09-23T21:12:04.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.246390+0000", 2023-09-23T21:12:04.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5637999999999996e-05 2023-09-23T21:12:04.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.249518+0000", 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003127402 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:04.251504+0000", 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001985746 2023-09-23T21:12:04.273 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:04.274 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:04.274 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:04.274 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:04.274 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T21:12:04.274 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 2 2023-09-23T21:12:04.274 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:12:04.408 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:12:04.442 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:12:04.543 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:04.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:04.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3612559914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:12:04.664 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:04.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3612559914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:12:04.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3612559914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:12:04.701 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:04.766 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:04.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:04.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:04.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:04.902 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:05.125 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:12:05.131 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:05.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:05.166 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:12:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:12:04] "GET /metrics HTTP/1.1" 200 36706 "" "Prometheus/2.43.0" 2023-09-23T21:12:05.240 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:05.268 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:05.340 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:05.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:05.390 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3612559914' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:12:05.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: osdmap e450: 8 total, 8 up, 8 in 2023-09-23T21:12:05.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:05 smithi123 ceph-mon[193903]: pgmap v1466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 517/196824 objects degraded (0.263%); 5.5 MiB/s, 99 objects/s recovering 2023-09-23T21:12:05.449 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:05.453 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:12:05.519 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:05.588 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:05.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3612559914' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: osdmap e450: 8 total, 8 up, 8 in 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[219709]: pgmap v1466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 517/196824 objects degraded (0.263%); 5.5 MiB/s, 99 objects/s recovering 2023-09-23T21:12:05.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3612559914' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:12:05.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: osdmap e450: 8 total, 8 up, 8 in 2023-09-23T21:12:05.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: failed to encode map e450 with expected crc 2023-09-23T21:12:05.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:05 smithi121 ceph-mon[211526]: pgmap v1466: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 517/196824 objects degraded (0.263%); 5.5 MiB/s, 99 objects/s recovering 2023-09-23T21:12:05.689 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:05.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:05.794 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:05.801 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:05.886 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:05.985 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:12:06.025 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:12:06.084 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:12:06.140 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:12:06.178 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:12:06.216 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:12:06.306 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:06.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:06.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:06.365 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:12:06.407 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:06.408 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:12:06.445 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.445 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.446 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.446 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.446 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.446 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.446 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.446 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:12:06.447 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:06.454 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:06.594 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:06.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:06.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2605465624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2605465624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:06.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:06.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2605465624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:06.846 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:07.186 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:07.186 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:07 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 517/196824 objects degraded (0.263%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:07 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:12:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:07 smithi123 ceph-mon[193903]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 517/190971 objects degraded (0.271%); 2.4 MiB/s, 49 objects/s recovering 2023-09-23T21:12:07.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:07 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 517/196824 objects degraded (0.263%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:07.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:07 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:12:07.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:07 smithi121 ceph-mon[211526]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 517/190971 objects degraded (0.271%); 2.4 MiB/s, 49 objects/s recovering 2023-09-23T21:12:07.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:07 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 517/196824 objects degraded (0.263%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:07.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:07 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:12:07.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:07 smithi121 ceph-mon[219709]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 517/190971 objects degraded (0.271%); 2.4 MiB/s, 49 objects/s recovering 2023-09-23T21:12:07.785 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:07.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:07.896 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:07.986 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:08.047 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:12:08.223 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:08.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:08.332 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:08.389 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:08.558 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:08.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:08.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:08.669 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:08.691 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:08.749 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:09.501 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:09.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:09.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:09.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:09.673 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:09.731 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:09.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:09.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:09.836 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:09.931 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:10.100 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:12:10.104 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:10.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:10.214 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:10.339 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:10.340 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:12:10.435 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:10 smithi123 ceph-mon[193903]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 73/189297 objects degraded (0.039%); 2.2 MiB/s, 94 objects/s recovering 2023-09-23T21:12:10.435 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1189830673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:10.435 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:10 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:10.621 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:10.634 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:10 smithi121 ceph-mon[211526]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 73/189297 objects degraded (0.039%); 2.2 MiB/s, 94 objects/s recovering 2023-09-23T21:12:10.634 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1189830673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:10.634 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:10 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:10.634 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:10 smithi121 ceph-mon[219709]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 73/189297 objects degraded (0.039%); 2.2 MiB/s, 94 objects/s recovering 2023-09-23T21:12:10.634 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1189830673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:10.635 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:10 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:10.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:10.736 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:10.771 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:10.831 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:10.831 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:10.969 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:10.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:10.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:11.074 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:11.102 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:11.334 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:12:11.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: osdmap e451: 8 total, 8 up, 8 in 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2281109167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:12:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:11 smithi123 ceph-mon[193903]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 73/183645 objects degraded (0.040%); 2.4 MiB/s, 105 objects/s recovering 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: osdmap e451: 8 total, 8 up, 8 in 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2281109167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:12:11.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[219709]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 73/183645 objects degraded (0.040%); 2.4 MiB/s, 105 objects/s recovering 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: osdmap e451: 8 total, 8 up, 8 in 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: failed to encode map e451 with expected crc 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2281109167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:12:11.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:11 smithi121 ceph-mon[211526]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 73/183645 objects degraded (0.040%); 2.4 MiB/s, 105 objects/s recovering 2023-09-23T21:12:11.943 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:11.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:11.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:11.963 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:12:12.047 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:12.117 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:12.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[211526]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[211526]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2281109167' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:12:12.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[211526]: osdmap e452: 8 total, 8 up, 8 in 2023-09-23T21:12:12.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[211526]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[211526]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.417 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 73/183645 objects degraded (0.040%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:12.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[219709]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[219709]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2281109167' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:12:12.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[219709]: osdmap e452: 8 total, 8 up, 8 in 2023-09-23T21:12:12.425 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[219709]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.425 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[219709]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.425 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 73/183645 objects degraded (0.040%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:12.439 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:12.459 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:12:12.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:12.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:12.564 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:12.698 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:12.702 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:12.707 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:12 smithi123 ceph-mon[193903]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:12 smithi123 ceph-mon[193903]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2281109167' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:12:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:12 smithi123 ceph-mon[193903]: osdmap e452: 8 total, 8 up, 8 in 2023-09-23T21:12:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:12 smithi123 ceph-mon[193903]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:12 smithi123 ceph-mon[193903]: failed to encode map e452 with expected crc 2023-09-23T21:12:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 73/183645 objects degraded (0.040%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:12:12.862 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:12.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:12.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:12.978 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:13.035 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.207 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:13.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:13.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:13.311 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:13.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:13.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:13.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:13.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[211526]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 73/182145 objects degraded (0.040%); 0 B/s, 55 objects/s recovering 2023-09-23T21:12:13.360 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.426 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:13.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:13.530 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:13.559 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.603 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:13.605 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:13.605 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:13.605 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:13.605 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:13 smithi123 ceph-mon[193903]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 73/182145 objects degraded (0.040%); 0 B/s, 55 objects/s recovering 2023-09-23T21:12:13.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:13.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:13.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:13.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:13 smithi121 ceph-mon[219709]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 73/182145 objects degraded (0.040%); 0 B/s, 55 objects/s recovering 2023-09-23T21:12:13.707 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:13.738 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:13.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:12:14.020 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:14.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:14.130 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:14.214 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:14.372 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:14.373 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:14.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:14.477 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:14.553 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:14.553 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:14.559 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:14.850 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:12:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:12:14] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.43.0" 2023-09-23T21:12:14.970 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:14.972 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:14.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:15.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:15.184 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:15.329 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:12:15.646 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:15 smithi121 ceph-mon[219709]: pgmap v1473: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 0 B/s, 64 objects/s recovering 2023-09-23T21:12:15.646 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:15 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 73/182145 objects degraded (0.040%), 1 pg degraded, 1 pg undersized) 2023-09-23T21:12:15.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:15 smithi121 ceph-mon[211526]: pgmap v1473: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 0 B/s, 64 objects/s recovering 2023-09-23T21:12:15.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:15 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 73/182145 objects degraded (0.040%), 1 pg degraded, 1 pg undersized) 2023-09-23T21:12:15.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:15 smithi123 ceph-mon[193903]: pgmap v1473: 89 pgs: 89 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 0 B/s, 64 objects/s recovering 2023-09-23T21:12:15.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:15 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 73/182145 objects degraded (0.040%), 1 pg degraded, 1 pg undersized) 2023-09-23T21:12:15.771 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:16.020 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:12:16.020 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:12:16.020 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.021 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.022 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:12:16.082 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:16.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:16.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:16.187 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:16.225 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:16.332 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:16.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:16.436 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:16.460 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1957766734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:16.461 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1957766734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:16.467 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:16.510 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:16.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:16.613 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:16.686 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1957766734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:17.338 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:17.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:17.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:17.443 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:17.455 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:12:17.455 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:12:17.455 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:12:17.455 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:12:17.455 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:12:17.527 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:17.623 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:17 smithi121 ceph-mon[219709]: pgmap v1474: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:17.891 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:17 smithi121 ceph-mon[211526]: pgmap v1474: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:17.911 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:17.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:17.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:17 smithi123 ceph-mon[193903]: pgmap v1474: 89 pgs: 89 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:18.016 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:18.096 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:18.279 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:12:18.454 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:18.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:18.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:18.567 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:18.650 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:18.701 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:12:18.949 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:18.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:19.066 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:19.105 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:19.106 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:12:19.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3453328458' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:12:19.256 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:19.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:19.360 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:19.383 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3453328458' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:12:19.383 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3453328458' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:12:19.455 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:19.915 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:12:20.117 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:12:20.137 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.137 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.137 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3453328458' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:12:20.138 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: osdmap e453: 8 total, 8 up, 8 in 2023-09-23T21:12:20.138 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.138 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: pgmap v1476: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:20.138 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2320466587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:20.138 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:20.375 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:12:20.375 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:12:20.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:12:20.375 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:12:20.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.375 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50076 11.5 11:a3fd1c51:::benchmark_data_smithi121_477989_object50075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.98083021800005, 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1126377949999999, 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50076 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392085+0000", 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392083+0000", 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:20.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392127+0000", 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3806e-05 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392141+0000", 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4394e-05 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392150+0000", 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1650000000000006e-06 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392234+0000", 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4027e-05 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392388+0000", 2023-09-23T21:12:20.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015483000000000001 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,6", 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.392554+0000", 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016592399999999999 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.398697+0000", 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061424319999999998 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.403213+0000", 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045163499999999997 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.403226+0000", 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2938999999999999e-05 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394392+0000", 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99116629099999998 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394406+0000", 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3896999999999999e-05 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504613+0000", 2023-09-23T21:12:20.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.110206864 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504722+0000", 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010918500000000001 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50104 11.5 11:ac5fad04:::benchmark_data_smithi121_477989_object50103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:12:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.94326439199995, 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0882670750000001, 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50104 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429650+0000", 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429649+0000", 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429803+0000", 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015389900000000001 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429823+0000", 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9340000000000001e-05 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429833+0000", 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0005e-05 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429893+0000", 2023-09-23T21:12:20.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0324e-05 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.429904+0000", 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0781000000000001e-05 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394433+0000", 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96452942900000005 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394443+0000", 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4879999999999996e-06 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504746+0000", 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.110303627 2023-09-23T21:12:20.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.504903+0000", 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015637600000000001 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.505086+0000", 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000183516 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.512945+0000", 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078586579999999993 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515525+0000", 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025799350000000002 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515589+0000", 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3974999999999996e-05 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515654+0000", 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4867000000000001e-05 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.517917+0000", 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022637120000000002 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.392 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50113 11.5 11:ab5e54a5:::benchmark_data_smithi121_477989_object50112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.93322575699995, 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0819687199999999, 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50113 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439689+0000", 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439688+0000", 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439777+0000", 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9221000000000005e-05 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439790+0000", 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3013999999999999e-05 2023-09-23T21:12:20.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439795+0000", 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2310000000000004e-06 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439871+0000", 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6036000000000002e-05 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.439883+0000", 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.186e-05 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394464+0000", 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95458152699999999 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394472+0000", 2023-09-23T21:12:20.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8790000000000003e-06 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.506903+0000", 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1124306429999999 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.507065+0000", 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016177499999999999 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.507239+0000", 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000174191 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515535+0000", 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0082961040000000003 2023-09-23T21:12:20.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.517971+0000", 2023-09-23T21:12:20.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024364360000000002 2023-09-23T21:12:20.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.519321+0000", 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001349814 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.519388+0000", 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6958999999999994e-05 2023-09-23T21:12:20.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.521658+0000", 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022694899999999999 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50122 11.5 11:a0d5e578:::benchmark_data_smithi121_477989_object50121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.74698131299999, 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.905001269, 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50122 2023-09-23T21:12:20.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625933+0000", 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625982+0000", 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9774999999999997e-05 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625994+0000", 2023-09-23T21:12:20.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1986e-05 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.625999+0000", 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9980000000000003e-06 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.626045+0000", 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5528999999999997e-05 2023-09-23T21:12:20.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.626052+0000", 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5159999999999997e-06 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394493+0000", 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76844040899999999 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394501+0000", 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5059999999999995e-06 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509037+0000", 2023-09-23T21:12:20.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1145361 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509179+0000", 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014180599999999999 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.509350+0000", 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017093699999999999 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515543+0000", 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061929389999999997 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.522051+0000", 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065082919999999997 2023-09-23T21:12:20.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.528556+0000", 2023-09-23T21:12:20.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065045950000000002 2023-09-23T21:12:20.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.528631+0000", 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4646000000000003e-05 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.530935+0000", 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023040959999999998 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.412 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:50126 11.5 11:a51a9ec8:::benchmark_data_smithi121_477989_object50125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 522.74409165999998, 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9041968090000001, 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.413 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50126 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628823+0000", 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628822+0000", 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628877+0000", 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5653999999999999e-05 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628891+0000", 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3808000000000001e-05 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628897+0000", 2023-09-23T21:12:20.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1460000000000001e-06 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628958+0000", 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0720000000000001e-05 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:37.628968+0000", 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6229999999999992e-06 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394522+0000", 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76555460600000003 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:38.394530+0000", 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9019999999999999e-06 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511058+0000", 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1165273579999999 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511192+0000", 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013458099999999999 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.511361+0000", 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016914299999999999 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.515550+0000", 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041891139999999999 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.522105+0000", 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065545459999999996 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.531002+0000", 2023-09-23T21:12:20.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088968529999999997 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.531064+0000", 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2463000000000001e-05 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:03:39.533020+0000", 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019555419999999998 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.421 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.422 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.422 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78881 11.4 11:2f950127:::benchmark_data_smithi121_477989_object78880:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:12:20.422 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:12:20.422 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 484.48759867500002, 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30758963700000003, 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78881 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:12:20.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885316+0000", 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885361+0000", 2023-09-23T21:12:20.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5769000000000002e-05 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885384+0000", 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3087000000000001e-05 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885390+0000", 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7540000000000003e-06 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.887935+0000", 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025446190000000001 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.888057+0000", 2023-09-23T21:12:20.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122161 2023-09-23T21:12:20.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.888200+0000", 2023-09-23T21:12:20.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014262500000000001 2023-09-23T21:12:20.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893701+0000", 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005501695 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.905399+0000", 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011697991 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192819+0000", 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.28741952999999998 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192867+0000", 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8605e-05 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192906+0000", 2023-09-23T21:12:20.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8302e-05 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78885 11.4 11:2a17f6c4:::benchmark_data_smithi121_477989_object78884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 484.48750968899998, 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30759584699999998, 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.432 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78885 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885405+0000", 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885437+0000", 2023-09-23T21:12:20.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2227999999999997e-05 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885444+0000", 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6939999999999995e-06 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.885449+0000", 2023-09-23T21:12:20.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3950000000000003e-06 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890915+0000", 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0054663869999999996 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.891027+0000", 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011214600000000001 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.891156+0000", 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012827499999999999 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893843+0000", 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026873719999999999 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.905447+0000", 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011604353 2023-09-23T21:12:20.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192935+0000", 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.28748794300000002 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192973+0000", 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7407999999999998e-05 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.193001+0000", 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8166000000000001e-05 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:78889 11.5 11:a04ebbe0:::benchmark_data_smithi121_477989_object78888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 484.486475412, 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.30786717499999999, 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.439 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 78889 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886439+0000", 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886438+0000", 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886479+0000", 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0985000000000001e-05 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886493+0000", 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4032999999999999e-05 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.886499+0000", 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1859999999999997e-06 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890610+0000", 2023-09-23T21:12:20.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041114760000000002 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890765+0000", 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000154184 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.890939+0000", 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017454800000000001 2023-09-23T21:12:20.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.893385+0000", 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002445984 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:15.899338+0000", 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059533499999999996 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192227+0000", 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.29288863399999998 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.192279+0000", 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2392999999999998e-05 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:16.194306+0000", 2023-09-23T21:12:20.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020270679999999999 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88133 11.5 11:a2e8b6ce:::benchmark_data_smithi121_477989_object88132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 471.62499880899998, 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.015216702, 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.446 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88133 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747916+0000", 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747914+0000", 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747958+0000", 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.333e-05 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747970+0000", 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2914999999999999e-05 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.747977+0000", 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4620000000000001e-06 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752486+0000", 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045088680000000001 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752629+0000", 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014352899999999999 2023-09-23T21:12:20.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.752804+0000", 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017513500000000001 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.776368+0000", 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023563562999999999 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.776380+0000", 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2476999999999999e-05 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763117+0000", 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98673686100000002 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763133+0000", 2023-09-23T21:12:20.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5291e-05 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88146 11.5 11.f29ad9f5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 471.60008099999999, 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99033201000000004, 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:12:20.453 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88146 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772834+0000", 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772833+0000", 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772873+0000", 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0082e-05 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772885+0000", 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2529e-05 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.772893+0000", 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3520000000000001e-06 2023-09-23T21:12:20.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.777718+0000", 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048257910000000003 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.777727+0000", 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4780000000000008e-06 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763156+0000", 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98542964799999999 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763166+0000", 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2620000000000004e-06 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36195.0:88149 11.5 11.fd969135 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 471.58377785200003, 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97405727799999997, 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36195", 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 88149 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789137+0000", 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789135+0000", 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789175+0000", 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9626000000000003e-05 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789193+0000", 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7737999999999999e-05 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789199+0000", 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4500000000000001e-06 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789265+0000", 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5572000000000002e-05 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:28.789281+0000", 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6612e-05 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763186+0000", 2023-09-23T21:12:20.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.973904412 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:29.763194+0000", 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5080000000000005e-06 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(12.e 445/433 [PushOp(12:75513fdb:::benchmark_data_smithi121_563784_object13419:head, version: 430'909, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:75513fdb:::benchmark_data_smithi121_563784_object13419:head@430'909, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:20.464 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:11:40.064200+0000", 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 40.308714704000003, 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025895397000000001, 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.064200+0000", 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.064200+0000", 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.064200+0000", 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.064248+0000", 2023-09-23T21:12:20.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8539999999999999e-05 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.064266+0000", 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7730000000000001e-05 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.064274+0000", 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8089999999999993e-06 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.087138+0000", 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022863794999999999 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.087149+0000", 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1617000000000001e-05 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:40.090095+0000", 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029461309999999998 2023-09-23T21:12:20.468 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:105664 12.e e446/433 12:763f1144:::benchmark_data_smithi121_563784_object15261:head v 446'6809, mlcod=430'1246)", 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:11:45.958990+0000", 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 34.413925012, 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025136985000000001, 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.958990+0000", 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.958990+0000", 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.958989+0000", 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.958994+0000", 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8670000000000002e-06 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959001+0000", 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2729999999999999e-06 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959005+0000", 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7770000000000002e-06 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959350+0000", 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00034422199999999999 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959369+0000", 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9426999999999999e-05 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.984085+0000", 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024716180000000001 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.984127+0000", 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1499999999999999e-05 2023-09-23T21:12:20.473 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:105665 12.e e446/433 12:7494f61f:::benchmark_data_smithi121_563784_object15262:head v 446'6810, mlcod=430'1246)", 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:11:45.959032+0000", 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 34.413883108999997, 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.025132473999999998, 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.474 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959032+0000", 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959032+0000", 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959031+0000", 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959035+0000", 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5729999999999999e-06 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959053+0000", 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7995999999999998e-05 2023-09-23T21:12:20.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.959057+0000", 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3810000000000001e-06 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.964136+0000", 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005078979 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.964157+0000", 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0679e-05 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.984134+0000", 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.019976958999999999 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:45.984164+0000", 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0103999999999998e-05 2023-09-23T21:12:20.478 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(12.e 447/433 [PushOp(12:7338c3bb:::benchmark_data_smithi121_563784_object30023:head, version: 431'1967, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:7338c3bb:::benchmark_data_smithi121_563784_object30023:head@431'1967, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:11:57.307489+0000", 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 23.065425628, 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027689973, 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.479 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307489+0000", 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307489+0000", 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307488+0000", 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307552+0000", 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3838999999999996e-05 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307585+0000", 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3132000000000001e-05 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307604+0000", 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9517e-05 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307793+0000", 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00018901599999999999 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.307798+0000", 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.242e-06 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:11:57.335179+0000", 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027380669999999999 2023-09-23T21:12:20.483 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(12.e 448/433 [PushOp(12:795e0ec5:::benchmark_data_smithi121_563784_object33260:head, version: 431'2164, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:795e0ec5:::benchmark_data_smithi121_563784_object33260:head@431'2164, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:00.178745+0000", 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 20.194169431999999, 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.105905924, 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178745+0000", 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178745+0000", 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178744+0000", 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178798+0000", 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3032000000000003e-05 2023-09-23T21:12:20.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178811+0000", 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3375e-05 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178819+0000", 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7910000000000005e-06 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178869+0000", 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0065000000000003e-05 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.178874+0000", 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9740000000000001e-06 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:00.284651+0000", 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10577750499999999 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:115020 12.e e449/433 12:7667ce99:::benchmark_data_smithi121_563784_object24617:head v 449'7394, mlcod=431'2279)", 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:01.562946+0000", 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 18.809968412, 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.092640344999999999, 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.562946+0000", 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.562946+0000", 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.562946+0000", 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.562953+0000", 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.708e-06 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.562963+0000", 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0518e-05 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.562967+0000", 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3309999999999997e-06 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.562995+0000", 2023-09-23T21:12:20.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7696999999999999e-05 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.563015+0000", 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9531999999999999e-05 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.565876+0000", 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00286105 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.655587+0000", 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.089710984999999993 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(12.e 449/433 [PushOp(12:7987cac6:::benchmark_data_smithi121_563784_object35192:head, version: 431'2281, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:7987cac6:::benchmark_data_smithi121_563784_object35192:head@431'2281, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:01.567911+0000", 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 18.80500357, 2023-09-23T21:12:20.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.091672692, 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.567911+0000", 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.567911+0000", 2023-09-23T21:12:20.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.567910+0000", 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.567976+0000", 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5772000000000007e-05 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.655553+0000", 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.087577038999999995 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.655586+0000", 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3680000000000003e-05 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.655710+0000", 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012358700000000001 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.655724+0000", 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3743e-05 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:01.659584+0000", 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038602749999999998 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:123349 12.2 e452/440 12:45910789:::benchmark_data_smithi121_563784_object32946:head v 452'7663, mlcod=452'7659)", 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:16.401480+0000", 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 3.9714350339999998, 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031599687000000001, 2023-09-23T21:12:20.499 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.401480+0000", 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.401480+0000", 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.401478+0000", 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.401494+0000", 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6518999999999999e-05 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.426380+0000", 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024885898 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.426389+0000", 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2199999999999998e-06 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.427151+0000", 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00076160599999999996 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.427170+0000", 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9258000000000001e-05 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.433043+0000", 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058728019999999999 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.433079+0000", 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6366999999999999e-05 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:123359 12.e e452/451 12:7bae47ca:::benchmark_data_smithi121_563784_object32956:head v 452'7918, mlcod=452'7916)", 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:16.403824+0000", 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 3.9690902440000002, 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.036663873, 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.403824+0000", 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.403824+0000", 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:12:20.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.403824+0000", 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.403832+0000", 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7499999999999992e-06 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:12:20.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.426182+0000", 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022349346999999999 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.426194+0000", 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2439000000000001e-05 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.427801+0000", 2023-09-23T21:12:20.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016065389999999999 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.427820+0000", 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9650999999999999e-05 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.440453+0000", 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012632197 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:12:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:16.440488+0000", 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5788999999999999e-05 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:12:20.510 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:12:20.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3453328458' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:12:20.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: osdmap e453: 8 total, 8 up, 8 in 2023-09-23T21:12:20.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: pgmap v1476: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:20.511 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2320466587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:20.512 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:20.512 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.512 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.512 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3453328458' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:12:20.512 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: osdmap e453: 8 total, 8 up, 8 in 2023-09-23T21:12:20.512 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: failed to encode map e453 with expected crc 2023-09-23T21:12:20.512 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: pgmap v1476: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:20.513 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2320466587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:20.513 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:20.514 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:12:20.569 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:20.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:20.683 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:20.713 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:20.758 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:20.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:20.861 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:20.896 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:21.292 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:21.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:21.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:21.302 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:12:21.355 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:21.355 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[219709]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[219709]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[219709]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[219709]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[219709]: osdmap e454: 8 total, 8 up, 8 in 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[211526]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[211526]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[211526]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[211526]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.356 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:21.357 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:21 smithi121 ceph-mon[211526]: osdmap e454: 8 total, 8 up, 8 in 2023-09-23T21:12:21.401 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:21 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:21 smithi123 ceph-mon[193903]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:21 smithi123 ceph-mon[193903]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:21 smithi123 ceph-mon[193903]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:21 smithi123 ceph-mon[193903]: failed to encode map e454 with expected crc 2023-09-23T21:12:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:21 smithi123 ceph-mon[193903]: osdmap e454: 8 total, 8 up, 8 in 2023-09-23T21:12:21.533 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:21.652 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:21.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:21.693 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:21.718 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:21.762 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:21.791 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:21.848 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:21.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:21.954 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:22.054 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:22.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:22 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:22.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:22 smithi121 ceph-mon[211526]: pgmap v1478: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:22.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:22 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:22.322 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:22 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:22.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:22 smithi121 ceph-mon[219709]: pgmap v1478: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:22.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:22 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:22.433 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:22.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:22.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:22 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:22 smithi123 ceph-mon[193903]: pgmap v1478: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 0 B/s, 9 objects/s recovering 2023-09-23T21:12:22.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:22 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:22.542 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:22.572 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:22.633 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:22.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:22.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:22.743 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:22.833 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:23.431 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:12:23.504 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:12:23.505 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:12:23.505 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:23.505 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:23.505 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:12:23.505 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:12:23.505 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:23.505 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:23 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:23.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:23 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:12:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:23 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:12:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:23 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:23 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:24.089 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:24.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:24.203 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:24.231 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.260 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:24.283 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:24.389 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:24.419 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:24 smithi121 ceph-mon[211526]: pgmap v1479: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T21:12:24.419 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:24 smithi121 ceph-mon[219709]: pgmap v1479: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T21:12:24.452 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.612 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:24.718 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:24.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:24 smithi123 ceph-mon[193903]: pgmap v1479: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-09-23T21:12:24.754 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.785 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:12:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:12:24] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.43.0" 2023-09-23T21:12:24.805 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:24.817 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:24.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:24.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:24.925 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:24.953 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:24.990 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:12:25.003 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:25.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:25.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:25.053 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:12:25.101 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:12:25.110 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:25.139 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:25.154 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:12:25.186 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:25.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:25.215 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:12:25.259 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:12:25.294 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:25.295 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:12:25.328 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:25.334 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:12:25.341 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.341 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.342 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.342 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.342 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.342 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.342 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.342 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:12:25.376 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1383483645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:25.376 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1383483645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:25.379 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:25.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:25.487 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:25.564 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1383483645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:26.301 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:12:26.527 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:26.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:26.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:26.641 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:26.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:26 smithi121 ceph-mon[211526]: pgmap v1480: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-09-23T21:12:26.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/353082295' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:12:26.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:26 smithi121 ceph-mon[219709]: pgmap v1480: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-09-23T21:12:26.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/353082295' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:12:26.708 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:26.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:26 smithi123 ceph-mon[193903]: pgmap v1480: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-09-23T21:12:26.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/353082295' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:12:27.533 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:27.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:27.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:27.638 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/353082295' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: osdmap e455: 8 total, 8 up, 8 in 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[211526]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/353082295' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: osdmap e455: 8 total, 8 up, 8 in 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:27 smithi121 ceph-mon[219709]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.711 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/353082295' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:12:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: osdmap e455: 8 total, 8 up, 8 in 2023-09-23T21:12:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: failed to encode map e455 with expected crc 2023-09-23T21:12:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:27 smithi123 ceph-mon[193903]: failed to encode map e455 with expected crc 2023-09-23T21:12:28.310 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:28 smithi121 ceph-mon[211526]: pgmap v1482: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T21:12:28.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:28.311 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:28.443 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:12:28.609 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:28.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:28.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:28 smithi121 ceph-mon[219709]: pgmap v1482: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T21:12:28.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:28.618 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:28.717 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:28.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:28 smithi123 ceph-mon[193903]: pgmap v1482: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-09-23T21:12:28.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:12:28.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:28.748 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:28.795 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:28.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:28.904 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:29.025 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:29.065 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:12:29.224 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:29.224 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:29.461 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:29.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:29.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:29.575 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:29.605 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:29.653 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:29.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:29.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:29.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:29.788 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:30.315 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:12:30.345 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:30.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:30.462 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:30.471 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:30.523 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:30.551 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:30 smithi121 ceph-mon[211526]: pgmap v1483: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T21:12:30.552 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:30 smithi121 ceph-mon[219709]: pgmap v1483: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T21:12:30.557 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:30.701 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:30.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:30.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:30 smithi123 ceph-mon[193903]: pgmap v1483: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-09-23T21:12:30.814 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:30.909 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:31.055 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:31.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:31.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:31.170 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:31.207 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:31.262 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:31.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:31.368 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:31.381 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:12:31.401 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:31.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1419704629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:31.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:31 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:31.422 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:31 smithi121 ceph-mon[211526]: pgmap v1484: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T21:12:31.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1419704629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:31.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:31 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:31.423 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:31 smithi121 ceph-mon[219709]: pgmap v1484: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T21:12:31.460 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:31.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1419704629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:31 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:12:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:31 smithi123 ceph-mon[193903]: pgmap v1484: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-09-23T21:12:31.571 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:31.602 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:31.668 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:31.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:31.773 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[211526]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:12:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[211526]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T21:12:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[211526]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[211526]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[211526]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[219709]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:12:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[219709]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T21:12:32.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[219709]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[219709]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:32 smithi121 ceph-mon[219709]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:32 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:32 smithi123 ceph-mon[193903]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:32 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:12:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:32 smithi123 ceph-mon[193903]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T21:12:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:32 smithi123 ceph-mon[193903]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:32 smithi123 ceph-mon[193903]: failed to encode map e456 with expected crc 2023-09-23T21:12:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:32 smithi123 ceph-mon[193903]: failed to encode map e456 with expected crc 2023-09-23T21:12:33.177 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:33.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:33.280 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:33.309 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:33.356 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:33 smithi121 ceph-mon[211526]: pgmap v1486: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T21:12:33.366 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:33.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:33.471 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:33.549 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:33.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:33 smithi121 ceph-mon[219709]: pgmap v1486: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T21:12:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:33 smithi123 ceph-mon[193903]: pgmap v1486: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-09-23T21:12:33.787 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:33.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:33.886 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:33.899 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:33.929 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:33.988 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:33.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:34.094 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:34.187 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:34.698 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:34.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:34.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:34.804 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:34.907 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:34.950 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:12:35.019 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:12:35.040 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:12:35.076 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:34 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4059540467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:35.076 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:12:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:12:34] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.43.0" 2023-09-23T21:12:35.076 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:34 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4059540467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:35.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:12:35.118 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:12:35.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:12:35.196 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:12:35.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:34 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4059540467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:35.250 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:12:35.287 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:12:35.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.306 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.306 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.306 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:12:35.427 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:12:36.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2748497567' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:12:36.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2748497567' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:35 smithi123 ceph-mon[193903]: pgmap v1488: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-09-23T21:12:36.273 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2748497567' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2748497567' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.277 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[219709]: pgmap v1488: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2748497567' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2748497567' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.278 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.279 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.279 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: failed to encode map e457 with expected crc 2023-09-23T21:12:36.279 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:35 smithi121 ceph-mon[211526]: pgmap v1488: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 486 op/s 2023-09-23T21:12:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:36.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:36.393 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:36.422 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:36.473 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:36.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:36.581 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:36.608 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:36.669 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:36.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:36.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:36.775 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:36.880 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:37.374 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:12:37.527 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:37.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:37.631 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:37 smithi121 ceph-mon[219709]: pgmap v1489: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T21:12:37.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:37.720 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:37.879 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:37.890 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:37 smithi121 ceph-mon[211526]: pgmap v1489: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T21:12:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:37.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:37.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:37 smithi123 ceph-mon[193903]: pgmap v1489: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-09-23T21:12:37.992 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:38.024 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:38.051 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:38.075 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:38.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:38.181 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:38.209 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:12:38.209 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:12:38.210 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:12:38.210 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:12:38.210 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:12:38.281 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:38.342 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:38.599 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:38.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:38.710 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:38.789 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:39.510 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:39.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:39.619 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:39.647 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:39.695 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:39.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:39.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:39.731 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:12:39.805 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:39.832 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:40.149 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:12:40.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:40 smithi121 ceph-mon[211526]: pgmap v1490: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-09-23T21:12:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:40 smithi121 ceph-mon[219709]: pgmap v1490: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-09-23T21:12:40.495 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:40.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:40.608 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:40.634 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:40 smithi123 ceph-mon[193903]: pgmap v1490: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-09-23T21:12:40.637 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:40.697 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:40.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:40.807 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:40.839 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:40.898 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:40.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:41.009 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:41.036 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:41.090 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:41.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:41.198 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:41.226 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:41.284 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:41.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:41.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:41.390 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:41.417 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:41.469 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:41.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:41.578 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:41.613 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:41.676 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:41.736 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:41.866 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:41.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:41.972 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:41.999 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:42.054 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:42.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:42.160 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:42.233 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:42.553 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:42 smithi121 ceph-mon[211526]: pgmap v1491: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T21:12:42.553 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1331095559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:42.554 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:42 smithi121 ceph-mon[219709]: pgmap v1491: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T21:12:42.554 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1331095559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:42.682 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:12:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:42 smithi123 ceph-mon[193903]: pgmap v1491: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T21:12:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1331095559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:12:42.870 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:42.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:42.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:42.978 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:43.082 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:43.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1331095559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/422087147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:12:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1331095559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/422087147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:12:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:43 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:12:43.713 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:12:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:12:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:12:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1331095559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:12:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T21:12:43.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: failed to encode map e458 with expected crc 2023-09-23T21:12:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/422087147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:12:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:43 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:12:44.559 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:44.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:44.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[211526]: failed to encode map e458 with expected crc 2023-09-23T21:12:44.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[211526]: pgmap v1493: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T21:12:44.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:12:44.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[211526]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T21:12:44.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[211526]: failed to encode map e459 with expected crc 2023-09-23T21:12:44.567 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[211526]: failed to encode map e459 with expected crc 2023-09-23T21:12:44.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[219709]: failed to encode map e458 with expected crc 2023-09-23T21:12:44.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[219709]: pgmap v1493: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T21:12:44.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:12:44.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[219709]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T21:12:44.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[219709]: failed to encode map e459 with expected crc 2023-09-23T21:12:44.568 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:44 smithi121 ceph-mon[219709]: failed to encode map e459 with expected crc 2023-09-23T21:12:44.668 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:44 smithi123 ceph-mon[193903]: failed to encode map e458 with expected crc 2023-09-23T21:12:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:44 smithi123 ceph-mon[193903]: pgmap v1493: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-09-23T21:12:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:44 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:12:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:44 smithi123 ceph-mon[193903]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T21:12:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:44 smithi123 ceph-mon[193903]: failed to encode map e459 with expected crc 2023-09-23T21:12:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:44 smithi123 ceph-mon[193903]: failed to encode map e459 with expected crc 2023-09-23T21:12:44.754 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:44.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:12:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:12:44] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.43.0" 2023-09-23T21:12:45.279 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:45.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:45.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:45.393 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:45.424 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:45 smithi121 ceph-mon[211526]: failed to encode map e459 with expected crc 2023-09-23T21:12:45.425 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:45 smithi121 ceph-mon[219709]: failed to encode map e459 with expected crc 2023-09-23T21:12:45.467 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:12:45.527 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:45.685 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:45.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:45.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:45 smithi123 ceph-mon[193903]: failed to encode map e459 with expected crc 2023-09-23T21:12:45.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:45.850 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:45.908 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:45.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:46.014 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:46.130 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:46.158 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:12:46.158 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88124 11.c 11:386a4abc:::benchmark_data_smithi121_477989_object88123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 497.41898813799997, 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6441045829999998, 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.159 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88124 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735063+0000", 2023-09-23T21:12:46.160 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735062+0000", 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735084+0000", 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.234e-05 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.161 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735099+0000", 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.508e-05 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735108+0000", 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3689999999999996e-06 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735157+0000", 2023-09-23T21:12:46.162 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8273999999999999e-05 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735257+0000", 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000100432 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.735419+0000", 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000162158 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.163 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.741925+0000", 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00650594 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.744875+0000", 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002949981 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757406+0000", 2023-09-23T21:12:46.164 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012530777999999999 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757417+0000", 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0699e-05 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770169+0000", 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0127527590000001 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.165 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770180+0000", 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0428000000000001e-05 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.379055+0000", 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6088747780000001 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.379167+0000", 2023-09-23T21:12:46.166 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112796 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88126 11.c 11:3f012f98:::benchmark_data_smithi121_477989_object88125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 497.41732108899998, 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6457891419999999, 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.167 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88126 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.168 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736730+0000", 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736728+0000", 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736758+0000", 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0073000000000002e-05 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.169 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736775+0000", 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6716000000000001e-05 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.736784+0000", 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1719999999999996e-06 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.170 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738664+0000", 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00187952 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738803+0000", 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013961799999999999 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.738967+0000", 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016343 2023-09-23T21:12:46.171 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.744883+0000", 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0059158769999999999 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757481+0000", 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012598689 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.172 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757491+0000", 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.098e-06 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770209+0000", 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.012718877 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770217+0000", 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1679999999999998e-06 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.173 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382403+0000", 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6121860369999998 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382519+0000", 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116488 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.174 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36195.0:88142 11.c 11:3d9b819d:::benchmark_data_smithi121_477989_object88141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 497.39771993300002, 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6474134190000003, 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36195", 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/964023155", 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 88142 2023-09-23T21:12:46.175 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756331+0000", 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.176 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756330+0000", 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756356+0000", 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6440000000000001e-05 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756382+0000", 2023-09-23T21:12:46.177 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5948999999999998e-05 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.756392+0000", 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5999999999999996e-06 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757514+0000", 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001122088 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.178 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:28.757521+0000", 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8099999999999998e-06 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770236+0000", 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0127141230000001 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:29.770241+0000", 2023-09-23T21:12:46.179 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9309999999999996e-06 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382568+0000", 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.612326293 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.382717+0000", 2023-09-23T21:12:46.181 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014953000000000001 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.383252+0000", 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00053481999999999998 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.388314+0000", 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050622560000000002 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.182 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.396573+0000", 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0082585120000000008 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.399933+0000", 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033602250000000001 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.399996+0000", 2023-09-23T21:12:46.183 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2965000000000004e-05 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:04:31.403745+0000", 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037484340000000001 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64073 12.7 12:ecfffd2a:::benchmark_data_smithi121_563784_object64072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.257742+0000", 2023-09-23T21:12:46.184 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 119.89630907, 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32869984099999999, 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64073 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.185 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257742+0000", 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257742+0000", 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257740+0000", 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:46.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257764+0000", 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3506000000000002e-05 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257777+0000", 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3035e-05 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.187 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257784+0000", 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.374e-06 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257831+0000", 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.638e-05 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257948+0000", 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000117759 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.188 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.258072+0000", 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012374400000000001 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.264437+0000", 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063651180000000003 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.189 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.264469+0000", 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1847000000000002e-05 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.586341+0000", 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32187152099999999 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.586397+0000", 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6854999999999997e-05 2023-09-23T21:12:46.190 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.586442+0000", 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4381000000000001e-05 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99892 12.6 12:6ec6e1ac:::benchmark_data_smithi121_563784_object9489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:12:46.191 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 69.500256897, 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.75238461400000001, 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99892 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:12:46.192 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.193 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653798+0000", 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3279999999999999e-06 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653806+0000", 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4330000000000004e-06 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653812+0000", 2023-09-23T21:12:46.194 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.088e-06 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653898+0000", 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5995999999999995e-05 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653908+0000", 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0057e-05 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.195 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.388228+0000", 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73431940200000001 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.388361+0000", 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013329600000000001 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.388479+0000", 2023-09-23T21:12:46.196 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011790000000000001 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402136+0000", 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013657463 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402579+0000", 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00044286800000000002 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.197 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406104+0000", 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035243060000000001 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406146+0000", 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2633000000000003e-05 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406179+0000", 2023-09-23T21:12:46.198 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2461e-05 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99922 12.6 12:65825bec:::benchmark_data_smithi121_563784_object9519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.734372+0000", 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 69.419679353000006, 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.671888288, 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.199 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99922 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734372+0000", 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.200 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734372+0000", 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734371+0000", 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734377+0000", 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.122e-06 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.201 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734390+0000", 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3426e-05 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734396+0000", 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1809999999999998e-06 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.202 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734451+0000", 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4364999999999999e-05 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734464+0000", 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3217999999999999e-05 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.391809+0000", 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65734495199999998 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.203 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.391934+0000", 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000125165 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.392044+0000", 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000109831 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402295+0000", 2023-09-23T21:12:46.204 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010250815 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402607+0000", 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00031272899999999998 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406188+0000", 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035809829999999998 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.205 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406228+0000", 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9366999999999998e-05 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406260+0000", 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2237000000000003e-05 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.206 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99923 12.6 12:6ca689b1:::benchmark_data_smithi121_563784_object9520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.741432+0000", 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 69.412619461000006, 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.664912222, 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99923 2023-09-23T21:12:46.207 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741432+0000", 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741432+0000", 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.208 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741429+0000", 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741442+0000", 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2733e-05 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741468+0000", 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6463000000000001e-05 2023-09-23T21:12:46.209 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741481+0000", 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2344000000000001e-05 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741551+0000", 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0297999999999997e-05 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.210 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741567+0000", 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6226e-05 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.395439+0000", 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65387172699999996 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.395545+0000", 2023-09-23T21:12:46.211 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106679 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.395652+0000", 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010650999999999999 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402425+0000", 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067725019999999997 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.212 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402637+0000", 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000212621 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406273+0000", 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036357099999999999 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406312+0000", 2023-09-23T21:12:46.213 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9579999999999997e-05 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406344+0000", 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1365000000000002e-05 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99931 12.6 12:6cc7051a:::benchmark_data_smithi121_563784_object9528:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.785517+0000", 2023-09-23T21:12:46.214 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 69.368534174000004, 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.62893938400000005, 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99931 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785517+0000", 2023-09-23T21:12:46.215 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785517+0000", 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785514+0000", 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999971 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.216 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785526+0000", 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2447e-05 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785553+0000", 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6906999999999998e-05 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785567+0000", 2023-09-23T21:12:46.217 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3652000000000001e-05 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785642+0000", 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4760999999999998e-05 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785657+0000", 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5546e-05 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.218 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.398524+0000", 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.61286635599999995 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.398647+0000", 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012347000000000001 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.398768+0000", 2023-09-23T21:12:46.219 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121278 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406353+0000", 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075846949999999998 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406412+0000", 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8974000000000002e-05 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.220 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.414368+0000", 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079556490000000004 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.414418+0000", 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9959000000000001e-05 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.414456+0000", 2023-09-23T21:12:46.221 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8525999999999997e-05 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:101360 12.6 12:6fee56fa:::benchmark_data_smithi121_563784_object10957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e445)", 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:39.430833+0000", 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 66.723218200999995, 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.202334984, 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.222 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 101360 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.430833+0000", 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.223 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.430833+0000", 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.430832+0000", 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.430837+0000", 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0699999999999997e-06 2023-09-23T21:12:46.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.430848+0000", 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0569e-05 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.430855+0000", 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1590000000000004e-06 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.225 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.430928+0000", 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2841000000000003e-05 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.431031+0000", 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010276999999999999 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.431123+0000", 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2053999999999997e-05 2023-09-23T21:12:46.226 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.435051+0000", 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003928534 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.443410+0000", 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0083583900000000003 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.227 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.633077+0000", 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.189667373 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.633126+0000", 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9530000000000002e-05 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.633168+0000", 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1381999999999998e-05 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.228 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:101375 12.6 12:6cda009c:::benchmark_data_smithi121_563784_object10972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e445)", 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:39.449500+0000", 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 66.704550815000005, 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.19852987599999999, 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.229 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 101375 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.449500+0000", 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.449500+0000", 2023-09-23T21:12:46.230 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.449499+0000", 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.449507+0000", 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1699999999999997e-06 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.231 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.449522+0000", 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4973e-05 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.449530+0000", 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6389999999999998e-06 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.450008+0000", 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00047735600000000002 2023-09-23T21:12:46.232 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.450132+0000", 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000124387 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.450238+0000", 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010602099999999999 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.233 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.454352+0000", 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0041137439999999999 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.455651+0000", 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001298953 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.647942+0000", 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.192291025 2023-09-23T21:12:46.234 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.647988+0000", 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6542999999999998e-05 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:39.648030+0000", 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1635999999999999e-05 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.235 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:102730 12.d 12:b95145bb:::benchmark_data_smithi121_563784_object12327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e445)", 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:41.541780+0000", 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 64.612271340999996, 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.20189761000000001, 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 102730 2023-09-23T21:12:46.236 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.541780+0000", 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.541780+0000", 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.237 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.541778+0000", 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.541785+0000", 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8000000000000001e-06 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.541797+0000", 2023-09-23T21:12:46.238 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2303999999999999e-05 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.541805+0000", 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9000000000000006e-06 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.544498+0000", 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026929760000000001 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.239 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.544625+0000", 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012736299999999999 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.544736+0000", 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110463 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.240 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.549167+0000", 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0044315040000000002 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.727734+0000", 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.17856714400000001 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.743599+0000", 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015864782000000001 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.241 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.743641+0000", 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1724000000000002e-05 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:41.743677+0000", 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6365000000000002e-05 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.242 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:105562 12.6 12:6421e75c:::benchmark_data_smithi121_563784_object15159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:45.625693+0000", 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 60.528358435000001, 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24364650800000001, 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 105562 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.243 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625693+0000", 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625693+0000", 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.244 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625691+0000", 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625699+0000", 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9910000000000002e-06 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625712+0000", 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3849000000000001e-05 2023-09-23T21:12:46.245 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625721+0000", 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5590000000000006e-06 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625786+0000", 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5050999999999998e-05 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.246 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625908+0000", 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121963 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.626023+0000", 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115346 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.633002+0000", 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0069786270000000003 2023-09-23T21:12:46.247 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.634017+0000", 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001015373 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.869250+0000", 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23523287900000001 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.248 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.869306+0000", 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5291000000000003e-05 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.869339+0000", 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3553000000000002e-05 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.249 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:105563 12.6 12:6eaeb932:::benchmark_data_smithi121_563784_object15160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:45.625735+0000", 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 60.528315747000001, 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243695785, 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 105563 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.250 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625735+0000", 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625735+0000", 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625735+0000", 2023-09-23T21:12:46.251 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625738+0000", 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.174e-06 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625745+0000", 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7089999999999996e-06 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.252 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.625751+0000", 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5189999999999998e-06 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.629492+0000", 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037409790000000002 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.629592+0000", 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010032899999999999 2023-09-23T21:12:46.253 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.629689+0000", 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6798000000000004e-05 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.633358+0000", 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036684069999999998 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.254 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.634092+0000", 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00073481499999999999 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.869369+0000", 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.235276492 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.255 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.869403+0000", 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4418999999999997e-05 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.869431+0000", 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7738000000000001e-05 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.256 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:105582 12.6 12:6288e697:::benchmark_data_smithi121_563784_object15179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:45.643648+0000", 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 60.510403508000003, 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25724708899999998, 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 105582 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.257 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643648+0000", 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643648+0000", 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643646+0000", 2023-09-23T21:12:46.258 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643654+0000", 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3399999999999998e-06 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817186+0000", 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.173532038 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.259 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817220+0000", 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4078999999999997e-05 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817316+0000", 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6385999999999995e-05 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817436+0000", 2023-09-23T21:12:46.260 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119918 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817510+0000", 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4039e-05 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.821079+0000", 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035684340000000001 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.261 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.821107+0000", 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8014999999999999e-05 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.900806+0000", 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.079699018999999996 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.900856+0000", 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0627999999999997e-05 2023-09-23T21:12:46.262 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.900895+0000", 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8126000000000001e-05 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116382 12.d 12:b1c268d1:::benchmark_data_smithi121_563784_object25979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.860298+0000", 2023-09-23T21:12:46.263 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 42.293753477999999, 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.244044026, 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116382 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.860298+0000", 2023-09-23T21:12:46.264 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.860298+0000", 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.860296+0000", 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.265 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.860303+0000", 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4230000000000002e-06 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.860315+0000", 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1939999999999999e-05 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.266 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.860322+0000", 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5660000000000001e-06 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.862297+0000", 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001974492 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.862422+0000", 2023-09-23T21:12:46.267 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012526700000000001 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3", 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.862531+0000", 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000108546 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866318+0000", 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037868799999999998 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.268 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874338+0000", 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.008020157 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.104251+0000", 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.22991282399999999 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.104305+0000", 2023-09-23T21:12:46.269 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4815999999999998e-05 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.104342+0000", 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6253e-05 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116400 12.7 12:e346ab9b:::benchmark_data_smithi121_563784_object25997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:12:46.270 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.881315+0000", 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 42.272736137000003, 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23429905000000001, 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116400 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.271 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881315+0000", 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881315+0000", 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881314+0000", 2023-09-23T21:12:46.272 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881321+0000", 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1169999999999998e-06 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881331+0000", 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0710000000000001e-05 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.273 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881339+0000", 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0290000000000005e-06 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881377+0000", 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7808999999999997e-05 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.274 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881461+0000", 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3702999999999995e-05 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.881531+0000", 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9816999999999999e-05 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.884438+0000", 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029075749999999999 2023-09-23T21:12:46.275 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.884958+0000", 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00051958699999999996 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.115525+0000", 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23056749200000001 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.276 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.115573+0000", 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7240999999999997e-05 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.115614+0000", 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1334000000000001e-05 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.277 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116903 12.6 12:6ad03ab1:::benchmark_data_smithi121_563784_object26500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:04.838727+0000", 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 41.315323679000002, 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.203424198, 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116903 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.278 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838727+0000", 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838727+0000", 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838726+0000", 2023-09-23T21:12:46.279 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838734+0000", 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0139999999999998e-06 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838744+0000", 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0713000000000001e-05 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.280 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838751+0000", 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9009999999999997e-06 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838790+0000", 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9146000000000001e-05 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838894+0000", 2023-09-23T21:12:46.281 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000103123 2023-09-23T21:12:46.282 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.282 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.282 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.282 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.838992+0000", 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8728999999999996e-05 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.842579+0000", 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035864209999999998 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.843456+0000", 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00087729799999999999 2023-09-23T21:12:46.283 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:05.042075+0000", 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.198618981 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:05.042115+0000", 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9780999999999997e-05 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.284 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:05.042152+0000", 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6733999999999998e-05 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:125334 12.6 12:6468b97f:::benchmark_data_smithi121_563784_object34931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:19.740114+0000", 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 26.413937345000001, 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.20598588100000001, 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.285 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 125334 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740114+0000", 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.286 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740114+0000", 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740112+0000", 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740120+0000", 2023-09-23T21:12:46.287 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2709999999999992e-06 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740136+0000", 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5276000000000001e-05 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740145+0000", 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1589999999999999e-06 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.288 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740193+0000", 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7747999999999998e-05 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740329+0000", 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013623500000000001 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.740438+0000", 2023-09-23T21:12:46.289 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000108736 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.744425+0000", 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0039876909999999998 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.745217+0000", 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00079135299999999996 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.290 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.946016+0000", 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.20079898400000001 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.946063+0000", 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7926999999999997e-05 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.946100+0000", 2023-09-23T21:12:46.291 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6093000000000002e-05 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:125337 12.6 12:68c6cec8:::benchmark_data_smithi121_563784_object34934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:19.748489+0000", 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 26.405561712000001, 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.21630482100000001, 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.292 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 125337 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.748489+0000", 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.293 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.748489+0000", 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.748488+0000", 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.748495+0000", 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8499999999999996e-06 2023-09-23T21:12:46.294 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.748508+0000", 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.292e-05 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.748515+0000", 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5800000000000003e-06 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.295 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.753617+0000", 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051020880000000003 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.753743+0000", 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012566399999999999 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.753862+0000", 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011901899999999999 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.296 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.897123+0000", 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.143260572 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.900208+0000", 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030853320000000001 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.297 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.964711+0000", 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.064503331999999997 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.964757+0000", 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5327000000000002e-05 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.964794+0000", 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7586999999999998e-05 2023-09-23T21:12:46.298 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:125354 12.6 12:68e98b77:::benchmark_data_smithi121_563784_object34951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:19.765233+0000", 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 26.388818500999999, 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.19965586199999999, 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:12:46.299 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 125354 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.765233+0000", 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.765233+0000", 2023-09-23T21:12:46.300 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.765231+0000", 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.765238+0000", 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.844e-06 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.301 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.897028+0000", 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.131790295 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.897043+0000", 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4279e-05 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:12:46.302 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.897148+0000", 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000105252 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.897256+0000", 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000108488 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.897347+0000", 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0945000000000004e-05 2023-09-23T21:12:46.303 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.902941+0000", 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0055938000000000003 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.902964+0000", 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2952e-05 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:12:46.304 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.964826+0000", 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.061862348999999997 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.964859+0000", 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2629000000000002e-05 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:19.964888+0000", 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9428e-05 2023-09-23T21:12:46.305 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.306 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.306 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.306 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:12:46.306 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:12:46.306 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:12:46.306 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:12:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:46.307 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:46.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:46.321 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:12:46.369 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:12:46.408 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:46.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:46 smithi121 ceph-mon[211526]: pgmap v1495: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 505 op/s 2023-09-23T21:12:46.420 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3901193410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:46.420 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:46 smithi121 ceph-mon[219709]: pgmap v1495: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 505 op/s 2023-09-23T21:12:46.420 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3901193410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:46.421 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:12:46.441 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:12:46.489 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:12:46.511 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:46.520 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:12:46.595 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:46.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:46.699 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:46.725 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:46 smithi123 ceph-mon[193903]: pgmap v1495: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 505 op/s 2023-09-23T21:12:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3901193410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:46.746 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:12:46.747 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:12:47.048 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:47.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:47.153 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:47.154 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:12:47.168 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:47.181 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:47.207 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:12:47.232 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:47.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:47.342 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:47.410 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:47.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:47.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:47.515 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:47.545 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:47.594 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:47.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:47.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:47.700 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:47.729 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:47.787 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:47.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:47.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:47.893 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:47.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:47 smithi121 ceph-mon[211526]: pgmap v1496: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T21:12:47.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:47 smithi121 ceph-mon[219709]: pgmap v1496: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T21:12:47.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:47 smithi123 ceph-mon[193903]: pgmap v1496: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T21:12:47.980 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:48.148 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:48.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:48.256 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:48.290 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:48.353 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:48.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:48.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:48.460 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:48.497 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:48.790 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:48.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:48.891 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:48.960 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.189 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:49.189 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:49.230 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:49.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:49.344 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:49.374 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.434 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:49.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:49.546 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:49.575 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.625 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:49.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:49.730 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:49.818 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:49.950 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:12:50.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:50 smithi121 ceph-mon[211526]: pgmap v1497: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s 2023-09-23T21:12:50.667 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:50 smithi121 ceph-mon[219709]: pgmap v1497: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s 2023-09-23T21:12:50.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:50 smithi123 ceph-mon[193903]: pgmap v1497: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s 2023-09-23T21:12:50.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:50.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:50.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:51.056 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:51.086 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:51.144 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:51.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:51.249 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:12:51.281 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:51.336 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:12:51.339 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:12:51.345 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2100251189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:12:51.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:51.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:12:51.447 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:51.473 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:51.518 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:51.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:51.624 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:51.669 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2100251189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:12:51.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2100251189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:12:51.688 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:52.335 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:12:52.335 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:12:52.384 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:52.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:52.490 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:52.527 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:52.653 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:52.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: pgmap v1498: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T21:12:52.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2100251189' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:12:52.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e460 with expected crc 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e460 with expected crc 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[211526]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:52.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:52.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: pgmap v1498: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T21:12:52.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2100251189' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:12:52.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T21:12:52.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e460 with expected crc 2023-09-23T21:12:52.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e460 with expected crc 2023-09-23T21:12:52.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T21:12:52.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:52 smithi121 ceph-mon[219709]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.663 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:52.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: pgmap v1498: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-09-23T21:12:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2100251189' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:12:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T21:12:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e460 with expected crc 2023-09-23T21:12:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e460 with expected crc 2023-09-23T21:12:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T21:12:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:52 smithi123 ceph-mon[193903]: failed to encode map e461 with expected crc 2023-09-23T21:12:52.760 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:12:52.761 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:12:52.762 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:52.805 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:53.335 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:53.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:53.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:53.444 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:53.484 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:53.557 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:53.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:53.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: osdmap e462: 8 total, 8 up, 8 in 2023-09-23T21:12:53.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: 12.d starting backfill to osd.7 from (0'0,0'0] MAX to 459'9055 2023-09-23T21:12:53.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: 12.7 starting backfill to osd.7 from (0'0,0'0] MAX to 459'9084 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: 12.1 starting backfill to osd.7 from (0'0,0'0] MAX to 459'8926 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: pgmap v1502: 89 pgs: 2 remapped+peering, 87 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3603053718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: osdmap e462: 8 total, 8 up, 8 in 2023-09-23T21:12:53.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.660 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: 12.d starting backfill to osd.7 from (0'0,0'0] MAX to 459'9055 2023-09-23T21:12:53.660 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: 12.7 starting backfill to osd.7 from (0'0,0'0] MAX to 459'9084 2023-09-23T21:12:53.660 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: 12.1 starting backfill to osd.7 from (0'0,0'0] MAX to 459'8926 2023-09-23T21:12:53.660 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.660 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: pgmap v1502: 89 pgs: 2 remapped+peering, 87 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-09-23T21:12:53.660 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3603053718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:53.673 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:53.685 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:12:53.800 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: osdmap e462: 8 total, 8 up, 8 in 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: 12.d starting backfill to osd.7 from (0'0,0'0] MAX to 459'9055 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: 12.7 starting backfill to osd.7 from (0'0,0'0] MAX to 459'9084 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: 12.1 starting backfill to osd.7 from (0'0,0'0] MAX to 459'8926 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: failed to encode map e462 with expected crc 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: pgmap v1502: 89 pgs: 2 remapped+peering, 87 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-09-23T21:12:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3603053718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:12:54.154 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:54.155 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:12:54.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:54.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:54.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:54.378 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:54 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:54 smithi123 ceph-mon[193903]: failed to encode map e463 with expected crc 2023-09-23T21:12:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:54 smithi123 ceph-mon[193903]: failed to encode map e463 with expected crc 2023-09-23T21:12:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3603053718' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:54 smithi123 ceph-mon[193903]: osdmap e463: 8 total, 8 up, 8 in 2023-09-23T21:12:55.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:12:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:12:54] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.43.0" 2023-09-23T21:12:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[211526]: failed to encode map e463 with expected crc 2023-09-23T21:12:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[211526]: failed to encode map e463 with expected crc 2023-09-23T21:12:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3603053718' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:55.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[211526]: osdmap e463: 8 total, 8 up, 8 in 2023-09-23T21:12:55.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:12:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[219709]: failed to encode map e463 with expected crc 2023-09-23T21:12:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[219709]: failed to encode map e463 with expected crc 2023-09-23T21:12:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3603053718' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:12:55.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:54 smithi121 ceph-mon[219709]: osdmap e463: 8 total, 8 up, 8 in 2023-09-23T21:12:55.308 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:55.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:55.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:55.420 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:55.454 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:55.494 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:55.513 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:55.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:55.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:55.623 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:55.718 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:55.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:55 smithi123 ceph-mon[193903]: failed to encode map e464 with expected crc 2023-09-23T21:12:55.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:55 smithi123 ceph-mon[193903]: osdmap e464: 8 total, 8 up, 8 in 2023-09-23T21:12:55.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:55 smithi123 ceph-mon[193903]: failed to encode map e464 with expected crc 2023-09-23T21:12:55.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:55 smithi123 ceph-mon[193903]: failed to encode map e464 with expected crc 2023-09-23T21:12:55.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:55 smithi123 ceph-mon[193903]: pgmap v1505: 89 pgs: 2 remapped+peering, 87 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 295 op/s 2023-09-23T21:12:55.887 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:55.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:55.992 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:56.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[211526]: failed to encode map e464 with expected crc 2023-09-23T21:12:56.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[211526]: osdmap e464: 8 total, 8 up, 8 in 2023-09-23T21:12:56.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[211526]: failed to encode map e464 with expected crc 2023-09-23T21:12:56.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[211526]: failed to encode map e464 with expected crc 2023-09-23T21:12:56.013 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[211526]: pgmap v1505: 89 pgs: 2 remapped+peering, 87 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 295 op/s 2023-09-23T21:12:56.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[219709]: failed to encode map e464 with expected crc 2023-09-23T21:12:56.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[219709]: osdmap e464: 8 total, 8 up, 8 in 2023-09-23T21:12:56.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[219709]: failed to encode map e464 with expected crc 2023-09-23T21:12:56.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[219709]: failed to encode map e464 with expected crc 2023-09-23T21:12:56.014 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:55 smithi121 ceph-mon[219709]: pgmap v1505: 89 pgs: 2 remapped+peering, 87 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 295 op/s 2023-09-23T21:12:56.031 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:56.102 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:56.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:56.128 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:56.209 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:12:56.283 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:56.947 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:12:56.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:56.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:12:56.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:12:56.994 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:12:57.033 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:12:57.053 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:57.073 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:12:57.113 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:12:57.125 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:57.156 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:12:57.205 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:12:57.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:12:57.269 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.269 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.269 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.269 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.269 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.269 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.270 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.270 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:12:57.444 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:12:57.535 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:57.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:57.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:57.651 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:12:57.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2195080153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:57.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:57 smithi121 ceph-mon[211526]: pgmap v1506: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 86 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 892/105648 objects misplaced (0.844%); 13 MiB/s, 214 objects/s recovering 2023-09-23T21:12:57.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2195080153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:57 smithi121 ceph-mon[219709]: pgmap v1506: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 86 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 892/105648 objects misplaced (0.844%); 13 MiB/s, 214 objects/s recovering 2023-09-23T21:12:57.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:57.679 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:57.727 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:12:57.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:57.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:12:57.833 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:12:57.919 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2195080153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:12:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:57 smithi123 ceph-mon[193903]: pgmap v1506: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 86 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 892/105648 objects misplaced (0.844%); 13 MiB/s, 214 objects/s recovering 2023-09-23T21:12:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:12:58.311 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:12:58.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:58.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:12:58.419 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:58.456 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:58.525 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:12:58.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:58.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:12:58.636 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:12:58.681 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:58.723 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:12:58.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:58.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:12:58.832 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:12:58.911 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:59.387 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:12:59.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:59.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:12:59.451 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:12:59.491 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:12:59.525 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:12:59.757 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:12:59.783 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:12:59.816 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:12:59.821 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[219709]: osdmap e465: 8 total, 8 up, 8 in 2023-09-23T21:12:59.821 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[219709]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.821 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[219709]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.821 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[219709]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.821 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[219709]: pgmap v1508: 89 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 86 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 5214/102591 objects misplaced (5.082%); 13 MiB/s, 200 objects/s recovering 2023-09-23T21:12:59.821 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[211526]: osdmap e465: 8 total, 8 up, 8 in 2023-09-23T21:12:59.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[211526]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[211526]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[211526]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:12:59 smithi121 ceph-mon[211526]: pgmap v1508: 89 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 86 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 5214/102591 objects misplaced (5.082%); 13 MiB/s, 200 objects/s recovering 2023-09-23T21:12:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:12:59.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:12:59.920 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:12:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:12:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:12:59.920 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:12:59.920 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:12:59.926 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:12:59.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:59 smithi123 ceph-mon[193903]: osdmap e465: 8 total, 8 up, 8 in 2023-09-23T21:12:59.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:59 smithi123 ceph-mon[193903]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:59 smithi123 ceph-mon[193903]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:59 smithi123 ceph-mon[193903]: failed to encode map e465 with expected crc 2023-09-23T21:12:59.963 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:12:59 smithi123 ceph-mon[193903]: pgmap v1508: 89 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 86 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 5214/102591 objects misplaced (5.082%); 13 MiB/s, 200 objects/s recovering 2023-09-23T21:12:59.965 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:00.025 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:00.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:00.130 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:00.171 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:00.235 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:00.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:00.336 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:00.349 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:13:00.380 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:00.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[219709]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[219709]: osdmap e466: 8 total, 8 up, 8 in 2023-09-23T21:13:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[219709]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[219709]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[211526]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[211526]: osdmap e466: 8 total, 8 up, 8 in 2023-09-23T21:13:00.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[211526]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[211526]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[211526]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[211526]: from='client.56953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:00.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[219709]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.918 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:00 smithi121 ceph-mon[219709]: from='client.56953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:00 smithi123 ceph-mon[193903]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:00 smithi123 ceph-mon[193903]: osdmap e466: 8 total, 8 up, 8 in 2023-09-23T21:13:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:00 smithi123 ceph-mon[193903]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:00 smithi123 ceph-mon[193903]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:00 smithi123 ceph-mon[193903]: failed to encode map e466 with expected crc 2023-09-23T21:13:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:00 smithi123 ceph-mon[193903]: from='client.56953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:01.083 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:01.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:01.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:01.190 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:01.251 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:01.944 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:01 smithi121 ceph-mon[219709]: pgmap v1510: 89 pgs: 2 active+remapped+backfill_wait, 87 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 4322/98268 objects misplaced (4.398%); 20 MiB/s, 315 objects/s recovering 2023-09-23T21:13:01.948 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:01 smithi121 ceph-mon[211526]: pgmap v1510: 89 pgs: 2 active+remapped+backfill_wait, 87 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 4322/98268 objects misplaced (4.398%); 20 MiB/s, 315 objects/s recovering 2023-09-23T21:13:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:01 smithi123 ceph-mon[193903]: pgmap v1510: 89 pgs: 2 active+remapped+backfill_wait, 87 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 4322/98268 objects misplaced (4.398%); 20 MiB/s, 315 objects/s recovering 2023-09-23T21:13:02.372 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1509,"stamp":"2023-09-23T21:12:59.662415+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2240092088,"num_objects":34197,"num_object_clones":0,"num_object_copies":102591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5214,"num_objects_unfound":0,"num_objects_dirty":34197,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":146929,"num_write_kb":5788046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10598,"num_bytes_recovered":651721820,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133345,"ondisk_log_size":133345,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12380732,"kb_used_data":6846972,"kb_used_omap":21,"kb_used_meta":5533482,"kb_avail":737580484,"statfs":{"total":767960285184,"available":755282415616,"internally_reserved":0,"allocated":7011299328,"data_stored":6996919355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":5666286051},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-183369728,"num_objects":-2798,"num_object_clones":0,"num_object_copies":-8394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5214,"num_objects_unfound":0,"num_objects_dirty":-2798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2798,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1206,"num_bytes_recovered":79495712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.006108"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":237,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675689+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:12:53.675689+0000","last_peered":"2023-09-23T21:12:53.675689+0000","last_clean":"2023-09-23T21:12:53.675689+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:12:53.675689+0000","last_undegraded":"2023-09-23T21:12:53.675689+0000","last_fullsized":"2023-09-23T21:12:53.675689+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.679847+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:12:53.679847+0000","last_peered":"2023-09-23T21:12:53.679847+0000","last_clean":"2023-09-23T21:12:53.679847+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:12:53.679847+0000","last_undegraded":"2023-09-23T21:12:53.679847+0000","last_fullsized":"2023-09-23T21:12:53.679847+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":379,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675530+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:12:53.675530+0000","last_peered":"2023-09-23T21:12:53.675530+0000","last_clean":"2023-09-23T21:12:53.675530+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:12:53.675530+0000","last_undegraded":"2023-09-23T21:12:53.675530+0000","last_fullsized":"2023-09-23T21:12:53.675530+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":113,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.691789+0000","last_change":"2023-09-23T21:12:51.650220+0000","last_active":"2023-09-23T21:12:54.691789+0000","last_peered":"2023-09-23T21:12:54.691789+0000","last_clean":"2023-09-23T21:12:54.691789+0000","last_became_active":"2023-09-23T21:12:51.649869+0000","last_became_peered":"2023-09-23T21:12:51.649869+0000","last_unstale":"2023-09-23T21:12:54.691789+0000","last_undegraded":"2023-09-23T21:12:54.691789+0000","last_fullsized":"2023-09-23T21:12:54.691789+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:04.107476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.678780+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:12:53.678780+0000","last_peered":"2023-09-23T21:12:53.678780+0000","last_clean":"2023-09-23T21:12:53.678780+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:12:53.678780+0000","last_undegraded":"2023-09-23T21:12:53.678780+0000","last_fullsized":"2023-09-23T21:12:53.678780+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":340,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675130+0000","last_change":"2023-09-23T21:09:44.161988+0000","last_active":"2023-09-23T21:12:53.675130+0000","last_peered":"2023-09-23T21:12:53.675130+0000","last_clean":"2023-09-23T21:12:53.675130+0000","last_became_active":"2023-09-23T21:09:44.161643+0000","last_became_peered":"2023-09-23T21:09:44.161643+0000","last_unstale":"2023-09-23T21:12:53.675130+0000","last_undegraded":"2023-09-23T21:12:53.675130+0000","last_fullsized":"2023-09-23T21:12:53.675130+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:17.083615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":104,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.691696+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:12:54.691696+0000","last_peered":"2023-09-23T21:12:54.691696+0000","last_clean":"2023-09-23T21:12:54.691696+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:12:54.691696+0000","last_undegraded":"2023-09-23T21:12:54.691696+0000","last_fullsized":"2023-09-23T21:12:54.691696+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":106,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.678713+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:12:53.678713+0000","last_peered":"2023-09-23T21:12:53.678713+0000","last_clean":"2023-09-23T21:12:53.678713+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:12:53.678713+0000","last_undegraded":"2023-09-23T21:12:53.678713+0000","last_fullsized":"2023-09-23T21:12:53.678713+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":97,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.669982+0000","last_change":"2023-09-23T21:10:15.584242+0000","last_active":"2023-09-23T21:12:53.669982+0000","last_peered":"2023-09-23T21:12:53.669982+0000","last_clean":"2023-09-23T21:12:53.669982+0000","last_became_active":"2023-09-23T21:10:15.583926+0000","last_became_peered":"2023-09-23T21:10:15.583926+0000","last_unstale":"2023-09-23T21:12:53.669982+0000","last_undegraded":"2023-09-23T21:12:53.669982+0000","last_fullsized":"2023-09-23T21:12:53.669982+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:15.759114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":115,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.679170+0000","last_change":"2023-09-23T21:12:51.652067+0000","last_active":"2023-09-23T21:12:53.679170+0000","last_peered":"2023-09-23T21:12:53.679170+0000","last_clean":"2023-09-23T21:12:53.679170+0000","last_became_active":"2023-09-23T21:12:51.651751+0000","last_became_peered":"2023-09-23T21:12:51.651751+0000","last_unstale":"2023-09-23T21:12:53.679170+0000","last_undegraded":"2023-09-23T21:12:53.679170+0000","last_fullsized":"2023-09-23T21:12:53.679170+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:03.235559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":19,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.671938+0000","last_change":"2023-09-23T21:12:51.657657+0000","last_active":"2023-09-23T21:12:53.671938+0000","last_peered":"2023-09-23T21:12:53.671938+0000","last_clean":"2023-09-23T21:12:53.671938+0000","last_became_active":"2023-09-23T21:12:51.641567+0000","last_became_peered":"2023-09-23T21:12:51.641567+0000","last_unstale":"2023-09-23T21:12:53.671938+0000","last_undegraded":"2023-09-23T21:12:53.671938+0000","last_fullsized":"2023-09-23T21:12:53.671938+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:00.366548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":98,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675514+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:12:53.675514+0000","last_peered":"2023-09-23T21:12:53.675514+0000","last_clean":"2023-09-23T21:12:53.675514+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:12:53.675514+0000","last_undegraded":"2023-09-23T21:12:53.675514+0000","last_fullsized":"2023-09-23T21:12:53.675514+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":51,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.671249+0000","last_change":"2023-09-23T21:10:15.594033+0000","last_active":"2023-09-23T21:12:53.671249+0000","last_peered":"2023-09-23T21:12:53.671249+0000","last_clean":"2023-09-23T21:12:53.671249+0000","last_became_active":"2023-09-23T21:10:15.579277+0000","last_became_peered":"2023-09-23T21:10:15.579277+0000","last_unstale":"2023-09-23T21:12:53.671249+0000","last_undegraded":"2023-09-23T21:12:53.671249+0000","last_fullsized":"2023-09-23T21:12:53.671249+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:42.561285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":382,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.682798+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:12:53.682798+0000","last_peered":"2023-09-23T21:12:53.682798+0000","last_clean":"2023-09-23T21:12:53.682798+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:12:53.682798+0000","last_undegraded":"2023-09-23T21:12:53.682798+0000","last_fullsized":"2023-09-23T21:12:53.682798+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":47,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.674251+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:12:53.674251+0000","last_peered":"2023-09-23T21:12:53.674251+0000","last_clean":"2023-09-23T21:12:53.674251+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:12:53.674251+0000","last_undegraded":"2023-09-23T21:12:53.674251+0000","last_fullsized":"2023-09-23T21:12:53.674251+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":124,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.680341+0000","last_change":"2023-09-23T21:10:15.590805+0000","last_active":"2023-09-23T21:12:53.680341+0000","last_peered":"2023-09-23T21:12:53.680341+0000","last_clean":"2023-09-23T21:12:53.680341+0000","last_became_active":"2023-09-23T21:10:15.590501+0000","last_became_peered":"2023-09-23T21:10:15.590501+0000","last_unstale":"2023-09-23T21:12:53.680341+0000","last_undegraded":"2023-09-23T21:12:53.680341+0000","last_fullsized":"2023-09-23T21:12:53.680341+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:27.143631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.671401+0000","last_change":"2023-09-23T21:12:51.643047+0000","last_active":"2023-09-23T21:12:53.671401+0000","last_peered":"2023-09-23T21:12:53.671401+0000","last_clean":"2023-09-23T21:12:53.671401+0000","last_became_active":"2023-09-23T21:12:51.642754+0000","last_became_peered":"2023-09-23T21:12:51.642754+0000","last_unstale":"2023-09-23T21:12:53.671401+0000","last_undegraded":"2023-09-23T21:12:53.671401+0000","last_fullsized":"2023-09-23T21:12:53.671401+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:31.430089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":41,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.673388+0000","last_change":"2023-09-23T21:10:15.585218+0000","last_active":"2023-09-23T21:12:53.673388+0000","last_peered":"2023-09-23T21:12:53.673388+0000","last_clean":"2023-09-23T21:12:53.673388+0000","last_became_active":"2023-09-23T21:10:15.584952+0000","last_became_peered":"2023-09-23T21:10:15.584952+0000","last_unstale":"2023-09-23T21:12:53.673388+0000","last_undegraded":"2023-09-23T21:12:53.673388+0000","last_fullsized":"2023-09-23T21:12:53.673388+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:55.016769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":41,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.673586+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:12:53.673586+0000","last_peered":"2023-09-23T21:12:53.673586+0000","last_clean":"2023-09-23T21:12:53.673586+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:12:53.673586+0000","last_undegraded":"2023-09-23T21:12:53.673586+0000","last_fullsized":"2023-09-23T21:12:53.673586+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":113,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.691857+0000","last_change":"2023-09-23T21:12:51.650915+0000","last_active":"2023-09-23T21:12:54.691857+0000","last_peered":"2023-09-23T21:12:54.691857+0000","last_clean":"2023-09-23T21:12:54.691857+0000","last_became_active":"2023-09-23T21:12:51.650550+0000","last_became_peered":"2023-09-23T21:12:51.650550+0000","last_unstale":"2023-09-23T21:12:54.691857+0000","last_undegraded":"2023-09-23T21:12:54.691857+0000","last_fullsized":"2023-09-23T21:12:54.691857+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:30.158548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":460,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.672441+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:12:53.672441+0000","last_peered":"2023-09-23T21:12:53.672441+0000","last_clean":"2023-09-23T21:12:53.672441+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:12:53.672441+0000","last_undegraded":"2023-09-23T21:12:53.672441+0000","last_fullsized":"2023-09-23T21:12:53.672441+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":124,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.679416+0000","last_change":"2023-09-23T21:12:51.645964+0000","last_active":"2023-09-23T21:12:53.679416+0000","last_peered":"2023-09-23T21:12:53.679416+0000","last_clean":"2023-09-23T21:12:53.679416+0000","last_became_active":"2023-09-23T21:12:51.645627+0000","last_became_peered":"2023-09-23T21:12:51.645627+0000","last_unstale":"2023-09-23T21:12:53.679416+0000","last_undegraded":"2023-09-23T21:12:53.679416+0000","last_fullsized":"2023-09-23T21:12:53.679416+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:53.725127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"464'9120","reported_seq":9166,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:55.939456+0000","last_change":"2023-09-23T21:09:45.077841+0000","last_active":"2023-09-23T21:12:55.939456+0000","last_peered":"2023-09-23T21:12:55.939456+0000","last_clean":"2023-09-23T21:12:55.939456+0000","last_became_active":"2023-09-23T21:09:45.077550+0000","last_became_peered":"2023-09-23T21:09:45.077550+0000","last_unstale":"2023-09-23T21:12:55.939456+0000","last_undegraded":"2023-09-23T21:12:55.939456+0000","last_fullsized":"2023-09-23T21:12:55.939456+0000","mapping_epoch":426,"log_start":"430'400","ondisk_log_start":"430'400","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8720,"log_dups_size":0,"ondisk_log_size":8720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:19.178954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138149888,"num_objects":2108,"num_object_clones":0,"num_object_copies":6324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9120,"num_write_kb":359296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":17,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.671804+0000","last_change":"2023-09-23T21:12:51.662783+0000","last_active":"2023-09-23T21:12:53.671804+0000","last_peered":"2023-09-23T21:12:53.671804+0000","last_clean":"2023-09-23T21:12:53.671804+0000","last_became_active":"2023-09-23T21:12:51.649440+0000","last_became_peered":"2023-09-23T21:12:51.649440+0000","last_unstale":"2023-09-23T21:12:53.671804+0000","last_undegraded":"2023-09-23T21:12:53.671804+0000","last_fullsized":"2023-09-23T21:12:53.671804+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:11.808407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.669502+0000","last_change":"2023-09-23T21:12:51.650942+0000","last_active":"2023-09-23T21:12:53.669502+0000","last_peered":"2023-09-23T21:12:53.669502+0000","last_clean":"2023-09-23T21:12:53.669502+0000","last_became_active":"2023-09-23T21:12:51.650556+0000","last_became_peered":"2023-09-23T21:12:51.650556+0000","last_unstale":"2023-09-23T21:12:53.669502+0000","last_undegraded":"2023-09-23T21:12:53.669502+0000","last_fullsized":"2023-09-23T21:12:53.669502+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:06.677094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"464'9039","reported_seq":11506,"reported_epoch":464,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T21:12:56.013140+0000","last_change":"2023-09-23T21:12:52.832316+0000","last_active":"2023-09-23T21:12:56.013140+0000","last_peered":"2023-09-23T21:12:56.013140+0000","last_clean":"2023-09-23T21:12:51.311277+0000","last_became_active":"2023-09-23T21:12:52.796692+0000","last_became_peered":"2023-09-23T21:12:52.796692+0000","last_unstale":"2023-09-23T21:12:56.013140+0000","last_undegraded":"2023-09-23T21:12:56.013140+0000","last_fullsized":"2023-09-23T21:12:56.013140+0000","mapping_epoch":461,"log_start":"430'400","ondisk_log_start":"430'400","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8639,"log_dups_size":0,"ondisk_log_size":8639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:00.802290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135593984,"num_objects":2069,"num_object_clones":0,"num_object_copies":6207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":892,"num_objects_unfound":0,"num_objects_dirty":2069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9039,"num_write_kb":355456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1204,"num_bytes_recovered":78905344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":2069}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":45,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.674006+0000","last_change":"2023-09-23T21:10:15.596183+0000","last_active":"2023-09-23T21:12:53.674006+0000","last_peered":"2023-09-23T21:12:53.674006+0000","last_clean":"2023-09-23T21:12:53.674006+0000","last_became_active":"2023-09-23T21:10:15.586221+0000","last_became_peered":"2023-09-23T21:10:15.586221+0000","last_unstale":"2023-09-23T21:12:53.674006+0000","last_undegraded":"2023-09-23T21:12:53.674006+0000","last_fullsized":"2023-09-23T21:12:53.674006+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:06.222859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":427,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.680826+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:12:54.680826+0000","last_peered":"2023-09-23T21:12:54.680826+0000","last_clean":"2023-09-23T21:12:54.680826+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:12:54.680826+0000","last_undegraded":"2023-09-23T21:12:54.680826+0000","last_fullsized":"2023-09-23T21:12:54.680826+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"464'9114","reported_seq":13817,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:55.440668+0000","last_change":"2023-09-23T21:11:08.638059+0000","last_active":"2023-09-23T21:12:55.440668+0000","last_peered":"2023-09-23T21:12:55.440668+0000","last_clean":"2023-09-23T21:12:55.440668+0000","last_became_active":"2023-09-23T21:11:08.630938+0000","last_became_peered":"2023-09-23T21:11:08.630938+0000","last_unstale":"2023-09-23T21:12:55.440668+0000","last_undegraded":"2023-09-23T21:12:55.440668+0000","last_fullsized":"2023-09-23T21:12:55.440668+0000","mapping_epoch":440,"log_start":"431'1800","ondisk_log_start":"431'1800","created":426,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7314,"log_dups_size":0,"ondisk_log_size":7314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:33.746589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136970240,"num_objects":2090,"num_object_clones":0,"num_object_copies":6270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9114,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2319,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":45,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.673486+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:12:53.673486+0000","last_peered":"2023-09-23T21:12:53.673486+0000","last_clean":"2023-09-23T21:12:53.673486+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:12:53.673486+0000","last_undegraded":"2023-09-23T21:12:53.673486+0000","last_fullsized":"2023-09-23T21:12:53.673486+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":362,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.672556+0000","last_change":"2023-09-23T21:12:51.650231+0000","last_active":"2023-09-23T21:12:53.672556+0000","last_peered":"2023-09-23T21:12:53.672556+0000","last_clean":"2023-09-23T21:12:53.672556+0000","last_became_active":"2023-09-23T21:12:51.649867+0000","last_became_peered":"2023-09-23T21:12:51.649867+0000","last_unstale":"2023-09-23T21:12:53.672556+0000","last_undegraded":"2023-09-23T21:12:53.672556+0000","last_fullsized":"2023-09-23T21:12:53.672556+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:56.792036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"465'9025","reported_seq":13602,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-09-23T21:12:58.952195+0000","last_change":"2023-09-23T21:10:39.657127+0000","last_active":"2023-09-23T21:12:58.952195+0000","last_peered":"2023-09-23T21:12:58.952195+0000","last_clean":"2023-09-23T21:12:58.952195+0000","last_became_active":"2023-09-23T21:10:39.652435+0000","last_became_peered":"2023-09-23T21:10:39.652435+0000","last_unstale":"2023-09-23T21:12:58.952195+0000","last_undegraded":"2023-09-23T21:12:58.952195+0000","last_fullsized":"2023-09-23T21:12:58.952195+0000","mapping_epoch":436,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9025,"log_dups_size":0,"ondisk_log_size":9025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:46.462306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135331840,"num_objects":2065,"num_object_clones":0,"num_object_copies":6195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9025,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2255,"num_bytes_recovered":147783680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.680418+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:12:53.680418+0000","last_peered":"2023-09-23T21:12:53.680418+0000","last_clean":"2023-09-23T21:12:53.680418+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:12:53.680418+0000","last_undegraded":"2023-09-23T21:12:53.680418+0000","last_fullsized":"2023-09-23T21:12:53.680418+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":113,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.692747+0000","last_change":"2023-09-23T21:09:44.163930+0000","last_active":"2023-09-23T21:12:54.692747+0000","last_peered":"2023-09-23T21:12:54.692747+0000","last_clean":"2023-09-23T21:12:54.692747+0000","last_became_active":"2023-09-23T21:09:44.163614+0000","last_became_peered":"2023-09-23T21:09:44.163614+0000","last_unstale":"2023-09-23T21:12:54.692747+0000","last_undegraded":"2023-09-23T21:12:54.692747+0000","last_fullsized":"2023-09-23T21:12:54.692747+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:26.549908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"464'9387","reported_seq":9433,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:55.972710+0000","last_change":"2023-09-23T21:09:45.040470+0000","last_active":"2023-09-23T21:12:55.972710+0000","last_peered":"2023-09-23T21:12:55.972710+0000","last_clean":"2023-09-23T21:12:55.972710+0000","last_became_active":"2023-09-23T21:09:45.040298+0000","last_became_peered":"2023-09-23T21:09:45.040298+0000","last_unstale":"2023-09-23T21:12:55.972710+0000","last_undegraded":"2023-09-23T21:12:55.972710+0000","last_fullsized":"2023-09-23T21:12:55.972710+0000","mapping_epoch":426,"log_start":"430'700","ondisk_log_start":"430'700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8687,"log_dups_size":0,"ondisk_log_size":8687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:06.840153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140443648,"num_objects":2143,"num_object_clones":0,"num_object_copies":6429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9387,"num_write_kb":368960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":123,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.700172+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:12:53.700172+0000","last_peered":"2023-09-23T21:12:53.700172+0000","last_clean":"2023-09-23T21:12:53.700172+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:12:53.700172+0000","last_undegraded":"2023-09-23T21:12:53.700172+0000","last_fullsized":"2023-09-23T21:12:53.700172+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":106,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.680256+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:12:53.680256+0000","last_peered":"2023-09-23T21:12:53.680256+0000","last_clean":"2023-09-23T21:12:53.680256+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:12:53.680256+0000","last_undegraded":"2023-09-23T21:12:53.680256+0000","last_fullsized":"2023-09-23T21:12:53.680256+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"464'9049","reported_seq":9095,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:55.996714+0000","last_change":"2023-09-23T21:09:45.041603+0000","last_active":"2023-09-23T21:12:55.996714+0000","last_peered":"2023-09-23T21:12:55.996714+0000","last_clean":"2023-09-23T21:12:55.996714+0000","last_became_active":"2023-09-23T21:09:45.041377+0000","last_became_peered":"2023-09-23T21:09:45.041377+0000","last_unstale":"2023-09-23T21:12:55.996714+0000","last_undegraded":"2023-09-23T21:12:55.996714+0000","last_fullsized":"2023-09-23T21:12:55.996714+0000","mapping_epoch":426,"log_start":"430'400","ondisk_log_start":"430'400","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8649,"log_dups_size":0,"ondisk_log_size":8649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:46.148257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138870784,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9049,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":390,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.672580+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:12:53.672580+0000","last_peered":"2023-09-23T21:12:53.672580+0000","last_clean":"2023-09-23T21:12:53.672580+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:12:53.672580+0000","last_undegraded":"2023-09-23T21:12:53.672580+0000","last_fullsized":"2023-09-23T21:12:53.672580+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":476,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T21:12:52.675422+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:12:52.675422+0000","last_peered":"2023-09-23T21:12:52.675422+0000","last_clean":"2023-09-23T21:12:52.675422+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:12:52.675422+0000","last_undegraded":"2023-09-23T21:12:52.675422+0000","last_fullsized":"2023-09-23T21:12:52.675422+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"464'9242","reported_seq":13927,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:57.246186+0000","last_change":"2023-09-23T21:11:37.353931+0000","last_active":"2023-09-23T21:12:57.246186+0000","last_peered":"2023-09-23T21:12:57.246186+0000","last_clean":"2023-09-23T21:12:57.246186+0000","last_became_active":"2023-09-23T21:11:37.344349+0000","last_became_peered":"2023-09-23T21:11:37.344349+0000","last_unstale":"2023-09-23T21:12:57.246186+0000","last_undegraded":"2023-09-23T21:12:57.246186+0000","last_fullsized":"2023-09-23T21:12:57.246186+0000","mapping_epoch":444,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7542,"log_dups_size":0,"ondisk_log_size":7542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:35.510223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141164544,"num_objects":2154,"num_object_clones":0,"num_object_copies":6462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9242,"num_write_kb":364672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2309,"num_bytes_recovered":151322624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":101,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.682365+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:12:53.682365+0000","last_peered":"2023-09-23T21:12:53.682365+0000","last_clean":"2023-09-23T21:12:53.682365+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:12:53.682365+0000","last_undegraded":"2023-09-23T21:12:53.682365+0000","last_fullsized":"2023-09-23T21:12:53.682365+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":503,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.672877+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:12:53.672877+0000","last_peered":"2023-09-23T21:12:53.672877+0000","last_clean":"2023-09-23T21:12:53.672877+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:12:53.672877+0000","last_undegraded":"2023-09-23T21:12:53.672877+0000","last_fullsized":"2023-09-23T21:12:53.672877+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"464'9251","reported_seq":9308,"reported_epoch":464,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:12:57.171461+0000","last_change":"2023-09-23T21:12:52.820481+0000","last_active":"2023-09-23T21:12:57.171461+0000","last_peered":"2023-09-23T21:12:57.171461+0000","last_clean":"2023-09-23T21:12:51.336402+0000","last_became_active":"2023-09-23T21:12:52.790762+0000","last_became_peered":"2023-09-23T21:12:52.790762+0000","last_unstale":"2023-09-23T21:12:57.171461+0000","last_undegraded":"2023-09-23T21:12:57.171461+0000","last_fullsized":"2023-09-23T21:12:57.171461+0000","mapping_epoch":461,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7551,"log_dups_size":0,"ondisk_log_size":7551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.461640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144113664,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2199,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9251,"num_write_kb":366400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":2199}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.678862+0000","last_change":"2023-09-23T21:10:15.584815+0000","last_active":"2023-09-23T21:12:53.678862+0000","last_peered":"2023-09-23T21:12:53.678862+0000","last_clean":"2023-09-23T21:12:53.678862+0000","last_became_active":"2023-09-23T21:10:15.584484+0000","last_became_peered":"2023-09-23T21:10:15.584484+0000","last_unstale":"2023-09-23T21:12:53.678862+0000","last_undegraded":"2023-09-23T21:12:53.678862+0000","last_fullsized":"2023-09-23T21:12:53.678862+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:58.137854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":212,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.679334+0000","last_change":"2023-09-23T21:09:44.164862+0000","last_active":"2023-09-23T21:12:53.679334+0000","last_peered":"2023-09-23T21:12:53.679334+0000","last_clean":"2023-09-23T21:12:53.679334+0000","last_became_active":"2023-09-23T21:09:44.164337+0000","last_became_peered":"2023-09-23T21:09:44.164337+0000","last_unstale":"2023-09-23T21:12:53.679334+0000","last_undegraded":"2023-09-23T21:12:53.679334+0000","last_fullsized":"2023-09-23T21:12:53.679334+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:57.424412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"463'9149","reported_seq":9196,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.630306+0000","last_change":"2023-09-23T21:09:45.033715+0000","last_active":"2023-09-23T21:12:54.630306+0000","last_peered":"2023-09-23T21:12:54.630306+0000","last_clean":"2023-09-23T21:12:54.630306+0000","last_became_active":"2023-09-23T21:09:45.033437+0000","last_became_peered":"2023-09-23T21:09:45.033437+0000","last_unstale":"2023-09-23T21:12:54.630306+0000","last_undegraded":"2023-09-23T21:12:54.630306+0000","last_fullsized":"2023-09-23T21:12:54.630306+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9149,"log_dups_size":0,"ondisk_log_size":9149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:15.193017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141754368,"num_objects":2163,"num_object_clones":0,"num_object_copies":6489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9149,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":370,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.680577+0000","last_change":"2023-09-23T21:10:15.588139+0000","last_active":"2023-09-23T21:12:53.680577+0000","last_peered":"2023-09-23T21:12:53.680577+0000","last_clean":"2023-09-23T21:12:53.680577+0000","last_became_active":"2023-09-23T21:10:15.587822+0000","last_became_peered":"2023-09-23T21:10:15.587822+0000","last_unstale":"2023-09-23T21:12:53.680577+0000","last_undegraded":"2023-09-23T21:12:53.680577+0000","last_fullsized":"2023-09-23T21:12:53.680577+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:47.570476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":129,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.669834+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:12:53.669834+0000","last_peered":"2023-09-23T21:12:53.669834+0000","last_clean":"2023-09-23T21:12:53.669834+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:12:53.669834+0000","last_undegraded":"2023-09-23T21:12:53.669834+0000","last_fullsized":"2023-09-23T21:12:53.669834+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":116,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675599+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:12:53.675599+0000","last_peered":"2023-09-23T21:12:53.675599+0000","last_clean":"2023-09-23T21:12:53.675599+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:12:53.675599+0000","last_undegraded":"2023-09-23T21:12:53.675599+0000","last_fullsized":"2023-09-23T21:12:53.675599+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"465'9283","reported_seq":9332,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.090135+0000","last_change":"2023-09-23T21:09:45.843731+0000","last_active":"2023-09-23T21:12:59.090135+0000","last_peered":"2023-09-23T21:12:59.090135+0000","last_clean":"2023-09-23T21:12:59.090135+0000","last_became_active":"2023-09-23T21:09:45.843440+0000","last_became_peered":"2023-09-23T21:09:45.843440+0000","last_unstale":"2023-09-23T21:12:59.090135+0000","last_undegraded":"2023-09-23T21:12:59.090135+0000","last_fullsized":"2023-09-23T21:12:59.090135+0000","mapping_epoch":426,"log_start":"430'200","ondisk_log_start":"430'200","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9083,"log_dups_size":0,"ondisk_log_size":9083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:17.052514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137953280,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9283,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":442,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675780+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:12:53.675780+0000","last_peered":"2023-09-23T21:12:53.675780+0000","last_clean":"2023-09-23T21:12:53.675780+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:12:53.675780+0000","last_undegraded":"2023-09-23T21:12:53.675780+0000","last_fullsized":"2023-09-23T21:12:53.675780+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":414,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.672883+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:12:53.672883+0000","last_peered":"2023-09-23T21:12:53.672883+0000","last_clean":"2023-09-23T21:12:53.672883+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:12:53.672883+0000","last_undegraded":"2023-09-23T21:12:53.672883+0000","last_fullsized":"2023-09-23T21:12:53.672883+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":98,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.680498+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:12:53.680498+0000","last_peered":"2023-09-23T21:12:53.680498+0000","last_clean":"2023-09-23T21:12:53.680498+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:12:53.680498+0000","last_undegraded":"2023-09-23T21:12:53.680498+0000","last_fullsized":"2023-09-23T21:12:53.680498+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"464'9161","reported_seq":9207,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:55.964979+0000","last_change":"2023-09-23T21:09:45.843960+0000","last_active":"2023-09-23T21:12:55.964979+0000","last_peered":"2023-09-23T21:12:55.964979+0000","last_clean":"2023-09-23T21:12:55.964979+0000","last_became_active":"2023-09-23T21:09:45.843562+0000","last_became_peered":"2023-09-23T21:09:45.843562+0000","last_unstale":"2023-09-23T21:12:55.964979+0000","last_undegraded":"2023-09-23T21:12:55.964979+0000","last_fullsized":"2023-09-23T21:12:55.964979+0000","mapping_epoch":426,"log_start":"430'500","ondisk_log_start":"430'500","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8661,"log_dups_size":0,"ondisk_log_size":8661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:10.329012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9161,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":442,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T21:12:52.674612+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:12:52.674612+0000","last_peered":"2023-09-23T21:12:52.674612+0000","last_clean":"2023-09-23T21:12:52.674612+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:12:52.674612+0000","last_undegraded":"2023-09-23T21:12:52.674612+0000","last_fullsized":"2023-09-23T21:12:52.674612+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":361,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675935+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:12:53.675935+0000","last_peered":"2023-09-23T21:12:53.675935+0000","last_clean":"2023-09-23T21:12:53.675935+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:12:53.675935+0000","last_undegraded":"2023-09-23T21:12:53.675935+0000","last_fullsized":"2023-09-23T21:12:53.675935+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":98,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.682877+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:12:53.682877+0000","last_peered":"2023-09-23T21:12:53.682877+0000","last_clean":"2023-09-23T21:12:53.682877+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:12:53.682877+0000","last_undegraded":"2023-09-23T21:12:53.682877+0000","last_fullsized":"2023-09-23T21:12:53.682877+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"464'9233","reported_seq":9288,"reported_epoch":464,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:12:57.232394+0000","last_change":"2023-09-23T21:12:52.821619+0000","last_active":"2023-09-23T21:12:57.232394+0000","last_peered":"2023-09-23T21:12:57.232394+0000","last_clean":"2023-09-23T21:12:51.339810+0000","last_became_active":"2023-09-23T21:12:52.789528+0000","last_became_peered":"2023-09-23T21:12:52.789528+0000","last_unstale":"2023-09-23T21:12:57.232394+0000","last_undegraded":"2023-09-23T21:12:57.232394+0000","last_fullsized":"2023-09-23T21:12:57.232394+0000","mapping_epoch":461,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7533,"log_dups_size":0,"ondisk_log_size":7533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:17.551283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139067416,"num_objects":2123,"num_object_clones":0,"num_object_copies":6369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2123,"num_objects_unfound":0,"num_objects_dirty":2123,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9233,"num_write_kb":363329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2,3],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2,3","objects":2123}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":289,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T21:12:52.674983+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:12:52.674983+0000","last_peered":"2023-09-23T21:12:52.674983+0000","last_clean":"2023-09-23T21:12:52.674983+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:12:52.674983+0000","last_undegraded":"2023-09-23T21:12:52.674983+0000","last_fullsized":"2023-09-23T21:12:52.674983+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":736,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.698525+0000","last_change":"2023-09-23T21:12:54.698525+0000","last_active":"2023-09-23T21:12:54.698525+0000","last_peered":"2023-09-23T21:12:54.698525+0000","last_clean":"2023-09-23T21:12:54.698525+0000","last_became_active":"2023-09-23T21:12:54.698074+0000","last_became_peered":"2023-09-23T21:12:54.698074+0000","last_unstale":"2023-09-23T21:12:54.698525+0000","last_undegraded":"2023-09-23T21:12:54.698525+0000","last_fullsized":"2023-09-23T21:12:54.698525+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:11.791067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":115,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.680893+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:12:53.680893+0000","last_peered":"2023-09-23T21:12:53.680893+0000","last_clean":"2023-09-23T21:12:53.680893+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:12:53.680893+0000","last_undegraded":"2023-09-23T21:12:53.680893+0000","last_fullsized":"2023-09-23T21:12:53.680893+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":362,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.673176+0000","last_change":"2023-09-23T21:10:15.582161+0000","last_active":"2023-09-23T21:12:53.673176+0000","last_peered":"2023-09-23T21:12:53.673176+0000","last_clean":"2023-09-23T21:12:53.673176+0000","last_became_active":"2023-09-23T21:10:15.581855+0000","last_became_peered":"2023-09-23T21:10:15.581855+0000","last_unstale":"2023-09-23T21:12:53.673176+0000","last_undegraded":"2023-09-23T21:12:53.673176+0000","last_fullsized":"2023-09-23T21:12:53.673176+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:19.873916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"465'9421","reported_seq":14282,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.177050+0000","last_change":"2023-09-23T21:12:11.354703+0000","last_active":"2023-09-23T21:12:59.177050+0000","last_peered":"2023-09-23T21:12:59.177050+0000","last_clean":"2023-09-23T21:12:59.177050+0000","last_became_active":"2023-09-23T21:12:11.333665+0000","last_became_peered":"2023-09-23T21:12:11.333665+0000","last_unstale":"2023-09-23T21:12:59.177050+0000","last_undegraded":"2023-09-23T21:12:59.177050+0000","last_fullsized":"2023-09-23T21:12:59.177050+0000","mapping_epoch":451,"log_start":"430'300","ondisk_log_start":"430'300","created":426,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9121,"log_dups_size":0,"ondisk_log_size":9121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:49.607296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140050432,"num_objects":2137,"num_object_clones":0,"num_object_copies":6411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9421,"num_write_kb":369856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2397,"num_bytes_recovered":117571584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":41,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.671234+0000","last_change":"2023-09-23T21:10:15.584319+0000","last_active":"2023-09-23T21:12:53.671234+0000","last_peered":"2023-09-23T21:12:53.671234+0000","last_clean":"2023-09-23T21:12:53.671234+0000","last_became_active":"2023-09-23T21:10:15.584030+0000","last_became_peered":"2023-09-23T21:10:15.584030+0000","last_unstale":"2023-09-23T21:12:53.671234+0000","last_undegraded":"2023-09-23T21:12:53.671234+0000","last_fullsized":"2023-09-23T21:12:53.671234+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:28.957511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":113,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.691930+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:12:54.691930+0000","last_peered":"2023-09-23T21:12:54.691930+0000","last_clean":"2023-09-23T21:12:54.691930+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:12:54.691930+0000","last_undegraded":"2023-09-23T21:12:54.691930+0000","last_fullsized":"2023-09-23T21:12:54.691930+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":104,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.691647+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:12:54.691647+0000","last_peered":"2023-09-23T21:12:54.691647+0000","last_clean":"2023-09-23T21:12:54.691647+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:12:54.691647+0000","last_undegraded":"2023-09-23T21:12:54.691647+0000","last_fullsized":"2023-09-23T21:12:54.691647+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"464'9161","reported_seq":9209,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:55.402189+0000","last_change":"2023-09-23T21:09:45.038161+0000","last_active":"2023-09-23T21:12:55.402189+0000","last_peered":"2023-09-23T21:12:55.402189+0000","last_clean":"2023-09-23T21:12:55.402189+0000","last_became_active":"2023-09-23T21:09:45.037907+0000","last_became_peered":"2023-09-23T21:09:45.037907+0000","last_unstale":"2023-09-23T21:12:55.402189+0000","last_undegraded":"2023-09-23T21:12:55.402189+0000","last_fullsized":"2023-09-23T21:12:55.402189+0000","mapping_epoch":426,"log_start":"431'1900","ondisk_log_start":"431'1900","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7261,"log_dups_size":0,"ondisk_log_size":7261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:37.785460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144637952,"num_objects":2207,"num_object_clones":0,"num_object_copies":6621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9161,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":41,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.672110+0000","last_change":"2023-09-23T21:10:15.584326+0000","last_active":"2023-09-23T21:12:53.672110+0000","last_peered":"2023-09-23T21:12:53.672110+0000","last_clean":"2023-09-23T21:12:53.672110+0000","last_became_active":"2023-09-23T21:10:15.583965+0000","last_became_peered":"2023-09-23T21:10:15.583965+0000","last_unstale":"2023-09-23T21:12:53.672110+0000","last_undegraded":"2023-09-23T21:12:53.672110+0000","last_fullsized":"2023-09-23T21:12:53.672110+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:21.120392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":363,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.681371+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:12:53.681371+0000","last_peered":"2023-09-23T21:12:53.681371+0000","last_clean":"2023-09-23T21:12:53.681371+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:12:53.681371+0000","last_undegraded":"2023-09-23T21:12:53.681371+0000","last_fullsized":"2023-09-23T21:12:53.681371+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":149,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.669660+0000","last_change":"2023-09-23T21:10:15.588413+0000","last_active":"2023-09-23T21:12:53.669660+0000","last_peered":"2023-09-23T21:12:53.669660+0000","last_clean":"2023-09-23T21:12:53.669660+0000","last_became_active":"2023-09-23T21:10:15.588077+0000","last_became_peered":"2023-09-23T21:10:15.588077+0000","last_unstale":"2023-09-23T21:12:53.669660+0000","last_undegraded":"2023-09-23T21:12:53.669660+0000","last_fullsized":"2023-09-23T21:12:53.669660+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:03.757383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"464'9095","reported_seq":9143,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:55.433932+0000","last_change":"2023-09-23T21:09:45.033672+0000","last_active":"2023-09-23T21:12:55.433932+0000","last_peered":"2023-09-23T21:12:55.433932+0000","last_clean":"2023-09-23T21:12:55.433932+0000","last_became_active":"2023-09-23T21:09:45.033296+0000","last_became_peered":"2023-09-23T21:09:45.033296+0000","last_unstale":"2023-09-23T21:12:55.433932+0000","last_undegraded":"2023-09-23T21:12:55.433932+0000","last_fullsized":"2023-09-23T21:12:55.433932+0000","mapping_epoch":426,"log_start":"431'1800","ondisk_log_start":"431'1800","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7295,"log_dups_size":0,"ondisk_log_size":7295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:26.841578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138346496,"num_objects":2111,"num_object_clones":0,"num_object_copies":6333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9095,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":136,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.679503+0000","last_change":"2023-09-23T21:10:15.593352+0000","last_active":"2023-09-23T21:12:53.679503+0000","last_peered":"2023-09-23T21:12:53.679503+0000","last_clean":"2023-09-23T21:12:53.679503+0000","last_became_active":"2023-09-23T21:10:15.589715+0000","last_became_peered":"2023-09-23T21:10:15.589715+0000","last_unstale":"2023-09-23T21:12:53.679503+0000","last_undegraded":"2023-09-23T21:12:53.679503+0000","last_fullsized":"2023-09-23T21:12:53.679503+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:24.563158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.680658+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:12:53.680658+0000","last_peered":"2023-09-23T21:12:53.680658+0000","last_clean":"2023-09-23T21:12:53.680658+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:12:53.680658+0000","last_undegraded":"2023-09-23T21:12:53.680658+0000","last_fullsized":"2023-09-23T21:12:53.680658+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":125,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.678601+0000","last_change":"2023-09-23T21:12:51.650884+0000","last_active":"2023-09-23T21:12:53.678601+0000","last_peered":"2023-09-23T21:12:53.678601+0000","last_clean":"2023-09-23T21:12:53.678601+0000","last_became_active":"2023-09-23T21:12:51.650560+0000","last_became_peered":"2023-09-23T21:12:51.650560+0000","last_unstale":"2023-09-23T21:12:53.678601+0000","last_undegraded":"2023-09-23T21:12:53.678601+0000","last_fullsized":"2023-09-23T21:12:53.678601+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:19.136242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"463'8899","reported_seq":8946,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.479348+0000","last_change":"2023-09-23T21:09:45.038290+0000","last_active":"2023-09-23T21:12:54.479348+0000","last_peered":"2023-09-23T21:12:54.479348+0000","last_clean":"2023-09-23T21:12:54.479348+0000","last_became_active":"2023-09-23T21:09:45.037976+0000","last_became_peered":"2023-09-23T21:09:45.037976+0000","last_unstale":"2023-09-23T21:12:54.479348+0000","last_undegraded":"2023-09-23T21:12:54.479348+0000","last_fullsized":"2023-09-23T21:12:54.479348+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8899,"log_dups_size":0,"ondisk_log_size":8899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:44.361487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143720448,"num_objects":2193,"num_object_clones":0,"num_object_copies":6579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8899,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":370,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.675272+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:12:53.675272+0000","last_peered":"2023-09-23T21:12:53.675272+0000","last_clean":"2023-09-23T21:12:53.675272+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:12:53.675272+0000","last_undegraded":"2023-09-23T21:12:53.675272+0000","last_fullsized":"2023-09-23T21:12:53.675272+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":108,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.691723+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:12:54.691723+0000","last_peered":"2023-09-23T21:12:54.691723+0000","last_clean":"2023-09-23T21:12:54.691723+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:12:54.691723+0000","last_undegraded":"2023-09-23T21:12:54.691723+0000","last_fullsized":"2023-09-23T21:12:54.691723+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":149,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.669577+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:12:53.669577+0000","last_peered":"2023-09-23T21:12:53.669577+0000","last_clean":"2023-09-23T21:12:53.669577+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:12:53.669577+0000","last_undegraded":"2023-09-23T21:12:53.669577+0000","last_fullsized":"2023-09-23T21:12:53.669577+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":122,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.692584+0000","last_change":"2023-09-23T21:10:15.588809+0000","last_active":"2023-09-23T21:12:54.692584+0000","last_peered":"2023-09-23T21:12:54.692584+0000","last_clean":"2023-09-23T21:12:54.692584+0000","last_became_active":"2023-09-23T21:10:15.588451+0000","last_became_peered":"2023-09-23T21:10:15.588451+0000","last_unstale":"2023-09-23T21:12:54.692584+0000","last_undegraded":"2023-09-23T21:12:54.692584+0000","last_fullsized":"2023-09-23T21:12:54.692584+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:58.509500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":497,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T21:12:52.675163+0000","last_change":"2023-09-23T21:09:44.163759+0000","last_active":"2023-09-23T21:12:52.675163+0000","last_peered":"2023-09-23T21:12:52.675163+0000","last_clean":"2023-09-23T21:12:52.675163+0000","last_became_active":"2023-09-23T21:09:44.163405+0000","last_became_peered":"2023-09-23T21:09:44.163405+0000","last_unstale":"2023-09-23T21:12:52.675163+0000","last_undegraded":"2023-09-23T21:12:52.675163+0000","last_fullsized":"2023-09-23T21:12:52.675163+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:52.346296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":126,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.682775+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:12:53.682775+0000","last_peered":"2023-09-23T21:12:53.682775+0000","last_clean":"2023-09-23T21:12:53.682775+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:12:53.682775+0000","last_undegraded":"2023-09-23T21:12:53.682775+0000","last_fullsized":"2023-09-23T21:12:53.682775+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":132,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.700255+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:12:53.700255+0000","last_peered":"2023-09-23T21:12:53.700255+0000","last_clean":"2023-09-23T21:12:53.700255+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:12:53.700255+0000","last_undegraded":"2023-09-23T21:12:53.700255+0000","last_fullsized":"2023-09-23T21:12:53.700255+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":345,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.673316+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:12:53.673316+0000","last_peered":"2023-09-23T21:12:53.673316+0000","last_clean":"2023-09-23T21:12:53.673316+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:12:53.673316+0000","last_undegraded":"2023-09-23T21:12:53.673316+0000","last_fullsized":"2023-09-23T21:12:53.673316+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.681007+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:12:53.681007+0000","last_peered":"2023-09-23T21:12:53.681007+0000","last_clean":"2023-09-23T21:12:53.681007+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:12:53.681007+0000","last_undegraded":"2023-09-23T21:12:53.681007+0000","last_fullsized":"2023-09-23T21:12:53.681007+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":107,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.679226+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:12:53.679226+0000","last_peered":"2023-09-23T21:12:53.679226+0000","last_clean":"2023-09-23T21:12:53.679226+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:12:53.679226+0000","last_undegraded":"2023-09-23T21:12:53.679226+0000","last_fullsized":"2023-09-23T21:12:53.679226+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.679038+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:12:53.679038+0000","last_peered":"2023-09-23T21:12:53.679038+0000","last_clean":"2023-09-23T21:12:53.679038+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:12:53.679038+0000","last_undegraded":"2023-09-23T21:12:53.679038+0000","last_fullsized":"2023-09-23T21:12:53.679038+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":110,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T21:12:54.691566+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:12:54.691566+0000","last_peered":"2023-09-23T21:12:54.691566+0000","last_clean":"2023-09-23T21:12:54.691566+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:12:54.691566+0000","last_undegraded":"2023-09-23T21:12:54.691566+0000","last_fullsized":"2023-09-23T21:12:54.691566+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":91,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T21:12:53.681508+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:12:53.681508+0000","last_peered":"2023-09-23T21:12:53.681508+0000","last_clean":"2023-09-23T21:12:53.681508+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:12:53.681508+0000","last_undegraded":"2023-09-23T21:12:53.681508+0000","last_fullsized":"2023-09-23T21:12:53.681508+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2239496216,"num_objects":34173,"num_object_clones":0,"num_object_copies":102519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5214,"num_objects_unfound":0,"num_objects_dirty":34173,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146629,"num_write_kb":5785601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10484,"num_bytes_recovered":647561216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6923366400,"data_stored":6923354184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133129,"ondisk_log_size":133129,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":28028,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201811,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1322936,"kb_used_data":668080,"kb_used_omap":0,"kb_used_meta":654848,"kb_avail":92422216,"statfs":{"total":95995035648,"available":94640349184,"internally_reserved":0,"allocated":684113920,"data_stored":682796536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":670564352},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":2.8570000000000002,"15min":1.5860000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":17.183,"5min":64.257000000000005,"15min":64.257000000000005},"last":3.2909999999999999},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.8149999999999999,"15min":1.5800000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":17.193000000000001,"5min":64.293000000000006,"15min":64.293000000000006},"last":3.3119999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:12:03 2023","interfaces":[{"interface":"back","average":{"1min":6.4729999999999999,"5min":6.4409999999999998,"15min":2.9689999999999999},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":76.007999999999996,"5min":114.16500000000001,"15min":114.16500000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":7.577,"5min":6.7690000000000001,"15min":3.125},"min":{"1min":0.36699999999999999,"5min":0.252,"15min":0.23799999999999999},"max":{"1min":75.926000000000002,"5min":114.372,"15min":114.372},"last":2.4969999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":8.4199999999999999,"5min":5.4859999999999998,"15min":3.6299999999999999},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":133.524,"5min":183.44200000000001,"15min":183.44200000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":8.7460000000000004,"5min":5.4530000000000003,"15min":3.5499999999999998},"min":{"1min":0.35999999999999999,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":133.55699999999999,"5min":183.46199999999999,"15min":183.46199999999999},"last":0.70499999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":12.335000000000001,"5min":5.2869999999999999,"15min":3.1840000000000002},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":118.15900000000001,"5min":118.15900000000001,"15min":118.15900000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":12.760999999999999,"5min":5.7560000000000002,"15min":3.2200000000000002},"min":{"1min":0.34799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":118.098,"5min":118.098,"15min":118.098},"last":3.7000000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.72499999999999998,"15min":0.78300000000000003},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":4.165,"5min":13.739000000000001,"15min":15.942},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.119,"5min":0.79100000000000004,"15min":0.70999999999999996},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.26500000000000001},"max":{"1min":6.8920000000000003,"5min":6.8920000000000003,"15min":16.725999999999999},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.93300000000000005,"15min":0.90500000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.1449999999999996,"5min":10.891,"15min":10.891},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.93799999999999994,"15min":0.81999999999999995},"min":{"1min":0.35799999999999998,"5min":0.33300000000000002,"15min":0.28399999999999997},"max":{"1min":3.9039999999999999,"5min":13.867000000000001,"15min":13.867000000000001},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":7.7460000000000004,"5min":2.8090000000000002,"15min":1.607},"min":{"1min":0.35799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":143.18700000000001,"5min":143.18700000000001,"15min":143.18700000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":7.6180000000000003,"5min":2.8559999999999999,"15min":1.5649999999999999},"min":{"1min":0.33800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":142.70400000000001,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.373}]}]},{"osd":6,"up_from":47,"seq":201863463431,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1581800,"kb_used_data":966944,"kb_used_omap":0,"kb_used_meta":614848,"kb_avail":92163352,"statfs":{"total":95995035648,"available":94375272448,"internally_reserved":0,"allocated":990150656,"data_stored":988849303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":629604352},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":10.954000000000001,"5min":3.444,"15min":2.536},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.20999999999999999},"max":{"1min":149.61600000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":4.6139999999999999},{"interface":"front","average":{"1min":10.253,"5min":3.1890000000000001,"15min":2.4430000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":149.303,"5min":149.303,"15min":149.303},"last":0.25}]},{"osd":1,"last update":"Sat Sep 23 21:12:21 2023","interfaces":[{"interface":"back","average":{"1min":11.156000000000001,"5min":4.4980000000000002,"15min":3.2869999999999999},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":129.37899999999999,"5min":129.37899999999999,"15min":129.37899999999999},"last":1.0840000000000001},{"interface":"front","average":{"1min":12.004,"5min":4.5469999999999997,"15min":3.2999999999999998},"min":{"1min":0.24399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":129.40100000000001,"5min":129.40100000000001,"15min":129.40100000000001},"last":1.0349999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":5.8659999999999997,"15min":5.1379999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":133.46600000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":2.2949999999999999},{"interface":"front","average":{"1min":10.975,"5min":5.3630000000000004,"15min":5.0069999999999997},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":133.33000000000001,"5min":133.33000000000001,"15min":199.08000000000001},"last":0.82699999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":15.163,"5min":5.6159999999999997,"15min":3.9039999999999999},"min":{"1min":0.30499999999999999,"5min":0.30399999999999999,"15min":0.26900000000000002},"max":{"1min":137.875,"5min":137.875,"15min":178.40000000000001},"last":2.8780000000000001},{"interface":"front","average":{"1min":13.106999999999999,"5min":5.2590000000000003,"15min":3.8620000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":137.60900000000001,"5min":137.60900000000001,"15min":178.26599999999999},"last":0.318}]},{"osd":4,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.748,"15min":0.86299999999999999},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.4489999999999998,"5min":9.0890000000000004,"15min":13.1},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.66100000000000003,"15min":0.73199999999999998},"min":{"1min":0.28299999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.78000000000000003,"5min":6.4400000000000004,"15min":6.4400000000000004},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.75700000000000001,"15min":0.876},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.419,"5min":9.0519999999999996,"15min":11.281000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.63800000000000001,"15min":0.85599999999999998},"min":{"1min":0.32200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.52300000000000002,"5min":6.0190000000000001,"15min":8.6229999999999993},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:12:21 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.76400000000000001,"15min":0.86799999999999999},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.60399999999999998,"5min":6.6299999999999999,"15min":7.8079999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.83099999999999996,"15min":0.90000000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0259999999999998,"5min":6.4749999999999996,"15min":7.1849999999999996},"last":0.434}]}]},{"osd":1,"up_from":276,"seq":1185410973878,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1388692,"kb_used_data":822540,"kb_used_omap":5,"kb_used_meta":566138,"kb_avail":92356460,"statfs":{"total":95995035648,"available":94573015040,"internally_reserved":0,"allocated":842280960,"data_stored":839998580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":579726051},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":8.4529999999999994,"5min":4.2279999999999998,"15min":1.837},"min":{"1min":0.23400000000000001,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":108.47,"5min":108.47,"15min":108.47},"last":2.02},{"interface":"front","average":{"1min":8.4969999999999999,"5min":4.0640000000000001,"15min":1.96},"min":{"1min":0.248,"5min":0.19600000000000001,"15min":0.193},"max":{"1min":109.532,"5min":109.532,"15min":109.532},"last":1.2070000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":9.7759999999999998,"5min":6.2039999999999997,"15min":3.0369999999999999},"min":{"1min":0.23200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":97.840999999999994,"5min":107.883,"15min":107.883},"last":1.1180000000000001},{"interface":"front","average":{"1min":8.5709999999999997,"5min":5.4649999999999999,"15min":2.6629999999999998},"min":{"1min":0.375,"5min":0.182,"15min":0.182},"max":{"1min":97.816999999999993,"5min":101.024,"15min":101.024},"last":2.0569999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":11.587999999999999,"5min":7.6180000000000003,"15min":4.7779999999999996},"min":{"1min":0.255,"5min":0.255,"15min":0.23899999999999999},"max":{"1min":99.120999999999995,"5min":99.120999999999995,"15min":172.44399999999999},"last":2.9129999999999998},{"interface":"front","average":{"1min":11.824,"5min":7.093,"15min":4.8769999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":101.849,"5min":101.849,"15min":172.58500000000001},"last":5.6040000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":1.861,"15min":1.393},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.728999999999999,"5min":11.728999999999999,"15min":17.006},"last":1.7789999999999999},{"interface":"front","average":{"1min":1.022,"5min":1.994,"15min":1.1839999999999999},"min":{"1min":0.29399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.8239999999999998,"5min":23.332999999999998,"15min":23.332999999999998},"last":2.0790000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":2.2949999999999999,"15min":1.4650000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.5800000000000001,"5min":16.196000000000002,"15min":17.155000000000001},"last":1.1759999999999999},{"interface":"front","average":{"1min":2.3839999999999999,"5min":2.1070000000000002,"15min":1.532},"min":{"1min":0.33800000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.760999999999999,"5min":11.760999999999999,"15min":17.076000000000001},"last":1.8120000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":2.391,"15min":1.9930000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.7549999999999999,"5min":16.222000000000001,"15min":127.995},"last":1.147},{"interface":"front","average":{"1min":2.528,"5min":2.173,"15min":2.0209999999999999},"min":{"1min":0.378,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.782999999999999,"5min":11.782999999999999,"15min":126.836},"last":1.883}]},{"osd":7,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":2.1219999999999999,"15min":2.0089999999999999},"min":{"1min":0.441,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":151.791},"last":1.849},{"interface":"front","average":{"1min":1.0669999999999999,"5min":2.1419999999999999,"15min":1.75},"min":{"1min":0.38100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.9409999999999998,"5min":23.376000000000001,"15min":151.809},"last":2.101}]}]},{"osd":0,"up_from":92,"seq":395136991618,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1663376,"kb_used_data":807304,"kb_used_omap":5,"kb_used_meta":855994,"kb_avail":92081776,"statfs":{"total":95995035648,"available":94291738624,"internally_reserved":0,"allocated":826679296,"data_stored":824515931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":876538490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:12:08 2023","interfaces":[{"interface":"back","average":{"1min":3.1989999999999998,"5min":3.524,"15min":2.4889999999999999},"min":{"1min":0.22,"5min":0.182,"15min":0.17799999999999999},"max":{"1min":11.782999999999999,"5min":53.262,"15min":61.341999999999999},"last":2.786},{"interface":"front","average":{"1min":2.9220000000000002,"5min":4.056,"15min":2.589},"min":{"1min":0.249,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.615,"5min":53.201999999999998,"15min":61.299999999999997},"last":4.4489999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":4.5199999999999996,"15min":2.8959999999999999},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.161},"max":{"1min":18.280000000000001,"5min":127.791,"15min":183.77099999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":2.5659999999999998,"5min":4.4240000000000004,"15min":3.0219999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.22600000000000001},"max":{"1min":10.513999999999999,"5min":127.953,"15min":183.78700000000001},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:12:00 2023","interfaces":[{"interface":"back","average":{"1min":3.9449999999999998,"5min":4.3879999999999999,"15min":2.948},"min":{"1min":0.30099999999999999,"5min":0.23999999999999999,"15min":0.183},"max":{"1min":20.489000000000001,"5min":86.989999999999995,"15min":186.143},"last":4.5090000000000003},{"interface":"front","average":{"1min":3.7450000000000001,"5min":4.2770000000000001,"15min":2.9950000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23000000000000001},"max":{"1min":23.494,"5min":87.043999999999997,"15min":186.97399999999999},"last":4.4180000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.613,"5min":1.7809999999999999,"15min":1.329},"min":{"1min":0.38,"5min":0.30299999999999999,"15min":0.29799999999999999},"max":{"1min":10.032,"5min":10.814,"15min":10.814},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.1680000000000001,"5min":1.657,"15min":1.042},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":11.753,"5min":11.753,"15min":11.753},"last":2.8090000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":1.8089999999999999,"15min":1.4019999999999999},"min":{"1min":0.34699999999999998,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":10.08,"5min":10.763,"15min":10.763},"last":0.66900000000000004},{"interface":"front","average":{"1min":2.4489999999999998,"5min":1.742,"15min":1.111},"min":{"1min":0.34000000000000002,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":11.589,"5min":11.589,"15min":11.589},"last":2.7200000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":1.8620000000000001,"15min":3.2989999999999999},"min":{"1min":0.35499999999999998,"5min":0.35199999999999998,"15min":0.30599999999999999},"max":{"1min":11.722,"5min":17.773,"15min":170.12700000000001},"last":2.8330000000000002},{"interface":"front","average":{"1min":2.1480000000000001,"5min":1.859,"15min":3.3879999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.28299999999999997},"max":{"1min":7.452,"5min":17.684999999999999,"15min":169.767},"last":0.42099999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:12:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":1.9690000000000001,"15min":1.234},"min":{"1min":0.41499999999999998,"5min":0.38200000000000001,"15min":0.33600000000000002},"max":{"1min":11.69,"5min":11.69,"15min":11.69},"last":2.7610000000000001},{"interface":"front","average":{"1min":2.3380000000000001,"5min":1.8839999999999999,"15min":1.2889999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.371},"max":{"1min":10.470000000000001,"5min":10.470000000000001,"15min":10.470000000000001},"last":0.45100000000000001}]}]},{"osd":2,"up_from":129,"seq":554050781508,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2272020,"kb_used_data":1423372,"kb_used_omap":7,"kb_used_meta":848504,"kb_avail":91473132,"statfs":{"total":95995035648,"available":93668487168,"internally_reserved":0,"allocated":1457532928,"data_stored":1455296803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":868868177},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":3.2629999999999999,"15min":1.8320000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17699999999999999},"max":{"1min":103.021,"5min":103.021,"15min":103.021},"last":0.33500000000000002},{"interface":"front","average":{"1min":8.0449999999999999,"5min":4.069,"15min":2.5099999999999998},"min":{"1min":0.22700000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":103.154,"5min":103.154,"15min":103.154},"last":0.42799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:12:12 2023","interfaces":[{"interface":"back","average":{"1min":6.7569999999999997,"5min":5.141,"15min":3.8519999999999999},"min":{"1min":0.22500000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":106.268,"5min":143.482,"15min":212.12},"last":2.9020000000000001},{"interface":"front","average":{"1min":8.0199999999999996,"5min":4.9980000000000002,"15min":3.7170000000000001},"min":{"1min":0.245,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":106.32299999999999,"5min":143.61000000000001,"15min":213.02799999999999},"last":0.70599999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:11:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8779999999999999,"5min":7.2880000000000003,"15min":4.1079999999999997},"min":{"1min":0.36799999999999999,"5min":0.216,"15min":0.21199999999999999},"max":{"1min":6.3949999999999996,"5min":143.50800000000001,"15min":143.50800000000001},"last":2.9239999999999999},{"interface":"front","average":{"1min":2.2229999999999999,"5min":7.3090000000000002,"15min":3.9369999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.215},"max":{"1min":6.0209999999999999,"5min":143.27799999999999,"15min":143.27799999999999},"last":1.1879999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":4.1120000000000001,"15min":2.0150000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":9.8170000000000002,"5min":225.80799999999999,"15min":225.80799999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.1579999999999999,"5min":4.9660000000000002,"15min":2.5670000000000002},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":8.109,"5min":225.79300000000001,"15min":225.79300000000001},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":3.54,"15min":2.1589999999999998},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.29999999999999999},"max":{"1min":8.1489999999999991,"5min":98.682000000000002,"15min":98.682000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.718,"5min":4.4610000000000003,"15min":2.2200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.9450000000000003,"5min":143.34700000000001,"15min":143.34700000000001},"last":2.843}]},{"osd":6,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":3.2490000000000001,"15min":2.5579999999999998},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":6.9969999999999999,"5min":143.41399999999999,"15min":175.72499999999999},"last":2.879},{"interface":"front","average":{"1min":2.2429999999999999,"5min":1.6579999999999999,"15min":1.9419999999999999},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":11.893000000000001,"5min":11.893000000000001,"15min":176.20400000000001},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":1.7210000000000001,"15min":1.3939999999999999},"min":{"1min":0.41599999999999998,"5min":0.36399999999999999,"15min":0.32700000000000001},"max":{"1min":9.843,"5min":11.811999999999999,"15min":12.609},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.1920000000000002,"5min":2.5960000000000001,"15min":1.8480000000000001},"min":{"1min":0.41399999999999998,"5min":0.38200000000000001,"15min":0.34499999999999997},"max":{"1min":8.1720000000000006,"5min":16.875,"15min":16.875},"last":0.52000000000000002}]}]},{"osd":3,"up_from":148,"seq":635655160102,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2052108,"kb_used_data":1335492,"kb_used_omap":2,"kb_used_meta":716605,"kb_avail":91693044,"statfs":{"total":95995035648,"available":93893677056,"internally_reserved":0,"allocated":1367543808,"data_stored":1365079472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":733803573},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":2.7850000000000001,"15min":1.8500000000000001},"min":{"1min":0.53400000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.696999999999999,"5min":17.425999999999998,"15min":44.963000000000001},"last":10.196999999999999},{"interface":"front","average":{"1min":4.3959999999999999,"5min":2.48,"15min":1.534},"min":{"1min":0.46000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":14.503,"5min":16.920999999999999,"15min":19.738},"last":2.028}]},{"osd":1,"last update":"Sat Sep 23 21:12:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6219999999999999,"5min":15.395,"15min":5.9349999999999996},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.20999999999999999},"max":{"1min":14.867000000000001,"5min":237.072,"15min":237.072},"last":1.8480000000000001},{"interface":"front","average":{"1min":4.3460000000000001,"5min":15.266999999999999,"15min":6.04},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":16.178000000000001,"5min":234.21899999999999,"15min":234.21899999999999},"last":1.405}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":6.3200000000000003,"5min":6.1280000000000001,"15min":3.0499999999999998},"min":{"1min":0.55900000000000005,"5min":0.23599999999999999,"15min":0.223},"max":{"1min":24.378,"5min":93.272000000000006,"15min":93.272000000000006},"last":1.5189999999999999},{"interface":"front","average":{"1min":4.2370000000000001,"5min":5.2320000000000002,"15min":2.6160000000000001},"min":{"1min":0.58599999999999997,"5min":0.22500000000000001,"15min":0.222},"max":{"1min":10.907,"5min":95.233999999999995,"15min":95.233999999999995},"last":0.36399999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1859999999999999,"5min":2.4780000000000002,"15min":2.6520000000000001},"min":{"1min":0.36399999999999999,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":15.763,"5min":15.763,"15min":111.36},"last":1.4350000000000001},{"interface":"front","average":{"1min":2.835,"5min":2.0510000000000002,"15min":2.367},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.6180000000000003,"5min":17.286999999999999,"15min":112.473},"last":1.4470000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":2.9809999999999999,"5min":2.105,"15min":1.5820000000000001},"min":{"1min":0.501,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.3710000000000004,"5min":16.855,"15min":36.488999999999997},"last":0.443},{"interface":"front","average":{"1min":4.0949999999999998,"5min":2.6400000000000001,"15min":1.9990000000000001},"min":{"1min":0.40699999999999997,"5min":0.315,"15min":0.315},"max":{"1min":15.789999999999999,"5min":15.789999999999999,"15min":36.295000000000002},"last":1.546}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0599999999999996,"5min":6.6529999999999996,"15min":3.2650000000000001},"min":{"1min":0.434,"5min":0.33500000000000002,"15min":0.312},"max":{"1min":15.718,"5min":304.22399999999999,"15min":304.22399999999999},"last":1.615},{"interface":"front","average":{"1min":2.827,"5min":6.0510000000000002,"15min":2.964},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":7.5839999999999996,"5min":304.15800000000002,"15min":304.15800000000002},"last":1.4870000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":3.282,"5min":2.1829999999999998,"15min":1.897},"min":{"1min":0.52100000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":8.3309999999999995,"5min":16.899000000000001,"15min":74.019000000000005},"last":0.47299999999999998},{"interface":"front","average":{"1min":4.8339999999999996,"5min":2.9039999999999999,"15min":2.3719999999999999},"min":{"1min":0.436,"5min":0.377,"15min":0.32900000000000001},"max":{"1min":15.816000000000001,"5min":15.816000000000001,"15min":73.364999999999995},"last":1.5820000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986721,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1156852,"kb_used_data":543788,"kb_used_omap":0,"kb_used_meta":613056,"kb_avail":92588300,"statfs":{"total":95995035648,"available":94810419200,"internally_reserved":0,"allocated":556838912,"data_stored":555525333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":627769344},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":11.087999999999999,"5min":3.4750000000000001,"15min":2.0830000000000002},"min":{"1min":0.219,"5min":0.20699999999999999,"15min":0.20399999999999999},"max":{"1min":226.751,"5min":226.751,"15min":226.751},"last":33.173000000000002},{"interface":"front","average":{"1min":11.529999999999999,"5min":3.589,"15min":2.0939999999999999},"min":{"1min":0.253,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":226.43299999999999,"5min":226.43299999999999,"15min":226.43299999999999},"last":33.018000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:12:16 2023","interfaces":[{"interface":"back","average":{"1min":5.1900000000000004,"5min":3.5350000000000001,"15min":4.306},"min":{"1min":0.33400000000000002,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":68.337000000000003,"5min":68.337000000000003,"15min":127.411},"last":41.613999999999997},{"interface":"front","average":{"1min":4.9500000000000002,"5min":3.2949999999999999,"15min":3.6509999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":68.293000000000006,"5min":68.293000000000006,"15min":118.075},"last":36.229999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":14.257,"5min":5.7480000000000002,"15min":4.4900000000000002},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":210.50299999999999,"5min":210.50299999999999,"15min":210.50299999999999},"last":0.502},{"interface":"front","average":{"1min":15.454000000000001,"5min":5.8390000000000004,"15min":4.3890000000000002},"min":{"1min":0.33100000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":210.59,"5min":210.59,"15min":210.59},"last":0.438}]},{"osd":3,"last update":"Sat Sep 23 21:12:16 2023","interfaces":[{"interface":"back","average":{"1min":23.369,"5min":7.9000000000000004,"15min":3.5030000000000001},"min":{"1min":0.42099999999999999,"5min":0.29399999999999998,"15min":0.27300000000000002},"max":{"1min":214.64099999999999,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":23.645,"5min":7.8780000000000001,"15min":3.5870000000000002},"min":{"1min":0.54700000000000004,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":214.54900000000001,"5min":214.54900000000001,"15min":214.54900000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.69699999999999995},"min":{"1min":0.28199999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.2069999999999999,"5min":6.0069999999999997,"15min":8.7089999999999996},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.66200000000000003,"15min":0.69999999999999996},"min":{"1min":0.27400000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.881,"5min":4.7140000000000004,"15min":7.0179999999999998},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":1.2549999999999999,"15min":0.98099999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79300000000000004,"5min":36.557000000000002,"15min":36.557000000000002},"last":0.371},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.125,"15min":0.97199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":2.5760000000000001,"5min":38.468000000000004,"15min":38.468000000000004},"last":0.35599999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.80500000000000005,"15min":1.4610000000000001},"min":{"1min":0.309,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.8319999999999999,"5min":4.7359999999999998,"15min":145.721},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.871,"15min":1.601},"min":{"1min":0.33800000000000002,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":0.61099999999999999,"5min":10.411,"15min":148.45699999999999},"last":0.42799999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725085,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942948,"kb_used_data":279452,"kb_used_omap":0,"kb_used_meta":663488,"kb_avail":92802204,"statfs":{"total":95995035648,"available":95029456896,"internally_reserved":0,"allocated":286158848,"data_stored":284857397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":679411712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":2.8399999999999999,"15min":1.4690000000000001},"min":{"1min":0.25700000000000001,"5min":0.217,"15min":0.217},"max":{"1min":20.023,"5min":82.280000000000001,"15min":82.280000000000001},"last":5.8890000000000002},{"interface":"front","average":{"1min":3.0569999999999999,"5min":2.9620000000000002,"15min":1.5860000000000001},"min":{"1min":0.252,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":19.853000000000002,"5min":85.236000000000004,"15min":85.236000000000004},"last":2.3210000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:12:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6819999999999999,"5min":5.2919999999999998,"15min":4.1429999999999998},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.711,"5min":125.23399999999999,"15min":233.77099999999999},"last":1.72},{"interface":"front","average":{"1min":3.3399999999999999,"5min":5.2590000000000003,"15min":4.0389999999999997},"min":{"1min":0.34100000000000003,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":10.724,"5min":125.249,"15min":233.75200000000001},"last":1.7}]},{"osd":2,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":4.1660000000000004,"15min":3.4990000000000001},"min":{"1min":0.35199999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":20.411999999999999,"5min":78.917000000000002,"15min":231.34700000000001},"last":2.0169999999999999},{"interface":"front","average":{"1min":2.8460000000000001,"5min":4.2640000000000002,"15min":3.5939999999999999},"min":{"1min":0.29999999999999999,"5min":0.28499999999999998,"15min":0.27300000000000002},"max":{"1min":11.509,"5min":78.888999999999996,"15min":231.381},"last":5.7039999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":7.5039999999999996,"15min":3.9809999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":10.053000000000001,"5min":263.887,"15min":263.887},"last":0.753},{"interface":"front","average":{"1min":1.6519999999999999,"5min":7.2450000000000001,"15min":3.8170000000000002},"min":{"1min":0.32200000000000001,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":4.218,"5min":263.70100000000002,"15min":263.70100000000002},"last":2.1890000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.66800000000000004,"15min":0.60199999999999998},"min":{"1min":0.29299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.0449999999999999,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.37},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.73199999999999998,"15min":0.72299999999999998},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9849999999999999,"5min":5.8490000000000002,"15min":10.792},"last":0.40200000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.708,"15min":1.333},"min":{"1min":0.35699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7440000000000002,"5min":83.451999999999998,"15min":83.451999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.677,"15min":1.3720000000000001},"min":{"1min":0.30399999999999999,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":3.9889999999999999,"5min":83.484999999999999,"15min":83.484999999999999},"last":0.433}]},{"osd":7,"last update":"Sat Sep 23 21:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":0.95399999999999996,"15min":0.875},"min":{"1min":0.33200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.7160000000000002,"5min":9.6329999999999991,"15min":9.6329999999999991},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.74199999999999999,"15min":0.64800000000000002},"min":{"1min":0.32200000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.0790000000000002,"5min":4.29,"15min":4.29},"last":0.56799999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":813957120,"data_stored":813957120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":831262720,"data_stored":831258648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1449857024,"data_stored":1449852952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1336938496,"data_stored":1336934424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":548470784,"data_stored":548470784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":278790144,"data_stored":278790144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":979566592,"data_stored":979566592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":684523520,"data_stored":684523520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:13:02.379 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:13:02.380 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:13:02.499 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:02.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:02.618 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:02.649 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:02.699 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:02.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:02.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:02.811 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:02.908 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:02.912 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:03.097 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:03.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:03.201 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:03.273 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:03.610 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:03.633 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:03.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:03.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:03.748 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:03.847 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:04.044 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:04.179 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:04.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:04.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:04.206 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:13:04.292 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:04.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:04 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:04.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:04 smithi121 ceph-mon[211526]: pgmap v1511: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 3157/96915 objects misplaced (3.257%); 23 MiB/s, 370 objects/s recovering 2023-09-23T21:13:04.394 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:04 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:04.394 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:04 smithi121 ceph-mon[219709]: pgmap v1511: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 3157/96915 objects misplaced (3.257%); 23 MiB/s, 370 objects/s recovering 2023-09-23T21:13:04.399 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:04 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:04.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:04 smithi123 ceph-mon[193903]: pgmap v1511: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 3157/96915 objects misplaced (3.257%); 23 MiB/s, 370 objects/s recovering 2023-09-23T21:13:05.128 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:13:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:13:04] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.43.0" 2023-09-23T21:13:05.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3949211002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[211526]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3949211002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[211526]: osdmap e467: 8 total, 8 up, 8 in 2023-09-23T21:13:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[211526]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[211526]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3949211002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:05.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:05.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[219709]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3949211002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:05.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[219709]: osdmap e467: 8 total, 8 up, 8 in 2023-09-23T21:13:05.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[219709]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:05 smithi121 ceph-mon[219709]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3949211002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:05 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:05 smithi123 ceph-mon[193903]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3949211002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:05 smithi123 ceph-mon[193903]: osdmap e467: 8 total, 8 up, 8 in 2023-09-23T21:13:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:05 smithi123 ceph-mon[193903]: failed to encode map e467 with expected crc 2023-09-23T21:13:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:05 smithi123 ceph-mon[193903]: failed to encode map e467 with expected crc 2023-09-23T21:13:06.161 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:06.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:06.284 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:06.383 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:06.424 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:06 smithi121 ceph-mon[219709]: failed to encode map e467 with expected crc 2023-09-23T21:13:06.425 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:06 smithi121 ceph-mon[219709]: pgmap v1513: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 3157/94410 objects misplaced (3.344%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:13:06.425 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:06 smithi121 ceph-mon[211526]: failed to encode map e467 with expected crc 2023-09-23T21:13:06.425 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:06 smithi121 ceph-mon[211526]: pgmap v1513: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 3157/94410 objects misplaced (3.344%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:13:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:06 smithi123 ceph-mon[193903]: failed to encode map e467 with expected crc 2023-09-23T21:13:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:06 smithi123 ceph-mon[193903]: pgmap v1513: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 3157/94410 objects misplaced (3.344%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:13:06.944 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:06.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:06.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:07.056 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:07.082 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:07.572 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:07.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:07.685 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:07.715 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:07.783 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:07.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:07.848 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:07 smithi121 ceph-mon[211526]: pgmap v1514: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 3157/90417 objects misplaced (3.492%); 14 MiB/s, 219 objects/s recovering 2023-09-23T21:13:07.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:07 smithi121 ceph-mon[219709]: pgmap v1514: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 3157/90417 objects misplaced (3.492%); 14 MiB/s, 219 objects/s recovering 2023-09-23T21:13:07.889 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:07.916 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:07.966 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:07.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:07.974 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:07 smithi123 ceph-mon[193903]: pgmap v1514: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 3157/90417 objects misplaced (3.492%); 14 MiB/s, 219 objects/s recovering 2023-09-23T21:13:08.075 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:08.107 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:08.161 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:08.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:08.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:08.266 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:08.325 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:13:08.357 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:13:08.386 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:08.402 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:13:08.470 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:13:08.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:13:08.561 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:13:08.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:13:08.638 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:13:08.646 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:13:08.656 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.656 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.657 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:13:08.781 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:08.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:08.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:08.892 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1780038312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:09.019 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1780038312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:09.020 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1780038312' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:09.027 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:09.231 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:09.231 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:09.649 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:09.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:09.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:09.765 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:09.801 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:09.853 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:09.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:09.958 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:09.961 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:09 smithi121 ceph-mon[211526]: pgmap v1515: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 2347/87477 objects misplaced (2.683%); 15 MiB/s, 244 objects/s recovering 2023-09-23T21:13:09.961 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:09 smithi121 ceph-mon[219709]: pgmap v1515: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 2347/87477 objects misplaced (2.683%); 15 MiB/s, 244 objects/s recovering 2023-09-23T21:13:09.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:09 smithi123 ceph-mon[193903]: pgmap v1515: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 2347/87477 objects misplaced (2.683%); 15 MiB/s, 244 objects/s recovering 2023-09-23T21:13:09.999 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:13:11.201 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:11.308 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:11.372 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:11.388 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.476 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:11.580 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:11.606 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.651 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:11.756 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:11.786 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:11.840 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:11.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:11.949 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:11 smithi123 ceph-mon[193903]: failed to encode map e468 with expected crc 2023-09-23T21:13:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:11 smithi123 ceph-mon[193903]: osdmap e468: 8 total, 8 up, 8 in 2023-09-23T21:13:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:11 smithi123 ceph-mon[193903]: failed to encode map e468 with expected crc 2023-09-23T21:13:11.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:11 smithi123 ceph-mon[193903]: failed to encode map e468 with expected crc 2023-09-23T21:13:11.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:11 smithi123 ceph-mon[193903]: pgmap v1517: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2347/82791 objects misplaced (2.835%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:13:12.000 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[211526]: failed to encode map e468 with expected crc 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[211526]: osdmap e468: 8 total, 8 up, 8 in 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[211526]: failed to encode map e468 with expected crc 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[211526]: failed to encode map e468 with expected crc 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[211526]: pgmap v1517: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2347/82791 objects misplaced (2.835%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[219709]: failed to encode map e468 with expected crc 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[219709]: osdmap e468: 8 total, 8 up, 8 in 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[219709]: failed to encode map e468 with expected crc 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[219709]: failed to encode map e468 with expected crc 2023-09-23T21:13:12.001 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:11 smithi121 ceph-mon[219709]: pgmap v1517: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2347/82791 objects misplaced (2.835%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T21:13:12.050 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:12.342 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:12.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:12.462 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:12.496 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:12.541 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:12.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:12.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:12.573 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:13:12.574 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:13:12.649 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:12.675 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:12.716 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:12.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:12.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:12.826 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:12.849 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:12 smithi123 ceph-mon[193903]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.849 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:12 smithi123 ceph-mon[193903]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.849 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:12 smithi123 ceph-mon[193903]: osdmap e469: 8 total, 8 up, 8 in 2023-09-23T21:13:12.849 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:12 smithi123 ceph-mon[193903]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.850 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:12 smithi123 ceph-mon[193903]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.850 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:12.851 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:12.915 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:12.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[211526]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[211526]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[211526]: osdmap e469: 8 total, 8 up, 8 in 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[211526]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[211526]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[219709]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[219709]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[219709]: osdmap e469: 8 total, 8 up, 8 in 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[219709]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.919 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[219709]: failed to encode map e469 with expected crc 2023-09-23T21:13:12.920 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:12.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:13.024 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:13.091 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:13.176 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:13:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:13 smithi123 ceph-mon[193903]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:13.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:13 smithi123 ceph-mon[193903]: pgmap v1519: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 1190/81453 objects misplaced (1.461%); 12 MiB/s, 198 objects/s recovering 2023-09-23T21:13:14.002 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:14.012 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:13 smithi121 ceph-mon[211526]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:14.012 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:13 smithi121 ceph-mon[211526]: pgmap v1519: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 1190/81453 objects misplaced (1.461%); 12 MiB/s, 198 objects/s recovering 2023-09-23T21:13:14.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:14.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:14.017 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:13 smithi121 ceph-mon[219709]: from='client.36885 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:14.017 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:13 smithi121 ceph-mon[219709]: pgmap v1519: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 1190/81453 objects misplaced (1.461%); 12 MiB/s, 198 objects/s recovering 2023-09-23T21:13:14.114 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:14.115 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:14.151 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:14.170 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1518,"stamp":"2023-09-23T21:13:11.711863+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1807554488,"num_objects":27597,"num_object_clones":0,"num_object_copies":82791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2347,"num_objects_unfound":0,"num_objects_dirty":27597,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":153529,"num_write_kb":5788046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12941,"num_bytes_recovered":805272668,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136645,"ondisk_log_size":136645,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11229672,"kb_used_data":5648872,"kb_used_omap":21,"kb_used_meta":5580010,"kb_avail":738731544,"statfs":{"total":767960285184,"available":756461101056,"internally_reserved":0,"allocated":5784444928,"data_stored":5769918342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":5713930723},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,5,1,5,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-308543488,"num_objects":-4708,"num_object_clones":0,"num_object_copies":-14124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-810,"num_objects_unfound":0,"num_objects_dirty":-4708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4708,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":587,"num_bytes_recovered":38469632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.542825"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":240,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674642+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:12:59.674642+0000","last_peered":"2023-09-23T21:12:59.674642+0000","last_clean":"2023-09-23T21:12:59.674642+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:12:59.674642+0000","last_undegraded":"2023-09-23T21:12:59.674642+0000","last_fullsized":"2023-09-23T21:12:59.674642+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":109,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667242+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:12:59.667242+0000","last_peered":"2023-09-23T21:12:59.667242+0000","last_clean":"2023-09-23T21:12:59.667242+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:12:59.667242+0000","last_undegraded":"2023-09-23T21:12:59.667242+0000","last_fullsized":"2023-09-23T21:12:59.667242+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":382,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674448+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:12:59.674448+0000","last_peered":"2023-09-23T21:12:59.674448+0000","last_clean":"2023-09-23T21:12:59.674448+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:12:59.674448+0000","last_undegraded":"2023-09-23T21:12:59.674448+0000","last_fullsized":"2023-09-23T21:12:59.674448+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":116,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.193633+0000","last_change":"2023-09-23T21:12:51.650220+0000","last_active":"2023-09-23T21:13:04.193633+0000","last_peered":"2023-09-23T21:13:04.193633+0000","last_clean":"2023-09-23T21:13:04.193633+0000","last_became_active":"2023-09-23T21:12:51.649869+0000","last_became_peered":"2023-09-23T21:12:51.649869+0000","last_unstale":"2023-09-23T21:13:04.193633+0000","last_undegraded":"2023-09-23T21:13:04.193633+0000","last_fullsized":"2023-09-23T21:13:04.193633+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:04.107476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":127,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.666697+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:12:59.666697+0000","last_peered":"2023-09-23T21:12:59.666697+0000","last_clean":"2023-09-23T21:12:59.666697+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:12:59.666697+0000","last_undegraded":"2023-09-23T21:12:59.666697+0000","last_fullsized":"2023-09-23T21:12:59.666697+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":344,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173311+0000","last_change":"2023-09-23T21:09:44.161988+0000","last_active":"2023-09-23T21:13:04.173311+0000","last_peered":"2023-09-23T21:13:04.173311+0000","last_clean":"2023-09-23T21:13:04.173311+0000","last_became_active":"2023-09-23T21:09:44.161643+0000","last_became_peered":"2023-09-23T21:09:44.161643+0000","last_unstale":"2023-09-23T21:13:04.173311+0000","last_undegraded":"2023-09-23T21:13:04.173311+0000","last_fullsized":"2023-09-23T21:13:04.173311+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:17.083615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":106,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.696111+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:12:59.696111+0000","last_peered":"2023-09-23T21:12:59.696111+0000","last_clean":"2023-09-23T21:12:59.696111+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:12:59.696111+0000","last_undegraded":"2023-09-23T21:12:59.696111+0000","last_fullsized":"2023-09-23T21:12:59.696111+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":109,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.666622+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:12:59.666622+0000","last_peered":"2023-09-23T21:12:59.666622+0000","last_clean":"2023-09-23T21:12:59.666622+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:12:59.666622+0000","last_undegraded":"2023-09-23T21:12:59.666622+0000","last_fullsized":"2023-09-23T21:12:59.666622+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.668053+0000","last_change":"2023-09-23T21:10:15.584242+0000","last_active":"2023-09-23T21:12:59.668053+0000","last_peered":"2023-09-23T21:12:59.668053+0000","last_clean":"2023-09-23T21:12:59.668053+0000","last_became_active":"2023-09-23T21:10:15.583926+0000","last_became_peered":"2023-09-23T21:10:15.583926+0000","last_unstale":"2023-09-23T21:12:59.668053+0000","last_undegraded":"2023-09-23T21:12:59.668053+0000","last_fullsized":"2023-09-23T21:12:59.668053+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:15.759114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":118,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667044+0000","last_change":"2023-09-23T21:12:51.652067+0000","last_active":"2023-09-23T21:12:59.667044+0000","last_peered":"2023-09-23T21:12:59.667044+0000","last_clean":"2023-09-23T21:12:59.667044+0000","last_became_active":"2023-09-23T21:12:51.651751+0000","last_became_peered":"2023-09-23T21:12:51.651751+0000","last_unstale":"2023-09-23T21:12:59.667044+0000","last_undegraded":"2023-09-23T21:12:59.667044+0000","last_fullsized":"2023-09-23T21:12:59.667044+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:03.235559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":22,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.671927+0000","last_change":"2023-09-23T21:12:51.657657+0000","last_active":"2023-09-23T21:12:59.671927+0000","last_peered":"2023-09-23T21:12:59.671927+0000","last_clean":"2023-09-23T21:12:59.671927+0000","last_became_active":"2023-09-23T21:12:51.641567+0000","last_became_peered":"2023-09-23T21:12:51.641567+0000","last_unstale":"2023-09-23T21:12:59.671927+0000","last_undegraded":"2023-09-23T21:12:59.671927+0000","last_fullsized":"2023-09-23T21:12:59.671927+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:00.366548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674411+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:12:59.674411+0000","last_peered":"2023-09-23T21:12:59.674411+0000","last_clean":"2023-09-23T21:12:59.674411+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:12:59.674411+0000","last_undegraded":"2023-09-23T21:12:59.674411+0000","last_fullsized":"2023-09-23T21:12:59.674411+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":54,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.671936+0000","last_change":"2023-09-23T21:10:15.594033+0000","last_active":"2023-09-23T21:12:59.671936+0000","last_peered":"2023-09-23T21:12:59.671936+0000","last_clean":"2023-09-23T21:12:59.671936+0000","last_became_active":"2023-09-23T21:10:15.579277+0000","last_became_peered":"2023-09-23T21:10:15.579277+0000","last_unstale":"2023-09-23T21:12:59.671936+0000","last_undegraded":"2023-09-23T21:12:59.671936+0000","last_fullsized":"2023-09-23T21:12:59.671936+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:42.561285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":385,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.694398+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:12:59.694398+0000","last_peered":"2023-09-23T21:12:59.694398+0000","last_clean":"2023-09-23T21:12:59.694398+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:12:59.694398+0000","last_undegraded":"2023-09-23T21:12:59.694398+0000","last_fullsized":"2023-09-23T21:12:59.694398+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":50,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.671725+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:12:59.671725+0000","last_peered":"2023-09-23T21:12:59.671725+0000","last_clean":"2023-09-23T21:12:59.671725+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:12:59.671725+0000","last_undegraded":"2023-09-23T21:12:59.671725+0000","last_fullsized":"2023-09-23T21:12:59.671725+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":127,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667606+0000","last_change":"2023-09-23T21:10:15.590805+0000","last_active":"2023-09-23T21:12:59.667606+0000","last_peered":"2023-09-23T21:12:59.667606+0000","last_clean":"2023-09-23T21:12:59.667606+0000","last_became_active":"2023-09-23T21:10:15.590501+0000","last_became_peered":"2023-09-23T21:10:15.590501+0000","last_unstale":"2023-09-23T21:12:59.667606+0000","last_undegraded":"2023-09-23T21:12:59.667606+0000","last_fullsized":"2023-09-23T21:12:59.667606+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:27.143631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.671622+0000","last_change":"2023-09-23T21:12:51.643047+0000","last_active":"2023-09-23T21:12:59.671622+0000","last_peered":"2023-09-23T21:12:59.671622+0000","last_clean":"2023-09-23T21:12:59.671622+0000","last_became_active":"2023-09-23T21:12:51.642754+0000","last_became_peered":"2023-09-23T21:12:51.642754+0000","last_unstale":"2023-09-23T21:12:59.671622+0000","last_undegraded":"2023-09-23T21:12:59.671622+0000","last_fullsized":"2023-09-23T21:12:59.671622+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:31.430089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":44,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.672068+0000","last_change":"2023-09-23T21:10:15.585218+0000","last_active":"2023-09-23T21:12:59.672068+0000","last_peered":"2023-09-23T21:12:59.672068+0000","last_clean":"2023-09-23T21:12:59.672068+0000","last_became_active":"2023-09-23T21:10:15.584952+0000","last_became_peered":"2023-09-23T21:10:15.584952+0000","last_unstale":"2023-09-23T21:12:59.672068+0000","last_undegraded":"2023-09-23T21:12:59.672068+0000","last_fullsized":"2023-09-23T21:12:59.672068+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:55.016769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.671724+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:12:59.671724+0000","last_peered":"2023-09-23T21:12:59.671724+0000","last_clean":"2023-09-23T21:12:59.671724+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:12:59.671724+0000","last_undegraded":"2023-09-23T21:12:59.671724+0000","last_fullsized":"2023-09-23T21:12:59.671724+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":115,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.696404+0000","last_change":"2023-09-23T21:12:51.650915+0000","last_active":"2023-09-23T21:12:59.696404+0000","last_peered":"2023-09-23T21:12:59.696404+0000","last_clean":"2023-09-23T21:12:59.696404+0000","last_became_active":"2023-09-23T21:12:51.650550+0000","last_became_peered":"2023-09-23T21:12:51.650550+0000","last_unstale":"2023-09-23T21:12:59.696404+0000","last_undegraded":"2023-09-23T21:12:59.696404+0000","last_fullsized":"2023-09-23T21:12:59.696404+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:30.158548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":464,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173180+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:13:04.173180+0000","last_peered":"2023-09-23T21:13:04.173180+0000","last_clean":"2023-09-23T21:13:04.173180+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:13:04.173180+0000","last_undegraded":"2023-09-23T21:13:04.173180+0000","last_fullsized":"2023-09-23T21:13:04.173180+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":127,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667346+0000","last_change":"2023-09-23T21:12:51.645964+0000","last_active":"2023-09-23T21:12:59.667346+0000","last_peered":"2023-09-23T21:12:59.667346+0000","last_clean":"2023-09-23T21:12:59.667346+0000","last_became_active":"2023-09-23T21:12:51.645627+0000","last_became_peered":"2023-09-23T21:12:51.645627+0000","last_unstale":"2023-09-23T21:12:59.667346+0000","last_undegraded":"2023-09-23T21:12:59.667346+0000","last_fullsized":"2023-09-23T21:12:59.667346+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:53.725127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"468'9597","reported_seq":9647,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T21:13:10.920040+0000","last_change":"2023-09-23T21:09:45.077841+0000","last_active":"2023-09-23T21:13:10.920040+0000","last_peered":"2023-09-23T21:13:10.920040+0000","last_clean":"2023-09-23T21:13:10.920040+0000","last_became_active":"2023-09-23T21:09:45.077550+0000","last_became_peered":"2023-09-23T21:09:45.077550+0000","last_unstale":"2023-09-23T21:13:10.920040+0000","last_undegraded":"2023-09-23T21:13:10.920040+0000","last_fullsized":"2023-09-23T21:13:10.920040+0000","mapping_epoch":426,"log_start":"430'700","ondisk_log_start":"430'700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8897,"log_dups_size":0,"ondisk_log_size":8897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:19.178954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106889216,"num_objects":1631,"num_object_clones":0,"num_object_copies":4893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9597,"num_write_kb":359296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":20,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.672261+0000","last_change":"2023-09-23T21:12:51.662783+0000","last_active":"2023-09-23T21:12:59.672261+0000","last_peered":"2023-09-23T21:12:59.672261+0000","last_clean":"2023-09-23T21:12:59.672261+0000","last_became_active":"2023-09-23T21:12:51.649440+0000","last_became_peered":"2023-09-23T21:12:51.649440+0000","last_unstale":"2023-09-23T21:12:59.672261+0000","last_undegraded":"2023-09-23T21:12:59.672261+0000","last_fullsized":"2023-09-23T21:12:59.672261+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:11.808407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":100,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667799+0000","last_change":"2023-09-23T21:12:51.650942+0000","last_active":"2023-09-23T21:12:59.667799+0000","last_peered":"2023-09-23T21:12:59.667799+0000","last_clean":"2023-09-23T21:12:59.667799+0000","last_became_active":"2023-09-23T21:12:51.650556+0000","last_became_peered":"2023-09-23T21:12:51.650556+0000","last_unstale":"2023-09-23T21:12:59.667799+0000","last_undegraded":"2023-09-23T21:12:59.667799+0000","last_fullsized":"2023-09-23T21:12:59.667799+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:06.677094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"468'9465","reported_seq":13687,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T21:13:11.008431+0000","last_change":"2023-09-23T21:12:59.694045+0000","last_active":"2023-09-23T21:13:11.008431+0000","last_peered":"2023-09-23T21:13:11.008431+0000","last_clean":"2023-09-23T21:13:11.008431+0000","last_became_active":"2023-09-23T21:12:59.672587+0000","last_became_peered":"2023-09-23T21:12:59.672587+0000","last_unstale":"2023-09-23T21:13:11.008431+0000","last_undegraded":"2023-09-23T21:13:11.008431+0000","last_fullsized":"2023-09-23T21:13:11.008431+0000","mapping_epoch":465,"log_start":"430'600","ondisk_log_start":"430'600","created":426,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8865,"log_dups_size":0,"ondisk_log_size":8865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.616121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9465,"num_write_kb":355456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2074,"num_bytes_recovered":135921664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":48,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.672070+0000","last_change":"2023-09-23T21:10:15.596183+0000","last_active":"2023-09-23T21:12:59.672070+0000","last_peered":"2023-09-23T21:12:59.672070+0000","last_clean":"2023-09-23T21:12:59.672070+0000","last_became_active":"2023-09-23T21:10:15.586221+0000","last_became_peered":"2023-09-23T21:10:15.586221+0000","last_unstale":"2023-09-23T21:12:59.672070+0000","last_undegraded":"2023-09-23T21:12:59.672070+0000","last_fullsized":"2023-09-23T21:12:59.672070+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:06.222859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":430,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674363+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:12:59.674363+0000","last_peered":"2023-09-23T21:12:59.674363+0000","last_clean":"2023-09-23T21:12:59.674363+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:12:59.674363+0000","last_undegraded":"2023-09-23T21:12:59.674363+0000","last_fullsized":"2023-09-23T21:12:59.674363+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"467'9593","reported_seq":14299,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:10.432153+0000","last_change":"2023-09-23T21:11:08.638059+0000","last_active":"2023-09-23T21:13:10.432153+0000","last_peered":"2023-09-23T21:13:10.432153+0000","last_clean":"2023-09-23T21:13:10.432153+0000","last_became_active":"2023-09-23T21:11:08.630938+0000","last_became_peered":"2023-09-23T21:11:08.630938+0000","last_unstale":"2023-09-23T21:13:10.432153+0000","last_undegraded":"2023-09-23T21:13:10.432153+0000","last_fullsized":"2023-09-23T21:13:10.432153+0000","mapping_epoch":440,"log_start":"431'2200","ondisk_log_start":"431'2200","created":426,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7393,"log_dups_size":0,"ondisk_log_size":7393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:33.746589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105578496,"num_objects":1611,"num_object_clones":0,"num_object_copies":4833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9593,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2319,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":48,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.672101+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:12:59.672101+0000","last_peered":"2023-09-23T21:12:59.672101+0000","last_clean":"2023-09-23T21:12:59.672101+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:12:59.672101+0000","last_undegraded":"2023-09-23T21:12:59.672101+0000","last_fullsized":"2023-09-23T21:12:59.672101+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":366,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173153+0000","last_change":"2023-09-23T21:12:51.650231+0000","last_active":"2023-09-23T21:13:04.173153+0000","last_peered":"2023-09-23T21:13:04.173153+0000","last_clean":"2023-09-23T21:13:04.173153+0000","last_became_active":"2023-09-23T21:12:51.649867+0000","last_became_peered":"2023-09-23T21:12:51.649867+0000","last_unstale":"2023-09-23T21:13:04.173153+0000","last_undegraded":"2023-09-23T21:13:04.173153+0000","last_fullsized":"2023-09-23T21:13:04.173153+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:56.792036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"467'9349","reported_seq":13928,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:09.152379+0000","last_change":"2023-09-23T21:10:39.657127+0000","last_active":"2023-09-23T21:13:09.152379+0000","last_peered":"2023-09-23T21:13:09.152379+0000","last_clean":"2023-09-23T21:13:09.152379+0000","last_became_active":"2023-09-23T21:10:39.652435+0000","last_became_peered":"2023-09-23T21:10:39.652435+0000","last_unstale":"2023-09-23T21:13:09.152379+0000","last_undegraded":"2023-09-23T21:13:09.152379+0000","last_fullsized":"2023-09-23T21:13:09.152379+0000","mapping_epoch":436,"log_start":"430'200","ondisk_log_start":"430'200","created":426,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9149,"log_dups_size":0,"ondisk_log_size":9149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:46.462306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114098176,"num_objects":1741,"num_object_clones":0,"num_object_copies":5223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9349,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2255,"num_bytes_recovered":147783680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":109,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667647+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:12:59.667647+0000","last_peered":"2023-09-23T21:12:59.667647+0000","last_clean":"2023-09-23T21:12:59.667647+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:12:59.667647+0000","last_undegraded":"2023-09-23T21:12:59.667647+0000","last_fullsized":"2023-09-23T21:12:59.667647+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":115,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.696356+0000","last_change":"2023-09-23T21:09:44.163930+0000","last_active":"2023-09-23T21:12:59.696356+0000","last_peered":"2023-09-23T21:12:59.696356+0000","last_clean":"2023-09-23T21:12:59.696356+0000","last_became_active":"2023-09-23T21:09:44.163614+0000","last_became_peered":"2023-09-23T21:09:44.163614+0000","last_unstale":"2023-09-23T21:12:59.696356+0000","last_undegraded":"2023-09-23T21:12:59.696356+0000","last_fullsized":"2023-09-23T21:12:59.696356+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:26.549908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"468'9859","reported_seq":9909,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T21:13:10.999930+0000","last_change":"2023-09-23T21:09:45.040470+0000","last_active":"2023-09-23T21:13:10.999930+0000","last_peered":"2023-09-23T21:13:10.999930+0000","last_clean":"2023-09-23T21:13:10.999930+0000","last_became_active":"2023-09-23T21:09:45.040298+0000","last_became_peered":"2023-09-23T21:09:45.040298+0000","last_unstale":"2023-09-23T21:13:10.999930+0000","last_undegraded":"2023-09-23T21:13:10.999930+0000","last_fullsized":"2023-09-23T21:13:10.999930+0000","mapping_epoch":426,"log_start":"430'1000","ondisk_log_start":"430'1000","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8859,"log_dups_size":0,"ondisk_log_size":8859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:06.840153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109510656,"num_objects":1671,"num_object_clones":0,"num_object_copies":5013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9859,"num_write_kb":368960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":126,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667341+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:12:59.667341+0000","last_peered":"2023-09-23T21:12:59.667341+0000","last_clean":"2023-09-23T21:12:59.667341+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:12:59.667341+0000","last_undegraded":"2023-09-23T21:12:59.667341+0000","last_fullsized":"2023-09-23T21:12:59.667341+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":109,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667567+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:12:59.667567+0000","last_peered":"2023-09-23T21:12:59.667567+0000","last_clean":"2023-09-23T21:12:59.667567+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:12:59.667567+0000","last_undegraded":"2023-09-23T21:12:59.667567+0000","last_fullsized":"2023-09-23T21:12:59.667567+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"468'9547","reported_seq":9597,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T21:13:11.025371+0000","last_change":"2023-09-23T21:09:45.041603+0000","last_active":"2023-09-23T21:13:11.025371+0000","last_peered":"2023-09-23T21:13:11.025371+0000","last_clean":"2023-09-23T21:13:11.025371+0000","last_became_active":"2023-09-23T21:09:45.041377+0000","last_became_peered":"2023-09-23T21:09:45.041377+0000","last_unstale":"2023-09-23T21:13:11.025371+0000","last_undegraded":"2023-09-23T21:13:11.025371+0000","last_fullsized":"2023-09-23T21:13:11.025371+0000","mapping_epoch":426,"log_start":"430'700","ondisk_log_start":"430'700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8847,"log_dups_size":0,"ondisk_log_size":8847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:46.148257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106233856,"num_objects":1621,"num_object_clones":0,"num_object_copies":4863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9547,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":394,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173200+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:13:04.173200+0000","last_peered":"2023-09-23T21:13:04.173200+0000","last_clean":"2023-09-23T21:13:04.173200+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:13:04.173200+0000","last_undegraded":"2023-09-23T21:13:04.173200+0000","last_fullsized":"2023-09-23T21:13:04.173200+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":480,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.663532+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:12:59.663532+0000","last_peered":"2023-09-23T21:12:59.663532+0000","last_clean":"2023-09-23T21:12:59.663532+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:12:59.663532+0000","last_undegraded":"2023-09-23T21:12:59.663532+0000","last_fullsized":"2023-09-23T21:12:59.663532+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"467'9537","reported_seq":14224,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:07.243801+0000","last_change":"2023-09-23T21:11:37.353931+0000","last_active":"2023-09-23T21:13:07.243801+0000","last_peered":"2023-09-23T21:13:07.243801+0000","last_clean":"2023-09-23T21:13:07.243801+0000","last_became_active":"2023-09-23T21:11:37.344349+0000","last_became_peered":"2023-09-23T21:11:37.344349+0000","last_unstale":"2023-09-23T21:13:07.243801+0000","last_undegraded":"2023-09-23T21:13:07.243801+0000","last_fullsized":"2023-09-23T21:13:07.243801+0000","mapping_epoch":444,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7837,"log_dups_size":0,"ondisk_log_size":7837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:35.510223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121831424,"num_objects":1859,"num_object_clones":0,"num_object_copies":5577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9537,"num_write_kb":364672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2309,"num_bytes_recovered":151322624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.694063+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:12:59.694063+0000","last_peered":"2023-09-23T21:12:59.694063+0000","last_clean":"2023-09-23T21:12:59.694063+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:12:59.694063+0000","last_undegraded":"2023-09-23T21:12:59.694063+0000","last_fullsized":"2023-09-23T21:12:59.694063+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":507,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173300+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:13:04.173300+0000","last_peered":"2023-09-23T21:13:04.173300+0000","last_clean":"2023-09-23T21:13:04.173300+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:13:04.173300+0000","last_undegraded":"2023-09-23T21:13:04.173300+0000","last_fullsized":"2023-09-23T21:13:04.173300+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"467'9585","reported_seq":12592,"reported_epoch":467,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T21:13:07.252131+0000","last_change":"2023-09-23T21:12:58.009421+0000","last_active":"2023-09-23T21:13:07.252131+0000","last_peered":"2023-09-23T21:13:07.252131+0000","last_clean":"2023-09-23T21:12:51.336402+0000","last_became_active":"2023-09-23T21:12:52.790762+0000","last_became_peered":"2023-09-23T21:12:52.790762+0000","last_unstale":"2023-09-23T21:13:07.252131+0000","last_undegraded":"2023-09-23T21:13:07.252131+0000","last_fullsized":"2023-09-23T21:13:07.252131+0000","mapping_epoch":461,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7885,"log_dups_size":0,"ondisk_log_size":7885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.461640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122224640,"num_objects":1865,"num_object_clones":0,"num_object_copies":5595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":536,"num_objects_unfound":0,"num_objects_dirty":1865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9585,"num_write_kb":366400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1473,"num_bytes_recovered":96534528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":1865}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":127,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.666781+0000","last_change":"2023-09-23T21:10:15.584815+0000","last_active":"2023-09-23T21:12:59.666781+0000","last_peered":"2023-09-23T21:12:59.666781+0000","last_clean":"2023-09-23T21:12:59.666781+0000","last_became_active":"2023-09-23T21:10:15.584484+0000","last_became_peered":"2023-09-23T21:10:15.584484+0000","last_unstale":"2023-09-23T21:12:59.666781+0000","last_undegraded":"2023-09-23T21:12:59.666781+0000","last_fullsized":"2023-09-23T21:12:59.666781+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:58.137854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":215,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667211+0000","last_change":"2023-09-23T21:09:44.164862+0000","last_active":"2023-09-23T21:12:59.667211+0000","last_peered":"2023-09-23T21:12:59.667211+0000","last_clean":"2023-09-23T21:12:59.667211+0000","last_became_active":"2023-09-23T21:09:44.164337+0000","last_became_peered":"2023-09-23T21:09:44.164337+0000","last_unstale":"2023-09-23T21:12:59.667211+0000","last_undegraded":"2023-09-23T21:12:59.667211+0000","last_fullsized":"2023-09-23T21:12:59.667211+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:57.424412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"467'9611","reported_seq":9662,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:09.618767+0000","last_change":"2023-09-23T21:09:45.033715+0000","last_active":"2023-09-23T21:13:09.618767+0000","last_peered":"2023-09-23T21:13:09.618767+0000","last_clean":"2023-09-23T21:13:09.618767+0000","last_became_active":"2023-09-23T21:09:45.033437+0000","last_became_peered":"2023-09-23T21:09:45.033437+0000","last_unstale":"2023-09-23T21:13:09.618767+0000","last_undegraded":"2023-09-23T21:13:09.618767+0000","last_fullsized":"2023-09-23T21:13:09.618767+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9611,"log_dups_size":0,"ondisk_log_size":9611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:15.193017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111476736,"num_objects":1701,"num_object_clones":0,"num_object_copies":5103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9611,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":373,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674641+0000","last_change":"2023-09-23T21:10:15.588139+0000","last_active":"2023-09-23T21:12:59.674641+0000","last_peered":"2023-09-23T21:12:59.674641+0000","last_clean":"2023-09-23T21:12:59.674641+0000","last_became_active":"2023-09-23T21:10:15.587822+0000","last_became_peered":"2023-09-23T21:10:15.587822+0000","last_unstale":"2023-09-23T21:12:59.674641+0000","last_undegraded":"2023-09-23T21:12:59.674641+0000","last_fullsized":"2023-09-23T21:12:59.674641+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:47.570476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":132,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667934+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:12:59.667934+0000","last_peered":"2023-09-23T21:12:59.667934+0000","last_clean":"2023-09-23T21:12:59.667934+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:12:59.667934+0000","last_undegraded":"2023-09-23T21:12:59.667934+0000","last_fullsized":"2023-09-23T21:12:59.667934+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":119,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674500+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:12:59.674500+0000","last_peered":"2023-09-23T21:12:59.674500+0000","last_clean":"2023-09-23T21:12:59.674500+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:12:59.674500+0000","last_undegraded":"2023-09-23T21:12:59.674500+0000","last_fullsized":"2023-09-23T21:12:59.674500+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"467'9594","reported_seq":9645,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:09.139740+0000","last_change":"2023-09-23T21:09:45.843731+0000","last_active":"2023-09-23T21:13:09.139740+0000","last_peered":"2023-09-23T21:13:09.139740+0000","last_clean":"2023-09-23T21:13:09.139740+0000","last_became_active":"2023-09-23T21:09:45.843440+0000","last_became_peered":"2023-09-23T21:09:45.843440+0000","last_unstale":"2023-09-23T21:13:09.139740+0000","last_undegraded":"2023-09-23T21:13:09.139740+0000","last_fullsized":"2023-09-23T21:13:09.139740+0000","mapping_epoch":426,"log_start":"430'500","ondisk_log_start":"430'500","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9094,"log_dups_size":0,"ondisk_log_size":9094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:17.052514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117571584,"num_objects":1794,"num_object_clones":0,"num_object_copies":5382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9594,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":445,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674720+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:12:59.674720+0000","last_peered":"2023-09-23T21:12:59.674720+0000","last_clean":"2023-09-23T21:12:59.674720+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:12:59.674720+0000","last_undegraded":"2023-09-23T21:12:59.674720+0000","last_fullsized":"2023-09-23T21:12:59.674720+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":418,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173268+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:13:04.173268+0000","last_peered":"2023-09-23T21:13:04.173268+0000","last_clean":"2023-09-23T21:13:04.173268+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:13:04.173268+0000","last_undegraded":"2023-09-23T21:13:04.173268+0000","last_fullsized":"2023-09-23T21:13:04.173268+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":101,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674580+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:12:59.674580+0000","last_peered":"2023-09-23T21:12:59.674580+0000","last_clean":"2023-09-23T21:12:59.674580+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:12:59.674580+0000","last_undegraded":"2023-09-23T21:12:59.674580+0000","last_fullsized":"2023-09-23T21:12:59.674580+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"468'9604","reported_seq":9654,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T21:13:10.998026+0000","last_change":"2023-09-23T21:09:45.843960+0000","last_active":"2023-09-23T21:13:10.998026+0000","last_peered":"2023-09-23T21:13:10.998026+0000","last_clean":"2023-09-23T21:13:10.998026+0000","last_became_active":"2023-09-23T21:09:45.843562+0000","last_became_peered":"2023-09-23T21:09:45.843562+0000","last_unstale":"2023-09-23T21:13:10.998026+0000","last_undegraded":"2023-09-23T21:13:10.998026+0000","last_fullsized":"2023-09-23T21:13:10.998026+0000","mapping_epoch":426,"log_start":"430'700","ondisk_log_start":"430'700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8904,"log_dups_size":0,"ondisk_log_size":8904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:10.329012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114294784,"num_objects":1744,"num_object_clones":0,"num_object_copies":5232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9604,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":446,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.662806+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:12:59.662806+0000","last_peered":"2023-09-23T21:12:59.662806+0000","last_clean":"2023-09-23T21:12:59.662806+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:12:59.662806+0000","last_undegraded":"2023-09-23T21:12:59.662806+0000","last_fullsized":"2023-09-23T21:12:59.662806+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":364,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674477+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:12:59.674477+0000","last_peered":"2023-09-23T21:12:59.674477+0000","last_clean":"2023-09-23T21:12:59.674477+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:12:59.674477+0000","last_undegraded":"2023-09-23T21:12:59.674477+0000","last_fullsized":"2023-09-23T21:12:59.674477+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.675054+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:12:59.675054+0000","last_peered":"2023-09-23T21:12:59.675054+0000","last_clean":"2023-09-23T21:12:59.675054+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:12:59.675054+0000","last_undegraded":"2023-09-23T21:12:59.675054+0000","last_fullsized":"2023-09-23T21:12:59.675054+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"467'9545","reported_seq":9604,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:13:07.245844+0000","last_change":"2023-09-23T21:12:52.821619+0000","last_active":"2023-09-23T21:13:07.245844+0000","last_peered":"2023-09-23T21:13:07.245844+0000","last_clean":"2023-09-23T21:12:51.339810+0000","last_became_active":"2023-09-23T21:12:52.789528+0000","last_became_peered":"2023-09-23T21:12:52.789528+0000","last_unstale":"2023-09-23T21:13:07.245844+0000","last_undegraded":"2023-09-23T21:13:07.245844+0000","last_fullsized":"2023-09-23T21:13:07.245844+0000","mapping_epoch":461,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7845,"log_dups_size":0,"ondisk_log_size":7845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:17.551283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118620184,"num_objects":1811,"num_object_clones":0,"num_object_copies":5433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1811,"num_objects_unfound":0,"num_objects_dirty":1811,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9545,"num_write_kb":363329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[1,2,3],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2,3","objects":1811}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":293,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.663395+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:12:59.663395+0000","last_peered":"2023-09-23T21:12:59.663395+0000","last_clean":"2023-09-23T21:12:59.663395+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:12:59.663395+0000","last_undegraded":"2023-09-23T21:12:59.663395+0000","last_fullsized":"2023-09-23T21:12:59.663395+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":739,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.671542+0000","last_change":"2023-09-23T21:12:54.698525+0000","last_active":"2023-09-23T21:12:59.671542+0000","last_peered":"2023-09-23T21:12:59.671542+0000","last_clean":"2023-09-23T21:12:59.671542+0000","last_became_active":"2023-09-23T21:12:54.698074+0000","last_became_peered":"2023-09-23T21:12:54.698074+0000","last_unstale":"2023-09-23T21:12:59.671542+0000","last_undegraded":"2023-09-23T21:12:59.671542+0000","last_fullsized":"2023-09-23T21:12:59.671542+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:11.791067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.175997+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:13:04.175997+0000","last_peered":"2023-09-23T21:13:04.175997+0000","last_clean":"2023-09-23T21:13:04.175997+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:13:04.175997+0000","last_undegraded":"2023-09-23T21:13:04.175997+0000","last_fullsized":"2023-09-23T21:13:04.175997+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":366,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173430+0000","last_change":"2023-09-23T21:10:15.582161+0000","last_active":"2023-09-23T21:13:04.173430+0000","last_peered":"2023-09-23T21:13:04.173430+0000","last_clean":"2023-09-23T21:13:04.173430+0000","last_became_active":"2023-09-23T21:10:15.581855+0000","last_became_peered":"2023-09-23T21:10:15.581855+0000","last_unstale":"2023-09-23T21:13:04.173430+0000","last_undegraded":"2023-09-23T21:13:04.173430+0000","last_fullsized":"2023-09-23T21:13:04.173430+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:19.873916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"467'9764","reported_seq":14627,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:09.144514+0000","last_change":"2023-09-23T21:12:11.354703+0000","last_active":"2023-09-23T21:13:09.144514+0000","last_peered":"2023-09-23T21:13:09.144514+0000","last_clean":"2023-09-23T21:13:09.144514+0000","last_became_active":"2023-09-23T21:12:11.333665+0000","last_became_peered":"2023-09-23T21:12:11.333665+0000","last_unstale":"2023-09-23T21:13:09.144514+0000","last_undegraded":"2023-09-23T21:13:09.144514+0000","last_fullsized":"2023-09-23T21:13:09.144514+0000","mapping_epoch":451,"log_start":"430'600","ondisk_log_start":"430'600","created":426,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9164,"log_dups_size":0,"ondisk_log_size":9164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:49.607296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117571584,"num_objects":1794,"num_object_clones":0,"num_object_copies":5382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9764,"num_write_kb":369856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2397,"num_bytes_recovered":117571584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.671569+0000","last_change":"2023-09-23T21:10:15.584319+0000","last_active":"2023-09-23T21:12:59.671569+0000","last_peered":"2023-09-23T21:12:59.671569+0000","last_clean":"2023-09-23T21:12:59.671569+0000","last_became_active":"2023-09-23T21:10:15.584030+0000","last_became_peered":"2023-09-23T21:10:15.584030+0000","last_unstale":"2023-09-23T21:12:59.671569+0000","last_undegraded":"2023-09-23T21:12:59.671569+0000","last_fullsized":"2023-09-23T21:12:59.671569+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:28.957511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":115,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.695787+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:12:59.695787+0000","last_peered":"2023-09-23T21:12:59.695787+0000","last_clean":"2023-09-23T21:12:59.695787+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:12:59.695787+0000","last_undegraded":"2023-09-23T21:12:59.695787+0000","last_fullsized":"2023-09-23T21:12:59.695787+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":106,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.696187+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:12:59.696187+0000","last_peered":"2023-09-23T21:12:59.696187+0000","last_clean":"2023-09-23T21:12:59.696187+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:12:59.696187+0000","last_undegraded":"2023-09-23T21:12:59.696187+0000","last_fullsized":"2023-09-23T21:12:59.696187+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"467'9661","reported_seq":9710,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:10.397058+0000","last_change":"2023-09-23T21:09:45.038161+0000","last_active":"2023-09-23T21:13:10.397058+0000","last_peered":"2023-09-23T21:13:10.397058+0000","last_clean":"2023-09-23T21:13:10.397058+0000","last_became_active":"2023-09-23T21:09:45.037907+0000","last_became_peered":"2023-09-23T21:09:45.037907+0000","last_unstale":"2023-09-23T21:13:10.397058+0000","last_undegraded":"2023-09-23T21:13:10.397058+0000","last_fullsized":"2023-09-23T21:13:10.397058+0000","mapping_epoch":426,"log_start":"431'2300","ondisk_log_start":"431'2300","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:37.785460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111869952,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9661,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":44,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.672018+0000","last_change":"2023-09-23T21:10:15.584326+0000","last_active":"2023-09-23T21:12:59.672018+0000","last_peered":"2023-09-23T21:12:59.672018+0000","last_clean":"2023-09-23T21:12:59.672018+0000","last_became_active":"2023-09-23T21:10:15.583965+0000","last_became_peered":"2023-09-23T21:10:15.583965+0000","last_unstale":"2023-09-23T21:12:59.672018+0000","last_undegraded":"2023-09-23T21:12:59.672018+0000","last_fullsized":"2023-09-23T21:12:59.672018+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:21.120392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":366,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674679+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:12:59.674679+0000","last_peered":"2023-09-23T21:12:59.674679+0000","last_clean":"2023-09-23T21:12:59.674679+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:12:59.674679+0000","last_undegraded":"2023-09-23T21:12:59.674679+0000","last_fullsized":"2023-09-23T21:12:59.674679+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":152,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667934+0000","last_change":"2023-09-23T21:10:15.588413+0000","last_active":"2023-09-23T21:12:59.667934+0000","last_peered":"2023-09-23T21:12:59.667934+0000","last_clean":"2023-09-23T21:12:59.667934+0000","last_became_active":"2023-09-23T21:10:15.588077+0000","last_became_peered":"2023-09-23T21:10:15.588077+0000","last_unstale":"2023-09-23T21:12:59.667934+0000","last_undegraded":"2023-09-23T21:12:59.667934+0000","last_fullsized":"2023-09-23T21:12:59.667934+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:03.757383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"467'9519","reported_seq":9570,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:10.376062+0000","last_change":"2023-09-23T21:09:45.033672+0000","last_active":"2023-09-23T21:13:10.376062+0000","last_peered":"2023-09-23T21:13:10.376062+0000","last_clean":"2023-09-23T21:13:10.376062+0000","last_became_active":"2023-09-23T21:09:45.033296+0000","last_became_peered":"2023-09-23T21:09:45.033296+0000","last_unstale":"2023-09-23T21:13:10.376062+0000","last_undegraded":"2023-09-23T21:13:10.376062+0000","last_fullsized":"2023-09-23T21:13:10.376062+0000","mapping_epoch":426,"log_start":"431'2200","ondisk_log_start":"431'2200","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7319,"log_dups_size":0,"ondisk_log_size":7319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:26.841578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110559232,"num_objects":1687,"num_object_clones":0,"num_object_copies":5061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9519,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":139,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667347+0000","last_change":"2023-09-23T21:10:15.593352+0000","last_active":"2023-09-23T21:12:59.667347+0000","last_peered":"2023-09-23T21:12:59.667347+0000","last_clean":"2023-09-23T21:12:59.667347+0000","last_became_active":"2023-09-23T21:10:15.589715+0000","last_became_peered":"2023-09-23T21:10:15.589715+0000","last_unstale":"2023-09-23T21:12:59.667347+0000","last_undegraded":"2023-09-23T21:12:59.667347+0000","last_fullsized":"2023-09-23T21:12:59.667347+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:24.563158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":101,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.674870+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:12:59.674870+0000","last_peered":"2023-09-23T21:12:59.674870+0000","last_clean":"2023-09-23T21:12:59.674870+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:12:59.674870+0000","last_undegraded":"2023-09-23T21:12:59.674870+0000","last_fullsized":"2023-09-23T21:12:59.674870+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":128,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.666550+0000","last_change":"2023-09-23T21:12:51.650884+0000","last_active":"2023-09-23T21:12:59.666550+0000","last_peered":"2023-09-23T21:12:59.666550+0000","last_clean":"2023-09-23T21:12:59.666550+0000","last_became_active":"2023-09-23T21:12:51.650560+0000","last_became_peered":"2023-09-23T21:12:51.650560+0000","last_unstale":"2023-09-23T21:12:59.666550+0000","last_undegraded":"2023-09-23T21:12:59.666550+0000","last_fullsized":"2023-09-23T21:12:59.666550+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:19.136242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"467'9399","reported_seq":9450,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:09.646875+0000","last_change":"2023-09-23T21:09:45.038290+0000","last_active":"2023-09-23T21:13:09.646875+0000","last_peered":"2023-09-23T21:13:09.646875+0000","last_clean":"2023-09-23T21:13:09.646875+0000","last_became_active":"2023-09-23T21:09:45.037976+0000","last_became_peered":"2023-09-23T21:09:45.037976+0000","last_unstale":"2023-09-23T21:13:09.646875+0000","last_undegraded":"2023-09-23T21:13:09.646875+0000","last_fullsized":"2023-09-23T21:13:09.646875+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9399,"log_dups_size":0,"ondisk_log_size":9399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:44.361487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110952448,"num_objects":1693,"num_object_clones":0,"num_object_copies":5079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9399,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":374,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173417+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:13:04.173417+0000","last_peered":"2023-09-23T21:13:04.173417+0000","last_clean":"2023-09-23T21:13:04.173417+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:13:04.173417+0000","last_undegraded":"2023-09-23T21:13:04.173417+0000","last_fullsized":"2023-09-23T21:13:04.173417+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":110,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.696263+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:12:59.696263+0000","last_peered":"2023-09-23T21:12:59.696263+0000","last_clean":"2023-09-23T21:12:59.696263+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:12:59.696263+0000","last_undegraded":"2023-09-23T21:12:59.696263+0000","last_fullsized":"2023-09-23T21:12:59.696263+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667773+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:12:59.667773+0000","last_peered":"2023-09-23T21:12:59.667773+0000","last_clean":"2023-09-23T21:12:59.667773+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:12:59.667773+0000","last_undegraded":"2023-09-23T21:12:59.667773+0000","last_fullsized":"2023-09-23T21:12:59.667773+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":124,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.696177+0000","last_change":"2023-09-23T21:10:15.588809+0000","last_active":"2023-09-23T21:12:59.696177+0000","last_peered":"2023-09-23T21:12:59.696177+0000","last_clean":"2023-09-23T21:12:59.696177+0000","last_became_active":"2023-09-23T21:10:15.588451+0000","last_became_peered":"2023-09-23T21:10:15.588451+0000","last_unstale":"2023-09-23T21:12:59.696177+0000","last_undegraded":"2023-09-23T21:12:59.696177+0000","last_fullsized":"2023-09-23T21:12:59.696177+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:58.509500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":501,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.663179+0000","last_change":"2023-09-23T21:09:44.163759+0000","last_active":"2023-09-23T21:12:59.663179+0000","last_peered":"2023-09-23T21:12:59.663179+0000","last_clean":"2023-09-23T21:12:59.663179+0000","last_became_active":"2023-09-23T21:09:44.163405+0000","last_became_peered":"2023-09-23T21:09:44.163405+0000","last_unstale":"2023-09-23T21:12:59.663179+0000","last_undegraded":"2023-09-23T21:12:59.663179+0000","last_fullsized":"2023-09-23T21:12:59.663179+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:52.346296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":129,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.694210+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:12:59.694210+0000","last_peered":"2023-09-23T21:12:59.694210+0000","last_clean":"2023-09-23T21:12:59.694210+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:12:59.694210+0000","last_undegraded":"2023-09-23T21:12:59.694210+0000","last_fullsized":"2023-09-23T21:12:59.694210+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667413+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:12:59.667413+0000","last_peered":"2023-09-23T21:12:59.667413+0000","last_clean":"2023-09-23T21:12:59.667413+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:12:59.667413+0000","last_undegraded":"2023-09-23T21:12:59.667413+0000","last_fullsized":"2023-09-23T21:12:59.667413+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":349,"reported_epoch":467,"state":"active+clean","last_fresh":"2023-09-23T21:13:04.173559+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:13:04.173559+0000","last_peered":"2023-09-23T21:13:04.173559+0000","last_clean":"2023-09-23T21:13:04.173559+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:13:04.173559+0000","last_undegraded":"2023-09-23T21:13:04.173559+0000","last_fullsized":"2023-09-23T21:13:04.173559+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667525+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:12:59.667525+0000","last_peered":"2023-09-23T21:12:59.667525+0000","last_clean":"2023-09-23T21:12:59.667525+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:12:59.667525+0000","last_undegraded":"2023-09-23T21:12:59.667525+0000","last_fullsized":"2023-09-23T21:12:59.667525+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":110,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.667363+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:12:59.667363+0000","last_peered":"2023-09-23T21:12:59.667363+0000","last_clean":"2023-09-23T21:12:59.667363+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:12:59.667363+0000","last_undegraded":"2023-09-23T21:12:59.667363+0000","last_fullsized":"2023-09-23T21:12:59.667363+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":118,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.666969+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:12:59.666969+0000","last_peered":"2023-09-23T21:12:59.666969+0000","last_clean":"2023-09-23T21:12:59.666969+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:12:59.666969+0000","last_undegraded":"2023-09-23T21:12:59.666969+0000","last_fullsized":"2023-09-23T21:12:59.666969+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":112,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.696025+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:12:59.696025+0000","last_peered":"2023-09-23T21:12:59.696025+0000","last_clean":"2023-09-23T21:12:59.696025+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:12:59.696025+0000","last_undegraded":"2023-09-23T21:12:59.696025+0000","last_fullsized":"2023-09-23T21:12:59.696025+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":94,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-09-23T21:12:59.693825+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:12:59.693825+0000","last_peered":"2023-09-23T21:12:59.693825+0000","last_clean":"2023-09-23T21:12:59.693825+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:12:59.693825+0000","last_undegraded":"2023-09-23T21:12:59.693825+0000","last_fullsized":"2023-09-23T21:12:59.693825+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1806958616,"num_objects":27573,"num_object_clones":0,"num_object_copies":82719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2347,"num_objects_unfound":0,"num_objects_dirty":27573,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153229,"num_write_kb":5785601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12827,"num_bytes_recovered":801112064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5699022848,"data_stored":5699010632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136429,"ondisk_log_size":136429,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":28028,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201813,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1328028,"kb_used_data":663060,"kb_used_omap":0,"kb_used_meta":664960,"kb_avail":92417124,"statfs":{"total":95995035648,"available":94635134976,"internally_reserved":0,"allocated":678973440,"data_stored":677643870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":680919040},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":2.8570000000000002,"15min":1.5860000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":17.183,"5min":64.257000000000005,"15min":64.257000000000005},"last":9.7520000000000007},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.8149999999999999,"15min":1.5800000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":17.193000000000001,"5min":64.293000000000006,"15min":64.293000000000006},"last":9.7639999999999993}]},{"osd":1,"last update":"Sat Sep 23 21:12:03 2023","interfaces":[{"interface":"back","average":{"1min":6.4729999999999999,"5min":6.4409999999999998,"15min":2.9689999999999999},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":76.007999999999996,"5min":114.16500000000001,"15min":114.16500000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":7.577,"5min":6.7690000000000001,"15min":3.125},"min":{"1min":0.36699999999999999,"5min":0.252,"15min":0.23799999999999999},"max":{"1min":75.926000000000002,"5min":114.372,"15min":114.372},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":8.4199999999999999,"5min":5.4859999999999998,"15min":3.6299999999999999},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":133.524,"5min":183.44200000000001,"15min":183.44200000000001},"last":1.7529999999999999},{"interface":"front","average":{"1min":8.7460000000000004,"5min":5.4530000000000003,"15min":3.5499999999999998},"min":{"1min":0.35999999999999999,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":133.55699999999999,"5min":183.46199999999999,"15min":183.46199999999999},"last":1.77}]},{"osd":3,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":12.335000000000001,"5min":5.2869999999999999,"15min":3.1840000000000002},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":118.15900000000001,"5min":118.15900000000001,"15min":118.15900000000001},"last":4.0659999999999998},{"interface":"front","average":{"1min":12.760999999999999,"5min":5.7560000000000002,"15min":3.2200000000000002},"min":{"1min":0.34799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":118.098,"5min":118.098,"15min":118.098},"last":14.814}]},{"osd":4,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.72499999999999998,"15min":0.78300000000000003},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":4.165,"5min":13.739000000000001,"15min":15.942},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.119,"5min":0.79100000000000004,"15min":0.70999999999999996},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.26500000000000001},"max":{"1min":6.8920000000000003,"5min":6.8920000000000003,"15min":16.725999999999999},"last":0.36599999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.93300000000000005,"15min":0.90500000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.1449999999999996,"5min":10.891,"15min":10.891},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.93799999999999994,"15min":0.81999999999999995},"min":{"1min":0.35799999999999998,"5min":0.33300000000000002,"15min":0.28399999999999997},"max":{"1min":3.9039999999999999,"5min":13.867000000000001,"15min":13.867000000000001},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":7.7460000000000004,"5min":2.8090000000000002,"15min":1.607},"min":{"1min":0.35799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":143.18700000000001,"5min":143.18700000000001,"15min":143.18700000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":7.6180000000000003,"5min":2.8559999999999999,"15min":1.5649999999999999},"min":{"1min":0.33800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":142.70400000000001,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.40899999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463434,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1382744,"kb_used_data":753424,"kb_used_omap":0,"kb_used_meta":629312,"kb_avail":92362408,"statfs":{"total":95995035648,"available":94579105792,"internally_reserved":0,"allocated":771506176,"data_stored":770188542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":644415488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":10.954000000000001,"5min":3.444,"15min":2.536},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.20999999999999999},"max":{"1min":149.61600000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":10.253,"5min":3.1890000000000001,"15min":2.4430000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":149.303,"5min":149.303,"15min":149.303},"last":0.29899999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:12:21 2023","interfaces":[{"interface":"back","average":{"1min":11.156000000000001,"5min":4.4980000000000002,"15min":3.2869999999999999},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":129.37899999999999,"5min":129.37899999999999,"15min":129.37899999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":12.004,"5min":4.5469999999999997,"15min":3.2999999999999998},"min":{"1min":0.24399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":129.40100000000001,"5min":129.40100000000001,"15min":129.40100000000001},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":5.8659999999999997,"15min":5.1379999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":133.46600000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":10.975,"5min":5.3630000000000004,"15min":5.0069999999999997},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":133.33000000000001,"5min":133.33000000000001,"15min":199.08000000000001},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":15.163,"5min":5.6159999999999997,"15min":3.9039999999999999},"min":{"1min":0.30499999999999999,"5min":0.30399999999999999,"15min":0.26900000000000002},"max":{"1min":137.875,"5min":137.875,"15min":178.40000000000001},"last":2.5329999999999999},{"interface":"front","average":{"1min":13.106999999999999,"5min":5.2590000000000003,"15min":3.8620000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":137.60900000000001,"5min":137.60900000000001,"15min":178.26599999999999},"last":2.4009999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.748,"15min":0.86299999999999999},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.4489999999999998,"5min":9.0890000000000004,"15min":13.1},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.66100000000000003,"15min":0.73199999999999998},"min":{"1min":0.28299999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.78000000000000003,"5min":6.4400000000000004,"15min":6.4400000000000004},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.75700000000000001,"15min":0.876},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.419,"5min":9.0519999999999996,"15min":11.281000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.63800000000000001,"15min":0.85599999999999998},"min":{"1min":0.32200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.52300000000000002,"5min":6.0190000000000001,"15min":8.6229999999999993},"last":0.75700000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:12:21 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.76400000000000001,"15min":0.86799999999999999},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.60399999999999998,"5min":6.6299999999999999,"15min":7.8079999999999998},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.83099999999999996,"15min":0.90000000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0259999999999998,"5min":6.4749999999999996,"15min":7.1849999999999996},"last":0.78600000000000003}]}]},{"osd":1,"up_from":276,"seq":1185410973880,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1254136,"kb_used_data":700272,"kb_used_omap":5,"kb_used_meta":553850,"kb_avail":92491016,"statfs":{"total":95995035648,"available":94710800384,"internally_reserved":0,"allocated":717078528,"data_stored":714783962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":567143139},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":8.4529999999999994,"5min":4.2279999999999998,"15min":1.837},"min":{"1min":0.23400000000000001,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":108.47,"5min":108.47,"15min":108.47},"last":1.4910000000000001},{"interface":"front","average":{"1min":8.4969999999999999,"5min":4.0640000000000001,"15min":1.96},"min":{"1min":0.248,"5min":0.19600000000000001,"15min":0.193},"max":{"1min":109.532,"5min":109.532,"15min":109.532},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":9.7759999999999998,"5min":6.2039999999999997,"15min":3.0369999999999999},"min":{"1min":0.23200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":97.840999999999994,"5min":107.883,"15min":107.883},"last":1.8029999999999999},{"interface":"front","average":{"1min":8.5709999999999997,"5min":5.4649999999999999,"15min":2.6629999999999998},"min":{"1min":0.375,"5min":0.182,"15min":0.182},"max":{"1min":97.816999999999993,"5min":101.024,"15min":101.024},"last":3.786}]},{"osd":3,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":11.587999999999999,"5min":7.6180000000000003,"15min":4.7779999999999996},"min":{"1min":0.255,"5min":0.255,"15min":0.23899999999999999},"max":{"1min":99.120999999999995,"5min":99.120999999999995,"15min":172.44399999999999},"last":0.37},{"interface":"front","average":{"1min":11.824,"5min":7.093,"15min":4.8769999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":101.849,"5min":101.849,"15min":172.58500000000001},"last":9.577}]},{"osd":4,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":1.861,"15min":1.393},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.728999999999999,"5min":11.728999999999999,"15min":17.006},"last":9.7110000000000003},{"interface":"front","average":{"1min":1.022,"5min":1.994,"15min":1.1839999999999999},"min":{"1min":0.29399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.8239999999999998,"5min":23.332999999999998,"15min":23.332999999999998},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":2.2949999999999999,"15min":1.4650000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.5800000000000001,"5min":16.196000000000002,"15min":17.155000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.3839999999999999,"5min":2.1070000000000002,"15min":1.532},"min":{"1min":0.33800000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.760999999999999,"5min":11.760999999999999,"15min":17.076000000000001},"last":9.7490000000000006}]},{"osd":6,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":2.391,"15min":1.9930000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.7549999999999999,"5min":16.222000000000001,"15min":127.995},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.528,"5min":2.173,"15min":2.0209999999999999},"min":{"1min":0.378,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.782999999999999,"5min":11.782999999999999,"15min":126.836},"last":9.7780000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:12:11 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":2.1219999999999999,"15min":2.0089999999999999},"min":{"1min":0.441,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":151.791},"last":9.8049999999999997},{"interface":"front","average":{"1min":1.0669999999999999,"5min":2.1419999999999999,"15min":1.75},"min":{"1min":0.38100000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.9409999999999998,"5min":23.376000000000001,"15min":151.809},"last":1.4590000000000001}]}]},{"osd":0,"up_from":92,"seq":395136991620,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1554620,"kb_used_data":690420,"kb_used_omap":5,"kb_used_meta":864186,"kb_avail":92190532,"statfs":{"total":95995035648,"available":94403104768,"internally_reserved":0,"allocated":706990080,"data_stored":704807123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":884927098},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:13:08 2023","interfaces":[{"interface":"back","average":{"1min":3.04,"5min":2.956,"15min":2.6579999999999999},"min":{"1min":0.23300000000000001,"5min":0.182,"15min":0.17799999999999999},"max":{"1min":8.1389999999999993,"5min":53.262,"15min":61.341999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":3.5030000000000001,"5min":3.4470000000000001,"15min":2.7799999999999998},"min":{"1min":0.251,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.266,"5min":53.201999999999998,"15min":61.299999999999997},"last":9.75}]},{"osd":2,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":9.343,"5min":5.2949999999999999,"15min":3.4870000000000001},"min":{"1min":0.27300000000000002,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":152.291,"5min":152.291,"15min":183.77099999999999},"last":0.81200000000000006},{"interface":"front","average":{"1min":9.375,"5min":5.2400000000000002,"15min":3.6059999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.22600000000000001},"max":{"1min":152.33099999999999,"5min":152.33099999999999,"15min":183.78700000000001},"last":9.0719999999999992}]},{"osd":3,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":4.1909999999999998,"5min":4.548,"15min":3.206},"min":{"1min":0.33500000000000002,"5min":0.23999999999999999,"15min":0.224},"max":{"1min":10.448,"5min":86.989999999999995,"15min":186.143},"last":0.83799999999999997},{"interface":"front","average":{"1min":3.2749999999999999,"5min":4.3959999999999999,"15min":3.177},"min":{"1min":0.30299999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.648999999999999,"5min":87.043999999999997,"15min":186.97399999999999},"last":7.5800000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":1.786,"15min":1.4390000000000001},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.29799999999999999},"max":{"1min":11.420999999999999,"5min":11.420999999999999,"15min":11.420999999999999},"last":3.9940000000000002},{"interface":"front","average":{"1min":2.0230000000000001,"5min":1.6619999999999999,"15min":1.1479999999999999},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":4.9989999999999997,"5min":11.753,"15min":11.753},"last":0.69199999999999995}]},{"osd":5,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.8049999999999999,"15min":1.5169999999999999},"min":{"1min":0.39000000000000001,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":11.375999999999999,"5min":11.375999999999999,"15min":11.375999999999999},"last":4.04},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.7210000000000001,"15min":1.216},"min":{"1min":0.373,"5min":0.313,"15min":0.313},"max":{"1min":5.032,"5min":11.589,"15min":11.589},"last":0.78700000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":1.859,"15min":3.403},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.9530000000000003,"5min":17.773,"15min":170.12700000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.7809999999999999,"15min":3.4169999999999998},"min":{"1min":0.39800000000000002,"5min":0.37,"15min":0.31900000000000001},"max":{"1min":4.9560000000000004,"5min":17.684999999999999,"15min":169.767},"last":7.6349999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.8480000000000001,"15min":1.3420000000000001},"min":{"1min":0.42999999999999999,"5min":0.38200000000000001,"15min":0.33600000000000002},"max":{"1min":6.5830000000000002,"5min":11.69,"15min":11.69},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.331,"5min":1.827,"15min":1.3300000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.379},"max":{"1min":5.0010000000000003,"5min":10.470000000000001,"15min":10.470000000000001},"last":7.657}]}]},{"osd":2,"up_from":129,"seq":554050781511,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1959316,"kb_used_data":1110412,"kb_used_omap":7,"kb_used_meta":848504,"kb_avail":91785836,"statfs":{"total":95995035648,"available":93988696064,"internally_reserved":0,"allocated":1137061888,"data_stored":1134794765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":868868177},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":3.2629999999999999,"15min":1.8320000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17699999999999999},"max":{"1min":103.021,"5min":103.021,"15min":103.021},"last":14.081},{"interface":"front","average":{"1min":8.0449999999999999,"5min":4.069,"15min":2.5099999999999998},"min":{"1min":0.22700000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":103.154,"5min":103.154,"15min":103.154},"last":18.879999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:12:12 2023","interfaces":[{"interface":"back","average":{"1min":6.7569999999999997,"5min":5.141,"15min":3.8519999999999999},"min":{"1min":0.22500000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":106.268,"5min":143.482,"15min":212.12},"last":0.495},{"interface":"front","average":{"1min":8.0199999999999996,"5min":4.9980000000000002,"15min":3.7170000000000001},"min":{"1min":0.245,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":106.32299999999999,"5min":143.61000000000001,"15min":213.02799999999999},"last":14.108000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4319999999999999,"5min":5.8810000000000002,"15min":4.3150000000000004},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":12.696999999999999,"5min":143.50800000000001,"15min":143.50800000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":4.2309999999999999,"5min":6.0300000000000002,"15min":4.1959999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.23799999999999999},"max":{"1min":12.502000000000001,"5min":143.27799999999999,"15min":143.27799999999999},"last":14.032999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":4.1120000000000001,"15min":2.0150000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":9.8170000000000002,"5min":225.80799999999999,"15min":225.80799999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.1579999999999999,"5min":4.9660000000000002,"15min":2.5670000000000002},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":8.109,"5min":225.79300000000001,"15min":225.79300000000001},"last":0.63500000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":3.54,"15min":2.1589999999999998},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.29999999999999999},"max":{"1min":8.1489999999999991,"5min":98.682000000000002,"15min":98.682000000000002},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.718,"5min":4.4610000000000003,"15min":2.2200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.9450000000000003,"5min":143.34700000000001,"15min":143.34700000000001},"last":0.56200000000000006}]},{"osd":6,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":3.2490000000000001,"15min":2.5579999999999998},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":6.9969999999999999,"5min":143.41399999999999,"15min":175.72499999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.2429999999999999,"5min":1.6579999999999999,"15min":1.9419999999999999},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":11.893000000000001,"5min":11.893000000000001,"15min":176.20400000000001},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":1.7210000000000001,"15min":1.3939999999999999},"min":{"1min":0.41599999999999998,"5min":0.36399999999999999,"15min":0.32700000000000001},"max":{"1min":9.843,"5min":11.811999999999999,"15min":12.609},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.1920000000000002,"5min":2.5960000000000001,"15min":1.8480000000000001},"min":{"1min":0.41399999999999998,"5min":0.38200000000000001,"15min":0.34499999999999997},"max":{"1min":8.1720000000000006,"5min":16.875,"15min":16.875},"last":0.68999999999999995}]}]},{"osd":3,"up_from":148,"seq":635655160105,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761420,"kb_used_data":1032196,"kb_used_omap":2,"kb_used_meta":728893,"kb_avail":91983732,"statfs":{"total":95995035648,"available":94191341568,"internally_reserved":0,"allocated":1056968704,"data_stored":1054473370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":746386485},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":2.7850000000000001,"15min":1.8500000000000001},"min":{"1min":0.53400000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.696999999999999,"5min":17.425999999999998,"15min":44.963000000000001},"last":1.1890000000000001},{"interface":"front","average":{"1min":4.3959999999999999,"5min":2.48,"15min":1.534},"min":{"1min":0.46000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":14.503,"5min":16.920999999999999,"15min":19.738},"last":3.274}]},{"osd":1,"last update":"Sat Sep 23 21:12:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6219999999999999,"5min":15.395,"15min":5.9349999999999996},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.20999999999999999},"max":{"1min":14.867000000000001,"5min":237.072,"15min":237.072},"last":147.12100000000001},{"interface":"front","average":{"1min":4.3460000000000001,"5min":15.266999999999999,"15min":6.04},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":16.178000000000001,"5min":234.21899999999999,"15min":234.21899999999999},"last":151.14699999999999}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":6.3200000000000003,"5min":6.1280000000000001,"15min":3.0499999999999998},"min":{"1min":0.55900000000000005,"5min":0.23599999999999999,"15min":0.223},"max":{"1min":24.378,"5min":93.272000000000006,"15min":93.272000000000006},"last":0.51500000000000001},{"interface":"front","average":{"1min":4.2370000000000001,"5min":5.2320000000000002,"15min":2.6160000000000001},"min":{"1min":0.58599999999999997,"5min":0.22500000000000001,"15min":0.222},"max":{"1min":10.907,"5min":95.233999999999995,"15min":95.233999999999995},"last":0.52600000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1859999999999999,"5min":2.4780000000000002,"15min":2.6520000000000001},"min":{"1min":0.36399999999999999,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":15.763,"5min":15.763,"15min":111.36},"last":0.28299999999999997},{"interface":"front","average":{"1min":2.835,"5min":2.0510000000000002,"15min":2.367},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.6180000000000003,"5min":17.286999999999999,"15min":112.473},"last":0.30099999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":2.9809999999999999,"5min":2.105,"15min":1.5820000000000001},"min":{"1min":0.501,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.3710000000000004,"5min":16.855,"15min":36.488999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":4.0949999999999998,"5min":2.6400000000000001,"15min":1.9990000000000001},"min":{"1min":0.40699999999999997,"5min":0.315,"15min":0.315},"max":{"1min":15.789999999999999,"5min":15.789999999999999,"15min":36.295000000000002},"last":0.34999999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0599999999999996,"5min":6.6529999999999996,"15min":3.2650000000000001},"min":{"1min":0.434,"5min":0.33500000000000002,"15min":0.312},"max":{"1min":15.718,"5min":304.22399999999999,"15min":304.22399999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.827,"5min":6.0510000000000002,"15min":2.964},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":7.5839999999999996,"5min":304.15800000000002,"15min":304.15800000000002},"last":0.33800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":3.282,"5min":2.1829999999999998,"15min":1.897},"min":{"1min":0.52100000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":8.3309999999999995,"5min":16.899000000000001,"15min":74.019000000000005},"last":0.499},{"interface":"front","average":{"1min":4.8339999999999996,"5min":2.9039999999999999,"15min":2.3719999999999999},"min":{"1min":0.436,"5min":0.377,"15min":0.32900000000000001},"max":{"1min":15.816000000000001,"5min":15.816000000000001,"15min":73.364999999999995},"last":0.374}]}]},{"osd":4,"up_from":31,"seq":133143986723,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081112,"kb_used_data":458768,"kb_used_omap":0,"kb_used_meta":622336,"kb_avail":92664040,"statfs":{"total":95995035648,"available":94887976960,"internally_reserved":0,"allocated":469778432,"data_stored":468452667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":637272064},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":11.087999999999999,"5min":3.4750000000000001,"15min":2.0830000000000002},"min":{"1min":0.219,"5min":0.20699999999999999,"15min":0.20399999999999999},"max":{"1min":226.751,"5min":226.751,"15min":226.751},"last":0.99299999999999999},{"interface":"front","average":{"1min":11.529999999999999,"5min":3.589,"15min":2.0939999999999999},"min":{"1min":0.253,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":226.43299999999999,"5min":226.43299999999999,"15min":226.43299999999999},"last":0.27200000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:12:16 2023","interfaces":[{"interface":"back","average":{"1min":5.1900000000000004,"5min":3.5350000000000001,"15min":4.306},"min":{"1min":0.33400000000000002,"5min":0.26700000000000002,"15min":0.23200000000000001},"max":{"1min":68.337000000000003,"5min":68.337000000000003,"15min":127.411},"last":2.8220000000000001},{"interface":"front","average":{"1min":4.9500000000000002,"5min":3.2949999999999999,"15min":3.6509999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":68.293000000000006,"5min":68.293000000000006,"15min":118.075},"last":1.117}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":14.257,"5min":5.7480000000000002,"15min":4.4900000000000002},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":210.50299999999999,"5min":210.50299999999999,"15min":210.50299999999999},"last":1.6319999999999999},{"interface":"front","average":{"1min":15.454000000000001,"5min":5.8390000000000004,"15min":4.3890000000000002},"min":{"1min":0.33100000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":210.59,"5min":210.59,"15min":210.59},"last":4.7880000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:12:16 2023","interfaces":[{"interface":"back","average":{"1min":23.369,"5min":7.9000000000000004,"15min":3.5030000000000001},"min":{"1min":0.42099999999999999,"5min":0.29399999999999998,"15min":0.27300000000000002},"max":{"1min":214.64099999999999,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":23.645,"5min":7.8780000000000001,"15min":3.5870000000000002},"min":{"1min":0.54700000000000004,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":214.54900000000001,"5min":214.54900000000001,"15min":214.54900000000001},"last":0.88100000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.69699999999999995},"min":{"1min":0.28199999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.2069999999999999,"5min":6.0069999999999997,"15min":8.7089999999999996},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.66200000000000003,"15min":0.69999999999999996},"min":{"1min":0.27400000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.881,"5min":4.7140000000000004,"15min":7.0179999999999998},"last":0.36199999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":1.2549999999999999,"15min":0.98099999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79300000000000004,"5min":36.557000000000002,"15min":36.557000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.125,"15min":0.97199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":2.5760000000000001,"5min":38.468000000000004,"15min":38.468000000000004},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.80500000000000005,"15min":1.4610000000000001},"min":{"1min":0.309,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.8319999999999999,"5min":4.7359999999999998,"15min":145.721},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.871,"15min":1.601},"min":{"1min":0.33800000000000002,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":0.61099999999999999,"5min":10.411,"15min":148.45699999999999},"last":0.53200000000000003}]}]},{"osd":5,"up_from":39,"seq":167503725087,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908296,"kb_used_data":240320,"kb_used_omap":0,"kb_used_meta":667968,"kb_avail":92836856,"statfs":{"total":95995035648,"available":95064940544,"internally_reserved":0,"allocated":246087680,"data_stored":244774043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":683999232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":2.8399999999999999,"15min":1.4690000000000001},"min":{"1min":0.25700000000000001,"5min":0.217,"15min":0.217},"max":{"1min":20.023,"5min":82.280000000000001,"15min":82.280000000000001},"last":8.6660000000000004},{"interface":"front","average":{"1min":3.0569999999999999,"5min":2.9620000000000002,"15min":1.5860000000000001},"min":{"1min":0.252,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":19.853000000000002,"5min":85.236000000000004,"15min":85.236000000000004},"last":16.977}]},{"osd":1,"last update":"Sat Sep 23 21:12:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6819999999999999,"5min":5.2919999999999998,"15min":4.1429999999999998},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.711,"5min":125.23399999999999,"15min":233.77099999999999},"last":12.032999999999999},{"interface":"front","average":{"1min":3.3399999999999999,"5min":5.2590000000000003,"15min":4.0389999999999997},"min":{"1min":0.34100000000000003,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":10.724,"5min":125.249,"15min":233.75200000000001},"last":12.016}]},{"osd":2,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":4.1660000000000004,"15min":3.4990000000000001},"min":{"1min":0.35199999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":20.411999999999999,"5min":78.917000000000002,"15min":231.34700000000001},"last":3.2200000000000002},{"interface":"front","average":{"1min":2.8460000000000001,"5min":4.2640000000000002,"15min":3.5939999999999999},"min":{"1min":0.29999999999999999,"5min":0.28499999999999998,"15min":0.27300000000000002},"max":{"1min":11.509,"5min":78.888999999999996,"15min":231.381},"last":4.1360000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":7.5039999999999996,"15min":3.9809999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":10.053000000000001,"5min":263.887,"15min":263.887},"last":8.5820000000000007},{"interface":"front","average":{"1min":1.6519999999999999,"5min":7.2450000000000001,"15min":3.8170000000000002},"min":{"1min":0.32200000000000001,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":4.218,"5min":263.70100000000002,"15min":263.70100000000002},"last":6.8949999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.66800000000000004,"15min":0.60199999999999998},"min":{"1min":0.29299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.0449999999999999,"5min":6.3940000000000001,"15min":12.739000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.73199999999999998,"15min":0.72299999999999998},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9849999999999999,"5min":5.8490000000000002,"15min":10.792},"last":0.38400000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.708,"15min":1.333},"min":{"1min":0.35699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7440000000000002,"5min":83.451999999999998,"15min":83.451999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.677,"15min":1.3720000000000001},"min":{"1min":0.30399999999999999,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":3.9889999999999999,"5min":83.484999999999999,"15min":83.484999999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:12:26 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":0.95399999999999996,"15min":0.875},"min":{"1min":0.33200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.7160000000000002,"5min":9.6329999999999991,"15min":9.6329999999999991},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.74199999999999999,"15min":0.64800000000000002},"min":{"1min":0.32200000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.0790000000000002,"5min":4.29,"15min":4.29},"last":0.52300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":692518912,"data_stored":692518912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":710217728,"data_stored":710213656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1130631168,"data_stored":1130627096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1029574656,"data_stored":1029570584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":461766656,"data_stored":461766656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":239206400,"data_stored":239206400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":764149760,"data_stored":764149760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":670957568,"data_stored":670957568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:13:14.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:13:14.197 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:14.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:14.303 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:14.336 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:14.399 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:14.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:14.507 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:14.515 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:14.628 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:14.873 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:13:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:13:14] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-09-23T21:13:15.030 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:15.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:15.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:15.142 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:15.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3870288347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:15.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:15.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3870288347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:15.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:15.185 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:13:15.349 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3870288347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:16.378 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:16.378 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:16.378 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[211526]: osdmap e470: 8 total, 8 up, 8 in 2023-09-23T21:13:16.378 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[211526]: failed to encode map e470 with expected crc 2023-09-23T21:13:16.378 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[211526]: failed to encode map e470 with expected crc 2023-09-23T21:13:16.379 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[211526]: pgmap v1521: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 1190/79338 objects misplaced (1.500%); 10 MiB/s, 166 objects/s recovering 2023-09-23T21:13:16.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:16.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:16.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[219709]: osdmap e470: 8 total, 8 up, 8 in 2023-09-23T21:13:16.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[219709]: failed to encode map e470 with expected crc 2023-09-23T21:13:16.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[219709]: failed to encode map e470 with expected crc 2023-09-23T21:13:16.379 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:16 smithi121 ceph-mon[219709]: pgmap v1521: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 1190/79338 objects misplaced (1.500%); 10 MiB/s, 166 objects/s recovering 2023-09-23T21:13:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:16 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:16 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:16 smithi123 ceph-mon[193903]: osdmap e470: 8 total, 8 up, 8 in 2023-09-23T21:13:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:16 smithi123 ceph-mon[193903]: failed to encode map e470 with expected crc 2023-09-23T21:13:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:16 smithi123 ceph-mon[193903]: failed to encode map e470 with expected crc 2023-09-23T21:13:16.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:16 smithi123 ceph-mon[193903]: pgmap v1521: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 1190/79338 objects misplaced (1.500%); 10 MiB/s, 166 objects/s recovering 2023-09-23T21:13:16.731 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:16.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:16.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:16.853 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:16.918 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:17.265 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:17.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:17.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:17.382 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:17.465 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:17.897 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:17.901 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:17 smithi121 ceph-mon[211526]: pgmap v1522: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 1190/75882 objects misplaced (1.568%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T21:13:17.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:17 smithi121 ceph-mon[219709]: pgmap v1522: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 1190/75882 objects misplaced (1.568%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T21:13:17.906 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:17.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:17.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:17 smithi123 ceph-mon[193903]: pgmap v1522: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 1190/75882 objects misplaced (1.568%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T21:13:18.030 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:18.109 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:18.608 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:18.609 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:18.796 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:18.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:18.903 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:18.936 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:18.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:18.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:18.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:19.048 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:13:19.094 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:19.218 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:19.737 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:13:19.737 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:13:19.737 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:13:19.746 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:13:19.762 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:19.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:19.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:19.784 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:13:19.831 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:13:19.861 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:13:19.878 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:19.880 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:13:19.881 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.881 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.881 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.882 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.882 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.882 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.882 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.882 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:13:19.903 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:19.963 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:19.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:20.074 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:20.141 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:20.374 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:20.386 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:20 smithi121 ceph-mon[211526]: pgmap v1523: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 471/72564 objects misplaced (0.649%); 13 MiB/s, 205 objects/s recovering 2023-09-23T21:13:20.386 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2981770037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:20.386 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:20 smithi121 ceph-mon[219709]: pgmap v1523: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 471/72564 objects misplaced (0.649%); 13 MiB/s, 205 objects/s recovering 2023-09-23T21:13:20.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2981770037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:20.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:20.408 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:13:20.489 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:20.571 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:20 smithi123 ceph-mon[193903]: pgmap v1523: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 471/72564 objects misplaced (0.649%); 13 MiB/s, 205 objects/s recovering 2023-09-23T21:13:20.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2981770037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:21.105 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:21.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:21.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:21.227 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:21.253 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:21.312 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:21.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:21.354 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:21.419 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:21.485 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:21.522 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:21.747 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:21.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:21.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:21.852 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:21.880 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:21.922 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:21.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:22.029 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:22.055 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.097 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:22.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:22.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:22.206 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:22.232 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.253 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:13:22.253 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2058,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":471,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":8,"osd_in_since":1695503414,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":89,"num_pools":5,"num_objects":22422,"data_bytes":1468405688,"bytes_used":10491768832,"bytes_avail":757468516352,"bytes_total":767960285184,"misplaced_objects":471,"misplaced_total":67266,"misplaced_ratio":0.0070020515565070033,"recovering_objects_per_sec":171,"recovering_bytes_per_sec":11260349,"recovering_keys_per_sec":0,"num_objects_recovered":1643,"num_bytes_recovered":107610136,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":541},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":31,"modified":"2023-09-23T21:11:55.199243+0000","services":{}},"progress_events":{"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"d9bd0069-e535-49e2-8586-c1545e56d2c4":{"message":"Global Recovery Event (15s)\n [===========================.] ","progress":0.98876404762268066,"add_to_ceph_s":true}}} 2023-09-23T21:13:22.274 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:22.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:22.380 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:22.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[211526]: pgmap v1524: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 471/67266 objects misplaced (0.700%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:13:22.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[211526]: osdmap e471: 8 total, 8 up, 8 in 2023-09-23T21:13:22.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[211526]: failed to encode map e471 with expected crc 2023-09-23T21:13:22.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[211526]: failed to encode map e471 with expected crc 2023-09-23T21:13:22.390 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3003167065' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:13:22.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[219709]: pgmap v1524: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 471/67266 objects misplaced (0.700%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:13:22.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[219709]: osdmap e471: 8 total, 8 up, 8 in 2023-09-23T21:13:22.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[219709]: failed to encode map e471 with expected crc 2023-09-23T21:13:22.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[219709]: failed to encode map e471 with expected crc 2023-09-23T21:13:22.391 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:22 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3003167065' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:13:22.409 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.457 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:22.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:22.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:22.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:22.665 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:22.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:22 smithi123 ceph-mon[193903]: pgmap v1524: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 471/67266 objects misplaced (0.700%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:13:22.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:22 smithi123 ceph-mon[193903]: osdmap e471: 8 total, 8 up, 8 in 2023-09-23T21:13:22.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:22 smithi123 ceph-mon[193903]: failed to encode map e471 with expected crc 2023-09-23T21:13:22.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:22 smithi123 ceph-mon[193903]: failed to encode map e471 with expected crc 2023-09-23T21:13:22.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:22 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3003167065' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:13:23.188 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:13:23.205 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:23.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:23.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:23.320 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:23.405 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:23.432 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.432 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: osdmap e472: 8 total, 8 up, 8 in 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.433 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[211526]: pgmap v1527: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 471/66252 objects misplaced (0.711%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:13:23.437 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.437 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: osdmap e472: 8 total, 8 up, 8 in 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.438 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:23 smithi121 ceph-mon[219709]: pgmap v1527: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 471/66252 objects misplaced (0.711%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:13:23.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: osdmap e472: 8 total, 8 up, 8 in 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: failed to encode map e472 with expected crc 2023-09-23T21:13:23.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:23 smithi123 ceph-mon[193903]: pgmap v1527: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 471/66252 objects misplaced (0.711%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:13:24.191 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:24.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:24.216 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:13:24.217 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:13:24.299 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:24.374 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:24.570 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:24.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:24.688 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:24.733 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:24.861 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:24.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:13:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:13:24] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-09-23T21:13:24.997 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:13:24.998 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:13:24.998 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:13:24.998 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:13:24.998 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:13:25.009 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:25.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:25.116 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:25.160 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:25.287 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:25.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:25.394 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:25.437 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:25.507 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:25.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:25.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:25.612 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:25.641 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:25.750 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:25.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:25.859 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:25.959 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:26.345 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:26 smithi121 ceph-mon[219709]: pgmap v1528: 89 pgs: 89 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 8.5 MiB/s, 136 objects/s recovering 2023-09-23T21:13:26.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/418088299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:26.346 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:26.404 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:13:26.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:26 smithi121 ceph-mon[211526]: pgmap v1528: 89 pgs: 89 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 8.5 MiB/s, 136 objects/s recovering 2023-09-23T21:13:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/418088299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:26.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:26 smithi123 ceph-mon[193903]: pgmap v1528: 89 pgs: 89 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 8.5 MiB/s, 136 objects/s recovering 2023-09-23T21:13:26.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/418088299' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:26.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:27.076 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:27.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:27.184 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:27.225 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:13:27.304 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:13:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: osdmap e473: 8 total, 8 up, 8 in 2023-09-23T21:13:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:27.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[219709]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:27.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:27.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:13:27.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:27.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: osdmap e473: 8 total, 8 up, 8 in 2023-09-23T21:13:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:27.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:27 smithi121 ceph-mon[211526]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:13:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:13:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: osdmap e473: 8 total, 8 up, 8 in 2023-09-23T21:13:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:27 smithi123 ceph-mon[193903]: failed to encode map e473 with expected crc 2023-09-23T21:13:27.821 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:27.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:27.933 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:27.971 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:28.025 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:28.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:28.132 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:28.162 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:28.192 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:28.219 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:28.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:28.328 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:28.356 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:28.405 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:28.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:28.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:28 smithi123 ceph-mon[193903]: pgmap v1530: 89 pgs: 89 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 4.7 MiB/s, 74 objects/s recovering 2023-09-23T21:13:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:28.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:28.514 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:28.594 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:28 smithi121 ceph-mon[211526]: pgmap v1530: 89 pgs: 89 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 4.7 MiB/s, 74 objects/s recovering 2023-09-23T21:13:28.594 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:28.594 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:28.595 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:28 smithi121 ceph-mon[219709]: pgmap v1530: 89 pgs: 89 active+clean; 1.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 4.7 MiB/s, 74 objects/s recovering 2023-09-23T21:13:28.595 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:28.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:28.597 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:28.692 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:28.717 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:28.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:28.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:28.765 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:13:28.819 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:13:28.825 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:28.862 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:13:28.896 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:28.924 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:13:28.962 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:13:28.997 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:13:29.063 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:13:29.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:13:29.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.137 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.138 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.138 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.138 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.138 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:13:29.142 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:29.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:29.255 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:29.361 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:29.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3646132742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:29.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3646132742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:29.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3646132742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:30.082 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:13:30.516 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:30.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:30.630 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:30.664 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:30 smithi123 ceph-mon[193903]: pgmap v1531: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:13:30.666 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:30.668 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:30 smithi121 ceph-mon[211526]: pgmap v1531: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:13:30.668 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:30 smithi121 ceph-mon[219709]: pgmap v1531: 89 pgs: 89 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 3.6 MiB/s, 57 objects/s recovering 2023-09-23T21:13:30.725 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:30.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:30.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:30.835 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:30.865 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:30.913 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:30.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:31.020 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:31.069 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:31.133 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:31.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:31.239 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:31.269 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:31.335 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:31.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:31.394 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:31 smithi121 ceph-mon[211526]: pgmap v1532: 89 pgs: 89 active+clean; 1.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 3.2 MiB/s, 50 objects/s recovering 2023-09-23T21:13:31.446 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:31.522 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:31.669 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:31 smithi121 ceph-mon[219709]: pgmap v1532: 89 pgs: 89 active+clean; 1.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 3.2 MiB/s, 50 objects/s recovering 2023-09-23T21:13:31.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:31 smithi123 ceph-mon[193903]: pgmap v1532: 89 pgs: 89 active+clean; 1.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 3.2 MiB/s, 50 objects/s recovering 2023-09-23T21:13:31.783 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:31.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:31.900 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:31.927 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:31.980 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:31.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:31.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:32.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:32.158 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:32.186 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.386 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:32.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:32.503 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:32.532 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.590 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:32.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:32.692 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:13:32.702 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:32.732 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:13:32.825 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:13:32.828 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:32.878 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:32.878 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:34.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:34 smithi121 ceph-mon[211526]: from='client.36915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:34.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:34 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:34.418 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:34 smithi121 ceph-mon[211526]: pgmap v1533: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:13:34.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:34 smithi121 ceph-mon[219709]: from='client.36915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:34.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:34 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:34.418 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:34 smithi121 ceph-mon[219709]: pgmap v1533: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:13:34.467 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:34.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:34.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:34 smithi123 ceph-mon[193903]: from='client.36915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:34 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:34 smithi123 ceph-mon[193903]: pgmap v1533: 89 pgs: 89 active+clean; 1.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-09-23T21:13:34.567 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:13:34.574 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:34.602 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:34.660 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:34.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:34.767 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:34.797 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:34.851 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:34.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:34.964 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:35.050 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:13:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:13:34] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.43.0" 2023-09-23T21:13:35.056 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:35.191 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:35.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:35.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:35.301 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:35.343 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:35.442 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:35.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:35.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:35.548 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:35.633 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:35.893 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:35.929 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1532,"stamp":"2023-09-23T21:13:31.270917+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1116084152,"num_objects":17046,"num_object_clones":0,"num_object_copies":51138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17046,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":164080,"num_write_kb":5788046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15033,"num_bytes_recovered":942308468,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139208,"ondisk_log_size":139208,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9231116,"kb_used_data":3639180,"kb_used_omap":21,"kb_used_meta":5591402,"kb_avail":740730100,"statfs":{"total":767960285184,"available":758507622400,"internally_reserved":0,"allocated":3726520320,"data_stored":3711824448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":5725596131},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,4,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-330170368,"num_objects":-5038,"num_object_clones":0,"num_object_copies":-15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-471,"num_objects_unfound":0,"num_objects_dirty":-5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5038,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":449,"num_bytes_recovered":29425664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.877454"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":246,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400786+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:13:22.400786+0000","last_peered":"2023-09-23T21:13:22.400786+0000","last_clean":"2023-09-23T21:13:22.400786+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:13:22.400786+0000","last_undegraded":"2023-09-23T21:13:22.400786+0000","last_fullsized":"2023-09-23T21:13:22.400786+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":115,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396737+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:13:22.396737+0000","last_peered":"2023-09-23T21:13:22.396737+0000","last_clean":"2023-09-23T21:13:22.396737+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:13:22.396737+0000","last_undegraded":"2023-09-23T21:13:22.396737+0000","last_fullsized":"2023-09-23T21:13:22.396737+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":388,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400683+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:13:22.400683+0000","last_peered":"2023-09-23T21:13:22.400683+0000","last_clean":"2023-09-23T21:13:22.400683+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:13:22.400683+0000","last_undegraded":"2023-09-23T21:13:22.400683+0000","last_fullsized":"2023-09-23T21:13:22.400683+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.493429+0000","last_change":"2023-09-23T21:12:51.650220+0000","last_active":"2023-09-23T21:13:22.493429+0000","last_peered":"2023-09-23T21:13:22.493429+0000","last_clean":"2023-09-23T21:13:22.493429+0000","last_became_active":"2023-09-23T21:12:51.649869+0000","last_became_peered":"2023-09-23T21:12:51.649869+0000","last_unstale":"2023-09-23T21:13:22.493429+0000","last_undegraded":"2023-09-23T21:13:22.493429+0000","last_fullsized":"2023-09-23T21:13:22.493429+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:04.107476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":133,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.395037+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:13:22.395037+0000","last_peered":"2023-09-23T21:13:22.395037+0000","last_clean":"2023-09-23T21:13:22.395037+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:13:22.395037+0000","last_undegraded":"2023-09-23T21:13:22.395037+0000","last_fullsized":"2023-09-23T21:13:22.395037+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":349,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482735+0000","last_change":"2023-09-23T21:09:44.161988+0000","last_active":"2023-09-23T21:13:22.482735+0000","last_peered":"2023-09-23T21:13:22.482735+0000","last_clean":"2023-09-23T21:13:22.482735+0000","last_became_active":"2023-09-23T21:09:44.161643+0000","last_became_peered":"2023-09-23T21:09:44.161643+0000","last_unstale":"2023-09-23T21:13:22.482735+0000","last_undegraded":"2023-09-23T21:13:22.482735+0000","last_fullsized":"2023-09-23T21:13:22.482735+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:17.083615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":112,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.492469+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:13:22.492469+0000","last_peered":"2023-09-23T21:13:22.492469+0000","last_clean":"2023-09-23T21:13:22.492469+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:13:22.492469+0000","last_undegraded":"2023-09-23T21:13:22.492469+0000","last_fullsized":"2023-09-23T21:13:22.492469+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.394945+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:13:22.394945+0000","last_peered":"2023-09-23T21:13:22.394945+0000","last_clean":"2023-09-23T21:13:22.394945+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:13:22.394945+0000","last_undegraded":"2023-09-23T21:13:22.394945+0000","last_fullsized":"2023-09-23T21:13:22.394945+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.397094+0000","last_change":"2023-09-23T21:10:15.584242+0000","last_active":"2023-09-23T21:13:22.397094+0000","last_peered":"2023-09-23T21:13:22.397094+0000","last_clean":"2023-09-23T21:13:22.397094+0000","last_became_active":"2023-09-23T21:10:15.583926+0000","last_became_peered":"2023-09-23T21:10:15.583926+0000","last_unstale":"2023-09-23T21:13:22.397094+0000","last_undegraded":"2023-09-23T21:13:22.397094+0000","last_fullsized":"2023-09-23T21:13:22.397094+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:15.759114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":124,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.395605+0000","last_change":"2023-09-23T21:12:51.652067+0000","last_active":"2023-09-23T21:13:22.395605+0000","last_peered":"2023-09-23T21:13:22.395605+0000","last_clean":"2023-09-23T21:13:22.395605+0000","last_became_active":"2023-09-23T21:12:51.651751+0000","last_became_peered":"2023-09-23T21:12:51.651751+0000","last_unstale":"2023-09-23T21:13:22.395605+0000","last_undegraded":"2023-09-23T21:13:22.395605+0000","last_fullsized":"2023-09-23T21:13:22.395605+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:03.235559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":28,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399500+0000","last_change":"2023-09-23T21:12:51.657657+0000","last_active":"2023-09-23T21:13:22.399500+0000","last_peered":"2023-09-23T21:13:22.399500+0000","last_clean":"2023-09-23T21:13:22.399500+0000","last_became_active":"2023-09-23T21:12:51.641567+0000","last_became_peered":"2023-09-23T21:12:51.641567+0000","last_unstale":"2023-09-23T21:13:22.399500+0000","last_undegraded":"2023-09-23T21:13:22.399500+0000","last_fullsized":"2023-09-23T21:13:22.399500+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:00.366548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":107,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400670+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:13:22.400670+0000","last_peered":"2023-09-23T21:13:22.400670+0000","last_clean":"2023-09-23T21:13:22.400670+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:13:22.400670+0000","last_undegraded":"2023-09-23T21:13:22.400670+0000","last_fullsized":"2023-09-23T21:13:22.400670+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":60,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399387+0000","last_change":"2023-09-23T21:10:15.594033+0000","last_active":"2023-09-23T21:13:22.399387+0000","last_peered":"2023-09-23T21:13:22.399387+0000","last_clean":"2023-09-23T21:13:22.399387+0000","last_became_active":"2023-09-23T21:10:15.579277+0000","last_became_peered":"2023-09-23T21:10:15.579277+0000","last_unstale":"2023-09-23T21:13:22.399387+0000","last_undegraded":"2023-09-23T21:13:22.399387+0000","last_fullsized":"2023-09-23T21:13:22.399387+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:42.561285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":391,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.489385+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:13:22.489385+0000","last_peered":"2023-09-23T21:13:22.489385+0000","last_clean":"2023-09-23T21:13:22.489385+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:13:22.489385+0000","last_undegraded":"2023-09-23T21:13:22.489385+0000","last_fullsized":"2023-09-23T21:13:22.489385+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":56,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399569+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:13:22.399569+0000","last_peered":"2023-09-23T21:13:22.399569+0000","last_clean":"2023-09-23T21:13:22.399569+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:13:22.399569+0000","last_undegraded":"2023-09-23T21:13:22.399569+0000","last_fullsized":"2023-09-23T21:13:22.399569+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":133,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396660+0000","last_change":"2023-09-23T21:10:15.590805+0000","last_active":"2023-09-23T21:13:22.396660+0000","last_peered":"2023-09-23T21:13:22.396660+0000","last_clean":"2023-09-23T21:13:22.396660+0000","last_became_active":"2023-09-23T21:10:15.590501+0000","last_became_peered":"2023-09-23T21:10:15.590501+0000","last_unstale":"2023-09-23T21:13:22.396660+0000","last_undegraded":"2023-09-23T21:13:22.396660+0000","last_fullsized":"2023-09-23T21:13:22.396660+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:27.143631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":22,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399429+0000","last_change":"2023-09-23T21:12:51.643047+0000","last_active":"2023-09-23T21:13:22.399429+0000","last_peered":"2023-09-23T21:13:22.399429+0000","last_clean":"2023-09-23T21:13:22.399429+0000","last_became_active":"2023-09-23T21:12:51.642754+0000","last_became_peered":"2023-09-23T21:12:51.642754+0000","last_unstale":"2023-09-23T21:13:22.399429+0000","last_undegraded":"2023-09-23T21:13:22.399429+0000","last_fullsized":"2023-09-23T21:13:22.399429+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:31.430089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":50,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399726+0000","last_change":"2023-09-23T21:10:15.585218+0000","last_active":"2023-09-23T21:13:22.399726+0000","last_peered":"2023-09-23T21:13:22.399726+0000","last_clean":"2023-09-23T21:13:22.399726+0000","last_became_active":"2023-09-23T21:10:15.584952+0000","last_became_peered":"2023-09-23T21:10:15.584952+0000","last_unstale":"2023-09-23T21:13:22.399726+0000","last_undegraded":"2023-09-23T21:13:22.399726+0000","last_fullsized":"2023-09-23T21:13:22.399726+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:55.016769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399239+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:13:22.399239+0000","last_peered":"2023-09-23T21:13:22.399239+0000","last_clean":"2023-09-23T21:13:22.399239+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:13:22.399239+0000","last_undegraded":"2023-09-23T21:13:22.399239+0000","last_fullsized":"2023-09-23T21:13:22.399239+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":121,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.493493+0000","last_change":"2023-09-23T21:12:51.650915+0000","last_active":"2023-09-23T21:13:22.493493+0000","last_peered":"2023-09-23T21:13:22.493493+0000","last_clean":"2023-09-23T21:13:22.493493+0000","last_became_active":"2023-09-23T21:12:51.650550+0000","last_became_peered":"2023-09-23T21:12:51.650550+0000","last_unstale":"2023-09-23T21:13:22.493493+0000","last_undegraded":"2023-09-23T21:13:22.493493+0000","last_fullsized":"2023-09-23T21:13:22.493493+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:30.158548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":469,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482649+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:13:22.482649+0000","last_peered":"2023-09-23T21:13:22.482649+0000","last_clean":"2023-09-23T21:13:22.482649+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:13:22.482649+0000","last_undegraded":"2023-09-23T21:13:22.482649+0000","last_fullsized":"2023-09-23T21:13:22.482649+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":133,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396554+0000","last_change":"2023-09-23T21:12:51.645964+0000","last_active":"2023-09-23T21:13:22.396554+0000","last_peered":"2023-09-23T21:13:22.396554+0000","last_clean":"2023-09-23T21:13:22.396554+0000","last_became_active":"2023-09-23T21:12:51.645627+0000","last_became_peered":"2023-09-23T21:12:51.645627+0000","last_unstale":"2023-09-23T21:13:22.396554+0000","last_undegraded":"2023-09-23T21:13:22.396554+0000","last_fullsized":"2023-09-23T21:13:22.396554+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:53.725127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"473'10222","reported_seq":10277,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:31.032866+0000","last_change":"2023-09-23T21:09:45.077841+0000","last_active":"2023-09-23T21:13:31.032866+0000","last_peered":"2023-09-23T21:13:31.032866+0000","last_clean":"2023-09-23T21:13:31.032866+0000","last_became_active":"2023-09-23T21:09:45.077550+0000","last_became_peered":"2023-09-23T21:09:45.077550+0000","last_unstale":"2023-09-23T21:13:31.032866+0000","last_undegraded":"2023-09-23T21:13:31.032866+0000","last_fullsized":"2023-09-23T21:13:31.032866+0000","mapping_epoch":426,"log_start":"430'1300","ondisk_log_start":"430'1300","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8922,"log_dups_size":0,"ondisk_log_size":8922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:19.178954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65929216,"num_objects":1006,"num_object_clones":0,"num_object_copies":3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10222,"num_write_kb":359296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":26,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399651+0000","last_change":"2023-09-23T21:12:51.662783+0000","last_active":"2023-09-23T21:13:22.399651+0000","last_peered":"2023-09-23T21:13:22.399651+0000","last_clean":"2023-09-23T21:13:22.399651+0000","last_became_active":"2023-09-23T21:12:51.649440+0000","last_became_peered":"2023-09-23T21:12:51.649440+0000","last_unstale":"2023-09-23T21:13:22.399651+0000","last_undegraded":"2023-09-23T21:13:22.399651+0000","last_fullsized":"2023-09-23T21:13:22.399651+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:11.808407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396811+0000","last_change":"2023-09-23T21:12:51.650942+0000","last_active":"2023-09-23T21:13:22.396811+0000","last_peered":"2023-09-23T21:13:22.396811+0000","last_clean":"2023-09-23T21:13:22.396811+0000","last_became_active":"2023-09-23T21:12:51.650556+0000","last_became_peered":"2023-09-23T21:12:51.650556+0000","last_unstale":"2023-09-23T21:13:22.396811+0000","last_undegraded":"2023-09-23T21:13:22.396811+0000","last_fullsized":"2023-09-23T21:13:22.396811+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:06.677094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"473'10093","reported_seq":14320,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:31.049318+0000","last_change":"2023-09-23T21:12:59.694045+0000","last_active":"2023-09-23T21:13:31.049318+0000","last_peered":"2023-09-23T21:13:31.049318+0000","last_clean":"2023-09-23T21:13:31.049318+0000","last_became_active":"2023-09-23T21:12:59.672587+0000","last_became_peered":"2023-09-23T21:12:59.672587+0000","last_unstale":"2023-09-23T21:13:31.049318+0000","last_undegraded":"2023-09-23T21:13:31.049318+0000","last_fullsized":"2023-09-23T21:13:31.049318+0000","mapping_epoch":465,"log_start":"430'1200","ondisk_log_start":"430'1200","created":426,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8893,"log_dups_size":0,"ondisk_log_size":8893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.616121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66519040,"num_objects":1015,"num_object_clones":0,"num_object_copies":3045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10093,"num_write_kb":355456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2074,"num_bytes_recovered":135921664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":54,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399555+0000","last_change":"2023-09-23T21:10:15.596183+0000","last_active":"2023-09-23T21:13:22.399555+0000","last_peered":"2023-09-23T21:13:22.399555+0000","last_clean":"2023-09-23T21:13:22.399555+0000","last_became_active":"2023-09-23T21:10:15.586221+0000","last_became_peered":"2023-09-23T21:10:15.586221+0000","last_unstale":"2023-09-23T21:13:22.399555+0000","last_undegraded":"2023-09-23T21:13:22.399555+0000","last_fullsized":"2023-09-23T21:13:22.399555+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:06.222859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":436,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400610+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:13:22.400610+0000","last_peered":"2023-09-23T21:13:22.400610+0000","last_clean":"2023-09-23T21:13:22.400610+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:13:22.400610+0000","last_undegraded":"2023-09-23T21:13:22.400610+0000","last_fullsized":"2023-09-23T21:13:22.400610+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"473'10218","reported_seq":14930,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:30.437072+0000","last_change":"2023-09-23T21:11:08.638059+0000","last_active":"2023-09-23T21:13:30.437072+0000","last_peered":"2023-09-23T21:13:30.437072+0000","last_clean":"2023-09-23T21:13:30.437072+0000","last_became_active":"2023-09-23T21:11:08.630938+0000","last_became_peered":"2023-09-23T21:11:08.630938+0000","last_unstale":"2023-09-23T21:13:30.437072+0000","last_undegraded":"2023-09-23T21:13:30.437072+0000","last_fullsized":"2023-09-23T21:13:30.437072+0000","mapping_epoch":440,"log_start":"434'2900","ondisk_log_start":"434'2900","created":426,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7318,"log_dups_size":0,"ondisk_log_size":7318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:33.746589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64618496,"num_objects":986,"num_object_clones":0,"num_object_copies":2958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10218,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2319,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":54,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399775+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:13:22.399775+0000","last_peered":"2023-09-23T21:13:22.399775+0000","last_clean":"2023-09-23T21:13:22.399775+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:13:22.399775+0000","last_undegraded":"2023-09-23T21:13:22.399775+0000","last_fullsized":"2023-09-23T21:13:22.399775+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":371,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482556+0000","last_change":"2023-09-23T21:12:51.650231+0000","last_active":"2023-09-23T21:13:22.482556+0000","last_peered":"2023-09-23T21:13:22.482556+0000","last_clean":"2023-09-23T21:13:22.482556+0000","last_became_active":"2023-09-23T21:12:51.649867+0000","last_became_peered":"2023-09-23T21:12:51.649867+0000","last_unstale":"2023-09-23T21:13:22.482556+0000","last_undegraded":"2023-09-23T21:13:22.482556+0000","last_fullsized":"2023-09-23T21:13:22.482556+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:56.792036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"473'10032","reported_seq":14617,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:29.080070+0000","last_change":"2023-09-23T21:10:39.657127+0000","last_active":"2023-09-23T21:13:29.080070+0000","last_peered":"2023-09-23T21:13:29.080070+0000","last_clean":"2023-09-23T21:13:29.080070+0000","last_became_active":"2023-09-23T21:10:39.652435+0000","last_became_peered":"2023-09-23T21:10:39.652435+0000","last_unstale":"2023-09-23T21:13:29.080070+0000","last_undegraded":"2023-09-23T21:13:29.080070+0000","last_fullsized":"2023-09-23T21:13:29.080070+0000","mapping_epoch":436,"log_start":"430'900","ondisk_log_start":"430'900","created":426,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9132,"log_dups_size":0,"ondisk_log_size":9132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:46.462306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69337088,"num_objects":1058,"num_object_clones":0,"num_object_copies":3174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10032,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2255,"num_bytes_recovered":147783680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":115,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396712+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:13:22.396712+0000","last_peered":"2023-09-23T21:13:22.396712+0000","last_clean":"2023-09-23T21:13:22.396712+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:13:22.396712+0000","last_undegraded":"2023-09-23T21:13:22.396712+0000","last_fullsized":"2023-09-23T21:13:22.396712+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.493052+0000","last_change":"2023-09-23T21:09:44.163930+0000","last_active":"2023-09-23T21:13:22.493052+0000","last_peered":"2023-09-23T21:13:22.493052+0000","last_clean":"2023-09-23T21:13:22.493052+0000","last_became_active":"2023-09-23T21:09:44.163614+0000","last_became_peered":"2023-09-23T21:09:44.163614+0000","last_unstale":"2023-09-23T21:13:22.493052+0000","last_undegraded":"2023-09-23T21:13:22.493052+0000","last_fullsized":"2023-09-23T21:13:22.493052+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:26.549908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"473'10495","reported_seq":10550,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:30.882370+0000","last_change":"2023-09-23T21:09:45.040470+0000","last_active":"2023-09-23T21:13:30.882370+0000","last_peered":"2023-09-23T21:13:30.882370+0000","last_clean":"2023-09-23T21:13:30.882370+0000","last_became_active":"2023-09-23T21:09:45.040298+0000","last_became_peered":"2023-09-23T21:09:45.040298+0000","last_unstale":"2023-09-23T21:13:30.882370+0000","last_undegraded":"2023-09-23T21:13:30.882370+0000","last_fullsized":"2023-09-23T21:13:30.882370+0000","mapping_epoch":426,"log_start":"431'1600","ondisk_log_start":"431'1600","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8895,"log_dups_size":0,"ondisk_log_size":8895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:06.840153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67829760,"num_objects":1035,"num_object_clones":0,"num_object_copies":3105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10495,"num_write_kb":368960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":132,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396468+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:13:22.396468+0000","last_peered":"2023-09-23T21:13:22.396468+0000","last_clean":"2023-09-23T21:13:22.396468+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:13:22.396468+0000","last_undegraded":"2023-09-23T21:13:22.396468+0000","last_fullsized":"2023-09-23T21:13:22.396468+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":115,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396602+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:13:22.396602+0000","last_peered":"2023-09-23T21:13:22.396602+0000","last_clean":"2023-09-23T21:13:22.396602+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:13:22.396602+0000","last_undegraded":"2023-09-23T21:13:22.396602+0000","last_fullsized":"2023-09-23T21:13:22.396602+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"473'10187","reported_seq":10242,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:30.968667+0000","last_change":"2023-09-23T21:09:45.041603+0000","last_active":"2023-09-23T21:13:30.968667+0000","last_peered":"2023-09-23T21:13:30.968667+0000","last_clean":"2023-09-23T21:13:30.968667+0000","last_became_active":"2023-09-23T21:09:45.041377+0000","last_became_peered":"2023-09-23T21:09:45.041377+0000","last_unstale":"2023-09-23T21:13:30.968667+0000","last_undegraded":"2023-09-23T21:13:30.968667+0000","last_fullsized":"2023-09-23T21:13:30.968667+0000","mapping_epoch":426,"log_start":"430'1300","ondisk_log_start":"430'1300","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8887,"log_dups_size":0,"ondisk_log_size":8887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:46.148257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64290816,"num_objects":981,"num_object_clones":0,"num_object_copies":2943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10187,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":399,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482631+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:13:22.482631+0000","last_peered":"2023-09-23T21:13:22.482631+0000","last_clean":"2023-09-23T21:13:22.482631+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:13:22.482631+0000","last_undegraded":"2023-09-23T21:13:22.482631+0000","last_fullsized":"2023-09-23T21:13:22.482631+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":486,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.394434+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:13:22.394434+0000","last_peered":"2023-09-23T21:13:22.394434+0000","last_clean":"2023-09-23T21:13:22.394434+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:13:22.394434+0000","last_undegraded":"2023-09-23T21:13:22.394434+0000","last_fullsized":"2023-09-23T21:13:22.394434+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"473'10215","reported_seq":14909,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:27.261916+0000","last_change":"2023-09-23T21:11:37.353931+0000","last_active":"2023-09-23T21:13:27.261916+0000","last_peered":"2023-09-23T21:13:27.261916+0000","last_clean":"2023-09-23T21:13:27.261916+0000","last_became_active":"2023-09-23T21:11:37.344349+0000","last_became_peered":"2023-09-23T21:11:37.344349+0000","last_unstale":"2023-09-23T21:13:27.261916+0000","last_undegraded":"2023-09-23T21:13:27.261916+0000","last_fullsized":"2023-09-23T21:13:27.261916+0000","mapping_epoch":444,"log_start":"431'1800","ondisk_log_start":"431'1800","created":426,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8415,"log_dups_size":0,"ondisk_log_size":8415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:35.510223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10215,"num_write_kb":364672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2309,"num_bytes_recovered":151322624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":110,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.488854+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:13:22.488854+0000","last_peered":"2023-09-23T21:13:22.488854+0000","last_clean":"2023-09-23T21:13:22.488854+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:13:22.488854+0000","last_undegraded":"2023-09-23T21:13:22.488854+0000","last_fullsized":"2023-09-23T21:13:22.488854+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":512,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482443+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:13:22.482443+0000","last_peered":"2023-09-23T21:13:22.482443+0000","last_clean":"2023-09-23T21:13:22.482443+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:13:22.482443+0000","last_undegraded":"2023-09-23T21:13:22.482443+0000","last_fullsized":"2023-09-23T21:13:22.482443+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"473'10264","reported_seq":14337,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:27.242556+0000","last_change":"2023-09-23T21:13:11.750840+0000","last_active":"2023-09-23T21:13:27.242556+0000","last_peered":"2023-09-23T21:13:27.242556+0000","last_clean":"2023-09-23T21:13:27.242556+0000","last_became_active":"2023-09-23T21:13:11.720815+0000","last_became_peered":"2023-09-23T21:13:11.720815+0000","last_unstale":"2023-09-23T21:13:27.242556+0000","last_undegraded":"2023-09-23T21:13:27.242556+0000","last_fullsized":"2023-09-23T21:13:27.242556+0000","mapping_epoch":468,"log_start":"431'1900","ondisk_log_start":"431'1900","created":426,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8364,"log_dups_size":0,"ondisk_log_size":8364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:35.959756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77725696,"num_objects":1186,"num_object_clones":0,"num_object_copies":3558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10264,"num_write_kb":366400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1997,"num_bytes_recovered":130875392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":134,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:25.485499+0000","last_change":"2023-09-23T21:10:15.584815+0000","last_active":"2023-09-23T21:13:25.485499+0000","last_peered":"2023-09-23T21:13:25.485499+0000","last_clean":"2023-09-23T21:13:25.485499+0000","last_became_active":"2023-09-23T21:10:15.584484+0000","last_became_peered":"2023-09-23T21:10:15.584484+0000","last_unstale":"2023-09-23T21:13:25.485499+0000","last_undegraded":"2023-09-23T21:13:25.485499+0000","last_fullsized":"2023-09-23T21:13:25.485499+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:58.137854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":221,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396589+0000","last_change":"2023-09-23T21:09:44.164862+0000","last_active":"2023-09-23T21:13:22.396589+0000","last_peered":"2023-09-23T21:13:22.396589+0000","last_clean":"2023-09-23T21:13:22.396589+0000","last_became_active":"2023-09-23T21:09:44.164337+0000","last_became_peered":"2023-09-23T21:09:44.164337+0000","last_unstale":"2023-09-23T21:13:22.396589+0000","last_undegraded":"2023-09-23T21:13:22.396589+0000","last_fullsized":"2023-09-23T21:13:22.396589+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:57.424412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"473'10237","reported_seq":10294,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:29.654505+0000","last_change":"2023-09-23T21:09:45.033715+0000","last_active":"2023-09-23T21:13:29.654505+0000","last_peered":"2023-09-23T21:13:29.654505+0000","last_clean":"2023-09-23T21:13:29.654505+0000","last_became_active":"2023-09-23T21:09:45.033437+0000","last_became_peered":"2023-09-23T21:09:45.033437+0000","last_unstale":"2023-09-23T21:13:29.654505+0000","last_undegraded":"2023-09-23T21:13:29.654505+0000","last_fullsized":"2023-09-23T21:13:29.654505+0000","mapping_epoch":426,"log_start":"430'200","ondisk_log_start":"430'200","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":10037,"log_dups_size":0,"ondisk_log_size":10037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:15.193017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70451200,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10237,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":379,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400943+0000","last_change":"2023-09-23T21:10:15.588139+0000","last_active":"2023-09-23T21:13:22.400943+0000","last_peered":"2023-09-23T21:13:22.400943+0000","last_clean":"2023-09-23T21:13:22.400943+0000","last_became_active":"2023-09-23T21:10:15.587822+0000","last_became_peered":"2023-09-23T21:10:15.587822+0000","last_unstale":"2023-09-23T21:13:22.400943+0000","last_undegraded":"2023-09-23T21:13:22.400943+0000","last_fullsized":"2023-09-23T21:13:22.400943+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:47.570476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.397101+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:13:22.397101+0000","last_peered":"2023-09-23T21:13:22.397101+0000","last_clean":"2023-09-23T21:13:22.397101+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:13:22.397101+0000","last_undegraded":"2023-09-23T21:13:22.397101+0000","last_fullsized":"2023-09-23T21:13:22.397101+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400762+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:13:22.400762+0000","last_peered":"2023-09-23T21:13:22.400762+0000","last_clean":"2023-09-23T21:13:22.400762+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:13:22.400762+0000","last_undegraded":"2023-09-23T21:13:22.400762+0000","last_fullsized":"2023-09-23T21:13:22.400762+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"473'10277","reported_seq":10333,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:29.210335+0000","last_change":"2023-09-23T21:09:45.843731+0000","last_active":"2023-09-23T21:13:29.210335+0000","last_peered":"2023-09-23T21:13:29.210335+0000","last_clean":"2023-09-23T21:13:29.210335+0000","last_became_active":"2023-09-23T21:09:45.843440+0000","last_became_peered":"2023-09-23T21:09:45.843440+0000","last_unstale":"2023-09-23T21:13:29.210335+0000","last_undegraded":"2023-09-23T21:13:29.210335+0000","last_fullsized":"2023-09-23T21:13:29.210335+0000","mapping_epoch":426,"log_start":"430'1100","ondisk_log_start":"430'1100","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9177,"log_dups_size":0,"ondisk_log_size":9177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:17.052514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72810496,"num_objects":1111,"num_object_clones":0,"num_object_copies":3333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10277,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":451,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400863+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:13:22.400863+0000","last_peered":"2023-09-23T21:13:22.400863+0000","last_clean":"2023-09-23T21:13:22.400863+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:13:22.400863+0000","last_undegraded":"2023-09-23T21:13:22.400863+0000","last_fullsized":"2023-09-23T21:13:22.400863+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":423,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482550+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:13:22.482550+0000","last_peered":"2023-09-23T21:13:22.482550+0000","last_clean":"2023-09-23T21:13:22.482550+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:13:22.482550+0000","last_undegraded":"2023-09-23T21:13:22.482550+0000","last_fullsized":"2023-09-23T21:13:22.482550+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":107,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400841+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:13:22.400841+0000","last_peered":"2023-09-23T21:13:22.400841+0000","last_clean":"2023-09-23T21:13:22.400841+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:13:22.400841+0000","last_undegraded":"2023-09-23T21:13:22.400841+0000","last_fullsized":"2023-09-23T21:13:22.400841+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"473'10273","reported_seq":10328,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:31.031771+0000","last_change":"2023-09-23T21:09:45.843960+0000","last_active":"2023-09-23T21:13:31.031771+0000","last_peered":"2023-09-23T21:13:31.031771+0000","last_clean":"2023-09-23T21:13:31.031771+0000","last_became_active":"2023-09-23T21:09:45.843562+0000","last_became_peered":"2023-09-23T21:09:45.843562+0000","last_unstale":"2023-09-23T21:13:31.031771+0000","last_undegraded":"2023-09-23T21:13:31.031771+0000","last_fullsized":"2023-09-23T21:13:31.031771+0000","mapping_epoch":426,"log_start":"431'1400","ondisk_log_start":"431'1400","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8873,"log_dups_size":0,"ondisk_log_size":8873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:10.329012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70451200,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10273,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":452,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.394126+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:13:22.394126+0000","last_peered":"2023-09-23T21:13:22.394126+0000","last_clean":"2023-09-23T21:13:22.394126+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:13:22.394126+0000","last_undegraded":"2023-09-23T21:13:22.394126+0000","last_fullsized":"2023-09-23T21:13:22.394126+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":370,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400687+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:13:22.400687+0000","last_peered":"2023-09-23T21:13:22.400687+0000","last_clean":"2023-09-23T21:13:22.400687+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:13:22.400687+0000","last_undegraded":"2023-09-23T21:13:22.400687+0000","last_fullsized":"2023-09-23T21:13:22.400687+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":107,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.400967+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:13:22.400967+0000","last_peered":"2023-09-23T21:13:22.400967+0000","last_clean":"2023-09-23T21:13:22.400967+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:13:22.400967+0000","last_undegraded":"2023-09-23T21:13:22.400967+0000","last_fullsized":"2023-09-23T21:13:22.400967+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"473'10249","reported_seq":13463,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:28.278806+0000","last_change":"2023-09-23T21:13:22.469840+0000","last_active":"2023-09-23T21:13:28.278806+0000","last_peered":"2023-09-23T21:13:28.278806+0000","last_clean":"2023-09-23T21:13:28.278806+0000","last_became_active":"2023-09-23T21:13:22.401293+0000","last_became_peered":"2023-09-23T21:13:22.401293+0000","last_unstale":"2023-09-23T21:13:28.278806+0000","last_undegraded":"2023-09-23T21:13:28.278806+0000","last_fullsized":"2023-09-23T21:13:28.278806+0000","mapping_epoch":471,"log_start":"431'2088","ondisk_log_start":"431'2088","created":426,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8161,"log_dups_size":0,"ondisk_log_size":8161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:23.955309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72482840,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10249,"num_write_kb":363329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1568,"num_bytes_recovered":102694936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":299,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.394448+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:13:22.394448+0000","last_peered":"2023-09-23T21:13:22.394448+0000","last_clean":"2023-09-23T21:13:22.394448+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:13:22.394448+0000","last_undegraded":"2023-09-23T21:13:22.394448+0000","last_fullsized":"2023-09-23T21:13:22.394448+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":745,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399066+0000","last_change":"2023-09-23T21:12:54.698525+0000","last_active":"2023-09-23T21:13:22.399066+0000","last_peered":"2023-09-23T21:13:22.399066+0000","last_clean":"2023-09-23T21:13:22.399066+0000","last_became_active":"2023-09-23T21:12:54.698074+0000","last_became_peered":"2023-09-23T21:12:54.698074+0000","last_unstale":"2023-09-23T21:13:22.399066+0000","last_undegraded":"2023-09-23T21:13:22.399066+0000","last_fullsized":"2023-09-23T21:13:22.399066+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:11.791067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":125,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396355+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:13:22.396355+0000","last_peered":"2023-09-23T21:13:22.396355+0000","last_clean":"2023-09-23T21:13:22.396355+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:13:22.396355+0000","last_undegraded":"2023-09-23T21:13:22.396355+0000","last_fullsized":"2023-09-23T21:13:22.396355+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":371,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482547+0000","last_change":"2023-09-23T21:10:15.582161+0000","last_active":"2023-09-23T21:13:22.482547+0000","last_peered":"2023-09-23T21:13:22.482547+0000","last_clean":"2023-09-23T21:13:22.482547+0000","last_became_active":"2023-09-23T21:10:15.581855+0000","last_became_peered":"2023-09-23T21:10:15.581855+0000","last_unstale":"2023-09-23T21:13:22.482547+0000","last_undegraded":"2023-09-23T21:13:22.482547+0000","last_fullsized":"2023-09-23T21:13:22.482547+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:19.873916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"473'10461","reported_seq":15330,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:29.143969+0000","last_change":"2023-09-23T21:12:11.354703+0000","last_active":"2023-09-23T21:13:29.143969+0000","last_peered":"2023-09-23T21:13:29.143969+0000","last_clean":"2023-09-23T21:13:29.143969+0000","last_became_active":"2023-09-23T21:12:11.333665+0000","last_became_peered":"2023-09-23T21:12:11.333665+0000","last_unstale":"2023-09-23T21:13:29.143969+0000","last_undegraded":"2023-09-23T21:13:29.143969+0000","last_fullsized":"2023-09-23T21:13:29.143969+0000","mapping_epoch":451,"log_start":"430'1300","ondisk_log_start":"430'1300","created":426,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9161,"log_dups_size":0,"ondisk_log_size":9161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:49.607296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10461,"num_write_kb":369856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2397,"num_bytes_recovered":117571584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":50,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399132+0000","last_change":"2023-09-23T21:10:15.584319+0000","last_active":"2023-09-23T21:13:22.399132+0000","last_peered":"2023-09-23T21:13:22.399132+0000","last_clean":"2023-09-23T21:13:22.399132+0000","last_became_active":"2023-09-23T21:10:15.584030+0000","last_became_peered":"2023-09-23T21:10:15.584030+0000","last_unstale":"2023-09-23T21:13:22.399132+0000","last_undegraded":"2023-09-23T21:13:22.399132+0000","last_fullsized":"2023-09-23T21:13:22.399132+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:28.957511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":122,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.492617+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:13:22.492617+0000","last_peered":"2023-09-23T21:13:22.492617+0000","last_clean":"2023-09-23T21:13:22.492617+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:13:22.492617+0000","last_undegraded":"2023-09-23T21:13:22.492617+0000","last_fullsized":"2023-09-23T21:13:22.492617+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":112,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.493283+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:13:22.493283+0000","last_peered":"2023-09-23T21:13:22.493283+0000","last_clean":"2023-09-23T21:13:22.493283+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:13:22.493283+0000","last_undegraded":"2023-09-23T21:13:22.493283+0000","last_fullsized":"2023-09-23T21:13:22.493283+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"473'10333","reported_seq":10390,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:30.418645+0000","last_change":"2023-09-23T21:09:45.038161+0000","last_active":"2023-09-23T21:13:30.418645+0000","last_peered":"2023-09-23T21:13:30.418645+0000","last_clean":"2023-09-23T21:13:30.418645+0000","last_became_active":"2023-09-23T21:09:45.037907+0000","last_became_peered":"2023-09-23T21:09:45.037907+0000","last_unstale":"2023-09-23T21:13:30.418645+0000","last_undegraded":"2023-09-23T21:13:30.418645+0000","last_fullsized":"2023-09-23T21:13:30.418645+0000","mapping_epoch":426,"log_start":"435'3000","ondisk_log_start":"435'3000","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7333,"log_dups_size":0,"ondisk_log_size":7333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:37.785460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67829760,"num_objects":1035,"num_object_clones":0,"num_object_copies":3105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10333,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":50,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.399661+0000","last_change":"2023-09-23T21:10:15.584326+0000","last_active":"2023-09-23T21:13:22.399661+0000","last_peered":"2023-09-23T21:13:22.399661+0000","last_clean":"2023-09-23T21:13:22.399661+0000","last_became_active":"2023-09-23T21:10:15.583965+0000","last_became_peered":"2023-09-23T21:10:15.583965+0000","last_unstale":"2023-09-23T21:13:22.399661+0000","last_undegraded":"2023-09-23T21:13:22.399661+0000","last_fullsized":"2023-09-23T21:13:22.399661+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:21.120392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":372,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.401062+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:13:22.401062+0000","last_peered":"2023-09-23T21:13:22.401062+0000","last_clean":"2023-09-23T21:13:22.401062+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:13:22.401062+0000","last_undegraded":"2023-09-23T21:13:22.401062+0000","last_fullsized":"2023-09-23T21:13:22.401062+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":158,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396918+0000","last_change":"2023-09-23T21:10:15.588413+0000","last_active":"2023-09-23T21:13:22.396918+0000","last_peered":"2023-09-23T21:13:22.396918+0000","last_clean":"2023-09-23T21:13:22.396918+0000","last_became_active":"2023-09-23T21:10:15.588077+0000","last_became_peered":"2023-09-23T21:10:15.588077+0000","last_unstale":"2023-09-23T21:13:22.396918+0000","last_undegraded":"2023-09-23T21:13:22.396918+0000","last_fullsized":"2023-09-23T21:13:22.396918+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:03.757383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"473'10179","reported_seq":10233,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:30.465187+0000","last_change":"2023-09-23T21:09:45.033672+0000","last_active":"2023-09-23T21:13:30.465187+0000","last_peered":"2023-09-23T21:13:30.465187+0000","last_clean":"2023-09-23T21:13:30.465187+0000","last_became_active":"2023-09-23T21:09:45.033296+0000","last_became_peered":"2023-09-23T21:09:45.033296+0000","last_unstale":"2023-09-23T21:13:30.465187+0000","last_undegraded":"2023-09-23T21:13:30.465187+0000","last_fullsized":"2023-09-23T21:13:30.465187+0000","mapping_epoch":426,"log_start":"434'2800","ondisk_log_start":"434'2800","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7379,"log_dups_size":0,"ondisk_log_size":7379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:26.841578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67305472,"num_objects":1027,"num_object_clones":0,"num_object_copies":3081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10179,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":146,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:25.485625+0000","last_change":"2023-09-23T21:10:15.593352+0000","last_active":"2023-09-23T21:13:25.485625+0000","last_peered":"2023-09-23T21:13:25.485625+0000","last_clean":"2023-09-23T21:13:25.485625+0000","last_became_active":"2023-09-23T21:10:15.589715+0000","last_became_peered":"2023-09-23T21:10:15.589715+0000","last_unstale":"2023-09-23T21:13:25.485625+0000","last_undegraded":"2023-09-23T21:13:25.485625+0000","last_fullsized":"2023-09-23T21:13:25.485625+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:24.563158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.401015+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:13:22.401015+0000","last_peered":"2023-09-23T21:13:22.401015+0000","last_clean":"2023-09-23T21:13:22.401015+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:13:22.401015+0000","last_undegraded":"2023-09-23T21:13:22.401015+0000","last_fullsized":"2023-09-23T21:13:22.401015+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":134,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.394945+0000","last_change":"2023-09-23T21:12:51.650884+0000","last_active":"2023-09-23T21:13:22.394945+0000","last_peered":"2023-09-23T21:13:22.394945+0000","last_clean":"2023-09-23T21:13:22.394945+0000","last_became_active":"2023-09-23T21:12:51.650560+0000","last_became_peered":"2023-09-23T21:12:51.650560+0000","last_unstale":"2023-09-23T21:13:22.394945+0000","last_undegraded":"2023-09-23T21:13:22.394945+0000","last_fullsized":"2023-09-23T21:13:22.394945+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:19.136242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"473'10045","reported_seq":10102,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:29.658604+0000","last_change":"2023-09-23T21:09:45.038290+0000","last_active":"2023-09-23T21:13:29.658604+0000","last_peered":"2023-09-23T21:13:29.658604+0000","last_clean":"2023-09-23T21:13:29.658604+0000","last_became_active":"2023-09-23T21:09:45.037976+0000","last_became_peered":"2023-09-23T21:09:45.037976+0000","last_unstale":"2023-09-23T21:13:29.658604+0000","last_undegraded":"2023-09-23T21:13:29.658604+0000","last_fullsized":"2023-09-23T21:13:29.658604+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:44.361487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68616192,"num_objects":1047,"num_object_clones":0,"num_object_copies":3141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10045,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":379,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482810+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:13:22.482810+0000","last_peered":"2023-09-23T21:13:22.482810+0000","last_clean":"2023-09-23T21:13:22.482810+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:13:22.482810+0000","last_undegraded":"2023-09-23T21:13:22.482810+0000","last_fullsized":"2023-09-23T21:13:22.482810+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":116,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.493361+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:13:22.493361+0000","last_peered":"2023-09-23T21:13:22.493361+0000","last_clean":"2023-09-23T21:13:22.493361+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:13:22.493361+0000","last_undegraded":"2023-09-23T21:13:22.493361+0000","last_fullsized":"2023-09-23T21:13:22.493361+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396806+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:13:22.396806+0000","last_peered":"2023-09-23T21:13:22.396806+0000","last_clean":"2023-09-23T21:13:22.396806+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:13:22.396806+0000","last_undegraded":"2023-09-23T21:13:22.396806+0000","last_fullsized":"2023-09-23T21:13:22.396806+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":130,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.492906+0000","last_change":"2023-09-23T21:10:15.588809+0000","last_active":"2023-09-23T21:13:22.492906+0000","last_peered":"2023-09-23T21:13:22.492906+0000","last_clean":"2023-09-23T21:13:22.492906+0000","last_became_active":"2023-09-23T21:10:15.588451+0000","last_became_peered":"2023-09-23T21:10:15.588451+0000","last_unstale":"2023-09-23T21:13:22.492906+0000","last_undegraded":"2023-09-23T21:13:22.492906+0000","last_fullsized":"2023-09-23T21:13:22.492906+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:58.509500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":507,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.394716+0000","last_change":"2023-09-23T21:09:44.163759+0000","last_active":"2023-09-23T21:13:22.394716+0000","last_peered":"2023-09-23T21:13:22.394716+0000","last_clean":"2023-09-23T21:13:22.394716+0000","last_became_active":"2023-09-23T21:09:44.163405+0000","last_became_peered":"2023-09-23T21:09:44.163405+0000","last_unstale":"2023-09-23T21:13:22.394716+0000","last_undegraded":"2023-09-23T21:13:22.394716+0000","last_fullsized":"2023-09-23T21:13:22.394716+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:52.346296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":135,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.489475+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:13:22.489475+0000","last_peered":"2023-09-23T21:13:22.489475+0000","last_clean":"2023-09-23T21:13:22.489475+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:13:22.489475+0000","last_undegraded":"2023-09-23T21:13:22.489475+0000","last_fullsized":"2023-09-23T21:13:22.489475+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":141,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396534+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:13:22.396534+0000","last_peered":"2023-09-23T21:13:22.396534+0000","last_clean":"2023-09-23T21:13:22.396534+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:13:22.396534+0000","last_undegraded":"2023-09-23T21:13:22.396534+0000","last_fullsized":"2023-09-23T21:13:22.396534+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":354,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.482946+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:13:22.482946+0000","last_peered":"2023-09-23T21:13:22.482946+0000","last_clean":"2023-09-23T21:13:22.482946+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:13:22.482946+0000","last_undegraded":"2023-09-23T21:13:22.482946+0000","last_fullsized":"2023-09-23T21:13:22.482946+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":133,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396441+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:13:22.396441+0000","last_peered":"2023-09-23T21:13:22.396441+0000","last_clean":"2023-09-23T21:13:22.396441+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:13:22.396441+0000","last_undegraded":"2023-09-23T21:13:22.396441+0000","last_fullsized":"2023-09-23T21:13:22.396441+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":116,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.396838+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:13:22.396838+0000","last_peered":"2023-09-23T21:13:22.396838+0000","last_clean":"2023-09-23T21:13:22.396838+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:13:22.396838+0000","last_undegraded":"2023-09-23T21:13:22.396838+0000","last_fullsized":"2023-09-23T21:13:22.396838+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.395470+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:13:22.395470+0000","last_peered":"2023-09-23T21:13:22.395470+0000","last_clean":"2023-09-23T21:13:22.395470+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:13:22.395470+0000","last_undegraded":"2023-09-23T21:13:22.395470+0000","last_fullsized":"2023-09-23T21:13:22.395470+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":119,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.492341+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:13:22.492341+0000","last_peered":"2023-09-23T21:13:22.492341+0000","last_clean":"2023-09-23T21:13:22.492341+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:13:22.492341+0000","last_undegraded":"2023-09-23T21:13:22.492341+0000","last_fullsized":"2023-09-23T21:13:22.492341+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T21:13:22.488830+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:13:22.488830+0000","last_peered":"2023-09-23T21:13:22.488830+0000","last_clean":"2023-09-23T21:13:22.488830+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:13:22.488830+0000","last_undegraded":"2023-09-23T21:13:22.488830+0000","last_fullsized":"2023-09-23T21:13:22.488830+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1115488280,"num_objects":17022,"num_object_clones":0,"num_object_copies":51066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17022,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163780,"num_write_kb":5785601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14919,"num_bytes_recovered":938147864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3624157184,"data_stored":3624140896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138992,"ondisk_log_size":138992,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":28028,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201817,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1188316,"kb_used_data":510740,"kb_used_omap":0,"kb_used_meta":677568,"kb_avail":92556836,"statfs":{"total":95995035648,"available":94778200064,"internally_reserved":0,"allocated":522997760,"data_stored":521651737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":693829632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":2.8570000000000002,"15min":1.5860000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":17.183,"5min":64.257000000000005,"15min":64.257000000000005},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.8149999999999999,"15min":1.5800000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":17.193000000000001,"5min":64.293000000000006,"15min":64.293000000000006},"last":0.29799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:13:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":4.6500000000000004,"15min":3.1110000000000002},"min":{"1min":0.247,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":16.684000000000001,"5min":112.355,"15min":114.16500000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":8.2799999999999994,"5min":6.0659999999999998,"15min":3.6469999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":117.637,"5min":117.637,"15min":117.637},"last":1.0529999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":8.4199999999999999,"5min":5.4859999999999998,"15min":3.6299999999999999},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":133.524,"5min":183.44200000000001,"15min":183.44200000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":8.7460000000000004,"5min":5.4530000000000003,"15min":3.5499999999999998},"min":{"1min":0.35999999999999999,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":133.55699999999999,"5min":183.46199999999999,"15min":183.46199999999999},"last":0.47899999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":12.335000000000001,"5min":5.2869999999999999,"15min":3.1840000000000002},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":118.15900000000001,"5min":118.15900000000001,"15min":118.15900000000001},"last":5.1479999999999997},{"interface":"front","average":{"1min":12.760999999999999,"5min":5.7560000000000002,"15min":3.2200000000000002},"min":{"1min":0.34799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":118.098,"5min":118.098,"15min":118.098},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.72499999999999998,"15min":0.78300000000000003},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":4.165,"5min":13.739000000000001,"15min":15.942},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.119,"5min":0.79100000000000004,"15min":0.70999999999999996},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.26500000000000001},"max":{"1min":6.8920000000000003,"5min":6.8920000000000003,"15min":16.725999999999999},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.93300000000000005,"15min":0.90500000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.1449999999999996,"5min":10.891,"15min":10.891},"last":0.378},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.93799999999999994,"15min":0.81999999999999995},"min":{"1min":0.35799999999999998,"5min":0.33300000000000002,"15min":0.28399999999999997},"max":{"1min":3.9039999999999999,"5min":13.867000000000001,"15min":13.867000000000001},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:12:27 2023","interfaces":[{"interface":"back","average":{"1min":7.7460000000000004,"5min":2.8090000000000002,"15min":1.607},"min":{"1min":0.35799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":143.18700000000001,"5min":143.18700000000001,"15min":143.18700000000001},"last":0.435},{"interface":"front","average":{"1min":7.6180000000000003,"5min":2.8559999999999999,"15min":1.5649999999999999},"min":{"1min":0.33800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":142.70400000000001,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.56799999999999995}]}]},{"osd":6,"up_from":47,"seq":201863463438,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1094088,"kb_used_data":461824,"kb_used_omap":0,"kb_used_meta":632256,"kb_avail":92651064,"statfs":{"total":95995035648,"available":94874689536,"internally_reserved":0,"allocated":472907776,"data_stored":471581856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":647430144},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":10.954000000000001,"5min":3.444,"15min":2.536},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.20999999999999999},"max":{"1min":149.61600000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":10.253,"5min":3.1890000000000001,"15min":2.4430000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":149.303,"5min":149.303,"15min":149.303},"last":1.417}]},{"osd":1,"last update":"Sat Sep 23 21:13:21 2023","interfaces":[{"interface":"back","average":{"1min":3.3399999999999999,"5min":4.4139999999999997,"15min":3.3919999999999999},"min":{"1min":0.26100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":25.122,"5min":129.37899999999999,"15min":129.37899999999999},"last":1.5369999999999999},{"interface":"front","average":{"1min":2.359,"5min":4.1790000000000003,"15min":3.3239999999999998},"min":{"1min":0.247,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":25.193999999999999,"5min":129.40100000000001,"15min":129.40100000000001},"last":0.54000000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":5.8659999999999997,"15min":5.1379999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":133.46600000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":10.975,"5min":5.3630000000000004,"15min":5.0069999999999997},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":133.33000000000001,"5min":133.33000000000001,"15min":199.08000000000001},"last":2.4500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":15.163,"5min":5.6159999999999997,"15min":3.9039999999999999},"min":{"1min":0.30499999999999999,"5min":0.30399999999999999,"15min":0.26900000000000002},"max":{"1min":137.875,"5min":137.875,"15min":178.40000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":13.106999999999999,"5min":5.2590000000000003,"15min":3.8620000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":137.60900000000001,"5min":137.60900000000001,"15min":178.26599999999999},"last":1.633}]},{"osd":4,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.748,"15min":0.86299999999999999},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.4489999999999998,"5min":9.0890000000000004,"15min":13.1},"last":2.4169999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.66100000000000003,"15min":0.73199999999999998},"min":{"1min":0.28299999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.78000000000000003,"5min":6.4400000000000004,"15min":6.4400000000000004},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.75700000000000001,"15min":0.876},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.419,"5min":9.0519999999999996,"15min":11.281000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.63800000000000001,"15min":0.85599999999999998},"min":{"1min":0.32200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.52300000000000002,"5min":6.0190000000000001,"15min":8.6229999999999993},"last":0.39600000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.69199999999999995,"15min":0.879},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.6230000000000002,"5min":6.6299999999999999,"15min":7.8079999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.872,"5min":0.82599999999999996,"15min":0.92400000000000004},"min":{"1min":0.38600000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.7549999999999999,"5min":6.4749999999999996,"15min":7.1849999999999996},"last":0.58099999999999996}]}]},{"osd":1,"up_from":276,"seq":1185410973884,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000584,"kb_used_data":450816,"kb_used_omap":5,"kb_used_meta":549754,"kb_avail":92744568,"statfs":{"total":95995035648,"available":94970437632,"internally_reserved":0,"allocated":461635584,"data_stored":459310168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":562948835},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3279999999999998,"5min":3.875,"15min":2.0329999999999999},"min":{"1min":0.245,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":16.262,"5min":108.47,"15min":108.47},"last":1.4970000000000001},{"interface":"front","average":{"1min":2.847,"5min":3.9239999999999999,"15min":2.1080000000000001},"min":{"1min":0.23499999999999999,"5min":0.19600000000000001,"15min":0.193},"max":{"1min":11.507999999999999,"5min":109.532,"15min":109.532},"last":2.5539999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":8.0250000000000004,"5min":6.8029999999999999,"15min":3.5390000000000001},"min":{"1min":0.26200000000000001,"5min":0.23200000000000001,"15min":0.20200000000000001},"max":{"1min":78.344999999999999,"5min":107.883,"15min":107.883},"last":2.593},{"interface":"front","average":{"1min":6.5629999999999997,"5min":5.6769999999999996,"15min":3.073},"min":{"1min":0.247,"5min":0.182,"15min":0.182},"max":{"1min":78.376999999999995,"5min":101.024,"15min":101.024},"last":5.3099999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":3.9049999999999998,"5min":7.3159999999999998,"15min":4.9930000000000003},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.23899999999999999},"max":{"1min":25.789000000000001,"5min":99.120999999999995,"15min":172.44399999999999},"last":1.23},{"interface":"front","average":{"1min":4.2930000000000001,"5min":7.2709999999999999,"15min":5.1280000000000001},"min":{"1min":0.35499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.9130000000000003,"5min":101.849,"15min":172.58500000000001},"last":150.38}]},{"osd":4,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":1.9950000000000001,"15min":1.5429999999999999},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.925000000000001,"5min":11.728999999999999,"15min":17.006},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.502,"5min":1.4990000000000001,"15min":1.2470000000000001},"min":{"1min":0.32300000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.0599999999999996,"5min":23.332999999999998,"15min":23.332999999999998},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.1030000000000002,"15min":1.5680000000000001},"min":{"1min":0.441,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.2210000000000001,"5min":16.196000000000002,"15min":17.155000000000001},"last":2.6230000000000002},{"interface":"front","average":{"1min":2.806,"5min":2.117,"15min":1.6850000000000001},"min":{"1min":0.46500000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.975,"5min":11.760999999999999,"15min":17.076000000000001},"last":5.1369999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":2.3029999999999999,"15min":2.1099999999999999},"min":{"1min":0.42199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.4480000000000004,"5min":16.222000000000001,"15min":127.995},"last":3.3879999999999999},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.1749999999999998,"15min":2.1739999999999999},"min":{"1min":0.46700000000000003,"5min":0.34300000000000003,"15min":0.31900000000000001},"max":{"1min":10.878,"5min":11.782999999999999,"15min":126.836},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":2.1160000000000001,"15min":2.169},"min":{"1min":0.42299999999999999,"5min":0.374,"15min":0.36299999999999999},"max":{"1min":10.951000000000001,"5min":11.680999999999999,"15min":151.791},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.573,"5min":1.6220000000000001,"15min":1.8160000000000001},"min":{"1min":0.432,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.0949999999999998,"5min":23.376000000000001,"15min":151.809},"last":1.1890000000000001}]}]},{"osd":0,"up_from":92,"seq":395136991624,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292744,"kb_used_data":432256,"kb_used_omap":5,"kb_used_meta":860090,"kb_avail":92452408,"statfs":{"total":95995035648,"available":94671265792,"internally_reserved":0,"allocated":442630144,"data_stored":440419994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":880732794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:13:08 2023","interfaces":[{"interface":"back","average":{"1min":3.04,"5min":2.956,"15min":2.6579999999999999},"min":{"1min":0.23300000000000001,"5min":0.182,"15min":0.17799999999999999},"max":{"1min":8.1389999999999993,"5min":53.262,"15min":61.341999999999999},"last":1.4339999999999999},{"interface":"front","average":{"1min":3.5030000000000001,"5min":3.4470000000000001,"15min":2.7799999999999998},"min":{"1min":0.251,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.266,"5min":53.201999999999998,"15min":61.299999999999997},"last":2.6549999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":9.343,"5min":5.2949999999999999,"15min":3.4870000000000001},"min":{"1min":0.27300000000000002,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":152.291,"5min":152.291,"15min":183.77099999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":9.375,"5min":5.2400000000000002,"15min":3.6059999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.22600000000000001},"max":{"1min":152.33099999999999,"5min":152.33099999999999,"15min":183.78700000000001},"last":10.82}]},{"osd":3,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":4.1909999999999998,"5min":4.548,"15min":3.206},"min":{"1min":0.33500000000000002,"5min":0.23999999999999999,"15min":0.224},"max":{"1min":10.448,"5min":86.989999999999995,"15min":186.143},"last":4.173},{"interface":"front","average":{"1min":3.2749999999999999,"5min":4.3959999999999999,"15min":3.177},"min":{"1min":0.30299999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.648999999999999,"5min":87.043999999999997,"15min":186.97399999999999},"last":4.2089999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":1.786,"15min":1.4390000000000001},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.29799999999999999},"max":{"1min":11.420999999999999,"5min":11.420999999999999,"15min":11.420999999999999},"last":3.0369999999999999},{"interface":"front","average":{"1min":2.0230000000000001,"5min":1.6619999999999999,"15min":1.1479999999999999},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":4.9989999999999997,"5min":11.753,"15min":11.753},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.8049999999999999,"15min":1.5169999999999999},"min":{"1min":0.39000000000000001,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":11.375999999999999,"5min":11.375999999999999,"15min":11.375999999999999},"last":3.0059999999999998},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.7210000000000001,"15min":1.216},"min":{"1min":0.373,"5min":0.313,"15min":0.313},"max":{"1min":5.032,"5min":11.589,"15min":11.589},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":1.859,"15min":3.403},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.9530000000000003,"5min":17.773,"15min":170.12700000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.7809999999999999,"15min":3.4169999999999998},"min":{"1min":0.39800000000000002,"5min":0.37,"15min":0.31900000000000001},"max":{"1min":4.9560000000000004,"5min":17.684999999999999,"15min":169.767},"last":2.698}]},{"osd":7,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.8480000000000001,"15min":1.3420000000000001},"min":{"1min":0.42999999999999999,"5min":0.38200000000000001,"15min":0.33600000000000002},"max":{"1min":6.5830000000000002,"5min":11.69,"15min":11.69},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.331,"5min":1.827,"15min":1.3300000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.379},"max":{"1min":5.0010000000000003,"5min":10.470000000000001,"15min":10.470000000000001},"last":2.726}]}]},{"osd":2,"up_from":129,"seq":554050781515,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1577316,"kb_used_data":710684,"kb_used_omap":7,"kb_used_meta":866552,"kb_avail":92167836,"statfs":{"total":95995035648,"available":94379864064,"internally_reserved":0,"allocated":727740416,"data_stored":725442422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":887349329},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":3.2629999999999999,"15min":1.8320000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17699999999999999},"max":{"1min":103.021,"5min":103.021,"15min":103.021},"last":0.79800000000000004},{"interface":"front","average":{"1min":8.0449999999999999,"5min":4.069,"15min":2.5099999999999998},"min":{"1min":0.22700000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":103.154,"5min":103.154,"15min":103.154},"last":3.5009999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:13:15 2023","interfaces":[{"interface":"back","average":{"1min":5.8019999999999996,"5min":5.4189999999999996,"15min":4.1900000000000004},"min":{"1min":0.39700000000000002,"5min":0.22500000000000001,"15min":0.188},"max":{"1min":54.521000000000001,"5min":143.482,"15min":212.12},"last":0.39100000000000001},{"interface":"front","average":{"1min":5.9470000000000001,"5min":5.6719999999999997,"15min":4.0860000000000003},"min":{"1min":0.223,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":54.598999999999997,"5min":143.61000000000001,"15min":213.02799999999999},"last":0.76400000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4319999999999999,"5min":5.8810000000000002,"15min":4.3150000000000004},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":12.696999999999999,"5min":143.50800000000001,"15min":143.50800000000001},"last":1.4319999999999999},{"interface":"front","average":{"1min":4.2309999999999999,"5min":6.0300000000000002,"15min":4.1959999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.23799999999999999},"max":{"1min":12.502000000000001,"5min":143.27799999999999,"15min":143.27799999999999},"last":1.327}]},{"osd":4,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":4.1120000000000001,"15min":2.0150000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":9.8170000000000002,"5min":225.80799999999999,"15min":225.80799999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":2.1579999999999999,"5min":4.9660000000000002,"15min":2.5670000000000002},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":8.109,"5min":225.79300000000001,"15min":225.79300000000001},"last":10.801}]},{"osd":5,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":3.54,"15min":2.1589999999999998},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.29999999999999999},"max":{"1min":8.1489999999999991,"5min":98.682000000000002,"15min":98.682000000000002},"last":10.839},{"interface":"front","average":{"1min":1.718,"5min":4.4610000000000003,"15min":2.2200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.9450000000000003,"5min":143.34700000000001,"15min":143.34700000000001},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":3.2490000000000001,"15min":2.5579999999999998},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":6.9969999999999999,"5min":143.41399999999999,"15min":175.72499999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.2429999999999999,"5min":1.6579999999999999,"15min":1.9419999999999999},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":11.893000000000001,"5min":11.893000000000001,"15min":176.20400000000001},"last":0.86699999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":1.7210000000000001,"15min":1.3939999999999999},"min":{"1min":0.41599999999999998,"5min":0.36399999999999999,"15min":0.32700000000000001},"max":{"1min":9.843,"5min":11.811999999999999,"15min":12.609},"last":0.84599999999999997},{"interface":"front","average":{"1min":2.1920000000000002,"5min":2.5960000000000001,"15min":1.8480000000000001},"min":{"1min":0.41399999999999998,"5min":0.38200000000000001,"15min":0.34499999999999997},"max":{"1min":8.1720000000000006,"5min":16.875,"15min":16.875},"last":10.864000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160109,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335068,"kb_used_data":618452,"kb_used_omap":2,"kb_used_meta":716605,"kb_avail":92410084,"statfs":{"total":95995035648,"available":94627926016,"internally_reserved":0,"allocated":633294848,"data_stored":630768643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":733803573},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":2.7850000000000001,"15min":1.8500000000000001},"min":{"1min":0.53400000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.696999999999999,"5min":17.425999999999998,"15min":44.963000000000001},"last":6.3570000000000002},{"interface":"front","average":{"1min":4.3959999999999999,"5min":2.48,"15min":1.534},"min":{"1min":0.46000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":14.503,"5min":16.920999999999999,"15min":19.738},"last":10.334}]},{"osd":1,"last update":"Sat Sep 23 21:13:15 2023","interfaces":[{"interface":"back","average":{"1min":13.276999999999999,"5min":15.555999999999999,"15min":6.7690000000000001},"min":{"1min":0.29499999999999998,"5min":0.27400000000000002,"15min":0.20999999999999999},"max":{"1min":147.12100000000001,"5min":237.072,"15min":237.072},"last":5.5739999999999998},{"interface":"front","average":{"1min":13.151,"5min":15.204000000000001,"15min":6.8479999999999999},"min":{"1min":0.308,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":151.14699999999999,"5min":234.21899999999999,"15min":234.21899999999999},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":6.3200000000000003,"5min":6.1280000000000001,"15min":3.0499999999999998},"min":{"1min":0.55900000000000005,"5min":0.23599999999999999,"15min":0.223},"max":{"1min":24.378,"5min":93.272000000000006,"15min":93.272000000000006},"last":0.215},{"interface":"front","average":{"1min":4.2370000000000001,"5min":5.2320000000000002,"15min":2.6160000000000001},"min":{"1min":0.58599999999999997,"5min":0.22500000000000001,"15min":0.222},"max":{"1min":10.907,"5min":95.233999999999995,"15min":95.233999999999995},"last":5.9710000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1859999999999999,"5min":2.4780000000000002,"15min":2.6520000000000001},"min":{"1min":0.36399999999999999,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":15.763,"5min":15.763,"15min":111.36},"last":0.32800000000000001},{"interface":"front","average":{"1min":2.835,"5min":2.0510000000000002,"15min":2.367},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.6180000000000003,"5min":17.286999999999999,"15min":112.473},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":2.9809999999999999,"5min":2.105,"15min":1.5820000000000001},"min":{"1min":0.501,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.3710000000000004,"5min":16.855,"15min":36.488999999999997},"last":1.238},{"interface":"front","average":{"1min":4.0949999999999998,"5min":2.6400000000000001,"15min":1.9990000000000001},"min":{"1min":0.40699999999999997,"5min":0.315,"15min":0.315},"max":{"1min":15.789999999999999,"5min":15.789999999999999,"15min":36.295000000000002},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0599999999999996,"5min":6.6529999999999996,"15min":3.2650000000000001},"min":{"1min":0.434,"5min":0.33500000000000002,"15min":0.312},"max":{"1min":15.718,"5min":304.22399999999999,"15min":304.22399999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.827,"5min":6.0510000000000002,"15min":2.964},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":7.5839999999999996,"5min":304.15800000000002,"15min":304.15800000000002},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":3.282,"5min":2.1829999999999998,"15min":1.897},"min":{"1min":0.52100000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":8.3309999999999995,"5min":16.899000000000001,"15min":74.019000000000005},"last":1.2749999999999999},{"interface":"front","average":{"1min":4.8339999999999996,"5min":2.9039999999999999,"15min":2.3719999999999999},"min":{"1min":0.436,"5min":0.377,"15min":0.32900000000000001},"max":{"1min":15.816000000000001,"5min":15.816000000000001,"15min":73.364999999999995},"last":1.9650000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986727,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":931732,"kb_used_data":299724,"kb_used_omap":0,"kb_used_meta":632000,"kb_avail":92813420,"statfs":{"total":95995035648,"available":95040942080,"internally_reserved":0,"allocated":306917376,"data_stored":305579230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":647168000},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":11.087999999999999,"5min":3.4750000000000001,"15min":2.0830000000000002},"min":{"1min":0.219,"5min":0.20699999999999999,"15min":0.20399999999999999},"max":{"1min":226.751,"5min":226.751,"15min":226.751},"last":6.9900000000000002},{"interface":"front","average":{"1min":11.529999999999999,"5min":3.589,"15min":2.0939999999999999},"min":{"1min":0.253,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":226.43299999999999,"5min":226.43299999999999,"15min":226.43299999999999},"last":4.4560000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:13:17 2023","interfaces":[{"interface":"back","average":{"1min":5.2830000000000004,"5min":4.0350000000000001,"15min":4.6050000000000004},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.23200000000000001},"max":{"1min":41.613999999999997,"5min":68.337000000000003,"15min":127.411},"last":6.1740000000000004},{"interface":"front","average":{"1min":3.2930000000000001,"5min":3.601,"15min":3.8290000000000002},"min":{"1min":0.27200000000000002,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":36.229999999999997,"5min":68.293000000000006,"15min":118.075},"last":5.9900000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":14.257,"5min":5.7480000000000002,"15min":4.4900000000000002},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":210.50299999999999,"5min":210.50299999999999,"15min":210.50299999999999},"last":7.7460000000000004},{"interface":"front","average":{"1min":15.454000000000001,"5min":5.8390000000000004,"15min":4.3890000000000002},"min":{"1min":0.33100000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":210.59,"5min":210.59,"15min":210.59},"last":7.891}]},{"osd":3,"last update":"Sat Sep 23 21:13:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":7.8520000000000003,"15min":3.6120000000000001},"min":{"1min":0.32700000000000001,"5min":0.29399999999999998,"15min":0.29299999999999998},"max":{"1min":9.2590000000000003,"5min":214.64099999999999,"15min":214.64099999999999},"last":1.032},{"interface":"front","average":{"1min":3.1779999999999999,"5min":8.1120000000000001,"15min":3.7639999999999998},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":13.16,"5min":214.54900000000001,"15min":214.54900000000001},"last":4.2990000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.69699999999999995},"min":{"1min":0.28199999999999997,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.2069999999999999,"5min":6.0069999999999997,"15min":8.7089999999999996},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.66200000000000003,"15min":0.69999999999999996},"min":{"1min":0.27400000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.881,"5min":4.7140000000000004,"15min":7.0179999999999998},"last":3.5510000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":1.2549999999999999,"15min":0.98099999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79300000000000004,"5min":36.557000000000002,"15min":36.557000000000002},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.125,"15min":0.97199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":2.5760000000000001,"5min":38.468000000000004,"15min":38.468000000000004},"last":3.9860000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:12:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.80500000000000005,"15min":1.4610000000000001},"min":{"1min":0.309,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.8319999999999999,"5min":4.7359999999999998,"15min":145.721},"last":3.601},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.871,"15min":1.601},"min":{"1min":0.33800000000000002,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":0.61099999999999999,"5min":10.411,"15min":148.45699999999999},"last":3.7189999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725091,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":811268,"kb_used_data":154684,"kb_used_omap":0,"kb_used_meta":656576,"kb_avail":92933884,"statfs":{"total":95995035648,"available":95164297216,"internally_reserved":0,"allocated":158396416,"data_stored":157070398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":672333824},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":2.8399999999999999,"15min":1.4690000000000001},"min":{"1min":0.25700000000000001,"5min":0.217,"15min":0.217},"max":{"1min":20.023,"5min":82.280000000000001,"15min":82.280000000000001},"last":1.788},{"interface":"front","average":{"1min":3.0569999999999999,"5min":2.9620000000000002,"15min":1.5860000000000001},"min":{"1min":0.252,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":19.853000000000002,"5min":85.236000000000004,"15min":85.236000000000004},"last":1.863}]},{"osd":1,"last update":"Sat Sep 23 21:12:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6819999999999999,"5min":5.2919999999999998,"15min":4.1429999999999998},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.711,"5min":125.23399999999999,"15min":233.77099999999999},"last":4.3680000000000003},{"interface":"front","average":{"1min":3.3399999999999999,"5min":5.2590000000000003,"15min":4.0389999999999997},"min":{"1min":0.34100000000000003,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":10.724,"5min":125.249,"15min":233.75200000000001},"last":1.847}]},{"osd":2,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":4.1660000000000004,"15min":3.4990000000000001},"min":{"1min":0.35199999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":20.411999999999999,"5min":78.917000000000002,"15min":231.34700000000001},"last":3.0230000000000001},{"interface":"front","average":{"1min":2.8460000000000001,"5min":4.2640000000000002,"15min":3.5939999999999999},"min":{"1min":0.29999999999999999,"5min":0.28499999999999998,"15min":0.27300000000000002},"max":{"1min":11.509,"5min":78.888999999999996,"15min":231.381},"last":3.8479999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":7.5039999999999996,"15min":3.9809999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":10.053000000000001,"5min":263.887,"15min":263.887},"last":11.597},{"interface":"front","average":{"1min":1.6519999999999999,"5min":7.2450000000000001,"15min":3.8170000000000002},"min":{"1min":0.32200000000000001,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":4.218,"5min":263.70100000000002,"15min":263.70100000000002},"last":2.6360000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.66800000000000004,"15min":0.60199999999999998},"min":{"1min":0.29299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.0449999999999999,"5min":6.3940000000000001,"15min":12.739000000000001},"last":7.5259999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.73199999999999998,"15min":0.72299999999999998},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9849999999999999,"5min":5.8490000000000002,"15min":10.792},"last":1.4319999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.708,"15min":1.333},"min":{"1min":0.35699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7440000000000002,"5min":83.451999999999998,"15min":83.451999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.677,"15min":1.3720000000000001},"min":{"1min":0.30399999999999999,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":3.9889999999999999,"5min":83.484999999999999,"15min":83.484999999999999},"last":4.4139999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:13:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.872,"15min":0.89900000000000002},"min":{"1min":0.39600000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.96,"5min":4.3399999999999999,"15min":9.6329999999999991},"last":1.6559999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.71499999999999997,"15min":0.66400000000000003},"min":{"1min":0.436,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.8140000000000001,"5min":3.0790000000000002,"15min":4.29},"last":1.8140000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":428867584,"data_stored":428867584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":455348224,"data_stored":455344152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":700911616,"data_stored":700907544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":614273024,"data_stored":614268952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":298057728,"data_stored":298057728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":151977984,"data_stored":151977984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":466616320,"data_stored":466616320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":508104704,"data_stored":508100632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:13:36.006 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:36.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:36.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:36.114 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:36.218 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:36.398 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:36.506 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:36.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:36.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:36.516 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:36 smithi121 ceph-mon[211526]: pgmap v1534: 89 pgs: 89 active+clean; 969 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T21:13:36.516 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:36 smithi121 ceph-mon[219709]: pgmap v1534: 89 pgs: 89 active+clean; 969 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T21:13:36.617 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:36.712 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:36.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:36 smithi123 ceph-mon[193903]: pgmap v1534: 89 pgs: 89 active+clean; 969 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-09-23T21:13:36.981 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:36.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:37.087 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:37.157 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3644475268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:37.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:37 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:37.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3644475268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:37.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:37 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:37.640 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:13:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3644475268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:37 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:13:38.639 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:38.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[211526]: pgmap v1535: 89 pgs: 89 active+clean; 878 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-09-23T21:13:38.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:38.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[211526]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:13:38.639 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[211526]: osdmap e474: 8 total, 8 up, 8 in 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[211526]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[211526]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[219709]: pgmap v1535: 89 pgs: 89 active+clean; 878 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[219709]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[219709]: osdmap e474: 8 total, 8 up, 8 in 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[219709]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.640 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:38 smithi121 ceph-mon[219709]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:38.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:38 smithi123 ceph-mon[193903]: pgmap v1535: 89 pgs: 89 active+clean; 878 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-09-23T21:13:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:38 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:38 smithi123 ceph-mon[193903]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:38 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:13:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:38 smithi123 ceph-mon[193903]: osdmap e474: 8 total, 8 up, 8 in 2023-09-23T21:13:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:38 smithi123 ceph-mon[193903]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:38 smithi123 ceph-mon[193903]: failed to encode map e474 with expected crc 2023-09-23T21:13:38.746 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:38.779 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:38.841 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:38.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:38.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:38.936 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:13:38.956 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:38.988 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:39.041 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:39.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:39.155 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:39.184 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:39.236 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:39.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:39.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:39.347 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:39.375 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:39.424 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:39.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:39.530 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:39.584 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:39.766 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:39.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:39.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:39.873 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:39.905 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:39.956 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:39.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:40.068 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:40.096 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:40.153 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:40.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:40.199 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:40.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:13:40.257 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:40.320 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:13:40.337 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:40.337 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:40.381 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:13:40.386 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:40.469 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:13:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:40 smithi123 ceph-mon[193903]: pgmap v1537: 89 pgs: 89 active+clean; 809 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T21:13:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:40 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:40 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1240116569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:40.515 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:13:40.544 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:13:40.594 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:13:40.643 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[211526]: pgmap v1537: 89 pgs: 89 active+clean; 809 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T21:13:40.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:40.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:40.644 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1240116569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:40.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[219709]: pgmap v1537: 89 pgs: 89 active+clean; 809 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T21:13:40.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:40.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:40.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1240116569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:40.645 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:13:40.646 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.646 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.647 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.647 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.647 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.647 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.647 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.647 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:13:40.748 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:40.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:40.850 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:40.938 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:41.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:13:41.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:13:41.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:41.406 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[211526]: pgmap v1538: 89 pgs: 89 active+clean; 693 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T21:13:41.407 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:13:41.407 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:13:41.407 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:41.407 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:41 smithi121 ceph-mon[219709]: pgmap v1538: 89 pgs: 89 active+clean; 693 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T21:13:41.478 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:13:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:41 smithi123 ceph-mon[193903]: pgmap v1538: 89 pgs: 89 active+clean; 693 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T21:13:42.686 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:13:42.795 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:42.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:42.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:42.902 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:42.931 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:42.982 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:42.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:43.081 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:43.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:43.180 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:43.522 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:43.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:43.638 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:43.715 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:43.874 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:43.882 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:43.883 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:43.883 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:43 smithi121 ceph-mon[211526]: pgmap v1539: 89 pgs: 89 active+clean; 671 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T21:13:43.883 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:43.883 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:43.883 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:43 smithi121 ceph-mon[219709]: pgmap v1539: 89 pgs: 89 active+clean; 671 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T21:13:43.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:43.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:13:43.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:43.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:43 smithi123 ceph-mon[193903]: pgmap v1539: 89 pgs: 89 active+clean; 671 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-09-23T21:13:43.985 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:44.067 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:44.175 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:13:44.176 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:44.177 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:13:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:44.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:44.284 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:44.320 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:44.387 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:44.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:44.483 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:44 smithi121 ceph-mon[211526]: from='client.56995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:44.483 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:44 smithi121 ceph-mon[219709]: from='client.56995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:44.492 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:44.582 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:44.687 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:13:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:44 smithi123 ceph-mon[193903]: from='client.56995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:13:45.010 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:45.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:45.035 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:13:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:13:44] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-09-23T21:13:45.136 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:45.165 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:45.231 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:45.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:45.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:45.328 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:45.335 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:45.365 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1539,"stamp":"2023-09-23T21:13:43.278853+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":703469496,"num_objects":10750,"num_object_clones":0,"num_object_copies":32250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10750,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":170376,"num_write_kb":5788046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15033,"num_bytes_recovered":942308468,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139604,"ondisk_log_size":139604,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7957328,"kb_used_data":2377616,"kb_used_omap":21,"kb_used_meta":5579562,"kb_avail":742003888,"statfs":{"total":767960285184,"available":759811981312,"internally_reserved":0,"allocated":2434678784,"data_stored":2419978949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":5713471971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-387973120,"num_objects":-5920,"num_object_clones":0,"num_object_copies":-17760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5920,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007383"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":247,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.366110+0000","last_change":"2023-09-23T21:05:24.130451+0000","last_active":"2023-09-23T21:13:26.366110+0000","last_peered":"2023-09-23T21:13:26.366110+0000","last_clean":"2023-09-23T21:13:26.366110+0000","last_became_active":"2023-09-23T21:05:24.130068+0000","last_became_peered":"2023-09-23T21:05:24.130068+0000","last_unstale":"2023-09-23T21:13:26.366110+0000","last_undegraded":"2023-09-23T21:13:26.366110+0000","last_fullsized":"2023-09-23T21:13:26.366110+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:33.510406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":116,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363232+0000","last_change":"2023-09-23T21:04:49.766348+0000","last_active":"2023-09-23T21:13:26.363232+0000","last_peered":"2023-09-23T21:13:26.363232+0000","last_clean":"2023-09-23T21:13:26.363232+0000","last_became_active":"2023-09-23T21:04:49.766066+0000","last_became_peered":"2023-09-23T21:04:49.766066+0000","last_unstale":"2023-09-23T21:13:26.363232+0000","last_undegraded":"2023-09-23T21:13:26.363232+0000","last_fullsized":"2023-09-23T21:13:26.363232+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:17.532705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":389,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.365833+0000","last_change":"2023-09-23T21:05:24.140209+0000","last_active":"2023-09-23T21:13:26.365833+0000","last_peered":"2023-09-23T21:13:26.365833+0000","last_clean":"2023-09-23T21:13:26.365833+0000","last_became_active":"2023-09-23T21:05:24.139843+0000","last_became_peered":"2023-09-23T21:05:24.139843+0000","last_unstale":"2023-09-23T21:13:26.365833+0000","last_undegraded":"2023-09-23T21:13:26.365833+0000","last_fullsized":"2023-09-23T21:13:26.365833+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:12.444616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.365669+0000","last_change":"2023-09-23T21:12:51.650220+0000","last_active":"2023-09-23T21:13:26.365669+0000","last_peered":"2023-09-23T21:13:26.365669+0000","last_clean":"2023-09-23T21:13:26.365669+0000","last_became_active":"2023-09-23T21:12:51.649869+0000","last_became_peered":"2023-09-23T21:12:51.649869+0000","last_unstale":"2023-09-23T21:13:26.365669+0000","last_undegraded":"2023-09-23T21:13:26.365669+0000","last_fullsized":"2023-09-23T21:13:26.365669+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:04.107476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":134,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.350417+0000","last_change":"2023-09-23T21:05:11.433135+0000","last_active":"2023-09-23T21:13:26.350417+0000","last_peered":"2023-09-23T21:13:26.350417+0000","last_clean":"2023-09-23T21:13:26.350417+0000","last_became_active":"2023-09-23T21:05:11.432854+0000","last_became_peered":"2023-09-23T21:05:11.432854+0000","last_unstale":"2023-09-23T21:13:26.350417+0000","last_undegraded":"2023-09-23T21:13:26.350417+0000","last_fullsized":"2023-09-23T21:13:26.350417+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.468091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":350,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356506+0000","last_change":"2023-09-23T21:09:44.161988+0000","last_active":"2023-09-23T21:13:26.356506+0000","last_peered":"2023-09-23T21:13:26.356506+0000","last_clean":"2023-09-23T21:13:26.356506+0000","last_became_active":"2023-09-23T21:09:44.161643+0000","last_became_peered":"2023-09-23T21:09:44.161643+0000","last_unstale":"2023-09-23T21:13:26.356506+0000","last_undegraded":"2023-09-23T21:13:26.356506+0000","last_fullsized":"2023-09-23T21:13:26.356506+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:17.083615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":113,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.366115+0000","last_change":"2023-09-23T21:04:58.426014+0000","last_active":"2023-09-23T21:13:26.366115+0000","last_peered":"2023-09-23T21:13:26.366115+0000","last_clean":"2023-09-23T21:13:26.366115+0000","last_became_active":"2023-09-23T21:04:58.425506+0000","last_became_peered":"2023-09-23T21:04:58.425506+0000","last_unstale":"2023-09-23T21:13:26.366115+0000","last_undegraded":"2023-09-23T21:13:26.366115+0000","last_fullsized":"2023-09-23T21:13:26.366115+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:25.756766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":116,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.350336+0000","last_change":"2023-09-23T21:04:41.458918+0000","last_active":"2023-09-23T21:13:26.350336+0000","last_peered":"2023-09-23T21:13:26.350336+0000","last_clean":"2023-09-23T21:13:26.350336+0000","last_became_active":"2023-09-23T21:04:41.457878+0000","last_became_peered":"2023-09-23T21:04:41.457878+0000","last_unstale":"2023-09-23T21:13:26.350336+0000","last_undegraded":"2023-09-23T21:13:26.350336+0000","last_fullsized":"2023-09-23T21:13:26.350336+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:27.283083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":107,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356866+0000","last_change":"2023-09-23T21:10:15.584242+0000","last_active":"2023-09-23T21:13:26.356866+0000","last_peered":"2023-09-23T21:13:26.356866+0000","last_clean":"2023-09-23T21:13:26.356866+0000","last_became_active":"2023-09-23T21:10:15.583926+0000","last_became_peered":"2023-09-23T21:10:15.583926+0000","last_unstale":"2023-09-23T21:13:26.356866+0000","last_undegraded":"2023-09-23T21:13:26.356866+0000","last_fullsized":"2023-09-23T21:13:26.356866+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:15.759114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":125,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.352818+0000","last_change":"2023-09-23T21:12:51.652067+0000","last_active":"2023-09-23T21:13:26.352818+0000","last_peered":"2023-09-23T21:13:26.352818+0000","last_clean":"2023-09-23T21:13:26.352818+0000","last_became_active":"2023-09-23T21:12:51.651751+0000","last_became_peered":"2023-09-23T21:12:51.651751+0000","last_unstale":"2023-09-23T21:13:26.352818+0000","last_undegraded":"2023-09-23T21:13:26.352818+0000","last_fullsized":"2023-09-23T21:13:26.352818+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:03.235559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":29,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363655+0000","last_change":"2023-09-23T21:12:51.657657+0000","last_active":"2023-09-23T21:13:26.363655+0000","last_peered":"2023-09-23T21:13:26.363655+0000","last_clean":"2023-09-23T21:13:26.363655+0000","last_became_active":"2023-09-23T21:12:51.641567+0000","last_became_peered":"2023-09-23T21:12:51.641567+0000","last_unstale":"2023-09-23T21:13:26.363655+0000","last_undegraded":"2023-09-23T21:13:26.363655+0000","last_fullsized":"2023-09-23T21:13:26.363655+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:00.366548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":108,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.365914+0000","last_change":"2023-09-23T21:05:11.430250+0000","last_active":"2023-09-23T21:13:26.365914+0000","last_peered":"2023-09-23T21:13:26.365914+0000","last_clean":"2023-09-23T21:13:26.365914+0000","last_became_active":"2023-09-23T21:05:11.424491+0000","last_became_peered":"2023-09-23T21:05:11.424491+0000","last_unstale":"2023-09-23T21:13:26.365914+0000","last_undegraded":"2023-09-23T21:13:26.365914+0000","last_fullsized":"2023-09-23T21:13:26.365914+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:51.185385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":61,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363723+0000","last_change":"2023-09-23T21:10:15.594033+0000","last_active":"2023-09-23T21:13:26.363723+0000","last_peered":"2023-09-23T21:13:26.363723+0000","last_clean":"2023-09-23T21:13:26.363723+0000","last_became_active":"2023-09-23T21:10:15.579277+0000","last_became_peered":"2023-09-23T21:10:15.579277+0000","last_unstale":"2023-09-23T21:13:26.363723+0000","last_undegraded":"2023-09-23T21:13:26.363723+0000","last_fullsized":"2023-09-23T21:13:26.363723+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:42.561285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":392,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.373951+0000","last_change":"2023-09-23T21:04:03.579487+0000","last_active":"2023-09-23T21:13:26.373951+0000","last_peered":"2023-09-23T21:13:26.373951+0000","last_clean":"2023-09-23T21:13:26.373951+0000","last_became_active":"2023-09-23T21:04:03.579239+0000","last_became_peered":"2023-09-23T21:04:03.579239+0000","last_unstale":"2023-09-23T21:13:26.373951+0000","last_undegraded":"2023-09-23T21:13:26.373951+0000","last_fullsized":"2023-09-23T21:13:26.373951+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:46.556433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":57,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363616+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:13:26.363616+0000","last_peered":"2023-09-23T21:13:26.363616+0000","last_clean":"2023-09-23T21:13:26.363616+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:13:26.363616+0000","last_undegraded":"2023-09-23T21:13:26.363616+0000","last_fullsized":"2023-09-23T21:13:26.363616+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":134,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.350548+0000","last_change":"2023-09-23T21:10:15.590805+0000","last_active":"2023-09-23T21:13:26.350548+0000","last_peered":"2023-09-23T21:13:26.350548+0000","last_clean":"2023-09-23T21:13:26.350548+0000","last_became_active":"2023-09-23T21:10:15.590501+0000","last_became_peered":"2023-09-23T21:10:15.590501+0000","last_unstale":"2023-09-23T21:13:26.350548+0000","last_undegraded":"2023-09-23T21:13:26.350548+0000","last_fullsized":"2023-09-23T21:13:26.350548+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:27.143631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":23,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363511+0000","last_change":"2023-09-23T21:12:51.643047+0000","last_active":"2023-09-23T21:13:26.363511+0000","last_peered":"2023-09-23T21:13:26.363511+0000","last_clean":"2023-09-23T21:13:26.363511+0000","last_became_active":"2023-09-23T21:12:51.642754+0000","last_became_peered":"2023-09-23T21:12:51.642754+0000","last_unstale":"2023-09-23T21:13:26.363511+0000","last_undegraded":"2023-09-23T21:13:26.363511+0000","last_fullsized":"2023-09-23T21:13:26.363511+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:31.430089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363790+0000","last_change":"2023-09-23T21:10:15.585218+0000","last_active":"2023-09-23T21:13:26.363790+0000","last_peered":"2023-09-23T21:13:26.363790+0000","last_clean":"2023-09-23T21:13:26.363790+0000","last_became_active":"2023-09-23T21:10:15.584952+0000","last_became_peered":"2023-09-23T21:10:15.584952+0000","last_unstale":"2023-09-23T21:13:26.363790+0000","last_undegraded":"2023-09-23T21:13:26.363790+0000","last_fullsized":"2023-09-23T21:13:26.363790+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:55.016769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":51,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363551+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:13:26.363551+0000","last_peered":"2023-09-23T21:13:26.363551+0000","last_clean":"2023-09-23T21:13:26.363551+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:13:26.363551+0000","last_undegraded":"2023-09-23T21:13:26.363551+0000","last_fullsized":"2023-09-23T21:13:26.363551+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":122,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.365732+0000","last_change":"2023-09-23T21:12:51.650915+0000","last_active":"2023-09-23T21:13:26.365732+0000","last_peered":"2023-09-23T21:13:26.365732+0000","last_clean":"2023-09-23T21:13:26.365732+0000","last_became_active":"2023-09-23T21:12:51.650550+0000","last_became_peered":"2023-09-23T21:12:51.650550+0000","last_unstale":"2023-09-23T21:13:26.365732+0000","last_undegraded":"2023-09-23T21:13:26.365732+0000","last_fullsized":"2023-09-23T21:13:26.365732+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:30.158548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":470,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356006+0000","last_change":"2023-09-23T21:05:34.238188+0000","last_active":"2023-09-23T21:13:26.356006+0000","last_peered":"2023-09-23T21:13:26.356006+0000","last_clean":"2023-09-23T21:13:26.356006+0000","last_became_active":"2023-09-23T21:05:34.183496+0000","last_became_peered":"2023-09-23T21:05:34.183496+0000","last_unstale":"2023-09-23T21:13:26.356006+0000","last_undegraded":"2023-09-23T21:13:26.356006+0000","last_fullsized":"2023-09-23T21:13:26.356006+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:27.857883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":134,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363049+0000","last_change":"2023-09-23T21:12:51.645964+0000","last_active":"2023-09-23T21:13:26.363049+0000","last_peered":"2023-09-23T21:13:26.363049+0000","last_clean":"2023-09-23T21:13:26.363049+0000","last_became_active":"2023-09-23T21:12:51.645627+0000","last_became_peered":"2023-09-23T21:12:51.645627+0000","last_unstale":"2023-09-23T21:13:26.363049+0000","last_undegraded":"2023-09-23T21:13:26.363049+0000","last_fullsized":"2023-09-23T21:13:26.363049+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:53.725127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"474'10592","reported_seq":10648,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:41.058034+0000","last_change":"2023-09-23T21:09:45.077841+0000","last_active":"2023-09-23T21:13:41.058034+0000","last_peered":"2023-09-23T21:13:41.058034+0000","last_clean":"2023-09-23T21:13:41.058034+0000","last_became_active":"2023-09-23T21:09:45.077550+0000","last_became_peered":"2023-09-23T21:09:45.077550+0000","last_unstale":"2023-09-23T21:13:41.058034+0000","last_undegraded":"2023-09-23T21:13:41.058034+0000","last_fullsized":"2023-09-23T21:13:41.058034+0000","mapping_epoch":426,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8892,"log_dups_size":0,"ondisk_log_size":8892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:19.178954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41680896,"num_objects":636,"num_object_clones":0,"num_object_copies":1908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10592,"num_write_kb":359296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":27,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363884+0000","last_change":"2023-09-23T21:12:51.662783+0000","last_active":"2023-09-23T21:13:26.363884+0000","last_peered":"2023-09-23T21:13:26.363884+0000","last_clean":"2023-09-23T21:13:26.363884+0000","last_became_active":"2023-09-23T21:12:51.649440+0000","last_became_peered":"2023-09-23T21:12:51.649440+0000","last_unstale":"2023-09-23T21:13:26.363884+0000","last_undegraded":"2023-09-23T21:13:26.363884+0000","last_fullsized":"2023-09-23T21:13:26.363884+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:11.808407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":107,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356429+0000","last_change":"2023-09-23T21:12:51.650942+0000","last_active":"2023-09-23T21:13:26.356429+0000","last_peered":"2023-09-23T21:13:26.356429+0000","last_clean":"2023-09-23T21:13:26.356429+0000","last_became_active":"2023-09-23T21:12:51.650556+0000","last_became_peered":"2023-09-23T21:12:51.650556+0000","last_unstale":"2023-09-23T21:13:26.356429+0000","last_undegraded":"2023-09-23T21:13:26.356429+0000","last_fullsized":"2023-09-23T21:13:26.356429+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:06.677094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"474'10476","reported_seq":14704,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:41.016813+0000","last_change":"2023-09-23T21:12:59.694045+0000","last_active":"2023-09-23T21:13:41.016813+0000","last_peered":"2023-09-23T21:13:41.016813+0000","last_clean":"2023-09-23T21:13:41.016813+0000","last_became_active":"2023-09-23T21:12:59.672587+0000","last_became_peered":"2023-09-23T21:12:59.672587+0000","last_unstale":"2023-09-23T21:13:41.016813+0000","last_undegraded":"2023-09-23T21:13:41.016813+0000","last_fullsized":"2023-09-23T21:13:41.016813+0000","mapping_epoch":465,"log_start":"431'1600","ondisk_log_start":"431'1600","created":426,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8876,"log_dups_size":0,"ondisk_log_size":8876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:02.616121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41418752,"num_objects":632,"num_object_clones":0,"num_object_copies":1896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10476,"num_write_kb":355456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2074,"num_bytes_recovered":135921664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":55,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363865+0000","last_change":"2023-09-23T21:10:15.596183+0000","last_active":"2023-09-23T21:13:26.363865+0000","last_peered":"2023-09-23T21:13:26.363865+0000","last_clean":"2023-09-23T21:13:26.363865+0000","last_became_active":"2023-09-23T21:10:15.586221+0000","last_became_peered":"2023-09-23T21:10:15.586221+0000","last_unstale":"2023-09-23T21:13:26.363865+0000","last_undegraded":"2023-09-23T21:13:26.363865+0000","last_fullsized":"2023-09-23T21:13:26.363865+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:26:06.222859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":437,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.365758+0000","last_change":"2023-09-23T21:04:41.431974+0000","last_active":"2023-09-23T21:13:26.365758+0000","last_peered":"2023-09-23T21:13:26.365758+0000","last_clean":"2023-09-23T21:13:26.365758+0000","last_became_active":"2023-09-23T21:04:41.431531+0000","last_became_peered":"2023-09-23T21:04:41.431531+0000","last_unstale":"2023-09-23T21:13:26.365758+0000","last_undegraded":"2023-09-23T21:13:26.365758+0000","last_fullsized":"2023-09-23T21:13:26.365758+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:44.929929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.2","version":"474'10541","reported_seq":15254,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:40.507570+0000","last_change":"2023-09-23T21:11:08.638059+0000","last_active":"2023-09-23T21:13:40.507570+0000","last_peered":"2023-09-23T21:13:40.507570+0000","last_clean":"2023-09-23T21:13:40.507570+0000","last_became_active":"2023-09-23T21:11:08.630938+0000","last_became_peered":"2023-09-23T21:11:08.630938+0000","last_unstale":"2023-09-23T21:13:40.507570+0000","last_undegraded":"2023-09-23T21:13:40.507570+0000","last_fullsized":"2023-09-23T21:13:40.507570+0000","mapping_epoch":440,"log_start":"435'3000","ondisk_log_start":"435'3000","created":426,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:33.746589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43450368,"num_objects":663,"num_object_clones":0,"num_object_copies":1989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10541,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2319,"num_bytes_recovered":151977984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":55,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363839+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:13:26.363839+0000","last_peered":"2023-09-23T21:13:26.363839+0000","last_clean":"2023-09-23T21:13:26.363839+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:13:26.363839+0000","last_undegraded":"2023-09-23T21:13:26.363839+0000","last_fullsized":"2023-09-23T21:13:26.363839+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":372,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356139+0000","last_change":"2023-09-23T21:12:51.650231+0000","last_active":"2023-09-23T21:13:26.356139+0000","last_peered":"2023-09-23T21:13:26.356139+0000","last_clean":"2023-09-23T21:13:26.356139+0000","last_became_active":"2023-09-23T21:12:51.649867+0000","last_became_peered":"2023-09-23T21:12:51.649867+0000","last_unstale":"2023-09-23T21:13:26.356139+0000","last_undegraded":"2023-09-23T21:13:26.356139+0000","last_fullsized":"2023-09-23T21:13:26.356139+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:56.792036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"474'10415","reported_seq":15001,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:39.149421+0000","last_change":"2023-09-23T21:10:39.657127+0000","last_active":"2023-09-23T21:13:39.149421+0000","last_peered":"2023-09-23T21:13:39.149421+0000","last_clean":"2023-09-23T21:13:39.149421+0000","last_became_active":"2023-09-23T21:10:39.652435+0000","last_became_peered":"2023-09-23T21:10:39.652435+0000","last_unstale":"2023-09-23T21:13:39.149421+0000","last_undegraded":"2023-09-23T21:13:39.149421+0000","last_fullsized":"2023-09-23T21:13:39.149421+0000","mapping_epoch":436,"log_start":"430'1300","ondisk_log_start":"430'1300","created":426,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9115,"log_dups_size":0,"ondisk_log_size":9115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:46.462306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44236800,"num_objects":675,"num_object_clones":0,"num_object_copies":2025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10415,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2255,"num_bytes_recovered":147783680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":116,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.350589+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:13:26.350589+0000","last_peered":"2023-09-23T21:13:26.350589+0000","last_clean":"2023-09-23T21:13:26.350589+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:13:26.350589+0000","last_undegraded":"2023-09-23T21:13:26.350589+0000","last_fullsized":"2023-09-23T21:13:26.350589+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":122,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.364501+0000","last_change":"2023-09-23T21:09:44.163930+0000","last_active":"2023-09-23T21:13:26.364501+0000","last_peered":"2023-09-23T21:13:26.364501+0000","last_clean":"2023-09-23T21:13:26.364501+0000","last_became_active":"2023-09-23T21:09:44.163614+0000","last_became_peered":"2023-09-23T21:09:44.163614+0000","last_unstale":"2023-09-23T21:13:26.364501+0000","last_undegraded":"2023-09-23T21:13:26.364501+0000","last_fullsized":"2023-09-23T21:13:26.364501+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:26.549908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"474'10872","reported_seq":10928,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:41.016627+0000","last_change":"2023-09-23T21:09:45.040470+0000","last_active":"2023-09-23T21:13:41.016627+0000","last_peered":"2023-09-23T21:13:41.016627+0000","last_clean":"2023-09-23T21:13:41.016627+0000","last_became_active":"2023-09-23T21:09:45.040298+0000","last_became_peered":"2023-09-23T21:09:45.040298+0000","last_unstale":"2023-09-23T21:13:41.016627+0000","last_undegraded":"2023-09-23T21:13:41.016627+0000","last_fullsized":"2023-09-23T21:13:41.016627+0000","mapping_epoch":426,"log_start":"431'2000","ondisk_log_start":"431'2000","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8872,"log_dups_size":0,"ondisk_log_size":8872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:06.840153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43122688,"num_objects":658,"num_object_clones":0,"num_object_copies":1974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10872,"num_write_kb":368960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":133,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.354013+0000","last_change":"2023-09-23T21:05:01.695466+0000","last_active":"2023-09-23T21:13:26.354013+0000","last_peered":"2023-09-23T21:13:26.354013+0000","last_clean":"2023-09-23T21:13:26.354013+0000","last_became_active":"2023-09-23T21:04:58.419120+0000","last_became_peered":"2023-09-23T21:04:58.419120+0000","last_unstale":"2023-09-23T21:13:26.354013+0000","last_undegraded":"2023-09-23T21:13:26.354013+0000","last_fullsized":"2023-09-23T21:13:26.354013+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:46.641908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":116,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.350453+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:13:26.350453+0000","last_peered":"2023-09-23T21:13:26.350453+0000","last_clean":"2023-09-23T21:13:26.350453+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:13:26.350453+0000","last_undegraded":"2023-09-23T21:13:26.350453+0000","last_fullsized":"2023-09-23T21:13:26.350453+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"474'10550","reported_seq":10606,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:41.045927+0000","last_change":"2023-09-23T21:09:45.041603+0000","last_active":"2023-09-23T21:13:41.045927+0000","last_peered":"2023-09-23T21:13:41.045927+0000","last_clean":"2023-09-23T21:13:41.045927+0000","last_became_active":"2023-09-23T21:09:45.041377+0000","last_became_peered":"2023-09-23T21:09:45.041377+0000","last_unstale":"2023-09-23T21:13:41.045927+0000","last_undegraded":"2023-09-23T21:13:41.045927+0000","last_fullsized":"2023-09-23T21:13:41.045927+0000","mapping_epoch":426,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8850,"log_dups_size":0,"ondisk_log_size":8850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:46.148257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40501248,"num_objects":618,"num_object_clones":0,"num_object_copies":1854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10550,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":400,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356226+0000","last_change":"2023-09-23T21:05:11.421379+0000","last_active":"2023-09-23T21:13:26.356226+0000","last_peered":"2023-09-23T21:13:26.356226+0000","last_clean":"2023-09-23T21:13:26.356226+0000","last_became_active":"2023-09-23T21:05:11.421011+0000","last_became_peered":"2023-09-23T21:05:11.421011+0000","last_unstale":"2023-09-23T21:13:26.356226+0000","last_undegraded":"2023-09-23T21:13:26.356226+0000","last_fullsized":"2023-09-23T21:13:26.356226+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:45.663465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":487,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.357387+0000","last_change":"2023-09-23T21:05:40.042468+0000","last_active":"2023-09-23T21:13:26.357387+0000","last_peered":"2023-09-23T21:13:26.357387+0000","last_clean":"2023-09-23T21:13:26.357387+0000","last_became_active":"2023-09-23T21:05:40.042155+0000","last_became_peered":"2023-09-23T21:05:40.042155+0000","last_unstale":"2023-09-23T21:13:26.357387+0000","last_undegraded":"2023-09-23T21:13:26.357387+0000","last_fullsized":"2023-09-23T21:13:26.357387+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:25.410050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"474'10763","reported_seq":15458,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:42.281310+0000","last_change":"2023-09-23T21:11:37.353931+0000","last_active":"2023-09-23T21:13:42.281310+0000","last_peered":"2023-09-23T21:13:42.281310+0000","last_clean":"2023-09-23T21:13:42.281310+0000","last_became_active":"2023-09-23T21:11:37.344349+0000","last_became_peered":"2023-09-23T21:11:37.344349+0000","last_unstale":"2023-09-23T21:13:42.281310+0000","last_undegraded":"2023-09-23T21:13:42.281310+0000","last_fullsized":"2023-09-23T21:13:42.281310+0000","mapping_epoch":444,"log_start":"434'2400","ondisk_log_start":"434'2400","created":426,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8363,"log_dups_size":0,"ondisk_log_size":8363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:35.510223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41484288,"num_objects":633,"num_object_clones":0,"num_object_copies":1899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10763,"num_write_kb":364672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2309,"num_bytes_recovered":151322624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.374087+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:13:26.374087+0000","last_peered":"2023-09-23T21:13:26.374087+0000","last_clean":"2023-09-23T21:13:26.374087+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:13:26.374087+0000","last_undegraded":"2023-09-23T21:13:26.374087+0000","last_fullsized":"2023-09-23T21:13:26.374087+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":513,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356268+0000","last_change":"2023-09-23T21:05:11.433076+0000","last_active":"2023-09-23T21:13:26.356268+0000","last_peered":"2023-09-23T21:13:26.356268+0000","last_clean":"2023-09-23T21:13:26.356268+0000","last_became_active":"2023-09-23T21:05:11.432705+0000","last_became_peered":"2023-09-23T21:05:11.432705+0000","last_unstale":"2023-09-23T21:13:26.356268+0000","last_undegraded":"2023-09-23T21:13:26.356268+0000","last_fullsized":"2023-09-23T21:13:26.356268+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:51.656462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"474'10809","reported_seq":14883,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:42.286217+0000","last_change":"2023-09-23T21:13:11.750840+0000","last_active":"2023-09-23T21:13:42.286217+0000","last_peered":"2023-09-23T21:13:42.286217+0000","last_clean":"2023-09-23T21:13:42.286217+0000","last_became_active":"2023-09-23T21:13:11.720815+0000","last_became_peered":"2023-09-23T21:13:11.720815+0000","last_unstale":"2023-09-23T21:13:42.286217+0000","last_undegraded":"2023-09-23T21:13:42.286217+0000","last_fullsized":"2023-09-23T21:13:42.286217+0000","mapping_epoch":468,"log_start":"434'2400","ondisk_log_start":"434'2400","created":426,"last_epoch_clean":469,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8409,"log_dups_size":0,"ondisk_log_size":8409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:35.959756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42008576,"num_objects":641,"num_object_clones":0,"num_object_copies":1923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10809,"num_write_kb":366400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1997,"num_bytes_recovered":130875392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":135,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.350493+0000","last_change":"2023-09-23T21:10:15.584815+0000","last_active":"2023-09-23T21:13:26.350493+0000","last_peered":"2023-09-23T21:13:26.350493+0000","last_clean":"2023-09-23T21:13:26.350493+0000","last_became_active":"2023-09-23T21:10:15.584484+0000","last_became_peered":"2023-09-23T21:10:15.584484+0000","last_unstale":"2023-09-23T21:13:26.350493+0000","last_undegraded":"2023-09-23T21:13:26.350493+0000","last_fullsized":"2023-09-23T21:13:26.350493+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:58.137854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":222,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363020+0000","last_change":"2023-09-23T21:09:44.164862+0000","last_active":"2023-09-23T21:13:26.363020+0000","last_peered":"2023-09-23T21:13:26.363020+0000","last_clean":"2023-09-23T21:13:26.363020+0000","last_became_active":"2023-09-23T21:09:44.164337+0000","last_became_peered":"2023-09-23T21:09:44.164337+0000","last_unstale":"2023-09-23T21:13:26.363020+0000","last_undegraded":"2023-09-23T21:13:26.363020+0000","last_fullsized":"2023-09-23T21:13:26.363020+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:57.424412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"474'10600","reported_seq":10658,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:39.636540+0000","last_change":"2023-09-23T21:09:45.033715+0000","last_active":"2023-09-23T21:13:39.636540+0000","last_peered":"2023-09-23T21:13:39.636540+0000","last_clean":"2023-09-23T21:13:39.636540+0000","last_became_active":"2023-09-23T21:09:45.033437+0000","last_became_peered":"2023-09-23T21:09:45.033437+0000","last_unstale":"2023-09-23T21:13:39.636540+0000","last_undegraded":"2023-09-23T21:13:39.636540+0000","last_fullsized":"2023-09-23T21:13:39.636540+0000","mapping_epoch":426,"log_start":"430'500","ondisk_log_start":"430'500","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:15.193017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46661632,"num_objects":712,"num_object_clones":0,"num_object_copies":2136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10600,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":380,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.366579+0000","last_change":"2023-09-23T21:10:15.588139+0000","last_active":"2023-09-23T21:13:26.366579+0000","last_peered":"2023-09-23T21:13:26.366579+0000","last_clean":"2023-09-23T21:13:26.366579+0000","last_became_active":"2023-09-23T21:10:15.587822+0000","last_became_peered":"2023-09-23T21:10:15.587822+0000","last_unstale":"2023-09-23T21:13:26.366579+0000","last_undegraded":"2023-09-23T21:13:26.366579+0000","last_fullsized":"2023-09-23T21:13:26.366579+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:47.570476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":139,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356774+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:13:26.356774+0000","last_peered":"2023-09-23T21:13:26.356774+0000","last_clean":"2023-09-23T21:13:26.356774+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:13:26.356774+0000","last_undegraded":"2023-09-23T21:13:26.356774+0000","last_fullsized":"2023-09-23T21:13:26.356774+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:37.284786+0000","last_change":"2023-09-23T21:05:34.180944+0000","last_active":"2023-09-23T21:13:37.284786+0000","last_peered":"2023-09-23T21:13:37.284786+0000","last_clean":"2023-09-23T21:13:37.284786+0000","last_became_active":"2023-09-23T21:05:34.180624+0000","last_became_peered":"2023-09-23T21:05:34.180624+0000","last_unstale":"2023-09-23T21:13:37.284786+0000","last_undegraded":"2023-09-23T21:13:37.284786+0000","last_fullsized":"2023-09-23T21:13:37.284786+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.405770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"474'10644","reported_seq":10702,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:39.101260+0000","last_change":"2023-09-23T21:09:45.843731+0000","last_active":"2023-09-23T21:13:39.101260+0000","last_peered":"2023-09-23T21:13:39.101260+0000","last_clean":"2023-09-23T21:13:39.101260+0000","last_became_active":"2023-09-23T21:09:45.843440+0000","last_became_peered":"2023-09-23T21:09:45.843440+0000","last_unstale":"2023-09-23T21:13:39.101260+0000","last_undegraded":"2023-09-23T21:13:39.101260+0000","last_fullsized":"2023-09-23T21:13:39.101260+0000","mapping_epoch":426,"log_start":"431'1500","ondisk_log_start":"431'1500","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9144,"log_dups_size":0,"ondisk_log_size":9144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:17.052514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48758784,"num_objects":744,"num_object_clones":0,"num_object_copies":2232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10644,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":452,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.366224+0000","last_change":"2023-09-23T21:03:38.412565+0000","last_active":"2023-09-23T21:13:26.366224+0000","last_peered":"2023-09-23T21:13:26.366224+0000","last_clean":"2023-09-23T21:13:26.366224+0000","last_became_active":"2023-09-23T21:03:38.411949+0000","last_became_peered":"2023-09-23T21:03:38.411949+0000","last_unstale":"2023-09-23T21:13:26.366224+0000","last_undegraded":"2023-09-23T21:13:26.366224+0000","last_fullsized":"2023-09-23T21:13:26.366224+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:21.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":424,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356071+0000","last_change":"2023-09-23T21:04:58.520387+0000","last_active":"2023-09-23T21:13:26.356071+0000","last_peered":"2023-09-23T21:13:26.356071+0000","last_clean":"2023-09-23T21:13:26.356071+0000","last_became_active":"2023-09-23T21:04:58.520050+0000","last_became_peered":"2023-09-23T21:04:58.520050+0000","last_unstale":"2023-09-23T21:13:26.356071+0000","last_undegraded":"2023-09-23T21:13:26.356071+0000","last_fullsized":"2023-09-23T21:13:26.356071+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:14.366424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:37.284822+0000","last_change":"2023-09-23T21:05:11.430590+0000","last_active":"2023-09-23T21:13:37.284822+0000","last_peered":"2023-09-23T21:13:37.284822+0000","last_clean":"2023-09-23T21:13:37.284822+0000","last_became_active":"2023-09-23T21:05:11.430304+0000","last_became_peered":"2023-09-23T21:05:11.430304+0000","last_unstale":"2023-09-23T21:13:37.284822+0000","last_undegraded":"2023-09-23T21:13:37.284822+0000","last_fullsized":"2023-09-23T21:13:37.284822+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:16.396697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"474'10675","reported_seq":10731,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:41.011855+0000","last_change":"2023-09-23T21:09:45.843960+0000","last_active":"2023-09-23T21:13:41.011855+0000","last_peered":"2023-09-23T21:13:41.011855+0000","last_clean":"2023-09-23T21:13:41.011855+0000","last_became_active":"2023-09-23T21:09:45.843562+0000","last_became_peered":"2023-09-23T21:09:45.843562+0000","last_unstale":"2023-09-23T21:13:41.011855+0000","last_undegraded":"2023-09-23T21:13:41.011855+0000","last_fullsized":"2023-09-23T21:13:41.011855+0000","mapping_epoch":426,"log_start":"431'1800","ondisk_log_start":"431'1800","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8875,"log_dups_size":0,"ondisk_log_size":8875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:10.329012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44105728,"num_objects":673,"num_object_clones":0,"num_object_copies":2019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10675,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":454,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:34.743055+0000","last_change":"2023-09-23T21:04:41.421960+0000","last_active":"2023-09-23T21:13:34.743055+0000","last_peered":"2023-09-23T21:13:34.743055+0000","last_clean":"2023-09-23T21:13:34.743055+0000","last_became_active":"2023-09-23T21:04:41.421620+0000","last_became_peered":"2023-09-23T21:04:41.421620+0000","last_unstale":"2023-09-23T21:13:34.743055+0000","last_undegraded":"2023-09-23T21:13:34.743055+0000","last_fullsized":"2023-09-23T21:13:34.743055+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.302247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":371,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.366951+0000","last_change":"2023-09-23T20:59:44.581263+0000","last_active":"2023-09-23T21:13:26.366951+0000","last_peered":"2023-09-23T21:13:26.366951+0000","last_clean":"2023-09-23T21:13:26.366951+0000","last_became_active":"2023-09-23T20:59:44.580990+0000","last_became_peered":"2023-09-23T20:59:44.580990+0000","last_unstale":"2023-09-23T21:13:26.366951+0000","last_undegraded":"2023-09-23T21:13:26.366951+0000","last_fullsized":"2023-09-23T21:13:26.366951+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:49.571594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.371469+0000","last_change":"2023-09-23T21:05:11.418428+0000","last_active":"2023-09-23T21:13:26.371469+0000","last_peered":"2023-09-23T21:13:26.371469+0000","last_clean":"2023-09-23T21:13:26.371469+0000","last_became_active":"2023-09-23T21:05:11.418076+0000","last_became_peered":"2023-09-23T21:05:11.418076+0000","last_unstale":"2023-09-23T21:13:26.371469+0000","last_undegraded":"2023-09-23T21:13:26.371469+0000","last_fullsized":"2023-09-23T21:13:26.371469+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:31.859277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"474'10630","reported_seq":13845,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:38.292553+0000","last_change":"2023-09-23T21:13:22.469840+0000","last_active":"2023-09-23T21:13:38.292553+0000","last_peered":"2023-09-23T21:13:38.292553+0000","last_clean":"2023-09-23T21:13:38.292553+0000","last_became_active":"2023-09-23T21:13:22.401293+0000","last_became_peered":"2023-09-23T21:13:22.401293+0000","last_unstale":"2023-09-23T21:13:38.292553+0000","last_undegraded":"2023-09-23T21:13:38.292553+0000","last_fullsized":"2023-09-23T21:13:38.292553+0000","mapping_epoch":471,"log_start":"434'2488","ondisk_log_start":"434'2488","created":426,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":8142,"log_dups_size":0,"ondisk_log_size":8142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:23.955309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47513624,"num_objects":726,"num_object_clones":0,"num_object_copies":2178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":726,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10630,"num_write_kb":363329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1568,"num_bytes_recovered":102694936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":301,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:34.743081+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:13:34.743081+0000","last_peered":"2023-09-23T21:13:34.743081+0000","last_clean":"2023-09-23T21:13:34.743081+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:13:34.743081+0000","last_undegraded":"2023-09-23T21:13:34.743081+0000","last_fullsized":"2023-09-23T21:13:34.743081+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":746,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363433+0000","last_change":"2023-09-23T21:12:54.698525+0000","last_active":"2023-09-23T21:13:26.363433+0000","last_peered":"2023-09-23T21:13:26.363433+0000","last_clean":"2023-09-23T21:13:26.363433+0000","last_became_active":"2023-09-23T21:12:54.698074+0000","last_became_peered":"2023-09-23T21:12:54.698074+0000","last_unstale":"2023-09-23T21:13:26.363433+0000","last_undegraded":"2023-09-23T21:13:26.363433+0000","last_fullsized":"2023-09-23T21:13:26.363433+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:11.791067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.351824+0000","last_change":"2023-09-23T21:05:34.232533+0000","last_active":"2023-09-23T21:13:26.351824+0000","last_peered":"2023-09-23T21:13:26.351824+0000","last_clean":"2023-09-23T21:13:26.351824+0000","last_became_active":"2023-09-23T21:05:34.232242+0000","last_became_peered":"2023-09-23T21:05:34.232242+0000","last_unstale":"2023-09-23T21:13:26.351824+0000","last_undegraded":"2023-09-23T21:13:26.351824+0000","last_fullsized":"2023-09-23T21:13:26.351824+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:38.596712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":372,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356223+0000","last_change":"2023-09-23T21:10:15.582161+0000","last_active":"2023-09-23T21:13:26.356223+0000","last_peered":"2023-09-23T21:13:26.356223+0000","last_clean":"2023-09-23T21:13:26.356223+0000","last_became_active":"2023-09-23T21:10:15.581855+0000","last_became_peered":"2023-09-23T21:10:15.581855+0000","last_unstale":"2023-09-23T21:13:26.356223+0000","last_undegraded":"2023-09-23T21:13:26.356223+0000","last_fullsized":"2023-09-23T21:13:26.356223+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:19.873916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"474'10844","reported_seq":15713,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:39.211473+0000","last_change":"2023-09-23T21:12:11.354703+0000","last_active":"2023-09-23T21:13:39.211473+0000","last_peered":"2023-09-23T21:13:39.211473+0000","last_clean":"2023-09-23T21:13:39.211473+0000","last_became_active":"2023-09-23T21:12:11.333665+0000","last_became_peered":"2023-09-23T21:12:11.333665+0000","last_unstale":"2023-09-23T21:13:39.211473+0000","last_undegraded":"2023-09-23T21:13:39.211473+0000","last_fullsized":"2023-09-23T21:13:39.211473+0000","mapping_epoch":451,"log_start":"431'1700","ondisk_log_start":"431'1700","created":426,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":9144,"log_dups_size":0,"ondisk_log_size":9144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:49.607296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46792704,"num_objects":714,"num_object_clones":0,"num_object_copies":2142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10844,"num_write_kb":369856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2397,"num_bytes_recovered":117571584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":51,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363416+0000","last_change":"2023-09-23T21:10:15.584319+0000","last_active":"2023-09-23T21:13:26.363416+0000","last_peered":"2023-09-23T21:13:26.363416+0000","last_clean":"2023-09-23T21:13:26.363416+0000","last_became_active":"2023-09-23T21:10:15.584030+0000","last_became_peered":"2023-09-23T21:10:15.584030+0000","last_unstale":"2023-09-23T21:13:26.363416+0000","last_undegraded":"2023-09-23T21:13:26.363416+0000","last_fullsized":"2023-09-23T21:13:26.363416+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:28.957511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":123,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363571+0000","last_change":"2023-09-23T21:05:24.123784+0000","last_active":"2023-09-23T21:13:26.363571+0000","last_peered":"2023-09-23T21:13:26.363571+0000","last_clean":"2023-09-23T21:13:26.363571+0000","last_became_active":"2023-09-23T21:05:24.123464+0000","last_became_peered":"2023-09-23T21:05:24.123464+0000","last_unstale":"2023-09-23T21:13:26.363571+0000","last_undegraded":"2023-09-23T21:13:26.363571+0000","last_fullsized":"2023-09-23T21:13:26.363571+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:43.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":114,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:37.370803+0000","last_change":"2023-09-23T21:04:58.671551+0000","last_active":"2023-09-23T21:13:37.370803+0000","last_peered":"2023-09-23T21:13:37.370803+0000","last_clean":"2023-09-23T21:13:37.370803+0000","last_became_active":"2023-09-23T21:04:58.671100+0000","last_became_peered":"2023-09-23T21:04:58.671100+0000","last_unstale":"2023-09-23T21:13:37.370803+0000","last_undegraded":"2023-09-23T21:13:37.370803+0000","last_fullsized":"2023-09-23T21:13:37.370803+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:57.455052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"474'10707","reported_seq":10765,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:40.445855+0000","last_change":"2023-09-23T21:09:45.038161+0000","last_active":"2023-09-23T21:13:40.445855+0000","last_peered":"2023-09-23T21:13:40.445855+0000","last_clean":"2023-09-23T21:13:40.445855+0000","last_became_active":"2023-09-23T21:09:45.037907+0000","last_became_peered":"2023-09-23T21:09:45.037907+0000","last_unstale":"2023-09-23T21:13:40.445855+0000","last_undegraded":"2023-09-23T21:13:40.445855+0000","last_fullsized":"2023-09-23T21:13:40.445855+0000","mapping_epoch":426,"log_start":"435'3200","ondisk_log_start":"435'3200","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7507,"log_dups_size":0,"ondisk_log_size":7507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:37.785460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43319296,"num_objects":661,"num_object_clones":0,"num_object_copies":1983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10707,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":51,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363749+0000","last_change":"2023-09-23T21:10:15.584326+0000","last_active":"2023-09-23T21:13:26.363749+0000","last_peered":"2023-09-23T21:13:26.363749+0000","last_clean":"2023-09-23T21:13:26.363749+0000","last_became_active":"2023-09-23T21:10:15.583965+0000","last_became_peered":"2023-09-23T21:10:15.583965+0000","last_unstale":"2023-09-23T21:13:26.363749+0000","last_undegraded":"2023-09-23T21:13:26.363749+0000","last_fullsized":"2023-09-23T21:13:26.363749+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:21.120392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":373,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.366674+0000","last_change":"2023-09-23T21:05:34.164470+0000","last_active":"2023-09-23T21:13:26.366674+0000","last_peered":"2023-09-23T21:13:26.366674+0000","last_clean":"2023-09-23T21:13:26.366674+0000","last_became_active":"2023-09-23T21:05:34.164146+0000","last_became_peered":"2023-09-23T21:05:34.164146+0000","last_unstale":"2023-09-23T21:13:26.366674+0000","last_undegraded":"2023-09-23T21:13:26.366674+0000","last_fullsized":"2023-09-23T21:13:26.366674+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:01.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":159,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356658+0000","last_change":"2023-09-23T21:10:15.588413+0000","last_active":"2023-09-23T21:13:26.356658+0000","last_peered":"2023-09-23T21:13:26.356658+0000","last_clean":"2023-09-23T21:13:26.356658+0000","last_became_active":"2023-09-23T21:10:15.588077+0000","last_became_peered":"2023-09-23T21:10:15.588077+0000","last_unstale":"2023-09-23T21:13:26.356658+0000","last_undegraded":"2023-09-23T21:13:26.356658+0000","last_fullsized":"2023-09-23T21:13:26.356658+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:03.757383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"474'10538","reported_seq":10596,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:40.459576+0000","last_change":"2023-09-23T21:09:45.033672+0000","last_active":"2023-09-23T21:13:40.459576+0000","last_peered":"2023-09-23T21:13:40.459576+0000","last_clean":"2023-09-23T21:13:40.459576+0000","last_became_active":"2023-09-23T21:09:45.033296+0000","last_became_peered":"2023-09-23T21:09:45.033296+0000","last_unstale":"2023-09-23T21:13:40.459576+0000","last_undegraded":"2023-09-23T21:13:40.459576+0000","last_fullsized":"2023-09-23T21:13:40.459576+0000","mapping_epoch":426,"log_start":"435'3000","ondisk_log_start":"435'3000","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:26.841578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43778048,"num_objects":668,"num_object_clones":0,"num_object_copies":2004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10538,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":147,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.363109+0000","last_change":"2023-09-23T21:10:15.593352+0000","last_active":"2023-09-23T21:13:26.363109+0000","last_peered":"2023-09-23T21:13:26.363109+0000","last_clean":"2023-09-23T21:13:26.363109+0000","last_became_active":"2023-09-23T21:10:15.589715+0000","last_became_peered":"2023-09-23T21:10:15.589715+0000","last_unstale":"2023-09-23T21:13:26.363109+0000","last_undegraded":"2023-09-23T21:13:26.363109+0000","last_fullsized":"2023-09-23T21:13:26.363109+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:24.563158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":108,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.371056+0000","last_change":"2023-09-23T21:05:11.421515+0000","last_active":"2023-09-23T21:13:26.371056+0000","last_peered":"2023-09-23T21:13:26.371056+0000","last_clean":"2023-09-23T21:13:26.371056+0000","last_became_active":"2023-09-23T21:05:11.421223+0000","last_became_peered":"2023-09-23T21:05:11.421223+0000","last_unstale":"2023-09-23T21:13:26.371056+0000","last_undegraded":"2023-09-23T21:13:26.371056+0000","last_fullsized":"2023-09-23T21:13:26.371056+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:40.494746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":135,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.350231+0000","last_change":"2023-09-23T21:12:51.650884+0000","last_active":"2023-09-23T21:13:26.350231+0000","last_peered":"2023-09-23T21:13:26.350231+0000","last_clean":"2023-09-23T21:13:26.350231+0000","last_became_active":"2023-09-23T21:12:51.650560+0000","last_became_peered":"2023-09-23T21:12:51.650560+0000","last_unstale":"2023-09-23T21:13:26.350231+0000","last_undegraded":"2023-09-23T21:13:26.350231+0000","last_fullsized":"2023-09-23T21:13:26.350231+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:19.136242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"474'10420","reported_seq":10478,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:39.712081+0000","last_change":"2023-09-23T21:09:45.038290+0000","last_active":"2023-09-23T21:13:39.712081+0000","last_peered":"2023-09-23T21:13:39.712081+0000","last_clean":"2023-09-23T21:13:39.712081+0000","last_became_active":"2023-09-23T21:09:45.037976+0000","last_became_peered":"2023-09-23T21:09:45.037976+0000","last_unstale":"2023-09-23T21:13:39.712081+0000","last_undegraded":"2023-09-23T21:13:39.712081+0000","last_fullsized":"2023-09-23T21:13:39.712081+0000","mapping_epoch":426,"log_start":"430'400","ondisk_log_start":"430'400","created":426,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:09:44.015845+0000","last_clean_scrub_stamp":"2023-09-23T21:09:44.015845+0000","objects_scrubbed":0,"log_size":10020,"log_dups_size":0,"ondisk_log_size":10020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:44.361487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44040192,"num_objects":672,"num_object_clones":0,"num_object_copies":2016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10420,"num_write_kb":354944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":380,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356577+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:13:26.356577+0000","last_peered":"2023-09-23T21:13:26.356577+0000","last_clean":"2023-09-23T21:13:26.356577+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:13:26.356577+0000","last_undegraded":"2023-09-23T21:13:26.356577+0000","last_fullsized":"2023-09-23T21:13:26.356577+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":118,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T21:13:37.370874+0000","last_change":"2023-09-23T21:05:01.696935+0000","last_active":"2023-09-23T21:13:37.370874+0000","last_peered":"2023-09-23T21:13:37.370874+0000","last_clean":"2023-09-23T21:13:37.370874+0000","last_became_active":"2023-09-23T21:04:58.672910+0000","last_became_peered":"2023-09-23T21:04:58.672910+0000","last_unstale":"2023-09-23T21:13:37.370874+0000","last_undegraded":"2023-09-23T21:13:37.370874+0000","last_fullsized":"2023-09-23T21:13:37.370874+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:23.766289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":159,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356478+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:13:26.356478+0000","last_peered":"2023-09-23T21:13:26.356478+0000","last_clean":"2023-09-23T21:13:26.356478+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:13:26.356478+0000","last_undegraded":"2023-09-23T21:13:26.356478+0000","last_fullsized":"2023-09-23T21:13:26.356478+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":131,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.364345+0000","last_change":"2023-09-23T21:10:15.588809+0000","last_active":"2023-09-23T21:13:26.364345+0000","last_peered":"2023-09-23T21:13:26.364345+0000","last_clean":"2023-09-23T21:13:26.364345+0000","last_became_active":"2023-09-23T21:10:15.588451+0000","last_became_peered":"2023-09-23T21:10:15.588451+0000","last_unstale":"2023-09-23T21:13:26.364345+0000","last_undegraded":"2023-09-23T21:13:26.364345+0000","last_fullsized":"2023-09-23T21:13:26.364345+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:58.509500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":508,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.355304+0000","last_change":"2023-09-23T21:09:44.163759+0000","last_active":"2023-09-23T21:13:26.355304+0000","last_peered":"2023-09-23T21:13:26.355304+0000","last_clean":"2023-09-23T21:13:26.355304+0000","last_became_active":"2023-09-23T21:09:44.163405+0000","last_became_peered":"2023-09-23T21:09:44.163405+0000","last_unstale":"2023-09-23T21:13:26.355304+0000","last_undegraded":"2023-09-23T21:13:26.355304+0000","last_fullsized":"2023-09-23T21:13:26.355304+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:52.346296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":136,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.373886+0000","last_change":"2023-09-23T21:04:03.577198+0000","last_active":"2023-09-23T21:13:26.373886+0000","last_peered":"2023-09-23T21:13:26.373886+0000","last_clean":"2023-09-23T21:13:26.373886+0000","last_became_active":"2023-09-23T21:04:03.576882+0000","last_became_peered":"2023-09-23T21:04:03.576882+0000","last_unstale":"2023-09-23T21:13:26.373886+0000","last_undegraded":"2023-09-23T21:13:26.373886+0000","last_fullsized":"2023-09-23T21:13:26.373886+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:30.938715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":142,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.354088+0000","last_change":"2023-09-23T21:05:24.127572+0000","last_active":"2023-09-23T21:13:26.354088+0000","last_peered":"2023-09-23T21:13:26.354088+0000","last_clean":"2023-09-23T21:13:26.354088+0000","last_became_active":"2023-09-23T21:05:24.127293+0000","last_became_peered":"2023-09-23T21:05:24.127293+0000","last_unstale":"2023-09-23T21:13:26.354088+0000","last_undegraded":"2023-09-23T21:13:26.354088+0000","last_fullsized":"2023-09-23T21:13:26.354088+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:24.488995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":355,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.356330+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:13:26.356330+0000","last_peered":"2023-09-23T21:13:26.356330+0000","last_clean":"2023-09-23T21:13:26.356330+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:13:26.356330+0000","last_undegraded":"2023-09-23T21:13:26.356330+0000","last_fullsized":"2023-09-23T21:13:26.356330+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":134,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.351953+0000","last_change":"2023-09-23T21:05:24.123790+0000","last_active":"2023-09-23T21:13:26.351953+0000","last_peered":"2023-09-23T21:13:26.351953+0000","last_clean":"2023-09-23T21:13:26.351953+0000","last_became_active":"2023-09-23T21:05:24.123492+0000","last_became_peered":"2023-09-23T21:05:24.123492+0000","last_unstale":"2023-09-23T21:13:26.351953+0000","last_undegraded":"2023-09-23T21:13:26.351953+0000","last_fullsized":"2023-09-23T21:13:26.351953+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:05.002887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":117,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.362955+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:13:26.362955+0000","last_peered":"2023-09-23T21:13:26.362955+0000","last_clean":"2023-09-23T21:13:26.362955+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:13:26.362955+0000","last_undegraded":"2023-09-23T21:13:26.362955+0000","last_fullsized":"2023-09-23T21:13:26.362955+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":125,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.352701+0000","last_change":"2023-09-23T21:04:58.424937+0000","last_active":"2023-09-23T21:13:26.352701+0000","last_peered":"2023-09-23T21:13:26.352701+0000","last_clean":"2023-09-23T21:13:26.352701+0000","last_became_active":"2023-09-23T21:04:58.424592+0000","last_became_peered":"2023-09-23T21:04:58.424592+0000","last_unstale":"2023-09-23T21:13:26.352701+0000","last_undegraded":"2023-09-23T21:13:26.352701+0000","last_fullsized":"2023-09-23T21:13:26.352701+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:33.521562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":120,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.365992+0000","last_change":"2023-09-23T21:04:58.460645+0000","last_active":"2023-09-23T21:13:26.365992+0000","last_peered":"2023-09-23T21:13:26.365992+0000","last_clean":"2023-09-23T21:13:26.365992+0000","last_became_active":"2023-09-23T21:04:58.437959+0000","last_became_peered":"2023-09-23T21:04:58.437959+0000","last_unstale":"2023-09-23T21:13:26.365992+0000","last_undegraded":"2023-09-23T21:13:26.365992+0000","last_fullsized":"2023-09-23T21:13:26.365992+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:41.934756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T21:13:26.373449+0000","last_change":"2023-09-23T21:05:24.130638+0000","last_active":"2023-09-23T21:13:26.373449+0000","last_peered":"2023-09-23T21:13:26.373449+0000","last_clean":"2023-09-23T21:13:26.373449+0000","last_became_active":"2023-09-23T21:05:24.130374+0000","last_became_peered":"2023-09-23T21:05:24.130374+0000","last_unstale":"2023-09-23T21:13:26.373449+0000","last_undegraded":"2023-09-23T21:13:26.373449+0000","last_fullsized":"2023-09-23T21:13:26.373449+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:41.172005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":702873624,"num_objects":10726,"num_object_clones":0,"num_object_copies":32178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10726,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170076,"num_write_kb":5785601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14919,"num_bytes_recovered":938147864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2326478848,"data_stored":2326462560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139388,"ondisk_log_size":139388,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":28028,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201819,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1017892,"kb_used_data":346844,"kb_used_omap":0,"kb_used_meta":671040,"kb_avail":92727260,"statfs":{"total":95995035648,"available":94952714240,"internally_reserved":0,"allocated":355168256,"data_stored":353822201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":687144960},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":2.8570000000000002,"15min":1.5860000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.20699999999999999},"max":{"1min":17.183,"5min":64.257000000000005,"15min":64.257000000000005},"last":0.23300000000000001},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.8149999999999999,"15min":1.5800000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":17.193000000000001,"5min":64.293000000000006,"15min":64.293000000000006},"last":0.39300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:13:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":4.6500000000000004,"15min":3.1110000000000002},"min":{"1min":0.247,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":16.684000000000001,"5min":112.355,"15min":114.16500000000001},"last":0.314},{"interface":"front","average":{"1min":8.2799999999999994,"5min":6.0659999999999998,"15min":3.6469999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":117.637,"5min":117.637,"15min":117.637},"last":0.29299999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":8.4199999999999999,"5min":5.4859999999999998,"15min":3.6299999999999999},"min":{"1min":0.30599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":133.524,"5min":183.44200000000001,"15min":183.44200000000001},"last":4.7779999999999996},{"interface":"front","average":{"1min":8.7460000000000004,"5min":5.4530000000000003,"15min":3.5499999999999998},"min":{"1min":0.35999999999999999,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":133.55699999999999,"5min":183.46199999999999,"15min":183.46199999999999},"last":6.2199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:12:57 2023","interfaces":[{"interface":"back","average":{"1min":12.335000000000001,"5min":5.2869999999999999,"15min":3.1840000000000002},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":118.15900000000001,"5min":118.15900000000001,"15min":118.15900000000001},"last":1.3759999999999999},{"interface":"front","average":{"1min":12.760999999999999,"5min":5.7560000000000002,"15min":3.2200000000000002},"min":{"1min":0.34799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":118.098,"5min":118.098,"15min":118.098},"last":6.2770000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:13:28 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.59899999999999998,"15min":0.78500000000000003},"min":{"1min":0.27300000000000002,"5min":0.27200000000000002,"15min":0.248},"max":{"1min":2.0379999999999998,"5min":4.165,"15min":15.942},"last":0.433},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.72799999999999998,"15min":0.70999999999999996},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26500000000000001},"max":{"1min":0.84299999999999997,"5min":6.8920000000000003,"15min":16.725999999999999},"last":1.214}]},{"osd":5,"last update":"Sat Sep 23 21:13:28 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.82999999999999996,"15min":0.91900000000000004},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.907,"5min":6.8970000000000002,"15min":10.891},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.81399999999999995,"15min":0.82999999999999996},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.28399999999999997},"max":{"1min":2.012,"5min":8.8219999999999992,"15min":13.867000000000001},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:13:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.6829999999999998,"15min":1.613},"min":{"1min":0.32700000000000001,"5min":0.313,"15min":0.313},"max":{"1min":2.0600000000000001,"5min":143.18700000000001,"15min":143.18700000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.499,"5min":2.6680000000000001,"15min":1.5669999999999999},"min":{"1min":0.36299999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.0289999999999999,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.373}]}]},{"osd":6,"up_from":47,"seq":201863463440,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923088,"kb_used_data":293256,"kb_used_omap":0,"kb_used_meta":629824,"kb_avail":92822064,"statfs":{"total":95995035648,"available":95049793536,"internally_reserved":0,"allocated":300294144,"data_stored":298968192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":644939776},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":10.954000000000001,"5min":3.444,"15min":2.536},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.20999999999999999},"max":{"1min":149.61600000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":2.4729999999999999},{"interface":"front","average":{"1min":10.253,"5min":3.1890000000000001,"15min":2.4430000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":149.303,"5min":149.303,"15min":149.303},"last":2.3679999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:13:21 2023","interfaces":[{"interface":"back","average":{"1min":3.3399999999999999,"5min":4.4139999999999997,"15min":3.3919999999999999},"min":{"1min":0.26100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":25.122,"5min":129.37899999999999,"15min":129.37899999999999},"last":3.323},{"interface":"front","average":{"1min":2.359,"5min":4.1790000000000003,"15min":3.3239999999999998},"min":{"1min":0.247,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":25.193999999999999,"5min":129.40100000000001,"15min":129.40100000000001},"last":2.3439999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":5.8659999999999997,"15min":5.1379999999999999},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.26800000000000002},"max":{"1min":133.46600000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":1.6890000000000001},{"interface":"front","average":{"1min":10.975,"5min":5.3630000000000004,"15min":5.0069999999999997},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.26200000000000001},"max":{"1min":133.33000000000001,"5min":133.33000000000001,"15min":199.08000000000001},"last":2.169}]},{"osd":3,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":15.163,"5min":5.6159999999999997,"15min":3.9039999999999999},"min":{"1min":0.30499999999999999,"5min":0.30399999999999999,"15min":0.26900000000000002},"max":{"1min":137.875,"5min":137.875,"15min":178.40000000000001},"last":4.4189999999999996},{"interface":"front","average":{"1min":13.106999999999999,"5min":5.2590000000000003,"15min":3.8620000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":137.60900000000001,"5min":137.60900000000001,"15min":178.26599999999999},"last":2.2749999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.748,"15min":0.86299999999999999},"min":{"1min":0.27300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.4489999999999998,"5min":9.0890000000000004,"15min":13.1},"last":0.311},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.66100000000000003,"15min":0.73199999999999998},"min":{"1min":0.28299999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.78000000000000003,"5min":6.4400000000000004,"15min":6.4400000000000004},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:12:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.75700000000000001,"15min":0.876},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.419,"5min":9.0519999999999996,"15min":11.281000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.63800000000000001,"15min":0.85599999999999998},"min":{"1min":0.32200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.52300000000000002,"5min":6.0190000000000001,"15min":8.6229999999999993},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:13:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.69199999999999995,"15min":0.879},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.6230000000000002,"5min":6.6299999999999999,"15min":7.8079999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.872,"5min":0.82599999999999996,"15min":0.92400000000000004},"min":{"1min":0.38600000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.7549999999999999,"5min":6.4749999999999996,"15min":7.1849999999999996},"last":0.376}]}]},{"osd":1,"up_from":276,"seq":1185410973887,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807056,"kb_used_data":249096,"kb_used_omap":5,"kb_used_meta":557946,"kb_avail":92938096,"statfs":{"total":95995035648,"available":95168610304,"internally_reserved":0,"allocated":255074304,"data_stored":252748856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":571337443},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3279999999999998,"5min":3.875,"15min":2.0329999999999999},"min":{"1min":0.245,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":16.262,"5min":108.47,"15min":108.47},"last":0.81100000000000005},{"interface":"front","average":{"1min":2.847,"5min":3.9239999999999999,"15min":2.1080000000000001},"min":{"1min":0.23499999999999999,"5min":0.19600000000000001,"15min":0.193},"max":{"1min":11.507999999999999,"5min":109.532,"15min":109.532},"last":0.442}]},{"osd":2,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":8.0250000000000004,"5min":6.8029999999999999,"15min":3.5390000000000001},"min":{"1min":0.26200000000000001,"5min":0.23200000000000001,"15min":0.20200000000000001},"max":{"1min":78.344999999999999,"5min":107.883,"15min":107.883},"last":0.35799999999999998},{"interface":"front","average":{"1min":6.5629999999999997,"5min":5.6769999999999996,"15min":3.073},"min":{"1min":0.247,"5min":0.182,"15min":0.182},"max":{"1min":78.376999999999995,"5min":101.024,"15min":101.024},"last":1.028}]},{"osd":3,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":3.9049999999999998,"5min":7.3159999999999998,"15min":4.9930000000000003},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.23899999999999999},"max":{"1min":25.789000000000001,"5min":99.120999999999995,"15min":172.44399999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":4.2930000000000001,"5min":7.2709999999999999,"15min":5.1280000000000001},"min":{"1min":0.35499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.9130000000000003,"5min":101.849,"15min":172.58500000000001},"last":0.68600000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":1.9950000000000001,"15min":1.5429999999999999},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":10.925000000000001,"5min":11.728999999999999,"15min":17.006},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.502,"5min":1.4990000000000001,"15min":1.2470000000000001},"min":{"1min":0.32300000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.0599999999999996,"5min":23.332999999999998,"15min":23.332999999999998},"last":0.73899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.1030000000000002,"15min":1.5680000000000001},"min":{"1min":0.441,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.2210000000000001,"5min":16.196000000000002,"15min":17.155000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.806,"5min":2.117,"15min":1.6850000000000001},"min":{"1min":0.46500000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":10.975,"5min":11.760999999999999,"15min":17.076000000000001},"last":0.86299999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":2.3029999999999999,"15min":2.1099999999999999},"min":{"1min":0.42199999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.4480000000000004,"5min":16.222000000000001,"15min":127.995},"last":0.65900000000000003},{"interface":"front","average":{"1min":2.7989999999999999,"5min":2.1749999999999998,"15min":2.1739999999999999},"min":{"1min":0.46700000000000003,"5min":0.34300000000000003,"15min":0.31900000000000001},"max":{"1min":10.878,"5min":11.782999999999999,"15min":126.836},"last":0.84199999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:13:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":2.1160000000000001,"15min":2.169},"min":{"1min":0.42299999999999999,"5min":0.374,"15min":0.36299999999999999},"max":{"1min":10.951000000000001,"5min":11.680999999999999,"15min":151.791},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.573,"5min":1.6220000000000001,"15min":1.8160000000000001},"min":{"1min":0.432,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.0949999999999998,"5min":23.376000000000001,"15min":151.809},"last":0.79100000000000004}]}]},{"osd":0,"up_from":92,"seq":395136991626,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143824,"kb_used_data":291848,"kb_used_omap":5,"kb_used_meta":851898,"kb_avail":92601328,"statfs":{"total":95995035648,"available":94823759872,"internally_reserved":0,"allocated":298852352,"data_stored":296642170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":872344186},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:13:08 2023","interfaces":[{"interface":"back","average":{"1min":3.04,"5min":2.956,"15min":2.6579999999999999},"min":{"1min":0.23300000000000001,"5min":0.182,"15min":0.17799999999999999},"max":{"1min":8.1389999999999993,"5min":53.262,"15min":61.341999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":3.5030000000000001,"5min":3.4470000000000001,"15min":2.7799999999999998},"min":{"1min":0.251,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.266,"5min":53.201999999999998,"15min":61.299999999999997},"last":0.29299999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":9.343,"5min":5.2949999999999999,"15min":3.4870000000000001},"min":{"1min":0.27300000000000002,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":152.291,"5min":152.291,"15min":183.77099999999999},"last":0.875},{"interface":"front","average":{"1min":9.375,"5min":5.2400000000000002,"15min":3.6059999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.22600000000000001},"max":{"1min":152.33099999999999,"5min":152.33099999999999,"15min":183.78700000000001},"last":0.85599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":4.1909999999999998,"5min":4.548,"15min":3.206},"min":{"1min":0.33500000000000002,"5min":0.23999999999999999,"15min":0.224},"max":{"1min":10.448,"5min":86.989999999999995,"15min":186.143},"last":0.53000000000000003},{"interface":"front","average":{"1min":3.2749999999999999,"5min":4.3959999999999999,"15min":3.177},"min":{"1min":0.30299999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.648999999999999,"5min":87.043999999999997,"15min":186.97399999999999},"last":0.51800000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":1.786,"15min":1.4390000000000001},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.29799999999999999},"max":{"1min":11.420999999999999,"5min":11.420999999999999,"15min":11.420999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.0230000000000001,"5min":1.6619999999999999,"15min":1.1479999999999999},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":4.9989999999999997,"5min":11.753,"15min":11.753},"last":0.59399999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.8049999999999999,"15min":1.5169999999999999},"min":{"1min":0.39000000000000001,"5min":0.29499999999999998,"15min":0.28699999999999998},"max":{"1min":11.375999999999999,"5min":11.375999999999999,"15min":11.375999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.7210000000000001,"15min":1.216},"min":{"1min":0.373,"5min":0.313,"15min":0.313},"max":{"1min":5.032,"5min":11.589,"15min":11.589},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":1.859,"15min":3.403},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.9530000000000003,"5min":17.773,"15min":170.12700000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.7809999999999999,"15min":3.4169999999999998},"min":{"1min":0.39800000000000002,"5min":0.37,"15min":0.31900000000000001},"max":{"1min":4.9560000000000004,"5min":17.684999999999999,"15min":169.767},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:13:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":1.8480000000000001,"15min":1.3420000000000001},"min":{"1min":0.42999999999999999,"5min":0.38200000000000001,"15min":0.33600000000000002},"max":{"1min":6.5830000000000002,"5min":11.69,"15min":11.69},"last":0.62},{"interface":"front","average":{"1min":1.331,"5min":1.827,"15min":1.3300000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.379},"max":{"1min":5.0010000000000003,"5min":10.470000000000001,"15min":10.470000000000001},"last":0.55200000000000005}]}]},{"osd":2,"up_from":129,"seq":554050781517,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1338476,"kb_used_data":476004,"kb_used_omap":7,"kb_used_meta":862456,"kb_avail":92406676,"statfs":{"total":95995035648,"available":94624436224,"internally_reserved":0,"allocated":487428096,"data_stored":485130070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":883155025},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":3.2629999999999999,"15min":1.8320000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17699999999999999},"max":{"1min":103.021,"5min":103.021,"15min":103.021},"last":1.129},{"interface":"front","average":{"1min":8.0449999999999999,"5min":4.069,"15min":2.5099999999999998},"min":{"1min":0.22700000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":103.154,"5min":103.154,"15min":103.154},"last":0.81000000000000005}]},{"osd":1,"last update":"Sat Sep 23 21:13:15 2023","interfaces":[{"interface":"back","average":{"1min":5.8019999999999996,"5min":5.4189999999999996,"15min":4.1900000000000004},"min":{"1min":0.39700000000000002,"5min":0.22500000000000001,"15min":0.188},"max":{"1min":54.521000000000001,"5min":143.482,"15min":212.12},"last":1.3560000000000001},{"interface":"front","average":{"1min":5.9470000000000001,"5min":5.6719999999999997,"15min":4.0860000000000003},"min":{"1min":0.223,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":54.598999999999997,"5min":143.61000000000001,"15min":213.02799999999999},"last":1.482}]},{"osd":3,"last update":"Sat Sep 23 21:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4319999999999999,"5min":5.8810000000000002,"15min":4.3150000000000004},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":12.696999999999999,"5min":143.50800000000001,"15min":143.50800000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":4.2309999999999999,"5min":6.0300000000000002,"15min":4.1959999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.23799999999999999},"max":{"1min":12.502000000000001,"5min":143.27799999999999,"15min":143.27799999999999},"last":1.5629999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":4.1120000000000001,"15min":2.0150000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":9.8170000000000002,"5min":225.80799999999999,"15min":225.80799999999999},"last":1.165},{"interface":"front","average":{"1min":2.1579999999999999,"5min":4.9660000000000002,"15min":2.5670000000000002},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":8.109,"5min":225.79300000000001,"15min":225.79300000000001},"last":0.78100000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":3.54,"15min":2.1589999999999998},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.29999999999999999},"max":{"1min":8.1489999999999991,"5min":98.682000000000002,"15min":98.682000000000002},"last":0.752},{"interface":"front","average":{"1min":1.718,"5min":4.4610000000000003,"15min":2.2200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.9450000000000003,"5min":143.34700000000001,"15min":143.34700000000001},"last":0.496}]},{"osd":6,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":3.2490000000000001,"15min":2.5579999999999998},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":6.9969999999999999,"5min":143.41399999999999,"15min":175.72499999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.2429999999999999,"5min":1.6579999999999999,"15min":1.9419999999999999},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":11.893000000000001,"5min":11.893000000000001,"15min":176.20400000000001},"last":1.1890000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":1.7210000000000001,"15min":1.3939999999999999},"min":{"1min":0.41599999999999998,"5min":0.36399999999999999,"15min":0.32700000000000001},"max":{"1min":9.843,"5min":11.811999999999999,"15min":12.609},"last":1.2130000000000001},{"interface":"front","average":{"1min":2.1920000000000002,"5min":2.5960000000000001,"15min":1.8480000000000001},"min":{"1min":0.41399999999999998,"5min":0.38200000000000001,"15min":0.34499999999999997},"max":{"1min":8.1720000000000006,"5min":16.875,"15min":16.875},"last":0.72499999999999998}]}]},{"osd":3,"up_from":148,"seq":635655160111,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129576,"kb_used_data":404768,"kb_used_omap":2,"kb_used_meta":724797,"kb_avail":92615576,"statfs":{"total":95995035648,"available":94838349824,"internally_reserved":0,"allocated":414482432,"data_stored":411952488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":742192181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0739999999999998,"5min":2.7850000000000001,"15min":1.8500000000000001},"min":{"1min":0.53400000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.696999999999999,"5min":17.425999999999998,"15min":44.963000000000001},"last":25.722999999999999},{"interface":"front","average":{"1min":4.3959999999999999,"5min":2.48,"15min":1.534},"min":{"1min":0.46000000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":14.503,"5min":16.920999999999999,"15min":19.738},"last":7.8040000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:13:15 2023","interfaces":[{"interface":"back","average":{"1min":13.276999999999999,"5min":15.555999999999999,"15min":6.7690000000000001},"min":{"1min":0.29499999999999998,"5min":0.27400000000000002,"15min":0.20999999999999999},"max":{"1min":147.12100000000001,"5min":237.072,"15min":237.072},"last":12.186999999999999},{"interface":"front","average":{"1min":13.151,"5min":15.204000000000001,"15min":6.8479999999999999},"min":{"1min":0.308,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":151.14699999999999,"5min":234.21899999999999,"15min":234.21899999999999},"last":4.4870000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":6.3200000000000003,"5min":6.1280000000000001,"15min":3.0499999999999998},"min":{"1min":0.55900000000000005,"5min":0.23599999999999999,"15min":0.223},"max":{"1min":24.378,"5min":93.272000000000006,"15min":93.272000000000006},"last":13.666},{"interface":"front","average":{"1min":4.2370000000000001,"5min":5.2320000000000002,"15min":2.6160000000000001},"min":{"1min":0.58599999999999997,"5min":0.22500000000000001,"15min":0.222},"max":{"1min":10.907,"5min":95.233999999999995,"15min":95.233999999999995},"last":12.127000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.1859999999999999,"5min":2.4780000000000002,"15min":2.6520000000000001},"min":{"1min":0.36399999999999999,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":15.763,"5min":15.763,"15min":111.36},"last":4.5659999999999998},{"interface":"front","average":{"1min":2.835,"5min":2.0510000000000002,"15min":2.367},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.6180000000000003,"5min":17.286999999999999,"15min":112.473},"last":13.644}]},{"osd":5,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":2.9809999999999999,"5min":2.105,"15min":1.5820000000000001},"min":{"1min":0.501,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.3710000000000004,"5min":16.855,"15min":36.488999999999997},"last":7.9050000000000002},{"interface":"front","average":{"1min":4.0949999999999998,"5min":2.6400000000000001,"15min":1.9990000000000001},"min":{"1min":0.40699999999999997,"5min":0.315,"15min":0.315},"max":{"1min":15.789999999999999,"5min":15.789999999999999,"15min":36.295000000000002},"last":4.6059999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0599999999999996,"5min":6.6529999999999996,"15min":3.2650000000000001},"min":{"1min":0.434,"5min":0.33500000000000002,"15min":0.312},"max":{"1min":15.718,"5min":304.22399999999999,"15min":304.22399999999999},"last":4.6390000000000002},{"interface":"front","average":{"1min":2.827,"5min":6.0510000000000002,"15min":2.964},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.312},"max":{"1min":7.5839999999999996,"5min":304.15800000000002,"15min":304.15800000000002},"last":13.609}]},{"osd":7,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":3.282,"5min":2.1829999999999998,"15min":1.897},"min":{"1min":0.52100000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":8.3309999999999995,"5min":16.899000000000001,"15min":74.019000000000005},"last":7.8680000000000003},{"interface":"front","average":{"1min":4.8339999999999996,"5min":2.9039999999999999,"15min":2.3719999999999999},"min":{"1min":0.436,"5min":0.377,"15min":0.32900000000000001},"max":{"1min":15.816000000000001,"5min":15.816000000000001,"15min":73.364999999999995},"last":4.9130000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986729,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":831132,"kb_used_data":210708,"kb_used_omap":0,"kb_used_meta":620416,"kb_avail":92914020,"statfs":{"total":95995035648,"available":95143956480,"internally_reserved":0,"allocated":215764992,"data_stored":214426814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":635305984},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":11.087999999999999,"5min":3.4750000000000001,"15min":2.0830000000000002},"min":{"1min":0.219,"5min":0.20699999999999999,"15min":0.20399999999999999},"max":{"1min":226.751,"5min":226.751,"15min":226.751},"last":1.3959999999999999},{"interface":"front","average":{"1min":11.529999999999999,"5min":3.589,"15min":2.0939999999999999},"min":{"1min":0.253,"5min":0.23000000000000001,"15min":0.223},"max":{"1min":226.43299999999999,"5min":226.43299999999999,"15min":226.43299999999999},"last":2.2730000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:13:17 2023","interfaces":[{"interface":"back","average":{"1min":5.2830000000000004,"5min":4.0350000000000001,"15min":4.6050000000000004},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.23200000000000001},"max":{"1min":41.613999999999997,"5min":68.337000000000003,"15min":127.411},"last":3.5880000000000001},{"interface":"front","average":{"1min":3.2930000000000001,"5min":3.601,"15min":3.8290000000000002},"min":{"1min":0.27200000000000002,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":36.229999999999997,"5min":68.293000000000006,"15min":118.075},"last":2.738}]},{"osd":2,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":14.257,"5min":5.7480000000000002,"15min":4.4900000000000002},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":210.50299999999999,"5min":210.50299999999999,"15min":210.50299999999999},"last":3.0830000000000002},{"interface":"front","average":{"1min":15.454000000000001,"5min":5.8390000000000004,"15min":4.3890000000000002},"min":{"1min":0.33100000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":210.59,"5min":210.59,"15min":210.59},"last":6.9089999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:13:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":7.8520000000000003,"15min":3.6120000000000001},"min":{"1min":0.32700000000000001,"5min":0.29399999999999998,"15min":0.29299999999999998},"max":{"1min":9.2590000000000003,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":3.1779999999999999,"5min":8.1120000000000001,"15min":3.7639999999999998},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":13.16,"5min":214.54900000000001,"15min":214.54900000000001},"last":1.103}]},{"osd":5,"last update":"Sat Sep 23 21:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.70499999999999996,"15min":0.70799999999999996},"min":{"1min":0.26800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.5139999999999998,"5min":4.9080000000000004,"15min":8.7089999999999996},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.66500000000000004,"15min":0.72399999999999998},"min":{"1min":0.34499999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.2729999999999997,"5min":5.2729999999999997,"15min":7.0179999999999998},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:12:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":1.2549999999999999,"15min":0.98099999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79300000000000004,"5min":36.557000000000002,"15min":36.557000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.125,"15min":0.97199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":2.5760000000000001,"5min":38.468000000000004,"15min":38.468000000000004},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.76100000000000001,"15min":1.4790000000000001},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.601,"5min":4.7359999999999998,"15min":145.721},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.80600000000000005,"15min":1.6160000000000001},"min":{"1min":0.34999999999999998,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":3.7189999999999999,"5min":10.411,"15min":148.45699999999999},"last":0.442}]}]},{"osd":5,"up_from":39,"seq":167503725093,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":766284,"kb_used_data":105092,"kb_used_omap":0,"kb_used_meta":661184,"kb_avail":92978868,"statfs":{"total":95995035648,"available":95210360832,"internally_reserved":0,"allocated":107614208,"data_stored":106288158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":677052416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":2.8399999999999999,"15min":1.4690000000000001},"min":{"1min":0.25700000000000001,"5min":0.217,"15min":0.217},"max":{"1min":20.023,"5min":82.280000000000001,"15min":82.280000000000001},"last":4.2140000000000004},{"interface":"front","average":{"1min":3.0569999999999999,"5min":2.9620000000000002,"15min":1.5860000000000001},"min":{"1min":0.252,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":19.853000000000002,"5min":85.236000000000004,"15min":85.236000000000004},"last":2.093}]},{"osd":1,"last update":"Sat Sep 23 21:13:32 2023","interfaces":[{"interface":"back","average":{"1min":2.629,"5min":3.923,"15min":4.2919999999999998},"min":{"1min":0.252,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":12.032999999999999,"5min":74.920000000000002,"15min":233.77099999999999},"last":2.3940000000000001},{"interface":"front","average":{"1min":2.4550000000000001,"5min":3.7530000000000001,"15min":4.1749999999999998},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.247},"max":{"1min":12.016,"5min":74.978999999999999,"15min":233.75200000000001},"last":2.3780000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":3.601,"5min":4.1660000000000004,"15min":3.4990000000000001},"min":{"1min":0.35199999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":20.411999999999999,"5min":78.917000000000002,"15min":231.34700000000001},"last":3.9209999999999998},{"interface":"front","average":{"1min":2.8460000000000001,"5min":4.2640000000000002,"15min":3.5939999999999999},"min":{"1min":0.29999999999999999,"5min":0.28499999999999998,"15min":0.27300000000000002},"max":{"1min":11.509,"5min":78.888999999999996,"15min":231.381},"last":7.4640000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":7.5039999999999996,"15min":3.9809999999999999},"min":{"1min":0.317,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":10.053000000000001,"5min":263.887,"15min":263.887},"last":2.8149999999999999},{"interface":"front","average":{"1min":1.6519999999999999,"5min":7.2450000000000001,"15min":3.8170000000000002},"min":{"1min":0.32200000000000001,"5min":0.30099999999999999,"15min":0.29499999999999998},"max":{"1min":4.218,"5min":263.70100000000002,"15min":263.70100000000002},"last":0.69299999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:13:32 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.62,"15min":0.63400000000000001},"min":{"1min":0.318,"5min":0.25,"15min":0.25},"max":{"1min":7.5259999999999998,"5min":7.5259999999999998,"15min":12.739000000000001},"last":0.314},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60199999999999998,"15min":0.73399999999999999},"min":{"1min":0.33300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.623,"5min":5.0369999999999999,"15min":10.792},"last":0.45300000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:12:50 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.708,"15min":1.333},"min":{"1min":0.35699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.7440000000000002,"5min":83.451999999999998,"15min":83.451999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.677,"15min":1.3720000000000001},"min":{"1min":0.30399999999999999,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":3.9889999999999999,"5min":83.484999999999999,"15min":83.484999999999999},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:13:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.872,"15min":0.89900000000000002},"min":{"1min":0.39600000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.96,"5min":4.3399999999999999,"15min":9.6329999999999991},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.71499999999999997,"15min":0.66400000000000003},"min":{"1min":0.436,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.8140000000000001,"5min":3.0790000000000002,"15min":4.29},"last":0.34899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":282722304,"data_stored":282722304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":245436416,"data_stored":245432344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":457576448,"data_stored":457572376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":400494592,"data_stored":400490520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":206897152,"data_stored":206897152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":101056512,"data_stored":101056512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":293732352,"data_stored":293732352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":338563072,"data_stored":338559000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:13:45.372 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:13:45.372 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:13:45.372 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:13:45.372 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.908046 2023-09-23T21:13:45.372 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.9080459151051512 2023-09-23T21:13:45.450 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:45.595 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:45 smithi121 ceph-mon[211526]: pgmap v1540: 89 pgs: 89 active+clean; 606 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T21:13:45.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:45 smithi121 ceph-mon[219709]: pgmap v1540: 89 pgs: 89 active+clean; 606 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T21:13:45.628 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:45.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:45.732 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:45.763 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:45 smithi123 ceph-mon[193903]: pgmap v1540: 89 pgs: 89 active+clean; 606 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-09-23T21:13:45.766 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:45.819 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:45.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:45.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:45.923 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:45.928 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:46.042 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:46.276 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:46.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:46.386 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:46.421 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:46.482 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:46.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:46.518 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:13:46.555 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4005989169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:46.555 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4005989169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:46.587 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:46.614 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:46.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4005989169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:13:47.205 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:47.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:47.315 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:47.351 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:47.412 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:47.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:47.520 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:47.547 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:47.589 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:47.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:47.695 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4005989169' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: osdmap e475: 8 total, 8 up, 8 in 2023-09-23T21:13:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:47 smithi123 ceph-mon[193903]: pgmap v1542: 89 pgs: 89 active+clean; 520 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T21:13:47.751 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:47.814 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:47.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4005989169' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: osdmap e475: 8 total, 8 up, 8 in 2023-09-23T21:13:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[211526]: pgmap v1542: 89 pgs: 89 active+clean; 520 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T21:13:47.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:47.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4005989169' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:13:47.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: osdmap e475: 8 total, 8 up, 8 in 2023-09-23T21:13:47.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: failed to encode map e475 with expected crc 2023-09-23T21:13:47.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:47 smithi121 ceph-mon[219709]: pgmap v1542: 89 pgs: 89 active+clean; 520 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T21:13:47.924 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:47.958 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:48.020 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:48.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:48.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:48.123 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:48.205 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:48.978 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:49.015 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:49.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:49.121 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:49.249 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:49.346 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:13:49.346 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:13:49.346 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:13:49.347 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:13:49.347 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:13:49.638 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:49.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:49.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:49.761 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:49.787 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:49.853 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:49.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:49.960 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:49.986 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:50.037 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:50.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:50.145 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:50.218 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:50.424 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:50.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:50.530 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:50.608 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:50 smithi121 ceph-mon[211526]: pgmap v1543: 89 pgs: 89 active+clean; 456 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T21:13:50.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:50 smithi121 ceph-mon[219709]: pgmap v1543: 89 pgs: 89 active+clean; 456 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T21:13:50.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:50.640 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:13:50.721 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:13:50.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:50 smithi123 ceph-mon[193903]: pgmap v1543: 89 pgs: 89 active+clean; 456 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-09-23T21:13:51.254 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:51.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:51.360 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:51.463 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:51.677 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:51.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:51.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:51.790 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:51.798 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:51.818 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:51.838 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:13:51.864 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:51.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:51.899 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:13:51.966 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:13:51.977 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:52.006 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:52.007 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:13:52.048 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:13:52.073 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:52.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:52.109 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:13:52.147 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:13:52.177 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:52.181 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:52.186 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:13:52.210 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.210 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.211 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.211 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.211 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.211 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.211 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.211 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:13:52.267 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:52 smithi121 ceph-mon[211526]: pgmap v1544: 89 pgs: 89 active+clean; 347 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T21:13:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/393775259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:52.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:52 smithi121 ceph-mon[219709]: pgmap v1544: 89 pgs: 89 active+clean; 347 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T21:13:52.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/393775259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:52.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:52 smithi123 ceph-mon[193903]: pgmap v1544: 89 pgs: 89 active+clean; 347 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2023-09-23T21:13:52.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/393775259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:13:52.803 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:52.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:52.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:52.911 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:13:53.023 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:53.425 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.3 primary-affinity to 0.908046 (8595092) 2023-09-23T21:13:53.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4020453081' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]: dispatch 2023-09-23T21:13:53.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:53 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]: dispatch 2023-09-23T21:13:53.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:53 smithi123 ceph-mon[193903]: pgmap v1545: 89 pgs: 89 active+clean; 322 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-09-23T21:13:53.760 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:13:53.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:53.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4020453081' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]: dispatch 2023-09-23T21:13:53.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:53 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]: dispatch 2023-09-23T21:13:53.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:53 smithi121 ceph-mon[211526]: pgmap v1545: 89 pgs: 89 active+clean; 322 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-09-23T21:13:53.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:13:53.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4020453081' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]: dispatch 2023-09-23T21:13:53.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:53 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]: dispatch 2023-09-23T21:13:53.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:53 smithi121 ceph-mon[219709]: pgmap v1545: 89 pgs: 89 active+clean; 322 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-09-23T21:13:53.870 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:53.949 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]': finished 2023-09-23T21:13:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: osdmap e476: 8 total, 8 up, 8 in 2023-09-23T21:13:54.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:54 smithi123 ceph-mon[193903]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T21:13:54.782 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]': finished 2023-09-23T21:13:54.782 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: osdmap e476: 8 total, 8 up, 8 in 2023-09-23T21:13:54.782 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.782 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.782 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.782 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[219709]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9080459151051512}]': finished 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: osdmap e476: 8 total, 8 up, 8 in 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.789 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: failed to encode map e476 with expected crc 2023-09-23T21:13:54.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:54 smithi121 ceph-mon[211526]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T21:13:54.790 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:54.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:54.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:54.892 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:54.922 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:54.983 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:54.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:54.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:55.088 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:55.112 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:13:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:13:54] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-09-23T21:13:55.115 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:55.448 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:55.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:55.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:55.569 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:55.600 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:55.658 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:55.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:55.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:55 smithi123 ceph-mon[193903]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:55 smithi123 ceph-mon[193903]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:55 smithi123 ceph-mon[193903]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:55 smithi123 ceph-mon[193903]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:55 smithi123 ceph-mon[193903]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:55 smithi123 ceph-mon[193903]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:55 smithi123 ceph-mon[193903]: pgmap v1548: 89 pgs: 89 active+clean; 249 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T21:13:55.766 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:13:55.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[211526]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[211526]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[211526]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[211526]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[211526]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.814 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[211526]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[211526]: pgmap v1548: 89 pgs: 89 active+clean; 249 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[219709]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[219709]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[219709]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[219709]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[219709]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[219709]: failed to encode map e477 with expected crc 2023-09-23T21:13:55.815 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:55 smithi121 ceph-mon[219709]: pgmap v1548: 89 pgs: 89 active+clean; 249 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T21:13:55.845 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:55.955 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:13:56.083 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:13:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:13:56.193 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:56.204 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:13:56.204 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:13:56.221 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.274 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:56.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:56.382 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:56.414 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.466 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:56.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:56.575 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:56.603 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.644 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:56.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:56.776 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:56.832 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:56.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:56.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:56.941 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:56.988 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:57.055 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:57.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:57.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:57.167 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:13:57.231 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:13:57.283 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:57.361 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:13:57.396 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:13:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:57.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:13:57.510 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:57.591 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:57.693 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:13:57.725 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:13:57.850 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1523022382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:57.850 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:57 smithi121 ceph-mon[211526]: pgmap v1549: 89 pgs: 89 active+clean; 179 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-09-23T21:13:57.850 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:57.854 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1523022382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:57.854 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:57 smithi121 ceph-mon[219709]: pgmap v1549: 89 pgs: 89 active+clean; 179 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-09-23T21:13:57.855 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:57.943 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1523022382' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:13:57.943 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:57 smithi123 ceph-mon[193903]: pgmap v1549: 89 pgs: 89 active+clean; 179 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-09-23T21:13:57.943 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:13:58.326 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:58.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:58.436 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:13:58.474 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:58.553 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:13:58.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:58.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:13:58.657 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:58.666 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1523022382' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.667 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[211526]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.691 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:58.753 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:58.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:58.861 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:13:58.892 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:58.892 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1523022382' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.893 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:58 smithi123 ceph-mon[193903]: failed to encode map e478 with expected crc 2023-09-23T21:13:58.897 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:58.958 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:13:58.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:58.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:13:59.068 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:13:59.095 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:13:59.095 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1523022382' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:13:59.095 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T21:13:59.095 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: failed to encode map e478 with expected crc 2023-09-23T21:13:59.095 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: failed to encode map e478 with expected crc 2023-09-23T21:13:59.095 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: failed to encode map e478 with expected crc 2023-09-23T21:13:59.096 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: failed to encode map e478 with expected crc 2023-09-23T21:13:59.096 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: failed to encode map e478 with expected crc 2023-09-23T21:13:59.096 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: failed to encode map e478 with expected crc 2023-09-23T21:13:59.096 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:58 smithi121 ceph-mon[219709]: failed to encode map e478 with expected crc 2023-09-23T21:13:59.098 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:59.384 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:13:59.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:59.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:13:59.502 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:13:59.536 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:59.658 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:13:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:59.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:13:59.767 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:59.804 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:13:59 smithi123 ceph-mon[193903]: pgmap v1551: 89 pgs: 89 active+clean; 116 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 549 op/s 2023-09-23T21:13:59.806 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:13:59.828 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 90400 objects 2023-09-23T21:13:59.828 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :156.573 2023-09-23T21:13:59.866 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:13:59.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:13:59.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:13:59.904 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 3, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:13:59.904 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:13:59.904 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:13:59.904 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-09-23T21:13:59.904 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T21:13:59.970 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:13:59.999 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:00.024 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:00.043 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:00.043 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:00.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:00.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:00.061 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:00.082 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:00.102 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:00.128 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:00.147 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:00.148 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:00.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:13:59 smithi121 ceph-mon[219709]: pgmap v1551: 89 pgs: 89 active+clean; 116 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 549 op/s 2023-09-23T21:14:00.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:13:59 smithi121 ceph-mon[211526]: pgmap v1551: 89 pgs: 89 active+clean; 116 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 549 op/s 2023-09-23T21:14:00.168 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:00.170 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.170 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.170 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.170 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.170 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.170 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.170 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.171 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:14:00.175 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:00.218 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:00.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:00.322 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:00.350 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:00.351 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:14:00.351 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:14:00.351 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:14:00.351 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:14:00.394 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:00.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:00.498 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:00.524 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:00.565 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:00.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:00.668 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:00.695 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:00.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2231612278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2231612278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:00.930 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:00.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:00.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2231612278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:00.982 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:01.034 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:01.060 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:01.106 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:01.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:01.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:01.129 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:14:01.210 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:01.243 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:01.286 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:01.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:01.328 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:14:01.389 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:01.417 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:01.475 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:01.514 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:01.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:01.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:01.646 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:01.694 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:01.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:01.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:01.798 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:01.824 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:01.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:01 smithi121 ceph-mon[219709]: pgmap v1552: 89 pgs: 89 active+clean; 20 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T21:14:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:01 smithi121 ceph-mon[211526]: pgmap v1552: 89 pgs: 89 active+clean; 20 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T21:14:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:01 smithi123 ceph-mon[193903]: pgmap v1552: 89 pgs: 89 active+clean; 20 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-09-23T21:14:02.127 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:02.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:02.171 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:02.234 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:02.261 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:02.304 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:02.408 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:02.434 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:02.481 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:02.585 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:02.613 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:02.657 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:02.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:02.761 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:02.789 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:02.941 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:02.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:02.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:02.985 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:14:02.985 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:14:02.985 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:14:02.985 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:14:02.985 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:14:02.997 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:14:03.021 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3989336820' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:14:03.022 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:14:03.022 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T21:14:03.046 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:03.073 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:03.118 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:03.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:03.222 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:03.252 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:03.297 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:03.298 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3989336820' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:14:03.298 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:14:03.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:03.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:03.362 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-09-23T21:14:03.362 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-09-23T21:14:03.386 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3989336820' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:14:03.386 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:14:03.401 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:03.429 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:03.529 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:03.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:03.620 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:14:03.633 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:03.660 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:03.959 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:04.115 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:04.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:04.220 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:04.248 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:04.318 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:04.343 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:04.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:14:04.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[211526]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T21:14:04.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[211526]: failed to encode map e479 with expected crc 2023-09-23T21:14:04.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[211526]: pgmap v1554: 89 pgs: 89 active+clean; 7.9 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2023-09-23T21:14:04.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:14:04.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[219709]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T21:14:04.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[219709]: failed to encode map e479 with expected crc 2023-09-23T21:14:04.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:04 smithi121 ceph-mon[219709]: pgmap v1554: 89 pgs: 89 active+clean; 7.9 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2023-09-23T21:14:04.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:04.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:04.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:14:04.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:04 smithi123 ceph-mon[193903]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T21:14:04.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:04 smithi123 ceph-mon[193903]: failed to encode map e479 with expected crc 2023-09-23T21:14:04.361 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:04 smithi123 ceph-mon[193903]: pgmap v1554: 89 pgs: 89 active+clean; 7.9 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 481 op/s 2023-09-23T21:14:04.369 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:04.449 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:04.479 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:04.576 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:04.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:04.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:04.679 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:04.707 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:04.861 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:04.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:04.965 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:04.993 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.012 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:14:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:14:04] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.43.0" 2023-09-23T21:14:05.039 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:05.049 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:14:05.145 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:05.172 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.249 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:05 smithi123 ceph-mon[193903]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.249 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:05 smithi123 ceph-mon[193903]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.249 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:05 smithi123 ceph-mon[193903]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.249 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:05 smithi123 ceph-mon[193903]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.249 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2393897491' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:05.266 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:05.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:05.369 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:05.399 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[211526]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[211526]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[211526]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[211526]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2393897491' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[219709]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[219709]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[219709]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[219709]: failed to encode map e479 with expected crc 2023-09-23T21:14:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2393897491' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:05.444 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:05.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:05.479 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73768 11.6 e365/343 11:608fae37:::benchmark_data_smithi121_477989_object73767:head v 365'4518, mlcod=365'4515)", 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.430536+0000", 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 596.04756365200001, 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042828863000000002, 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.480 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430536+0000", 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430536+0000", 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430535+0000", 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430569+0000", 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3970999999999997e-05 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.482 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430581+0000", 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2292e-05 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430587+0000", 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0299999999999999e-06 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430625+0000", 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8036e-05 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.430642+0000", 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7461000000000001e-05 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.473297+0000", 2023-09-23T21:14:05.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042654408999999997 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.473364+0000", 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7594000000000004e-05 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73771 11.7 e365/343 11:efacb03e:::benchmark_data_smithi121_477989_object73770:head v 365'4702, mlcod=365'4700)", 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.431350+0000", 2023-09-23T21:14:05.485 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 596.04674959700003, 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.047820113999999997, 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431350+0000", 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431350+0000", 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431349+0000", 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431439+0000", 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0464000000000007e-05 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431457+0000", 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7408999999999999e-05 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431463+0000", 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0599999999999996e-06 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431520+0000", 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6932000000000003e-05 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.431538+0000", 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7935999999999999e-05 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479111+0000", 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.047573794000000003 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479170+0000", 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8316000000000002e-05 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:73777 11.7 e365/343 11:e307afbb:::benchmark_data_smithi121_477989_object73776:head v 365'4703, mlcod=365'4700)", 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:09.437194+0000", 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 596.04090486200005, 2023-09-23T21:14:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042000286999999997, 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437194+0000", 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437194+0000", 2023-09-23T21:14:05.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437193+0000", 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437310+0000", 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117335 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437320+0000", 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0520000000000007e-06 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.437324+0000", 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8960000000000002e-06 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.457544+0000", 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020220005999999999 2023-09-23T21:14:05.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.457566+0000", 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1832999999999999e-05 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479174+0000", 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021607695 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:09.479195+0000", 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0701000000000001e-05 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75115 11.7 e365/343 11:e673a9a4:::benchmark_data_smithi121_477989_object75114:head v 365'4791, mlcod=365'4788)", 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.098286+0000", 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 594.37981351300004, 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.079072325999999998, 2023-09-23T21:14:05.495 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098286+0000", 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098286+0000", 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098284+0000", 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098331+0000", 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6903999999999997e-05 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098348+0000", 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6639000000000001e-05 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098355+0000", 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0650000000000001e-06 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098401+0000", 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5549e-05 2023-09-23T21:14:05.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.098418+0000", 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7513999999999999e-05 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100712+0000", 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022936300000000001 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177358+0000", 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.076646251999999998 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75121 11.8 e365/343 11:16e12dc2:::benchmark_data_smithi121_477989_object75120:head v 365'4741, mlcod=365'4739)", 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.100430+0000", 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 594.377669728, 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.087722467999999998, 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.500 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100430+0000", 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100430+0000", 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100428+0000", 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100457+0000", 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8529000000000001e-05 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100473+0000", 2023-09-23T21:14:05.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5502000000000002e-05 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100480+0000", 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4490000000000002e-06 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100559+0000", 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9246999999999995e-05 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.100583+0000", 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3499e-05 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188104+0000", 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.087521579000000002 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188152+0000", 2023-09-23T21:14:05.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7685000000000002e-05 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75128 11.d e365/364 11:b4c8a7c0:::benchmark_data_smithi121_477989_object75127:head v 365'4669, mlcod=365'4667)", 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.110280+0000", 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 594.36781907399995, 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.069341358000000006, 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.505 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110280+0000", 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110280+0000", 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110279+0000", 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.110313+0000", 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4057000000000003e-05 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176778+0000", 2023-09-23T21:14:05.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.066465381000000004 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176793+0000", 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4304000000000001e-05 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177003+0000", 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021034100000000001 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177027+0000", 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4105e-05 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179579+0000", 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025522380000000001 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179622+0000", 2023-09-23T21:14:05.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2051000000000001e-05 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75143 11.8 e365/343 11:1870ba7c:::benchmark_data_smithi121_477989_object75142:head v 365'4742, mlcod=365'4739)", 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.127972+0000", 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 594.35012727900005, 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.060207736999999997, 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.127972+0000", 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.127972+0000", 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.127971+0000", 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.128003+0000", 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2271000000000003e-05 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176780+0000", 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.048777648 2023-09-23T21:14:05.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176792+0000", 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1612e-05 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179249+0000", 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024567289999999999 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.179272+0000", 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3546000000000001e-05 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188155+0000", 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088824309999999997 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.188180+0000", 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4926000000000002e-05 2023-09-23T21:14:05.514 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:75146 11.7 e365/343 11:e7f09222:::benchmark_data_smithi121_477989_object75145:head v 365'4792, mlcod=365'4789)", 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:04:11.133924+0000", 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 594.344174856, 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.052426268999999998, 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.515 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133924+0000", 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133924+0000", 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133923+0000", 2023-09-23T21:14:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.133986+0000", 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2927000000000002e-05 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176800+0000", 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.042814011999999999 2023-09-23T21:14:05.517 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.176845+0000", 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4521999999999998e-05 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177391+0000", 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00054616599999999995 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.177410+0000", 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9324999999999999e-05 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.186297+0000", 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0088871019999999992 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:04:11.186351+0000", 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3195000000000001e-05 2023-09-23T21:14:05.519 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:94174 11.6 e402/375 11:6b7068d0:::benchmark_data_smithi121_477989_object5985:head v 402'5748, mlcod=402'5745)", 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:44.144799+0000", 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 441.33330024999998, 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.126705347, 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.520 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144799+0000", 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144799+0000", 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.521 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144798+0000", 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144807+0000", 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6500000000000008e-06 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144820+0000", 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2224e-05 2023-09-23T21:14:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144826+0000", 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0399999999999998e-06 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144871+0000", 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5735000000000002e-05 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.523 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.144898+0000", 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6435e-05 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.271480+0000", 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.12658245100000001 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.271504+0000", 2023-09-23T21:14:05.524 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4170999999999999e-05 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:94192 11.8 e402/386 11:1f65d4ab:::benchmark_data_smithi121_477989_object6003:head v 402'5957, mlcod=402'5955)", 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:44.188483+0000", 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 441.28961652800001, 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.077938652999999997, 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.525 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188483+0000", 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188483+0000", 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.526 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188482+0000", 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.188506+0000", 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3886000000000001e-05 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263882+0000", 2023-09-23T21:14:05.527 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.075376705000000002 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263897+0000", 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4396000000000001e-05 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263939+0000", 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2661999999999999e-05 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.528 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.263963+0000", 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3879e-05 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.266378+0000", 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002415169 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.529 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:44.266421+0000", 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2985999999999998e-05 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:100873 11.6 e405/375 11:602c3a22:::benchmark_data_smithi121_477989_object12684:head v 405'6177, mlcod=405'6175)", 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:06:58.129773+0000", 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 427.34832652599999, 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040138322999999997, 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.530 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129773+0000", 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129773+0000", 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.531 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129772+0000", 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129778+0000", 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0020000000000003e-06 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.532 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129787+0000", 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1230000000000006e-06 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129791+0000", 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2679999999999996e-06 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129842+0000", 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0748000000000003e-05 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.533 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.129859+0000", 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7144999999999999e-05 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.169864+0000", 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040004524999999999 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.534 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:06:58.169911+0000", 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7222000000000003e-05 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:102759 11.8 e406/386 11:1f479d3e:::benchmark_data_smithi121_477989_object14570:head v 406'6487, mlcod=406'6483)", 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:02.054425+0000", 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 423.42367414900002, 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040732209999999998, 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.535 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054425+0000", 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054425+0000", 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054424+0000", 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054433+0000", 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0559999999999994e-06 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.537 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054448+0000", 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5193e-05 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054456+0000", 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1189999999999999e-06 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054511+0000", 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5461999999999999e-05 2023-09-23T21:14:05.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.054540+0000", 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9091e-05 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095107+0000", 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.040567021000000002 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:02.095157+0000", 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0207999999999998e-05 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:109584 11.d e407/364 11:bb1f75d3:::benchmark_data_smithi121_477989_object21395:head v 407'6837, mlcod=407'6834)", 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:15.609395+0000", 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 409.868704057, 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.071550276999999995, 2023-09-23T21:14:05.540 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609395+0000", 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609395+0000", 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609394+0000", 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609409+0000", 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4746000000000001e-05 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609425+0000", 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5625999999999999e-05 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609432+0000", 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1930000000000004e-06 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609479+0000", 2023-09-23T21:14:05.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7236e-05 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.609500+0000", 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0567000000000001e-05 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612284+0000", 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027845119999999998 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680945+0000", 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.068661429999999996 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:109575 11.e e407/382 11:7af111bd:::benchmark_data_smithi121_477989_object21386:head v 407'6939, mlcod=0'0)", 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:15.612355+0000", 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 409.86574416000002, 2023-09-23T21:14:05.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.075662072999999996, 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612355+0000", 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612355+0000", 2023-09-23T21:14:05.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612354+0000", 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.612362+0000", 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8080000000000005e-06 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680918+0000", 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.068555560000000001 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680928+0000", 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0455000000000001e-05 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.681061+0000", 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000133273 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.681082+0000", 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0496000000000002e-05 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.687981+0000", 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0068987909999999996 2023-09-23T21:14:05.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.688017+0000", 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6429e-05 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36195.0:109588 11.d e407/364 11:be08a181:::benchmark_data_smithi121_477989_object21399:head v 407'6838, mlcod=407'6834)", 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:07:15.624386+0000", 2023-09-23T21:14:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 409.853713089, 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.062509666000000005, 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624386+0000", 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624386+0000", 2023-09-23T21:14:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624385+0000", 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.624394+0000", 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4530000000000002e-06 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680913+0000", 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.056519579 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680926+0000", 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2864000000000001e-05 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.553 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680975+0000", 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9079999999999998e-05 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.680994+0000", 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9083e-05 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.686856+0000", 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058616989999999997 2023-09-23T21:14:05.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:07:15.686896+0000", 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9870000000000003e-05 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:45566 12.2 e434/433 12:441f91eb:::benchmark_data_smithi121_563784_object45565:head v 434'2868, mlcod=0'0)", 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:10:24.928117+0000", 2023-09-23T21:14:05.555 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 220.549981938, 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25272826199999998, 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928117+0000", 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928117+0000", 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928117+0000", 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928179+0000", 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2428999999999995e-05 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928187+0000", 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0760000000000003e-06 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928192+0000", 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.42e-06 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.558 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928223+0000", 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1337999999999997e-05 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.928240+0000", 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6660999999999999e-05 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.180799+0000", 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25255934600000002 2023-09-23T21:14:05.559 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.180846+0000", 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6569000000000001e-05 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:45568 12.7 e434/426 12:e11f8aaf:::benchmark_data_smithi121_563784_object45567:head v 434'2812, mlcod=434'2809)", 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:10:24.929906+0000", 2023-09-23T21:14:05.560 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 220.54819329, 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25221269099999999, 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.929906+0000", 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.929906+0000", 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.929905+0000", 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.929943+0000", 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8309999999999997e-05 2023-09-23T21:14:05.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177378+0000", 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24743467599999999 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177385+0000", 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4109999999999999e-06 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177499+0000", 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113521 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177519+0000", 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9825000000000001e-05 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.182091+0000", 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045725990000000001 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.182119+0000", 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.72e-05 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:45560 12.e e434/433 12:7ad95309:::benchmark_data_smithi121_563784_object45559:head v 434'2911, mlcod=0'0)", 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:10:24.930479+0000", 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 220.54761995499999, 2023-09-23T21:14:05.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.25091336600000003, 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.930479+0000", 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.930479+0000", 2023-09-23T21:14:05.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.930478+0000", 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.930519+0000", 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0374999999999999e-05 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177366+0000", 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24684758800000001 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177376+0000", 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2709999999999998e-06 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177420+0000", 2023-09-23T21:14:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4181000000000003e-05 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177454+0000", 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3787e-05 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.181362+0000", 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039080039999999996 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.181393+0000", 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1004000000000003e-05 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.36681.0:45595 12.2 e434/433 12:4d76ac44:::benchmark_data_smithi121_563784_object45594:head v 434'2869, mlcod=0'0)", 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:10:24.954907+0000", 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 220.523192693, 2023-09-23T21:14:05.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.225970692, 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.954907+0000", 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.954907+0000", 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:05.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.954900+0000", 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999933 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:24.954938+0000", 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8177999999999999e-05 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177363+0000", 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.222425073 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.177375+0000", 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2289999999999999e-05 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.178687+0000", 2023-09-23T21:14:05.573 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0013118380000000001 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.178711+0000", 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4447000000000002e-05 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.180853+0000", 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021411400000000001 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:05.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:10:25.180877+0000", 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4604999999999999e-05 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:05.575 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:14:05.575 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:05.602 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:05.745 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:05.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:05.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:05.848 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:05.877 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:06.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2393897491' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: pgmap v1556: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 331 op/s 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[219709]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2393897491' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: pgmap v1556: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 331 op/s 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:06 smithi121 ceph-mon[211526]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.203 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:14:06.271 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:06.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:06.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:06.375 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:06.403 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:06.448 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:06.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:06.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:14:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2393897491' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T21:14:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: pgmap v1556: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 331 op/s 2023-09-23T21:14:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:06 smithi123 ceph-mon[193903]: failed to encode map e480 with expected crc 2023-09-23T21:14:06.544 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:06.552 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:06.583 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:06.735 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:06.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:06.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:06.838 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:06.866 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.009 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:07.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:07.142 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.175 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:07.188 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:07.194 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:07.212 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:07.231 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:07.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:07.267 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:07.284 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:07.292 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:07.303 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.305 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:07.319 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.563 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:07.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:07.668 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:07.752 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_7' removed 2023-09-23T21:14:07.780 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:07.818 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2350675032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:07.818 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/950498797' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:14:07.819 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:07 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:14:07.819 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:07 smithi123 ceph-mon[193903]: pgmap v1557: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 231 op/s 2023-09-23T21:14:07.824 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:07.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:07.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2350675032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/950498797' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:14:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:14:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[219709]: pgmap v1557: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 231 op/s 2023-09-23T21:14:07.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2350675032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/950498797' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:14:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:14:07.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:07 smithi121 ceph-mon[211526]: pgmap v1557: 89 pgs: 89 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 231 op/s 2023-09-23T21:14:07.928 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:07.957 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:08.016 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:08.016 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:08.056 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:08.160 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:08.186 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:08.465 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:14:08.553 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:14:08.554 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:14:08.554 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-09-23T21:14:08.554 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_8 16 2023-09-23T21:14:08.564 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:08.671 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:08.697 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:08.719 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:14:08.744 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:08.852 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:08.879 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:08.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[211526]: failed to encode map e481 with expected crc 2023-09-23T21:14:08.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[211526]: failed to encode map e481 with expected crc 2023-09-23T21:14:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:14:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[211526]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T21:14:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[219709]: failed to encode map e481 with expected crc 2023-09-23T21:14:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[219709]: failed to encode map e481 with expected crc 2023-09-23T21:14:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:14:08.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:08 smithi121 ceph-mon[219709]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T21:14:08.926 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:08.929 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:08 smithi123 ceph-mon[193903]: failed to encode map e481 with expected crc 2023-09-23T21:14:08.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:08 smithi123 ceph-mon[193903]: failed to encode map e481 with expected crc 2023-09-23T21:14:08.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:08 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:14:08.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:08 smithi123 ceph-mon[193903]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T21:14:08.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:08.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:08.978 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:09.032 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:09.033 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:09.060 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:09.071 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:09.103 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:09.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:09.129 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 2, 3, 7] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:14:09.130 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:14:09.130 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 4, 5, 6, 2, 3, 7] 2023-09-23T21:14:09.130 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-23T21:14:09.207 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:09.234 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:09.276 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:09.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:09.379 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:09.406 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:09.651 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:09.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:09.754 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:09.782 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:09 smithi121 ceph-mon[211526]: failed to encode map e481 with expected crc 2023-09-23T21:14:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:09 smithi121 ceph-mon[211526]: pgmap v1559: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:09 smithi121 ceph-mon[219709]: failed to encode map e481 with expected crc 2023-09-23T21:14:09.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:09 smithi121 ceph-mon[219709]: pgmap v1559: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:09.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:09 smithi123 ceph-mon[193903]: failed to encode map e481 with expected crc 2023-09-23T21:14:09.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:09 smithi123 ceph-mon[193903]: pgmap v1559: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:10.131 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:10.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:10.232 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:10.262 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:10.264 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:14:10.331 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:14:10 smithi123 conmon[172845]: logger=cleanup t=2023-09-23T21:14:10.259587134Z level=info msg="Completed cleanup jobs" duration=11.63683ms 2023-09-23T21:14:10.542 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:10.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:10.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:10.549 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:10.550 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:10.646 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:10.676 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:10.731 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:10.770 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:10.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:10.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:10.819 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:10.873 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:10.910 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.062 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:11.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:11.166 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:11.197 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.253 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:14:11.452 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:11.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:11.559 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:11.591 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.597 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:11.645 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_8' created 2023-09-23T21:14:11.646 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:11.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:11.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:11 smithi121 ceph-mon[211526]: pgmap v1560: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:11.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/9428086' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T21:14:11.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:11 smithi121 ceph-mon[219709]: pgmap v1560: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:11.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/9428086' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T21:14:11.750 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:11.784 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:11.861 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:11 smithi123 ceph-mon[193903]: pgmap v1560: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:11.861 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/9428086' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-09-23T21:14:12.633 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:12.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:12.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:12.641 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:14:12.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:12.767 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:12.860 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/9428086' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3718681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:12.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[219709]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/9428086' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T21:14:12.862 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3718681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:12.863 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:12 smithi121 ceph-mon[211526]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T21:14:12.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/9428086' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-09-23T21:14:12.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T21:14:12.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: failed to encode map e482 with expected crc 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3718681' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:12.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:12 smithi123 ceph-mon[193903]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T21:14:12.911 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:12.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:13.015 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:13.041 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.084 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:13.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:13.104 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:13.104 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:13.188 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:13.214 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.357 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:13.440 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-09-23T21:14:13.461 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:13.487 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.529 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:13.633 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:13.660 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.760 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:13.761 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:13 smithi123 ceph-mon[193903]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.761 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:13 smithi123 ceph-mon[193903]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.762 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:13 smithi123 ceph-mon[193903]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.762 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:13 smithi123 ceph-mon[193903]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.762 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:13 smithi123 ceph-mon[193903]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.762 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:13 smithi123 ceph-mon[193903]: pgmap v1563: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:13.809 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:14:13.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[219709]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[219709]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[219709]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[219709]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[219709]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[219709]: pgmap v1563: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[211526]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[211526]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[211526]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[211526]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[211526]: failed to encode map e483 with expected crc 2023-09-23T21:14:13.829 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:13 smithi121 ceph-mon[211526]: pgmap v1563: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:13.864 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:13.891 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:13.934 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:13.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:14.039 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:14.060 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:14.067 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:14.079 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:14.110 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:14.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:14.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:14.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:14.137 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:14.149 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:14.158 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:14.163 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:14.182 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:14.203 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:14.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:14.221 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:14.223 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.223 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.224 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.224 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.224 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.224 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.224 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.224 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:14:14.248 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:14.364 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:14.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:14.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:14.468 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:14.495 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:14.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:14:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:14:14] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.43.0" 2023-09-23T21:14:14.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:14 smithi121 ceph-mon[211526]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T21:14:14.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2537434510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:14.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:14 smithi121 ceph-mon[219709]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T21:14:14.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2537434510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:14 smithi123 ceph-mon[193903]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T21:14:14.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2537434510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:15.092 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:15.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:15.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:15.231 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:15.377 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:15.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:15.480 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:15.507 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:15.648 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 conmon[211495]: 2023-09-23T21:14:15.335+0000 7fdc50c66700 -1 mon.a@0(leader).osd e484 definitely_dead 0 2023-09-23T21:14:15.665 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.0. 2023-09-23T21:14:15.733 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:14:15.734 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:14:15.734 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:14:15.734 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:14:15.904 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:15.904 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.904 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.904 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.904 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.905 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: pgmap v1565: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:15.905 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4170927066' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T21:14:15.905 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T21:14:15.905 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:15 smithi123 ceph-mon[193903]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T21:14:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: pgmap v1565: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4170927066' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T21:14:15.906 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T21:14:15.907 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[211526]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T21:14:15.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.907 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: failed to encode map e484 with expected crc 2023-09-23T21:14:15.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: pgmap v1565: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2023-09-23T21:14:15.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4170927066' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T21:14:15.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T21:14:15.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:15 smithi121 ceph-mon[219709]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T21:14:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:15.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:15.909 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:14:16.010 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:16.041 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:16.236 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:16.318 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:14:16.341 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:16.386 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:16.429 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:16.534 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:16.563 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:16.609 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:16.665 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:16 smithi123 ceph-mon[193903]: failed to encode map e485 with expected crc 2023-09-23T21:14:16.665 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:16 smithi123 ceph-mon[193903]: failed to encode map e485 with expected crc 2023-09-23T21:14:16.665 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:16 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T21:14:16.665 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:16 smithi123 ceph-mon[193903]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T21:14:16.679 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:16.718 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:16.823 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:16.850 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[219709]: failed to encode map e485 with expected crc 2023-09-23T21:14:16.850 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[219709]: failed to encode map e485 with expected crc 2023-09-23T21:14:16.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T21:14:16.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[219709]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T21:14:16.851 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[211526]: failed to encode map e485 with expected crc 2023-09-23T21:14:16.851 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[211526]: failed to encode map e485 with expected crc 2023-09-23T21:14:16.851 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T21:14:16.851 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:16 smithi121 ceph-mon[211526]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T21:14:16.854 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:16.956 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:16.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:17.062 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:17.090 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:17.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:14:17.670 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-09-23T21:14:17.831 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:17.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:17.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:17.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/180448931' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:14:17.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[211526]: pgmap v1568: 89 pgs: 2 peering, 1 creating+peering, 86 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/180448931' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:14:17.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:17 smithi121 ceph-mon[219709]: pgmap v1568: 89 pgs: 2 peering, 1 creating+peering, 86 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:14:17.935 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:17.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T21:14:17.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.962 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.963 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.963 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.963 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: failed to encode map e486 with expected crc 2023-09-23T21:14:17.963 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/180448931' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:14:17.963 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:14:17.963 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:17 smithi123 ceph-mon[193903]: pgmap v1568: 89 pgs: 2 peering, 1 creating+peering, 86 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T21:14:17.966 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:18.013 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:18.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:18.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:18.148 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:18.198 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:14:18.198 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:14:18.198 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:14:18.199 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:14:18.199 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:14:18.202 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:18.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:18.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:18.306 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:18.335 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:18.428 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:18.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:18.475 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:18.532 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:18.562 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:18.750 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:18.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:18.825 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[211526]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[211526]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[211526]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[211526]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[219709]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[219709]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[219709]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T21:14:18.826 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:18 smithi121 ceph-mon[219709]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.855 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:18.885 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:18.921 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:18 smithi123 ceph-mon[193903]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.921 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:18 smithi123 ceph-mon[193903]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:14:18.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:18 smithi123 ceph-mon[193903]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T21:14:18.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:18 smithi123 ceph-mon[193903]: failed to encode map e487 with expected crc 2023-09-23T21:14:18.932 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:18.940 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:14:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:18.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:19.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:19.070 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:19.114 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:19.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:19.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:19.248 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:19.292 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:19.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:19.335 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:19.395 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:19.413 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-09-23T21:14:19.422 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:19.423 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:14:19.423 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:14:19.464 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:19.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:19.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:19.568 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:19.596 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:19.639 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:19.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:19.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T21:14:19.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:14:19.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_621349 2023-09-23T21:14:19.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:14:19.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:14:19.700 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[211526]: failed to encode map e488 with expected crc 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[211526]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[211526]: failed to encode map e488 with expected crc 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3723663576' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[211526]: pgmap v1571: 89 pgs: 1 active+undersized+remapped, 2 peering, 86 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[219709]: failed to encode map e488 with expected crc 2023-09-23T21:14:19.733 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[219709]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T21:14:19.734 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[219709]: failed to encode map e488 with expected crc 2023-09-23T21:14:19.734 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3723663576' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:19.734 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:19.734 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:19 smithi121 ceph-mon[219709]: pgmap v1571: 89 pgs: 1 active+undersized+remapped, 2 peering, 86 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T21:14:19.742 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:19.799 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:19 smithi123 ceph-mon[193903]: failed to encode map e488 with expected crc 2023-09-23T21:14:19.799 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:19 smithi123 ceph-mon[193903]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T21:14:19.799 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:19 smithi123 ceph-mon[193903]: failed to encode map e488 with expected crc 2023-09-23T21:14:19.800 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3723663576' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:19.800 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:19 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:19.800 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:19 smithi123 ceph-mon[193903]: pgmap v1571: 89 pgs: 1 active+undersized+remapped, 2 peering, 86 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T21:14:19.804 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:19.949 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:19.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:19.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:20.055 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:20.091 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:20.132 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:20.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:20.236 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:20.261 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:20.370 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:20.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:20.476 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:20.525 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:20.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 2677 2661 166.303 166.312 0.00316113 0.0059829 2023-09-23T21:14:20.673 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:20.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:20.778 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:20.848 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:20.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T21:14:20.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[211526]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:20.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T21:14:20.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:20 smithi121 ceph-mon[219709]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.958 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:20.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:20.964 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:20.964 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:20 smithi123 ceph-mon[193903]: failed to encode map e489 with expected crc 2023-09-23T21:14:20.973 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:21.010 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:21.043 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:21.065 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:21.075 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:21.113 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.115 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:21.158 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:21.191 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:21.211 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:21.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:21.226 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:21.232 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.232 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.232 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.232 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.233 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.233 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.233 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.233 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:21.317 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:21.370 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.449 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:21.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:21.487 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:21.487 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:21.532 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 6, 2, 3, 7] out_osds: [0] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:14:21.532 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:14:21.532 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 4, 5, 6, 2, 3, 7] 2023-09-23T21:14:21.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T21:14:21.553 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:21.626 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 5886 5870 183.424 200.562 0.00557177 0.00543622 2023-09-23T21:14:21.730 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:21.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:21.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:21.834 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:21.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3818014717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:21.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:21 smithi121 ceph-mon[211526]: pgmap v1573: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 83 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 18 MiB/s wr, 284 op/s; 124 KiB/s, 1 objects/s recovering 2023-09-23T21:14:21.881 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3818014717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:21.881 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:21 smithi121 ceph-mon[219709]: pgmap v1573: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 83 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 18 MiB/s wr, 284 op/s; 124 KiB/s, 1 objects/s recovering 2023-09-23T21:14:21.884 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3818014717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:21 smithi123 ceph-mon[193903]: pgmap v1573: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 83 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 18 MiB/s wr, 284 op/s; 124 KiB/s, 1 objects/s recovering 2023-09-23T21:14:22.525 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:22.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:22.600 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:14:22.636 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:22.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 8173 8157 169.923 142.938 0.00355998 0.00587682 2023-09-23T21:14:22.724 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:22.735 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:22.839 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:22.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:22.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:22.943 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:22.991 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.082 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:23.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:23.189 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:23.215 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.296 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:23.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:23.456 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.523 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:23.629 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:23.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 10844 10828 169.173 166.938 0.00474581 0.00590364 2023-09-23T21:14:23.683 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:23.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:23.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:24.026 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:14:24.081 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:24.107 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:24.162 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:24.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:24.266 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:24.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:24 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:24.332 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:24 smithi123 ceph-mon[193903]: pgmap v1574: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 111 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s; 96 KiB/s, 1 objects/s recovering 2023-09-23T21:14:24.340 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:24 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:24.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:24 smithi121 ceph-mon[219709]: pgmap v1574: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 111 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s; 96 KiB/s, 1 objects/s recovering 2023-09-23T21:14:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:24 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:24.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:24 smithi121 ceph-mon[211526]: pgmap v1574: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 111 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 18 MiB/s wr, 293 op/s; 96 KiB/s, 1 objects/s recovering 2023-09-23T21:14:24.442 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:24.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:24.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:24.503 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:24.554 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:24.607 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:24.638 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:14:24.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 12541 12525 156.549 106.062 0.00656174 0.00637987 2023-09-23T21:14:24.672 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:24.712 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:24.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:24.820 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:24.876 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:24.982 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:24.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:25.090 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:25.145 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.164 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:14:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:14:24] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.43.0" 2023-09-23T21:14:25.289 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:25.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:25.399 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:25.446 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.571 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:25.661 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 14776 14760 153.736 139.688 0.00124323 0.00639481 2023-09-23T21:14:25.676 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:25.718 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:25.734 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:25 smithi121 conmon[211495]: 2023-09-23T21:14:25.617+0000 7fdc50c66700 -1 mon.a@0(leader).osd e489 definitely_dead 0 2023-09-23T21:14:25.763 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:25.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:25.866 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:25.892 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:26.007 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:26.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:26.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:26.113 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:26.200 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:26.303 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:26.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:26.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:26.388 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.1. 2023-09-23T21:14:26.410 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:26.427 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:26.436 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:26.477 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:26.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:26 smithi123 ceph-mon[193903]: pgmap v1575: 89 pgs: 89 active+clean; 475 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s; 76 KiB/s, 1 objects/s recovering 2023-09-23T21:14:26.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2825216273' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:14:26.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:14:26.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:26 smithi123 ceph-mon[193903]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T21:14:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:26.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:26.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:26.606 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[219709]: pgmap v1575: 89 pgs: 89 active+clean; 475 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s; 76 KiB/s, 1 objects/s recovering 2023-09-23T21:14:26.606 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2825216273' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:14:26.606 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:14:26.606 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[219709]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T21:14:26.606 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[211526]: pgmap v1575: 89 pgs: 89 active+clean; 475 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s; 76 KiB/s, 1 objects/s recovering 2023-09-23T21:14:26.607 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2825216273' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:14:26.607 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T21:14:26.607 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:26 smithi121 ceph-mon[211526]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T21:14:26.608 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:26.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 16005 15989 142.733 76.8125 0.00146495 0.00676555 2023-09-23T21:14:27.246 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:27.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:27.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:27.351 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:27.381 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:27.424 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:27.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: osdmap e490: 8 total, 8 up, 6 in 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: osdmap e491: 8 total, 8 up, 6 in 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:27 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.496 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:27.496 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: osdmap e490: 8 total, 8 up, 6 in 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.518 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: osdmap e491: 8 total, 8 up, 6 in 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.519 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: osdmap e490: 8 total, 8 up, 6 in 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e490 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: osdmap e491: 8 total, 8 up, 6 in 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.520 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:27 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:27.527 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:27.555 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:27.662 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 16005 15989 124.893 0 - 0.00676555 2023-09-23T21:14:27.754 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:27.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:27.857 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:27.921 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:28.018 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:28.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:28.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:28.126 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:28.183 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:28.283 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:28.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:28.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:28.388 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:28.430 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:28.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: failed to encode map e491 with expected crc 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: pgmap v1578: 89 pgs: 89 active+clean; 757 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 99 MiB/s wr, 1.59k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: osdmap e492: 8 total, 8 up, 6 in 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2904288812' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:28.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:28 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:28.611 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:28.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:28.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 17513 17497 121.485 47.125 0.013875 0.00822389 2023-09-23T21:14:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: failed to encode map e491 with expected crc 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: pgmap v1578: 89 pgs: 89 active+clean; 757 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 99 MiB/s wr, 1.59k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: osdmap e492: 8 total, 8 up, 6 in 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2904288812' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: failed to encode map e491 with expected crc 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: pgmap v1578: 89 pgs: 89 active+clean; 757 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 99 MiB/s wr, 1.59k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: osdmap e492: 8 total, 8 up, 6 in 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: failed to encode map e492 with expected crc 2023-09-23T21:14:28.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2904288812' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:28.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:28 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:14:28.694 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:14:28.700 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:14:28.717 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:28.759 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:29.275 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:29.632 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:29 smithi123 ceph-mon[193903]: failed to encode map e492 with expected crc 2023-09-23T21:14:29.632 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:29 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:29.632 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:29 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:29.632 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:29 smithi123 ceph-mon[193903]: osdmap e493: 8 total, 8 up, 6 in 2023-09-23T21:14:29.632 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:29 smithi123 ceph-mon[193903]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.632 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:29 smithi123 ceph-mon[193903]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.632 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:29 smithi123 ceph-mon[193903]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[211526]: failed to encode map e492 with expected crc 2023-09-23T21:14:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[211526]: osdmap e493: 8 total, 8 up, 6 in 2023-09-23T21:14:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[211526]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[211526]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[211526]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 18872 18856 117.83 84.9375 0.00994163 0.00847984 2023-09-23T21:14:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[219709]: failed to encode map e492 with expected crc 2023-09-23T21:14:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:14:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[219709]: osdmap e493: 8 total, 8 up, 6 in 2023-09-23T21:14:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[219709]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[219709]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:29 smithi121 ceph-mon[219709]: failed to encode map e493 with expected crc 2023-09-23T21:14:29.739 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:29.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:29.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:29.890 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:29.929 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:29.992 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:29.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:29.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:30.031 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:30.057 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:30.082 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:30.097 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:30.109 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:30.131 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.148 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:30.180 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:30.196 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:30.209 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:30.240 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:30.242 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.242 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.243 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.243 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.243 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.243 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.243 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.243 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:14:30.299 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:30.340 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.450 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:30.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:30.555 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:30.587 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[211526]: failed to encode map e493 with expected crc 2023-09-23T21:14:30.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[211526]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 985 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 127 MiB/s wr, 2.04k op/s; 1957/47328 objects degraded (4.135%); 941/47328 objects misplaced (1.988%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:14:30.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 1957/47328 objects degraded (4.135%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:30.588 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3747686907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:30.588 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[219709]: failed to encode map e493 with expected crc 2023-09-23T21:14:30.588 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[219709]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 985 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 127 MiB/s wr, 2.04k op/s; 1957/47328 objects degraded (4.135%); 941/47328 objects misplaced (1.988%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:14:30.588 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 1957/47328 objects degraded (4.135%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:30.588 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3747686907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:30.591 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.649 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:30.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:30.662 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 16 20638 20622 117.151 110.375 0.0023991 0.00851248 2023-09-23T21:14:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:30 smithi123 ceph-mon[193903]: failed to encode map e493 with expected crc 2023-09-23T21:14:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:30 smithi123 ceph-mon[193903]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 985 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 127 MiB/s wr, 2.04k op/s; 1957/47328 objects degraded (4.135%); 941/47328 objects misplaced (1.988%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T21:14:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:30 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 1957/47328 objects degraded (4.135%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3747686907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:30.752 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:30.782 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:30.832 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:30.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:30.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:30.935 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:30.969 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:31.244 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:31.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:31.348 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:31.411 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:31.554 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:31.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:31.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 22036 22020 114.67 87.375 0.0094225 0.0087144 2023-09-23T21:14:31.666 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:31.740 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:31.814 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:14:31.853 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:31.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:31.959 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:32.024 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:32.257 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:32.277 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:32.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:32.392 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:32.434 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 3, 7] out_osds: [0, 1] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:14:32.435 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:14:32.435 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T21:14:32.435 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:14:32.435 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:14:32.435 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T21:14:32.480 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:32.619 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:32 smithi123 ceph-mon[193903]: pgmap v1582: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 2966/56262 objects degraded (5.272%); 1626/56262 objects misplaced (2.890%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:14:32.650 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:32.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:32.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:32.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:32 smithi121 ceph-mon[219709]: pgmap v1582: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 2966/56262 objects degraded (5.272%); 1626/56262 objects misplaced (2.890%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:14:32.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 23611 23595 113.42 98.4375 0.014969 0.00880953 2023-09-23T21:14:32.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:32 smithi121 ceph-mon[211526]: pgmap v1582: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 2966/56262 objects degraded (5.272%); 1626/56262 objects misplaced (2.890%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:14:32.761 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:32.837 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:33.002 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:33.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:33.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:33.095 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:14:33.095 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:14:33.095 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:14:33.095 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:14:33.095 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:14:33.113 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:33.210 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:33.317 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:14:33.362 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:33.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:33.466 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:33.530 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:33.618 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:33.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:33.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 25137 25121 112.131 95.375 0.00427143 0.00891222 2023-09-23T21:14:33.725 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:33.759 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:34.144 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:34.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:34.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:34.259 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:34.331 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:34.353 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:34.404 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:34.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:34 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:34.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:34 smithi121 ceph-mon[211526]: pgmap v1583: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 62 MiB/s wr, 998 op/s; 2966/56262 objects degraded (5.272%); 1626/56262 objects misplaced (2.890%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:14:34.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:34 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:34.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:34 smithi121 ceph-mon[219709]: pgmap v1583: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 62 MiB/s wr, 998 op/s; 2966/56262 objects degraded (5.272%); 1626/56262 objects misplaced (2.890%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:14:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:34 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:34.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:34 smithi123 ceph-mon[193903]: pgmap v1583: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 62 MiB/s wr, 998 op/s; 2966/56262 objects degraded (5.272%); 1626/56262 objects misplaced (2.890%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T21:14:34.498 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:34.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:34.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:34.602 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:34.652 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:34.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 26822 26806 111.676 105.312 0.0103705 0.00894795 2023-09-23T21:14:35.127 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:14:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:14:34] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.43.0" 2023-09-23T21:14:35.151 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:35.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:35.224 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:14:35.256 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:35.294 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:14:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/854815288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:35 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:35.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:35 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:35.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/854815288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:35.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:35 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:35.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:35 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/854815288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:35.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:35 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:35.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:35 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:14:35.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 28315 28299 110.527 93.3125 0.00877917 0.00901355 2023-09-23T21:14:35.770 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:14:36.217 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:36.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:36.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:36.323 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:36.359 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:36.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[211526]: pgmap v1584: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 2966/70662 objects degraded (4.197%); 1034/70662 objects misplaced (1.463%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T21:14:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[211526]: failed to encode map e494 with expected crc 2023-09-23T21:14:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[211526]: failed to encode map e494 with expected crc 2023-09-23T21:14:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[211526]: osdmap e494: 8 total, 8 up, 6 in 2023-09-23T21:14:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2966/70662 objects degraded (4.197%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[219709]: pgmap v1584: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 2966/70662 objects degraded (4.197%); 1034/70662 objects misplaced (1.463%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T21:14:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[219709]: failed to encode map e494 with expected crc 2023-09-23T21:14:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[219709]: failed to encode map e494 with expected crc 2023-09-23T21:14:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[219709]: osdmap e494: 8 total, 8 up, 6 in 2023-09-23T21:14:36.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2966/70662 objects degraded (4.197%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:36.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 29377 29361 107.93 66.375 0.00303977 0.00913316 2023-09-23T21:14:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:36 smithi123 ceph-mon[193903]: pgmap v1584: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 89 MiB/s wr, 1.43k op/s; 2966/70662 objects degraded (4.197%); 1034/70662 objects misplaced (1.463%); 7.9 MiB/s, 127 objects/s recovering 2023-09-23T21:14:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:36 smithi123 ceph-mon[193903]: failed to encode map e494 with expected crc 2023-09-23T21:14:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:36 smithi123 ceph-mon[193903]: failed to encode map e494 with expected crc 2023-09-23T21:14:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:36 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:36 smithi123 ceph-mon[193903]: osdmap e494: 8 total, 8 up, 6 in 2023-09-23T21:14:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2966/70662 objects degraded (4.197%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:37.352 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:37.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:37.455 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:37.490 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[211526]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[211526]: osdmap e495: 8 total, 8 up, 6 in 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[211526]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[211526]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[211526]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[211526]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[211526]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[219709]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[219709]: osdmap e495: 8 total, 8 up, 6 in 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[219709]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[219709]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[219709]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[219709]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:37 smithi121 ceph-mon[219709]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.657 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:37.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:37.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 29614 29598 102.757 14.8125 0.00230718 0.0096424 2023-09-23T21:14:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:37 smithi123 ceph-mon[193903]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:37 smithi123 ceph-mon[193903]: osdmap e495: 8 total, 8 up, 6 in 2023-09-23T21:14:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:37 smithi123 ceph-mon[193903]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:37 smithi123 ceph-mon[193903]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:37 smithi123 ceph-mon[193903]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:37 smithi123 ceph-mon[193903]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:37 smithi123 ceph-mon[193903]: failed to encode map e495 with expected crc 2023-09-23T21:14:37.762 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:37.793 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:37.840 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:37.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:37.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:37.946 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:37.974 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.071 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:38.176 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:38.204 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.249 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:38.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:38.320 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:14:38.320 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:14:38.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:14:38.320 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:14:38.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.320 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159372 12.d 12:b6c49147:::benchmark_data_smithi121_563784_object68969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.893865446999996, 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.04656902, 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159372 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423455+0000", 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423460+0000", 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5589999999999996e-06 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423473+0000", 2023-09-23T21:14:38.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2984e-05 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423479+0000", 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0290000000000002e-06 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423555+0000", 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5654000000000004e-05 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423563+0000", 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4989999999999994e-06 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432320+0000", 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.008756972 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432467+0000", 2023-09-23T21:14:38.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014657299999999999 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432861+0000", 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039391600000000001 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442657+0000", 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.009796674 2023-09-23T21:14:38.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469909+0000", 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027251573000000001 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469942+0000", 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3423000000000002e-05 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469983+0000", 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1127000000000001e-05 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470025+0000", 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1706999999999999e-05 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.328 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159374 12.d 12:bcbc6391:::benchmark_data_smithi121_563784_object68971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.886858529999998, 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0397036749999999, 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159374 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430462+0000", 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430471+0000", 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8060000000000001e-06 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430485+0000", 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4771e-05 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430492+0000", 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2400000000000004e-06 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430543+0000", 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1551999999999997e-05 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430554+0000", 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0668e-05 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436375+0000", 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0058211749999999 2023-09-23T21:14:38.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436460+0000", 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4902999999999997e-05 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436540+0000", 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9652000000000002e-05 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447278+0000", 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010738077 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470065+0000", 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022787136999999999 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470102+0000", 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6761000000000003e-05 2023-09-23T21:14:38.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470134+0000", 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2085999999999997e-05 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470167+0000", 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3135e-05 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.338 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159390 12.d 12:b1ecff3e:::benchmark_data_smithi121_563784_object68987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.863070725, 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.016060076, 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.339 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159390 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454249+0000", 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454265+0000", 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5506999999999999e-05 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454282+0000", 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6966e-05 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454288+0000", 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4189999999999999e-06 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454345+0000", 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6897000000000001e-05 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454353+0000", 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2530000000000004e-06 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438507+0000", 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98415336099999995 2023-09-23T21:14:38.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438592+0000", 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5316999999999996e-05 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438669+0000", 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6503999999999998e-05 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447287+0000", 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0086180560000000007 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470085+0000", 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.02279871 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470250+0000", 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016454100000000001 2023-09-23T21:14:38.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470281+0000", 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0571000000000003e-05 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470311+0000", 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0451999999999999e-05 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.346 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159394 12.d 12:b55656ba:::benchmark_data_smithi121_563784_object68991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.858370418000007, 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.011503713, 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.347 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159394 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458950+0000", 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458958+0000", 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4789999999999996e-06 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458972+0000", 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.31e-05 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458978+0000", 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6930000000000001e-06 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.459041+0000", 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3157999999999994e-05 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.459049+0000", 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0639999999999994e-06 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440532+0000", 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98148217000000004 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440613+0000", 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0908000000000005e-05 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440689+0000", 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6810000000000005e-05 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457467+0000", 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016777255000000001 2023-09-23T21:14:38.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470372+0000", 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012905020999999999 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470399+0000", 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7229999999999998e-05 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470424+0000", 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5514e-05 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470455+0000", 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0576e-05 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159428 12.d 12:b4378ef8:::benchmark_data_smithi121_563784_object69025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.810871250999995, 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96414785400000003, 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159428 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506449+0000", 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506456+0000", 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9489999999999999e-06 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506471+0000", 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5101e-05 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506478+0000", 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8709999999999999e-06 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506530+0000", 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2335e-05 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506539+0000", 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6319999999999997e-06 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442702+0000", 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93616331100000005 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442785+0000", 2023-09-23T21:14:38.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2917000000000006e-05 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442859+0000", 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4169000000000001e-05 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457477+0000", 2023-09-23T21:14:38.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014618133 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470489+0000", 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013011224 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470543+0000", 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4104999999999998e-05 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470575+0000", 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1878000000000002e-05 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470598+0000", 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3657999999999999e-05 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159431 12.d 12:b9860c03:::benchmark_data_smithi121_563784_object69028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.810829169000002, 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96419124899999997, 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159431 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.363 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506495+0000", 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6230000000000001e-06 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506501+0000", 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.879e-06 2023-09-23T21:14:38.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506503+0000", 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7319999999999998e-06 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506557+0000", 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3786000000000003e-05 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506561+0000", 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7440000000000001e-06 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445001+0000", 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93843964400000002 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445133+0000", 2023-09-23T21:14:38.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013199800000000001 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445286+0000", 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015309000000000001 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462371+0000", 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.017085625 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470516+0000", 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0081449550000000006 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470630+0000", 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113372 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470654+0000", 2023-09-23T21:14:38.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4085000000000001e-05 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470684+0000", 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0071000000000001e-05 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159432 12.d 12:b2bd39fd:::benchmark_data_smithi121_563784_object69029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.370 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.794865689999995, 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94835160100000004, 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159432 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522454+0000", 2023-09-23T21:14:38.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522466+0000", 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1997000000000001e-05 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522497+0000", 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1341999999999999e-05 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522511+0000", 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3417e-05 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522631+0000", 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120647 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522648+0000", 2023-09-23T21:14:38.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7096999999999999e-05 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447334+0000", 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92468556300000004 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447475+0000", 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014071400000000001 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447625+0000", 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015043700000000001 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462380+0000", 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014754949 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470712+0000", 2023-09-23T21:14:38.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083322190000000001 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470763+0000", 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0980999999999999e-05 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470789+0000", 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6191999999999999e-05 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470808+0000", 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8159999999999999e-05 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159457 12.d 12:b7a9054f:::benchmark_data_smithi121_563784_object69054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.570139+0000", 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.747183049, 2023-09-23T21:14:38.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90074547599999999, 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159457 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570139+0000", 2023-09-23T21:14:38.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570139+0000", 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570137+0000", 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570143+0000", 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1940000000000003e-06 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570170+0000", 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7053999999999999e-05 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570177+0000", 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8020000000000003e-06 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570304+0000", 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012738100000000001 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.570313+0000", 2023-09-23T21:14:38.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8319999999999995e-06 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.449606+0000", 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.87929242600000002 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.449681+0000", 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5712999999999998e-05 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.449751+0000", 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9597000000000005e-05 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462386+0000", 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012634714999999999 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470739+0000", 2023-09-23T21:14:38.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083534219999999992 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470831+0000", 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2190999999999999e-05 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470857+0000", 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5199e-05 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470884+0000", 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7517000000000001e-05 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159464 12.d 12:b0a6c534:::benchmark_data_smithi121_563784_object69061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.575578+0000", 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.741744079, 2023-09-23T21:14:38.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89673405399999995, 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159464 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575578+0000", 2023-09-23T21:14:38.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575578+0000", 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575576+0000", 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575582+0000", 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2720000000000003e-06 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575599+0000", 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6571000000000001e-05 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575607+0000", 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5669999999999997e-06 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575663+0000", 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6032000000000002e-05 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.575673+0000", 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0794e-05 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.452158+0000", 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.87648446899999999 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.452245+0000", 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6664999999999998e-05 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.452344+0000", 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9536000000000004e-05 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462391+0000", 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010046487999999999 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.471179+0000", 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0087886460000000007 2023-09-23T21:14:38.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.472248+0000", 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0010684410000000001 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.472290+0000", 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2432999999999998e-05 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.472312+0000", 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1467999999999998e-05 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159469 12.d 12:b2aecdc1:::benchmark_data_smithi121_563784_object69066:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.589391+0000", 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.727931076999994, 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88300230000000002, 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.394 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159469 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589391+0000", 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589391+0000", 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589389+0000", 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589397+0000", 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5939999999999996e-06 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589414+0000", 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7588000000000002e-05 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589421+0000", 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0990000000000001e-06 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589468+0000", 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6248000000000002e-05 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.589474+0000", 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6290000000000004e-06 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.454819+0000", 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86534458199999997 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.454907+0000", 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7676000000000006e-05 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.455001+0000", 2023-09-23T21:14:38.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3936000000000003e-05 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.466003+0000", 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011002349 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.471201+0000", 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051985499999999997 2023-09-23T21:14:38.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.472336+0000", 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011347270000000001 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.472363+0000", 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6974999999999998e-05 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.472393+0000", 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9685000000000001e-05 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159502 12.d 12:b84f85e0:::benchmark_data_smithi121_563784_object69099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.659006+0000", 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.658315779000006, 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82039565800000003, 2023-09-23T21:14:38.402 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159502 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659006+0000", 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659006+0000", 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659004+0000", 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659011+0000", 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1849999999999998e-06 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659026+0000", 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4158999999999999e-05 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659032+0000", 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4169999999999997e-06 2023-09-23T21:14:38.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659085+0000", 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3278e-05 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.659092+0000", 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7989999999999996e-06 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457536+0000", 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.79844421399999999 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457696+0000", 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000159884 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457857+0000", 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016054900000000001 2023-09-23T21:14:38.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.466023+0000", 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0081664549999999995 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479189+0000", 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013166155000000001 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479311+0000", 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121469 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479358+0000", 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7638e-05 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479402+0000", 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3084999999999998e-05 2023-09-23T21:14:38.409 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159565 12.d 12:b6b3e09b:::benchmark_data_smithi121_563784_object69162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.762400+0000", 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 76.554922038000001, 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.71708817999999996, 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.410 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159565 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762400+0000", 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762400+0000", 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762398+0000", 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762404+0000", 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5330000000000002e-06 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762420+0000", 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6228000000000001e-05 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762427+0000", 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.742e-06 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762482+0000", 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5770000000000003e-05 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.762492+0000", 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0047999999999999e-05 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.460373+0000", 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.69788088400000003 2023-09-23T21:14:38.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.460490+0000", 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000116255 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.460573+0000", 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3731000000000002e-05 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469576+0000", 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0090028999999999994 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479220+0000", 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0096433360000000006 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479438+0000", 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021875099999999999 2023-09-23T21:14:38.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479465+0000", 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6735000000000001e-05 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.479488+0000", 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2738000000000002e-05 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.417 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:173462 12.d 12:b2a7d54e:::benchmark_data_smithi121_563784_object83059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:46.558350+0000", 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 51.758971705, 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.18665585000000001, 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.418 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 173462 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558350+0000", 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558350+0000", 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558349+0000", 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558355+0000", 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.19e-06 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558368+0000", 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3523000000000001e-05 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558374+0000", 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4829999999999999e-06 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558426+0000", 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2354000000000001e-05 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558563+0000", 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000137047 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.558699+0000", 2023-09-23T21:14:38.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013575600000000001 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.561242+0000", 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0025427650000000002 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.563679+0000", 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002436751 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.744920+0000", 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.181241607 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.744961+0000", 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0580000000000001e-05 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.745006+0000", 2023-09-23T21:14:38.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5049999999999997e-05 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:173485 12.d 12:b7d9d5c9:::benchmark_data_smithi121_563784_object83082:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:46.645383+0000", 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 51.671938937999997, 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.10395027699999999, 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.425 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 173485 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645383+0000", 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645383+0000", 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645381+0000", 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645391+0000", 2023-09-23T21:14:38.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5359999999999998e-06 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645408+0000", 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6903000000000001e-05 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645414+0000", 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.037e-06 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645457+0000", 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3742000000000002e-05 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645601+0000", 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014363900000000001 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.645695+0000", 2023-09-23T21:14:38.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3831000000000003e-05 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.648349+0000", 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026543920000000002 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.650483+0000", 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002133551 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.749233+0000", 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.098750058000000002 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.749290+0000", 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6879000000000002e-05 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:46.749333+0000", 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3133999999999997e-05 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:174031 12.d 12:b4ceec06:::benchmark_data_smithi121_563784_object83628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:47.530539+0000", 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 50.786782791999997, 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.168481992, 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.432 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 174031 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530539+0000", 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530539+0000", 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530537+0000", 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530546+0000", 2023-09-23T21:14:38.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7229999999999993e-06 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530565+0000", 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9516000000000001e-05 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530571+0000", 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8479999999999997e-06 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530626+0000", 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5124999999999998e-05 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530777+0000", 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015025300000000001 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:14:38.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.530894+0000", 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011724500000000001 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.533825+0000", 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029307169999999998 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.535511+0000", 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001686295 2023-09-23T21:14:38.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.698954+0000", 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.163442697 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.698993+0000", 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9079000000000003e-05 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:47.699021+0000", 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8112000000000001e-05 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:1191 13.e 13:7499457c:::benchmark_data_smithi121_621349_object1190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:20.071089+0000", 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 18.246233, 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.19652344599999999, 2023-09-23T21:14:38.439 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 1191 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071089+0000", 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071089+0000", 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071088+0000", 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071113+0000", 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5789e-05 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071126+0000", 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.217e-05 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071132+0000", 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4810000000000003e-06 2023-09-23T21:14:38.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071191+0000", 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9088000000000001e-05 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071329+0000", 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013815100000000001 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.071466+0000", 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013694900000000001 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.072806+0000", 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0013397610000000001 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.073874+0000", 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0010684259999999999 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.267506+0000", 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.19363129300000001 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.267571+0000", 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5525999999999996e-05 2023-09-23T21:14:38.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.267612+0000", 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0824000000000001e-05 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.446 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:1203 13.7 13:ef18e30e:::benchmark_data_smithi121_621349_object1202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:20.075271+0000", 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 18.242050522, 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.18904794599999999, 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 1203 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075271+0000", 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075271+0000", 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075270+0000", 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075323+0000", 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2787000000000001e-05 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075335+0000", 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1765e-05 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075339+0000", 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6709999999999998e-06 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075383+0000", 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3164000000000002e-05 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075483+0000", 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100561 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,6", 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.075577+0000", 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3737000000000001e-05 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.076004+0000", 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00042699600000000002 2023-09-23T21:14:38.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.076207+0000", 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00020306100000000001 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.264205+0000", 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.18799836 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.264275+0000", 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9499999999999995e-05 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:20.264319+0000", 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4212000000000003e-05 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.453 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:24496 13.0 13:08321653:::benchmark_data_smithi121_621349_object24495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:33.248645+0000", 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 5.068676838, 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023920522999999999, 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 24496 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.248645+0000", 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.248645+0000", 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.248644+0000", 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.250342+0000", 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001698017 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.250356+0000", 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4242e-05 2023-09-23T21:14:38.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.250362+0000", 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5509999999999996e-06 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.250654+0000", 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00029196799999999998 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.250797+0000", 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000142776 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.250961+0000", 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016352700000000001 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.253184+0000", 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022229120000000001 2023-09-23T21:14:38.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.263214+0000", 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010030704999999999 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.271502+0000", 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0082880790000000003 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.271566+0000", 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3471000000000003e-05 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:33.272565+0000", 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00099938799999999993 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29143 13.7 13:eb501746:::benchmark_data_smithi121_621349_object29142:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.322708+0000", 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 1.9946138960000002, 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.023266826000000001, 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29143 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.322708+0000", 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.322708+0000", 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.322707+0000", 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.322740+0000", 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2898999999999998e-05 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.322754+0000", 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4571e-05 2023-09-23T21:14:38.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.322761+0000", 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5370000000000003e-06 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.322904+0000", 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143005 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.323024+0000", 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120318 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,6", 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.323166+0000", 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000142397 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.332361+0000", 2023-09-23T21:14:38.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0091946809999999997 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.343832+0000", 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011470413 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.345885+0000", 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020537469999999999 2023-09-23T21:14:38.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.345938+0000", 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2228000000000002e-05 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.345975+0000", 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6996999999999998e-05 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.467 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(13.a 495/491 [PushOp(13:54c64426:::benchmark_data_smithi121_621349_object3909:head, version: 489'274, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(13:54c64426:::benchmark_data_smithi121_621349_object3909:head@489'274, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.316801+0000", 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 1.0005207759999999, 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.090687657000000005, 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316801+0000", 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316801+0000", 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316800+0000", 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:14:38.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316862+0000", 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1383000000000005e-05 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316874+0000", 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2763e-05 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316881+0000", 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0960000000000003e-06 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316927+0000", 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6031e-05 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.316932+0000", 2023-09-23T21:14:38.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.639e-06 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.407489+0000", 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.090556434000000005 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:14:38.472 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:14:38.473 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:38.473 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:14:38.474 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:14:38.474 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:14:38.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[219709]: pgmap v1587: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 2966/79407 objects degraded (3.735%); 544/79407 objects misplaced (0.685%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:14:38.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[219709]: osdmap e496: 8 total, 8 up, 6 in 2023-09-23T21:14:38.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[219709]: failed to encode map e496 with expected crc 2023-09-23T21:14:38.474 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[219709]: failed to encode map e496 with expected crc 2023-09-23T21:14:38.475 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:38.475 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:38.475 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:38.482 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:38.524 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:38.546 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.554 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:38.590 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:38.624 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:38.627 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.628 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.628 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.628 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.628 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.628 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.628 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.628 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:38.651 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[211526]: pgmap v1587: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 2966/79407 objects degraded (3.735%); 544/79407 objects misplaced (0.685%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:14:38.651 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[211526]: osdmap e496: 8 total, 8 up, 6 in 2023-09-23T21:14:38.651 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[211526]: failed to encode map e496 with expected crc 2023-09-23T21:14:38.651 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:38 smithi121 ceph-mon[211526]: failed to encode map e496 with expected crc 2023-09-23T21:14:38.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 30056 30040 98.8028 27.625 0.00243075 0.0101017 2023-09-23T21:14:38.673 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:38.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:38 smithi123 ceph-mon[193903]: pgmap v1587: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 2966/79407 objects degraded (3.735%); 544/79407 objects misplaced (0.685%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:14:38.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:38 smithi123 ceph-mon[193903]: osdmap e496: 8 total, 8 up, 6 in 2023-09-23T21:14:38.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:38 smithi123 ceph-mon[193903]: failed to encode map e496 with expected crc 2023-09-23T21:14:38.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:38 smithi123 ceph-mon[193903]: failed to encode map e496 with expected crc 2023-09-23T21:14:38.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:38.777 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:38.802 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:38.842 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:38.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:38.946 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:39.012 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:39.219 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:39.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:39.266 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:14:39.324 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:39.399 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:39.513 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:39.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:39.524 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:39 smithi123 ceph-mon[193903]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.524 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:39 smithi123 ceph-mon[193903]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.524 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:39 smithi123 ceph-mon[193903]: osdmap e497: 8 total, 8 up, 6 in 2023-09-23T21:14:39.524 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:39 smithi123 ceph-mon[193903]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.524 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:39 smithi123 ceph-mon[193903]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.525 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1479542669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:39.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[219709]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[219709]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[219709]: osdmap e497: 8 total, 8 up, 6 in 2023-09-23T21:14:39.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[219709]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[219709]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.582 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1479542669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:39.584 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[211526]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.584 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[211526]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.584 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[211526]: osdmap e497: 8 total, 8 up, 6 in 2023-09-23T21:14:39.584 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[211526]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.584 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[211526]: failed to encode map e497 with expected crc 2023-09-23T21:14:39.584 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1479542669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:39.625 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:39.663 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:14:39.664420+0000 min lat: 0.00124323 max lat: 1.4251 avg lat: 0.0103061 2023-09-23T21:14:39.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:14:39.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 31053 31037 96.9781 62.3125 0.0138692 0.0103061 2023-09-23T21:14:39.678 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:39.721 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:39.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:39.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:39.733 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:14:39.828 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:39.871 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:40.359 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:40.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:40.470 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:40.499 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:40.524 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:40.533 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:40.656 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:40.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:40 smithi121 ceph-mon[211526]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 85 MiB/s wr, 1.35k op/s; 2966/86901 objects degraded (3.413%); 195/86901 objects misplaced (0.224%); 13 MiB/s, 209 objects/s recovering 2023-09-23T21:14:40.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:40 smithi121 ceph-mon[219709]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 85 MiB/s wr, 1.35k op/s; 2966/86901 objects degraded (3.413%); 195/86901 objects misplaced (0.224%); 13 MiB/s, 209 objects/s recovering 2023-09-23T21:14:40.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 32643 32627 97.0918 99.375 0.00714287 0.0102924 2023-09-23T21:14:40.666 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:40 smithi123 ceph-mon[193903]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 85 MiB/s wr, 1.35k op/s; 2966/86901 objects degraded (3.413%); 195/86901 objects misplaced (0.224%); 13 MiB/s, 209 objects/s recovering 2023-09-23T21:14:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:40.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:40.768 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:40.832 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:40.975 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:40.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:40.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:41.086 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:41.137 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:41.295 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:41.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:41.356 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:41 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:14:41.401 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:41.465 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:41.611 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:41.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:41.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:41 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:14:41.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:41 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:14:41.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 34080 34064 96.7606 89.8125 0.0116683 0.0103275 2023-09-23T21:14:41.725 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:41.764 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:41.980 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:41.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:42.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:42.111 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:42.174 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:42.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:42.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:42.278 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:42.303 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:14:42.337 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:42.468 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:42.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:42.574 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:42.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:42 smithi121 ceph-mon[211526]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 1958/93525 objects degraded (2.094%); 288/93525 objects misplaced (0.308%); 18 MiB/s, 292 objects/s recovering 2023-09-23T21:14:42.597 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1958/93525 objects degraded (2.094%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:42.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:42 smithi121 ceph-mon[219709]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 1958/93525 objects degraded (2.094%); 288/93525 objects misplaced (0.308%); 18 MiB/s, 292 objects/s recovering 2023-09-23T21:14:42.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1958/93525 objects degraded (2.094%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:42.599 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:42.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 35094 35078 95.3089 63.375 0.00910386 0.0104857 2023-09-23T21:14:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:42 smithi123 ceph-mon[193903]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 1958/93525 objects degraded (2.094%); 288/93525 objects misplaced (0.308%); 18 MiB/s, 292 objects/s recovering 2023-09-23T21:14:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1958/93525 objects degraded (2.094%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:43.061 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:43.128 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:43.167 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:43.197 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.244 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:43.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:43.349 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:43.379 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.433 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:43.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:43.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: osdmap e498: 8 total, 8 up, 6 in 2023-09-23T21:14:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:43 smithi123 ceph-mon[193903]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 49 MiB/s wr, 784 op/s; 1958/93525 objects degraded (2.094%); 288/93525 objects misplaced (0.308%); 13 MiB/s, 210 objects/s recovering 2023-09-23T21:14:43.540 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:43.574 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:43.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:43.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: osdmap e498: 8 total, 8 up, 6 in 2023-09-23T21:14:43.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.646 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[211526]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 49 MiB/s wr, 784 op/s; 1958/93525 objects degraded (2.094%); 288/93525 objects misplaced (0.308%); 13 MiB/s, 210 objects/s recovering 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: osdmap e498: 8 total, 8 up, 6 in 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: failed to encode map e498 with expected crc 2023-09-23T21:14:43.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:43 smithi121 ceph-mon[219709]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 49 MiB/s wr, 784 op/s; 1958/93525 objects degraded (2.094%); 288/93525 objects misplaced (0.308%); 13 MiB/s, 210 objects/s recovering 2023-09-23T21:14:43.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 35425 35409 92.1998 20.6875 0.00373874 0.0104994 2023-09-23T21:14:43.670 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:43.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:43.776 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:43.838 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:43.961 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:43.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:43.980 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:43.981 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:44.072 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:44.164 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:44.296 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:44.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:44.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:44.406 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:44.492 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:44.572 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:44.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:44.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 36490 36474 91.1741 66.5625 0.00955029 0.0109622 2023-09-23T21:14:44.676 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:44.689 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:14:44.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:44 smithi123 ceph-mon[193903]: failed to encode map e498 with expected crc 2023-09-23T21:14:44.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:44 smithi123 ceph-mon[193903]: osdmap e499: 8 total, 8 up, 6 in 2023-09-23T21:14:44.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:44 smithi123 ceph-mon[193903]: failed to encode map e499 with expected crc 2023-09-23T21:14:44.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:44 smithi123 ceph-mon[193903]: failed to encode map e499 with expected crc 2023-09-23T21:14:44.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:44 smithi123 ceph-mon[193903]: failed to encode map e499 with expected crc 2023-09-23T21:14:44.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3439746892' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:44.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:44 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:44.751 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:44.953 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:44.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:44.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:45.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[219709]: failed to encode map e498 with expected crc 2023-09-23T21:14:45.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[219709]: osdmap e499: 8 total, 8 up, 6 in 2023-09-23T21:14:45.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[219709]: failed to encode map e499 with expected crc 2023-09-23T21:14:45.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[219709]: failed to encode map e499 with expected crc 2023-09-23T21:14:45.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[219709]: failed to encode map e499 with expected crc 2023-09-23T21:14:45.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3439746892' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:45.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:45.038 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:14:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:14:44] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.43.0" 2023-09-23T21:14:45.038 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[211526]: failed to encode map e498 with expected crc 2023-09-23T21:14:45.038 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[211526]: osdmap e499: 8 total, 8 up, 6 in 2023-09-23T21:14:45.038 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[211526]: failed to encode map e499 with expected crc 2023-09-23T21:14:45.038 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[211526]: failed to encode map e499 with expected crc 2023-09-23T21:14:45.039 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[211526]: failed to encode map e499 with expected crc 2023-09-23T21:14:45.039 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3439746892' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:45.039 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:44 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:14:45.064 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:45.101 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:45.253 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 3, 7] out_osds: [0, 1] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:14:45.253 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:14:45.254 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_8 2023-09-23T21:14:45.254 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:14:45.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 37612 37596 90.3643 70.125 0.00647964 0.0110608 2023-09-23T21:14:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:45 smithi123 ceph-mon[193903]: failed to encode map e499 with expected crc 2023-09-23T21:14:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:45 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:45 smithi123 ceph-mon[193903]: failed to encode map e500 with expected crc 2023-09-23T21:14:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:45 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:45 smithi123 ceph-mon[193903]: osdmap e500: 8 total, 8 up, 6 in 2023-09-23T21:14:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:45 smithi123 ceph-mon[193903]: failed to encode map e500 with expected crc 2023-09-23T21:14:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:45 smithi123 ceph-mon[193903]: pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s; 954/104004 objects degraded (0.917%); 749/104004 objects misplaced (0.720%); 15 MiB/s, 242 objects/s recovering 2023-09-23T21:14:46.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[211526]: failed to encode map e499 with expected crc 2023-09-23T21:14:46.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[211526]: failed to encode map e500 with expected crc 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[211526]: osdmap e500: 8 total, 8 up, 6 in 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[211526]: failed to encode map e500 with expected crc 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[211526]: pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s; 954/104004 objects degraded (0.917%); 749/104004 objects misplaced (0.720%); 15 MiB/s, 242 objects/s recovering 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[219709]: failed to encode map e499 with expected crc 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:46.096 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[219709]: failed to encode map e500 with expected crc 2023-09-23T21:14:46.097 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:14:46.097 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[219709]: osdmap e500: 8 total, 8 up, 6 in 2023-09-23T21:14:46.097 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[219709]: failed to encode map e500 with expected crc 2023-09-23T21:14:46.097 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:45 smithi121 ceph-mon[219709]: pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s; 954/104004 objects degraded (0.917%); 749/104004 objects misplaced (0.720%); 15 MiB/s, 242 objects/s recovering 2023-09-23T21:14:46.167 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:46.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:46.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:46.198 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:14:46.273 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:46.339 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:46.453 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:46.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:46.566 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:46.589 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:46.633 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:46.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 38953 38937 90.1214 83.8125 0.00633398 0.0110912 2023-09-23T21:14:46.687 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:46 smithi123 ceph-mon[193903]: failed to encode map e500 with expected crc 2023-09-23T21:14:46.688 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:46 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:46.688 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:46 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:46.688 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 954/104004 objects degraded (0.917%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:46.689 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[211526]: failed to encode map e500 with expected crc 2023-09-23T21:14:46.689 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:46.689 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:46.689 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 954/104004 objects degraded (0.917%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:46.761 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:46.787 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:46.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:46.793 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:46.827 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:46.859 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:46.890 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:46.894 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:46.929 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:46.962 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:46.968 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:46.995 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:46.997 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:14:46.997 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:14:46.997 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:14:46.998 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:14:46.998 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:14:46.998 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:14:46.998 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:14:46.998 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:14:47.027 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[219709]: failed to encode map e500 with expected crc 2023-09-23T21:14:47.027 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:47.027 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:47.027 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 954/104004 objects degraded (0.917%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:47.098 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:47.210 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:47.274 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:47.389 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:47.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:47.502 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:47.552 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:47.614 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:47.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:47.664 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 40253 40237 89.8043 81.25 0.00408553 0.0111107 2023-09-23T21:14:47.721 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:47.748 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:47.859 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:47.865 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2455859193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:47.865 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:47 smithi123 ceph-mon[193903]: pgmap v1597: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 56 MiB/s wr, 890 op/s; 954/109554 objects degraded (0.871%); 749/109554 objects misplaced (0.684%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:14:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:47.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:47.967 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:48.002 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2455859193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:48.002 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:47 smithi121 ceph-mon[211526]: pgmap v1597: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 56 MiB/s wr, 890 op/s; 954/109554 objects degraded (0.871%); 749/109554 objects misplaced (0.684%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:14:48.003 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2455859193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:48.003 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:47 smithi121 ceph-mon[219709]: pgmap v1597: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 56 MiB/s wr, 890 op/s; 954/109554 objects degraded (0.871%); 749/109554 objects misplaced (0.684%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:14:48.006 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:48.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:14:48.579 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:48.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:48.651 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:48.669 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 41631 41615 89.6614 86.125 0.00834651 0.0111477 2023-09-23T21:14:48.684 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:48.743 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:48.847 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:48.867 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:48.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:48.974 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:49.046 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:49.202 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:49.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:49.316 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:49.377 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:49.482 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:49.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:49.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:49.592 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:49.640 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:49.669 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 43038 43022 89.6037 87.9375 0.00683516 0.0111547 2023-09-23T21:14:50.045 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:14:50.050 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:14:50.126 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:50.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:50.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:50.230 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:50.302 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:50.364 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:50 smithi121 ceph-mon[211526]: pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 954/118404 objects degraded (0.806%); 113/118404 objects misplaced (0.095%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:14:50.384 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:50.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:50.489 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:50.532 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:50.597 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:50.600 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:50 smithi123 ceph-mon[193903]: pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 954/118404 objects degraded (0.806%); 113/118404 objects misplaced (0.095%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:14:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:50.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:50.610 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:14:50.636 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:50 smithi121 ceph-mon[219709]: pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 954/118404 objects degraded (0.806%); 113/118404 objects misplaced (0.095%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:14:50.669 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 44159 44143 88.9733 70.0625 0.00246415 0.0111871 2023-09-23T21:14:50.703 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:50.732 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:51.396 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:51.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:51.503 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:51.541 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:51.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[211526]: from='client.37059 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:51.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[211526]: osdmap e501: 8 total, 8 up, 6 in 2023-09-23T21:14:51.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[211526]: failed to encode map e501 with expected crc 2023-09-23T21:14:51.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[211526]: failed to encode map e501 with expected crc 2023-09-23T21:14:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[219709]: from='client.37059 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[219709]: osdmap e501: 8 total, 8 up, 6 in 2023-09-23T21:14:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[219709]: failed to encode map e501 with expected crc 2023-09-23T21:14:51.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:51 smithi121 ceph-mon[219709]: failed to encode map e501 with expected crc 2023-09-23T21:14:51.669 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 44601 44585 87.0565 27.625 0.0203027 0.0114802 2023-09-23T21:14:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:51 smithi123 ceph-mon[193903]: from='client.37059 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:14:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:51 smithi123 ceph-mon[193903]: osdmap e501: 8 total, 8 up, 6 in 2023-09-23T21:14:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:51 smithi123 ceph-mon[193903]: failed to encode map e501 with expected crc 2023-09-23T21:14:51.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:51 smithi123 ceph-mon[193903]: failed to encode map e501 with expected crc 2023-09-23T21:14:51.876 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:51.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:51.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:51.983 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:14:52.020 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:52.209 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1598,"stamp":"2023-09-23T21:14:49.299577+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2585597856,"num_objects":39468,"num_object_clones":0,"num_object_copies":118404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":954,"num_objects_misplaced":113,"num_objects_unfound":0,"num_objects_dirty":39468,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":39744,"num_write_kb":2526861,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4073,"num_bytes_recovered":263552680,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39660,"ondisk_log_size":39660,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11832924,"kb_used_data":7450092,"kb_used_omap":10,"kb_used_meta":4382773,"kb_avail":550637988,"statfs":{"total":575970213888,"available":563853299712,"internally_reserved":0,"allocated":7628894208,"data_stored":7618535085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11130,"internal_metadata":4487959686},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":543490048,"num_objects":8293,"num_object_clones":0,"num_object_copies":24879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1004,"num_objects_misplaced":-175,"num_objects_unfound":0,"num_objects_dirty":8293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8293,"num_write_kb":530752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1179,"num_bytes_recovered":77266944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.622845"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":101,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.686277+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:14:43.686277+0000","last_peered":"2023-09-23T21:14:43.686277+0000","last_clean":"2023-09-23T21:14:43.686277+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:14:43.686277+0000","last_undegraded":"2023-09-23T21:14:43.686277+0000","last_fullsized":"2023-09-23T21:14:43.686277+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":152,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.687576+0000","last_change":"2023-09-23T21:14:26.692334+0000","last_active":"2023-09-23T21:14:44.687576+0000","last_peered":"2023-09-23T21:14:44.687576+0000","last_clean":"2023-09-23T21:14:44.687576+0000","last_became_active":"2023-09-23T21:14:26.692006+0000","last_became_peered":"2023-09-23T21:14:26.692006+0000","last_unstale":"2023-09-23T21:14:44.687576+0000","last_undegraded":"2023-09-23T21:14:44.687576+0000","last_fullsized":"2023-09-23T21:14:44.687576+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:49.195156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":307,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680356+0000","last_change":"2023-09-23T21:14:26.680465+0000","last_active":"2023-09-23T21:14:43.680356+0000","last_peered":"2023-09-23T21:14:43.680356+0000","last_clean":"2023-09-23T21:14:43.680356+0000","last_became_active":"2023-09-23T21:14:26.680145+0000","last_became_peered":"2023-09-23T21:14:26.680145+0000","last_unstale":"2023-09-23T21:14:43.680356+0000","last_undegraded":"2023-09-23T21:14:43.680356+0000","last_fullsized":"2023-09-23T21:14:43.680356+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:27.959630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":137,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689080+0000","last_change":"2023-09-23T21:14:26.689393+0000","last_active":"2023-09-23T21:14:43.689080+0000","last_peered":"2023-09-23T21:14:43.689080+0000","last_clean":"2023-09-23T21:14:43.689080+0000","last_became_active":"2023-09-23T21:14:26.689070+0000","last_became_peered":"2023-09-23T21:14:26.689070+0000","last_unstale":"2023-09-23T21:14:43.689080+0000","last_undegraded":"2023-09-23T21:14:43.689080+0000","last_fullsized":"2023-09-23T21:14:43.689080+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:42.767382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":179,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680458+0000","last_change":"2023-09-23T21:14:26.677137+0000","last_active":"2023-09-23T21:14:43.680458+0000","last_peered":"2023-09-23T21:14:43.680458+0000","last_clean":"2023-09-23T21:14:43.680458+0000","last_became_active":"2023-09-23T21:14:26.676823+0000","last_became_peered":"2023-09-23T21:14:26.676823+0000","last_unstale":"2023-09-23T21:14:43.680458+0000","last_undegraded":"2023-09-23T21:14:43.680458+0000","last_fullsized":"2023-09-23T21:14:43.680458+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:46.494803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":386,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.709013+0000","last_change":"2023-09-23T21:14:26.688060+0000","last_active":"2023-09-23T21:14:43.709013+0000","last_peered":"2023-09-23T21:14:43.709013+0000","last_clean":"2023-09-23T21:14:43.709013+0000","last_became_active":"2023-09-23T21:14:26.687635+0000","last_became_peered":"2023-09-23T21:14:26.687635+0000","last_unstale":"2023-09-23T21:14:43.709013+0000","last_undegraded":"2023-09-23T21:14:43.709013+0000","last_fullsized":"2023-09-23T21:14:43.709013+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:05.201856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":64,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.696974+0000","last_change":"2023-09-23T21:14:16.668716+0000","last_active":"2023-09-23T21:14:43.696974+0000","last_peered":"2023-09-23T21:14:43.696974+0000","last_clean":"2023-09-23T21:14:43.696974+0000","last_became_active":"2023-09-23T21:14:16.668449+0000","last_became_peered":"2023-09-23T21:14:16.668449+0000","last_unstale":"2023-09-23T21:14:43.696974+0000","last_undegraded":"2023-09-23T21:14:43.696974+0000","last_fullsized":"2023-09-23T21:14:43.696974+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:24.128504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":152,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680356+0000","last_change":"2023-09-23T21:14:26.678550+0000","last_active":"2023-09-23T21:14:43.680356+0000","last_peered":"2023-09-23T21:14:43.680356+0000","last_clean":"2023-09-23T21:14:43.680356+0000","last_became_active":"2023-09-23T21:14:26.678230+0000","last_became_peered":"2023-09-23T21:14:26.678230+0000","last_unstale":"2023-09-23T21:14:43.680356+0000","last_undegraded":"2023-09-23T21:14:43.680356+0000","last_fullsized":"2023-09-23T21:14:43.680356+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:46.735135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":143,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.696520+0000","last_change":"2023-09-23T21:14:26.694422+0000","last_active":"2023-09-23T21:14:43.696520+0000","last_peered":"2023-09-23T21:14:43.696520+0000","last_clean":"2023-09-23T21:14:43.696520+0000","last_became_active":"2023-09-23T21:14:26.694100+0000","last_became_peered":"2023-09-23T21:14:26.694100+0000","last_unstale":"2023-09-23T21:14:43.696520+0000","last_undegraded":"2023-09-23T21:14:43.696520+0000","last_fullsized":"2023-09-23T21:14:43.696520+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:00.817476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680696+0000","last_change":"2023-09-23T21:12:51.652067+0000","last_active":"2023-09-23T21:14:43.680696+0000","last_peered":"2023-09-23T21:14:43.680696+0000","last_clean":"2023-09-23T21:14:43.680696+0000","last_became_active":"2023-09-23T21:12:51.651751+0000","last_became_peered":"2023-09-23T21:12:51.651751+0000","last_unstale":"2023-09-23T21:14:43.680696+0000","last_undegraded":"2023-09-23T21:14:43.680696+0000","last_fullsized":"2023-09-23T21:14:43.680696+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:03.235559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":55,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688485+0000","last_change":"2023-09-23T21:12:51.657657+0000","last_active":"2023-09-23T21:14:43.688485+0000","last_peered":"2023-09-23T21:14:43.688485+0000","last_clean":"2023-09-23T21:14:43.688485+0000","last_became_active":"2023-09-23T21:12:51.641567+0000","last_became_peered":"2023-09-23T21:12:51.641567+0000","last_unstale":"2023-09-23T21:14:43.688485+0000","last_undegraded":"2023-09-23T21:14:43.688485+0000","last_fullsized":"2023-09-23T21:14:43.688485+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:00.366548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":274,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.707930+0000","last_change":"2023-09-23T21:14:26.686907+0000","last_active":"2023-09-23T21:14:43.707930+0000","last_peered":"2023-09-23T21:14:43.707930+0000","last_clean":"2023-09-23T21:14:43.707930+0000","last_became_active":"2023-09-23T21:14:26.686514+0000","last_became_peered":"2023-09-23T21:14:26.686514+0000","last_unstale":"2023-09-23T21:14:43.707930+0000","last_undegraded":"2023-09-23T21:14:43.707930+0000","last_fullsized":"2023-09-23T21:14:43.707930+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:16.339519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":87,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688470+0000","last_change":"2023-09-23T21:10:15.594033+0000","last_active":"2023-09-23T21:14:43.688470+0000","last_peered":"2023-09-23T21:14:43.688470+0000","last_clean":"2023-09-23T21:14:43.688470+0000","last_became_active":"2023-09-23T21:10:15.579277+0000","last_became_peered":"2023-09-23T21:10:15.579277+0000","last_unstale":"2023-09-23T21:14:43.688470+0000","last_undegraded":"2023-09-23T21:14:43.688470+0000","last_fullsized":"2023-09-23T21:14:43.688470+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:42.561285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":428,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689332+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:14:43.689332+0000","last_peered":"2023-09-23T21:14:43.689332+0000","last_clean":"2023-09-23T21:14:43.689332+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:14:43.689332+0000","last_undegraded":"2023-09-23T21:14:43.689332+0000","last_fullsized":"2023-09-23T21:14:43.689332+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":83,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688479+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:14:43.688479+0000","last_peered":"2023-09-23T21:14:43.688479+0000","last_clean":"2023-09-23T21:14:43.688479+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:14:43.688479+0000","last_undegraded":"2023-09-23T21:14:43.688479+0000","last_fullsized":"2023-09-23T21:14:43.688479+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":170,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680751+0000","last_change":"2023-09-23T21:14:26.676357+0000","last_active":"2023-09-23T21:14:43.680751+0000","last_peered":"2023-09-23T21:14:43.680751+0000","last_clean":"2023-09-23T21:14:43.680751+0000","last_became_active":"2023-09-23T21:14:26.675906+0000","last_became_peered":"2023-09-23T21:14:26.675906+0000","last_unstale":"2023-09-23T21:14:43.680751+0000","last_undegraded":"2023-09-23T21:14:43.680751+0000","last_fullsized":"2023-09-23T21:14:43.680751+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:51.301880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688272+0000","last_change":"2023-09-23T21:14:26.678085+0000","last_active":"2023-09-23T21:14:43.688272+0000","last_peered":"2023-09-23T21:14:43.688272+0000","last_clean":"2023-09-23T21:14:43.688272+0000","last_became_active":"2023-09-23T21:14:26.677710+0000","last_became_peered":"2023-09-23T21:14:26.677710+0000","last_unstale":"2023-09-23T21:14:43.688272+0000","last_undegraded":"2023-09-23T21:14:43.688272+0000","last_fullsized":"2023-09-23T21:14:43.688272+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:42.625210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688744+0000","last_change":"2023-09-23T21:10:15.585218+0000","last_active":"2023-09-23T21:14:43.688744+0000","last_peered":"2023-09-23T21:14:43.688744+0000","last_clean":"2023-09-23T21:14:43.688744+0000","last_became_active":"2023-09-23T21:10:15.584952+0000","last_became_peered":"2023-09-23T21:10:15.584952+0000","last_unstale":"2023-09-23T21:14:43.688744+0000","last_undegraded":"2023-09-23T21:14:43.688744+0000","last_fullsized":"2023-09-23T21:14:43.688744+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:55.016769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688073+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:14:43.688073+0000","last_peered":"2023-09-23T21:14:43.688073+0000","last_clean":"2023-09-23T21:14:43.688073+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:14:43.688073+0000","last_undegraded":"2023-09-23T21:14:43.688073+0000","last_fullsized":"2023-09-23T21:14:43.688073+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":126,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680972+0000","last_change":"2023-09-23T21:14:16.666291+0000","last_active":"2023-09-23T21:14:43.680972+0000","last_peered":"2023-09-23T21:14:43.680972+0000","last_clean":"2023-09-23T21:14:43.680972+0000","last_became_active":"2023-09-23T21:14:16.665943+0000","last_became_peered":"2023-09-23T21:14:16.665943+0000","last_unstale":"2023-09-23T21:14:43.680972+0000","last_undegraded":"2023-09-23T21:14:43.680972+0000","last_fullsized":"2023-09-23T21:14:43.680972+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:46.233638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":516,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708207+0000","last_change":"2023-09-23T21:14:16.683109+0000","last_active":"2023-09-23T21:14:43.708207+0000","last_peered":"2023-09-23T21:14:43.708207+0000","last_clean":"2023-09-23T21:14:43.708207+0000","last_became_active":"2023-09-23T21:14:16.672619+0000","last_became_peered":"2023-09-23T21:14:16.672619+0000","last_unstale":"2023-09-23T21:14:43.708207+0000","last_undegraded":"2023-09-23T21:14:43.708207+0000","last_fullsized":"2023-09-23T21:14:43.708207+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:15.106005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":170,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.681031+0000","last_change":"2023-09-23T21:14:26.684346+0000","last_active":"2023-09-23T21:14:43.681031+0000","last_peered":"2023-09-23T21:14:43.681031+0000","last_clean":"2023-09-23T21:14:43.681031+0000","last_became_active":"2023-09-23T21:14:26.683593+0000","last_became_peered":"2023-09-23T21:14:26.683593+0000","last_unstale":"2023-09-23T21:14:43.681031+0000","last_undegraded":"2023-09-23T21:14:43.681031+0000","last_fullsized":"2023-09-23T21:14:43.681031+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:42.875671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"500'2339","reported_seq":2365,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:46.059891+0000","last_change":"2023-09-23T21:14:12.654110+0000","last_active":"2023-09-23T21:14:46.059891+0000","last_peered":"2023-09-23T21:14:46.059891+0000","last_clean":"2023-09-23T21:14:46.059891+0000","last_became_active":"2023-09-23T21:14:12.653807+0000","last_became_peered":"2023-09-23T21:14:12.653807+0000","last_unstale":"2023-09-23T21:14:46.059891+0000","last_undegraded":"2023-09-23T21:14:46.059891+0000","last_fullsized":"2023-09-23T21:14:46.059891+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2339,"log_dups_size":0,"ondisk_log_size":2339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:44.519331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153288704,"num_objects":2339,"num_object_clones":0,"num_object_copies":7017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2339,"num_write_kb":149696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":81,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689527+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:14:43.689527+0000","last_peered":"2023-09-23T21:14:43.689527+0000","last_clean":"2023-09-23T21:14:43.689527+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:14:43.689527+0000","last_undegraded":"2023-09-23T21:14:43.689527+0000","last_fullsized":"2023-09-23T21:14:43.689527+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.692524+0000","last_change":"2023-09-23T21:12:51.650942+0000","last_active":"2023-09-23T21:14:43.692524+0000","last_peered":"2023-09-23T21:14:43.692524+0000","last_clean":"2023-09-23T21:14:43.692524+0000","last_became_active":"2023-09-23T21:12:51.650556+0000","last_became_peered":"2023-09-23T21:12:51.650556+0000","last_unstale":"2023-09-23T21:14:43.692524+0000","last_undegraded":"2023-09-23T21:14:43.692524+0000","last_fullsized":"2023-09-23T21:14:43.692524+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:06.677094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"500'2525","reported_seq":2561,"reported_epoch":500,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:48.376183+0000","last_change":"2023-09-23T21:14:27.747096+0000","last_active":"2023-09-23T21:14:48.376183+0000","last_peered":"2023-09-23T21:14:48.376183+0000","last_clean":"2023-09-23T21:14:26.370501+0000","last_became_active":"2023-09-23T21:14:27.723147+0000","last_became_peered":"2023-09-23T21:14:27.723147+0000","last_unstale":"2023-09-23T21:14:48.376183+0000","last_undegraded":"2023-09-23T21:14:27.674349+0000","last_fullsized":"2023-09-23T21:14:27.668778+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2525,"log_dups_size":0,"ondisk_log_size":2525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:46.722866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2525,"num_write_kb":161600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":952},{"shards":"2,7","objects":1573}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":95,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689163+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:14:43.689163+0000","last_peered":"2023-09-23T21:14:43.689163+0000","last_clean":"2023-09-23T21:14:43.689163+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:14:43.689163+0000","last_undegraded":"2023-09-23T21:14:43.689163+0000","last_fullsized":"2023-09-23T21:14:43.689163+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":343,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680436+0000","last_change":"2023-09-23T21:14:26.680462+0000","last_active":"2023-09-23T21:14:43.680436+0000","last_peered":"2023-09-23T21:14:43.680436+0000","last_clean":"2023-09-23T21:14:43.680436+0000","last_became_active":"2023-09-23T21:14:26.679721+0000","last_became_peered":"2023-09-23T21:14:26.679721+0000","last_unstale":"2023-09-23T21:14:43.680436+0000","last_undegraded":"2023-09-23T21:14:43.680436+0000","last_fullsized":"2023-09-23T21:14:43.680436+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.861519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"500'2404","reported_seq":2430,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:45.511471+0000","last_change":"2023-09-23T21:14:16.667881+0000","last_active":"2023-09-23T21:14:45.511471+0000","last_peered":"2023-09-23T21:14:45.511471+0000","last_clean":"2023-09-23T21:14:45.511471+0000","last_became_active":"2023-09-23T21:14:16.667552+0000","last_became_peered":"2023-09-23T21:14:16.667552+0000","last_unstale":"2023-09-23T21:14:45.511471+0000","last_undegraded":"2023-09-23T21:14:45.511471+0000","last_fullsized":"2023-09-23T21:14:45.511471+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2404,"log_dups_size":0,"ondisk_log_size":2404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:21.697900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157548544,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2404,"num_write_kb":153856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":81,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688814+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:14:43.688814+0000","last_peered":"2023-09-23T21:14:43.688814+0000","last_clean":"2023-09-23T21:14:43.688814+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:14:43.688814+0000","last_undegraded":"2023-09-23T21:14:43.688814+0000","last_fullsized":"2023-09-23T21:14:43.688814+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":408,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708163+0000","last_change":"2023-09-23T21:14:26.687082+0000","last_active":"2023-09-23T21:14:43.708163+0000","last_peered":"2023-09-23T21:14:43.708163+0000","last_clean":"2023-09-23T21:14:43.708163+0000","last_became_active":"2023-09-23T21:14:26.686618+0000","last_became_peered":"2023-09-23T21:14:26.686618+0000","last_unstale":"2023-09-23T21:14:43.708163+0000","last_undegraded":"2023-09-23T21:14:43.708163+0000","last_fullsized":"2023-09-23T21:14:43.708163+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:22.797585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"500'2320","reported_seq":2347,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:45.528370+0000","last_change":"2023-09-23T21:14:12.651043+0000","last_active":"2023-09-23T21:14:45.528370+0000","last_peered":"2023-09-23T21:14:45.528370+0000","last_clean":"2023-09-23T21:14:45.528370+0000","last_became_active":"2023-09-23T21:14:12.650774+0000","last_became_peered":"2023-09-23T21:14:12.650774+0000","last_unstale":"2023-09-23T21:14:45.528370+0000","last_undegraded":"2023-09-23T21:14:45.528370+0000","last_fullsized":"2023-09-23T21:14:45.528370+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2320,"log_dups_size":0,"ondisk_log_size":2320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:22.395010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152043520,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2320,"num_write_kb":148480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680824+0000","last_change":"2023-09-23T21:04:41.410710+0000","last_active":"2023-09-23T21:14:43.680824+0000","last_peered":"2023-09-23T21:14:43.680824+0000","last_clean":"2023-09-23T21:14:43.680824+0000","last_became_active":"2023-09-23T21:04:41.407621+0000","last_became_peered":"2023-09-23T21:04:41.407621+0000","last_unstale":"2023-09-23T21:14:43.680824+0000","last_undegraded":"2023-09-23T21:14:43.680824+0000","last_fullsized":"2023-09-23T21:14:43.680824+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:16.031414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":344,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689112+0000","last_change":"2023-09-23T21:14:26.690574+0000","last_active":"2023-09-23T21:14:43.689112+0000","last_peered":"2023-09-23T21:14:43.689112+0000","last_clean":"2023-09-23T21:14:43.689112+0000","last_became_active":"2023-09-23T21:14:26.690290+0000","last_became_peered":"2023-09-23T21:14:26.690290+0000","last_unstale":"2023-09-23T21:14:43.689112+0000","last_undegraded":"2023-09-23T21:14:43.689112+0000","last_fullsized":"2023-09-23T21:14:43.689112+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:04.755073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.5","version":"500'2585","reported_seq":2621,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.380972+0000","last_change":"2023-09-23T21:14:16.666900+0000","last_active":"2023-09-23T21:14:48.380972+0000","last_peered":"2023-09-23T21:14:48.380972+0000","last_clean":"2023-09-23T21:14:48.380972+0000","last_became_active":"2023-09-23T21:14:16.666715+0000","last_became_peered":"2023-09-23T21:14:16.666715+0000","last_unstale":"2023-09-23T21:14:48.380972+0000","last_undegraded":"2023-09-23T21:14:48.380972+0000","last_fullsized":"2023-09-23T21:14:48.380972+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2585,"log_dups_size":0,"ondisk_log_size":2585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:54.530150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410560,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2585,"num_write_kb":165440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":186,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.681576+0000","last_change":"2023-09-23T21:14:26.690682+0000","last_active":"2023-09-23T21:14:43.681576+0000","last_peered":"2023-09-23T21:14:43.681576+0000","last_clean":"2023-09-23T21:14:43.681576+0000","last_became_active":"2023-09-23T21:14:26.674538+0000","last_became_peered":"2023-09-23T21:14:26.674538+0000","last_unstale":"2023-09-23T21:14:43.681576+0000","last_undegraded":"2023-09-23T21:14:43.681576+0000","last_fullsized":"2023-09-23T21:14:43.681576+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:16.032594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":143,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680674+0000","last_change":"2023-09-23T21:04:41.410911+0000","last_active":"2023-09-23T21:14:43.680674+0000","last_peered":"2023-09-23T21:14:43.680674+0000","last_clean":"2023-09-23T21:14:43.680674+0000","last_became_active":"2023-09-23T21:04:41.408087+0000","last_became_peered":"2023-09-23T21:04:41.408087+0000","last_unstale":"2023-09-23T21:14:43.680674+0000","last_undegraded":"2023-09-23T21:14:43.680674+0000","last_fullsized":"2023-09-23T21:14:43.680674+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:06.540769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"500'2467","reported_seq":4595,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:46.065004+0000","last_change":"2023-09-23T21:14:38.353994+0000","last_active":"2023-09-23T21:14:46.065004+0000","last_peered":"2023-09-23T21:14:46.065004+0000","last_clean":"2023-09-23T21:14:46.065004+0000","last_became_active":"2023-09-23T21:14:38.336701+0000","last_became_peered":"2023-09-23T21:14:38.336701+0000","last_unstale":"2023-09-23T21:14:46.065004+0000","last_undegraded":"2023-09-23T21:14:46.065004+0000","last_fullsized":"2023-09-23T21:14:46.065004+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2467,"log_dups_size":0,"ondisk_log_size":2467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:41.236492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161677312,"num_objects":2467,"num_object_clones":0,"num_object_copies":7401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2467,"num_write_kb":157888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1041,"num_bytes_recovered":68222976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":436,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708245+0000","last_change":"2023-09-23T21:14:26.694605+0000","last_active":"2023-09-23T21:14:43.708245+0000","last_peered":"2023-09-23T21:14:43.708245+0000","last_clean":"2023-09-23T21:14:43.708245+0000","last_became_active":"2023-09-23T21:14:26.693553+0000","last_became_peered":"2023-09-23T21:14:26.693553+0000","last_unstale":"2023-09-23T21:14:43.708245+0000","last_undegraded":"2023-09-23T21:14:43.708245+0000","last_fullsized":"2023-09-23T21:14:43.708245+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:44.258040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":522,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684188+0000","last_change":"2023-09-23T21:14:26.681722+0000","last_active":"2023-09-23T21:14:43.684188+0000","last_peered":"2023-09-23T21:14:43.684188+0000","last_clean":"2023-09-23T21:14:43.684188+0000","last_became_active":"2023-09-23T21:14:26.681480+0000","last_became_peered":"2023-09-23T21:14:26.681480+0000","last_unstale":"2023-09-23T21:14:43.684188+0000","last_undegraded":"2023-09-23T21:14:43.684188+0000","last_fullsized":"2023-09-23T21:14:43.684188+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:30.216045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"500'2552","reported_seq":2575,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.389962+0000","last_change":"2023-09-23T21:14:12.649536+0000","last_active":"2023-09-23T21:14:48.389962+0000","last_peered":"2023-09-23T21:14:48.389962+0000","last_clean":"2023-09-23T21:14:48.389962+0000","last_became_active":"2023-09-23T21:14:12.649267+0000","last_became_peered":"2023-09-23T21:14:12.649267+0000","last_unstale":"2023-09-23T21:14:48.389962+0000","last_undegraded":"2023-09-23T21:14:48.389962+0000","last_fullsized":"2023-09-23T21:14:48.389962+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2552,"log_dups_size":0,"ondisk_log_size":2552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:47.149744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167247872,"num_objects":2552,"num_object_clones":0,"num_object_copies":7656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2552,"num_write_kb":163328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":137,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.690033+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:14:43.690033+0000","last_peered":"2023-09-23T21:14:43.690033+0000","last_clean":"2023-09-23T21:14:43.690033+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:14:43.690033+0000","last_undegraded":"2023-09-23T21:14:43.690033+0000","last_fullsized":"2023-09-23T21:14:43.690033+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":549,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708590+0000","last_change":"2023-09-23T21:14:26.696260+0000","last_active":"2023-09-23T21:14:43.708590+0000","last_peered":"2023-09-23T21:14:43.708590+0000","last_clean":"2023-09-23T21:14:43.708590+0000","last_became_active":"2023-09-23T21:14:26.695433+0000","last_became_peered":"2023-09-23T21:14:26.695433+0000","last_unstale":"2023-09-23T21:14:43.708590+0000","last_undegraded":"2023-09-23T21:14:43.708590+0000","last_fullsized":"2023-09-23T21:14:43.708590+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:18.226824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"500'2502","reported_seq":2528,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.352894+0000","last_change":"2023-09-23T21:14:12.651167+0000","last_active":"2023-09-23T21:14:48.352894+0000","last_peered":"2023-09-23T21:14:48.352894+0000","last_clean":"2023-09-23T21:14:48.352894+0000","last_became_active":"2023-09-23T21:14:12.650746+0000","last_became_peered":"2023-09-23T21:14:12.650746+0000","last_unstale":"2023-09-23T21:14:48.352894+0000","last_undegraded":"2023-09-23T21:14:48.352894+0000","last_fullsized":"2023-09-23T21:14:48.352894+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2502,"log_dups_size":0,"ondisk_log_size":2502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:40.162522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163971072,"num_objects":2502,"num_object_clones":0,"num_object_copies":7506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2502,"num_write_kb":160128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":170,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680548+0000","last_change":"2023-09-23T21:14:26.676307+0000","last_active":"2023-09-23T21:14:43.680548+0000","last_peered":"2023-09-23T21:14:43.680548+0000","last_clean":"2023-09-23T21:14:43.680548+0000","last_became_active":"2023-09-23T21:14:26.675974+0000","last_became_peered":"2023-09-23T21:14:26.675974+0000","last_unstale":"2023-09-23T21:14:43.680548+0000","last_undegraded":"2023-09-23T21:14:43.680548+0000","last_fullsized":"2023-09-23T21:14:43.680548+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:21.521773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":258,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680959+0000","last_change":"2023-09-23T21:14:26.684532+0000","last_active":"2023-09-23T21:14:43.680959+0000","last_peered":"2023-09-23T21:14:43.680959+0000","last_clean":"2023-09-23T21:14:43.680959+0000","last_became_active":"2023-09-23T21:14:26.684092+0000","last_became_peered":"2023-09-23T21:14:26.684092+0000","last_unstale":"2023-09-23T21:14:43.680959+0000","last_undegraded":"2023-09-23T21:14:43.680959+0000","last_fullsized":"2023-09-23T21:14:43.680959+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:38.075689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"500'2405","reported_seq":2432,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:45.511465+0000","last_change":"2023-09-23T21:14:12.651083+0000","last_active":"2023-09-23T21:14:45.511465+0000","last_peered":"2023-09-23T21:14:45.511465+0000","last_clean":"2023-09-23T21:14:45.511465+0000","last_became_active":"2023-09-23T21:14:12.650816+0000","last_became_peered":"2023-09-23T21:14:12.650816+0000","last_unstale":"2023-09-23T21:14:45.511465+0000","last_undegraded":"2023-09-23T21:14:45.511465+0000","last_fullsized":"2023-09-23T21:14:45.511465+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2405,"log_dups_size":0,"ondisk_log_size":2405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:11.604259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157614080,"num_objects":2405,"num_object_clones":0,"num_object_copies":7215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2405,"num_write_kb":153920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":414,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688393+0000","last_change":"2023-09-23T21:14:26.676384+0000","last_active":"2023-09-23T21:14:43.688393+0000","last_peered":"2023-09-23T21:14:43.688393+0000","last_clean":"2023-09-23T21:14:43.688393+0000","last_became_active":"2023-09-23T21:14:26.676089+0000","last_became_peered":"2023-09-23T21:14:26.676089+0000","last_unstale":"2023-09-23T21:14:43.688393+0000","last_undegraded":"2023-09-23T21:14:43.688393+0000","last_fullsized":"2023-09-23T21:14:43.688393+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:07.512037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":165,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.696389+0000","last_change":"2023-09-23T21:04:49.769420+0000","last_active":"2023-09-23T21:14:43.696389+0000","last_peered":"2023-09-23T21:14:43.696389+0000","last_clean":"2023-09-23T21:14:43.696389+0000","last_became_active":"2023-09-23T21:04:49.769041+0000","last_became_peered":"2023-09-23T21:04:49.769041+0000","last_unstale":"2023-09-23T21:14:43.696389+0000","last_undegraded":"2023-09-23T21:14:43.696389+0000","last_fullsized":"2023-09-23T21:14:43.696389+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:23.200560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.692592+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:14:43.692592+0000","last_peered":"2023-09-23T21:14:43.692592+0000","last_clean":"2023-09-23T21:14:43.692592+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:14:43.692592+0000","last_undegraded":"2023-09-23T21:14:43.692592+0000","last_fullsized":"2023-09-23T21:14:43.692592+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"500'2642","reported_seq":4705,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.375820+0000","last_change":"2023-09-23T21:14:37.425346+0000","last_active":"2023-09-23T21:14:48.375820+0000","last_peered":"2023-09-23T21:14:48.375820+0000","last_clean":"2023-09-23T21:14:48.375820+0000","last_became_active":"2023-09-23T21:14:37.408591+0000","last_became_peered":"2023-09-23T21:14:37.408591+0000","last_unstale":"2023-09-23T21:14:48.375820+0000","last_undegraded":"2023-09-23T21:14:48.375820+0000","last_fullsized":"2023-09-23T21:14:48.375820+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2642,"log_dups_size":0,"ondisk_log_size":2642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:59.854875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173146112,"num_objects":2642,"num_object_clones":0,"num_object_copies":7926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2642,"num_write_kb":169088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1008,"num_bytes_recovered":66060288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":341,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708085+0000","last_change":"2023-09-23T21:14:26.687190+0000","last_active":"2023-09-23T21:14:43.708085+0000","last_peered":"2023-09-23T21:14:43.708085+0000","last_clean":"2023-09-23T21:14:43.708085+0000","last_became_active":"2023-09-23T21:14:26.686904+0000","last_became_peered":"2023-09-23T21:14:26.686904+0000","last_unstale":"2023-09-23T21:14:43.708085+0000","last_undegraded":"2023-09-23T21:14:43.708085+0000","last_fullsized":"2023-09-23T21:14:43.708085+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:44.270746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":470,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708015+0000","last_change":"2023-09-23T21:14:26.695819+0000","last_active":"2023-09-23T21:14:43.708015+0000","last_peered":"2023-09-23T21:14:43.708015+0000","last_clean":"2023-09-23T21:14:43.708015+0000","last_became_active":"2023-09-23T21:14:26.695505+0000","last_became_peered":"2023-09-23T21:14:26.695505+0000","last_unstale":"2023-09-23T21:14:43.708015+0000","last_undegraded":"2023-09-23T21:14:43.708015+0000","last_fullsized":"2023-09-23T21:14:43.708015+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:05.224230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":186,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684404+0000","last_change":"2023-09-23T21:14:26.687497+0000","last_active":"2023-09-23T21:14:43.684404+0000","last_peered":"2023-09-23T21:14:43.684404+0000","last_clean":"2023-09-23T21:14:43.684404+0000","last_became_active":"2023-09-23T21:14:26.687028+0000","last_became_peered":"2023-09-23T21:14:26.687028+0000","last_unstale":"2023-09-23T21:14:43.684404+0000","last_undegraded":"2023-09-23T21:14:43.684404+0000","last_fullsized":"2023-09-23T21:14:43.684404+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:10.978138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"500'2518","reported_seq":2544,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.395941+0000","last_change":"2023-09-23T21:14:12.651062+0000","last_active":"2023-09-23T21:14:48.395941+0000","last_peered":"2023-09-23T21:14:48.395941+0000","last_clean":"2023-09-23T21:14:48.395941+0000","last_became_active":"2023-09-23T21:14:12.650805+0000","last_became_peered":"2023-09-23T21:14:12.650805+0000","last_unstale":"2023-09-23T21:14:48.395941+0000","last_undegraded":"2023-09-23T21:14:48.395941+0000","last_fullsized":"2023-09-23T21:14:48.395941+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2518,"log_dups_size":0,"ondisk_log_size":2518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:43.223080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165019648,"num_objects":2518,"num_object_clones":0,"num_object_copies":7554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2518,"num_write_kb":161152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":498,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684769+0000","last_change":"2023-09-23T21:14:26.674068+0000","last_active":"2023-09-23T21:14:43.684769+0000","last_peered":"2023-09-23T21:14:43.684769+0000","last_clean":"2023-09-23T21:14:43.684769+0000","last_became_active":"2023-09-23T21:14:26.673679+0000","last_became_peered":"2023-09-23T21:14:26.673679+0000","last_unstale":"2023-09-23T21:14:43.684769+0000","last_undegraded":"2023-09-23T21:14:43.684769+0000","last_fullsized":"2023-09-23T21:14:43.684769+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:06.016762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":227,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684161+0000","last_change":"2023-09-23T21:14:26.683103+0000","last_active":"2023-09-23T21:14:43.684161+0000","last_peered":"2023-09-23T21:14:43.684161+0000","last_clean":"2023-09-23T21:14:43.684161+0000","last_became_active":"2023-09-23T21:14:26.682853+0000","last_became_peered":"2023-09-23T21:14:26.682853+0000","last_unstale":"2023-09-23T21:14:43.684161+0000","last_undegraded":"2023-09-23T21:14:43.684161+0000","last_fullsized":"2023-09-23T21:14:43.684161+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:46.052476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":143,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689276+0000","last_change":"2023-09-23T21:14:26.686199+0000","last_active":"2023-09-23T21:14:43.689276+0000","last_peered":"2023-09-23T21:14:43.689276+0000","last_clean":"2023-09-23T21:14:43.689276+0000","last_became_active":"2023-09-23T21:14:26.685857+0000","last_became_peered":"2023-09-23T21:14:26.685857+0000","last_unstale":"2023-09-23T21:14:43.689276+0000","last_undegraded":"2023-09-23T21:14:43.689276+0000","last_fullsized":"2023-09-23T21:14:43.689276+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:40.218146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"500'2607","reported_seq":4423,"reported_epoch":500,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:14:48.403791+0000","last_change":"2023-09-23T21:14:42.074482+0000","last_active":"2023-09-23T21:14:48.403791+0000","last_peered":"2023-09-23T21:14:48.403791+0000","last_clean":"2023-09-23T21:14:26.332298+0000","last_became_active":"2023-09-23T21:14:27.718532+0000","last_became_peered":"2023-09-23T21:14:27.718532+0000","last_unstale":"2023-09-23T21:14:48.403791+0000","last_undegraded":"2023-09-23T21:14:48.403791+0000","last_fullsized":"2023-09-23T21:14:27.666376+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2607,"log_dups_size":0,"ondisk_log_size":2607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:47.056251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170852352,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":113,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2607,"num_write_kb":166848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":890,"num_bytes_recovered":58327040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":113},{"shards":"2,5","objects":2494}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":327,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684571+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:14:43.684571+0000","last_peered":"2023-09-23T21:14:43.684571+0000","last_clean":"2023-09-23T21:14:43.684571+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:14:43.684571+0000","last_undegraded":"2023-09-23T21:14:43.684571+0000","last_fullsized":"2023-09-23T21:14:43.684571+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":808,"reported_epoch":499,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:14:43.688120+0000","last_change":"2023-09-23T21:14:27.746982+0000","last_active":"2023-09-23T21:14:43.688120+0000","last_peered":"2023-09-23T21:14:43.688120+0000","last_clean":"2023-09-23T21:14:19.705775+0000","last_became_active":"2023-09-23T21:14:27.746640+0000","last_became_peered":"2023-09-23T21:14:27.746640+0000","last_unstale":"2023-09-23T21:14:43.688120+0000","last_undegraded":"2023-09-23T21:14:27.675740+0000","last_fullsized":"2023-09-23T21:14:27.675112+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:47.772598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6],"avail_no_missing":["7","1","6"],"object_location_counts":[{"shards":"1,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.692871+0000","last_change":"2023-09-23T21:13:54.414963+0000","last_active":"2023-09-23T21:14:43.692871+0000","last_peered":"2023-09-23T21:14:43.692871+0000","last_clean":"2023-09-23T21:14:43.692871+0000","last_became_active":"2023-09-23T21:13:54.414651+0000","last_became_peered":"2023-09-23T21:13:54.414651+0000","last_unstale":"2023-09-23T21:14:43.692871+0000","last_undegraded":"2023-09-23T21:14:43.692871+0000","last_fullsized":"2023-09-23T21:14:43.692871+0000","mapping_epoch":476,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:36.717944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":398,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708534+0000","last_change":"2023-09-23T21:10:15.582161+0000","last_active":"2023-09-23T21:14:43.708534+0000","last_peered":"2023-09-23T21:14:43.708534+0000","last_clean":"2023-09-23T21:14:43.708534+0000","last_became_active":"2023-09-23T21:10:15.581855+0000","last_became_peered":"2023-09-23T21:10:15.581855+0000","last_unstale":"2023-09-23T21:14:43.708534+0000","last_undegraded":"2023-09-23T21:14:43.708534+0000","last_fullsized":"2023-09-23T21:14:43.708534+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:19.873916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"500'2403","reported_seq":2429,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:46.044795+0000","last_change":"2023-09-23T21:14:12.651677+0000","last_active":"2023-09-23T21:14:46.044795+0000","last_peered":"2023-09-23T21:14:46.044795+0000","last_clean":"2023-09-23T21:14:46.044795+0000","last_became_active":"2023-09-23T21:14:12.651343+0000","last_became_peered":"2023-09-23T21:14:12.651343+0000","last_unstale":"2023-09-23T21:14:46.044795+0000","last_undegraded":"2023-09-23T21:14:46.044795+0000","last_fullsized":"2023-09-23T21:14:46.044795+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2403,"log_dups_size":0,"ondisk_log_size":2403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:50.047066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157483008,"num_objects":2403,"num_object_clones":0,"num_object_copies":7209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2403,"num_write_kb":153792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":97,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688295+0000","last_change":"2023-09-23T21:14:26.683262+0000","last_active":"2023-09-23T21:14:43.688295+0000","last_peered":"2023-09-23T21:14:43.688295+0000","last_clean":"2023-09-23T21:14:43.688295+0000","last_became_active":"2023-09-23T21:14:26.682908+0000","last_became_peered":"2023-09-23T21:14:26.682908+0000","last_unstale":"2023-09-23T21:14:43.688295+0000","last_undegraded":"2023-09-23T21:14:43.688295+0000","last_fullsized":"2023-09-23T21:14:43.688295+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:52.972272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680611+0000","last_change":"2023-09-23T21:14:16.665607+0000","last_active":"2023-09-23T21:14:43.680611+0000","last_peered":"2023-09-23T21:14:43.680611+0000","last_clean":"2023-09-23T21:14:43.680611+0000","last_became_active":"2023-09-23T21:14:16.665181+0000","last_became_peered":"2023-09-23T21:14:16.665181+0000","last_unstale":"2023-09-23T21:14:43.680611+0000","last_undegraded":"2023-09-23T21:14:43.680611+0000","last_fullsized":"2023-09-23T21:14:43.680611+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:15.811179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":408,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684508+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:14:43.684508+0000","last_peered":"2023-09-23T21:14:43.684508+0000","last_clean":"2023-09-23T21:14:43.684508+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:14:43.684508+0000","last_undegraded":"2023-09-23T21:14:43.684508+0000","last_fullsized":"2023-09-23T21:14:43.684508+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"500'2320","reported_seq":4384,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.993186+0000","last_change":"2023-09-23T21:14:43.693596+0000","last_active":"2023-09-23T21:14:44.993186+0000","last_peered":"2023-09-23T21:14:44.993186+0000","last_clean":"2023-09-23T21:14:44.993186+0000","last_became_active":"2023-09-23T21:14:43.690785+0000","last_became_peered":"2023-09-23T21:14:43.690785+0000","last_unstale":"2023-09-23T21:14:44.993186+0000","last_undegraded":"2023-09-23T21:14:44.993186+0000","last_fullsized":"2023-09-23T21:14:44.993186+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2320,"log_dups_size":0,"ondisk_log_size":2320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:00.973146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152043520,"num_objects":2320,"num_object_clones":0,"num_object_copies":6960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2320,"num_write_kb":148480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1010,"num_bytes_recovered":66191360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":97,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688626+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:14:43.688626+0000","last_peered":"2023-09-23T21:14:43.688626+0000","last_clean":"2023-09-23T21:14:43.688626+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:14:43.688626+0000","last_undegraded":"2023-09-23T21:14:43.688626+0000","last_fullsized":"2023-09-23T21:14:43.688626+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":294,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684480+0000","last_change":"2023-09-23T21:14:26.690090+0000","last_active":"2023-09-23T21:14:43.684480+0000","last_peered":"2023-09-23T21:14:43.684480+0000","last_clean":"2023-09-23T21:14:43.684480+0000","last_became_active":"2023-09-23T21:14:26.689793+0000","last_became_peered":"2023-09-23T21:14:26.689793+0000","last_unstale":"2023-09-23T21:14:43.684480+0000","last_undegraded":"2023-09-23T21:14:43.684480+0000","last_fullsized":"2023-09-23T21:14:43.684480+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:21.747940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":205,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.692725+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:14:43.692725+0000","last_peered":"2023-09-23T21:14:43.692725+0000","last_clean":"2023-09-23T21:14:43.692725+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:14:43.692725+0000","last_undegraded":"2023-09-23T21:14:43.692725+0000","last_fullsized":"2023-09-23T21:14:43.692725+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"500'2547","reported_seq":2583,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:48.389983+0000","last_change":"2023-09-23T21:14:16.668838+0000","last_active":"2023-09-23T21:14:48.389983+0000","last_peered":"2023-09-23T21:14:48.389983+0000","last_clean":"2023-09-23T21:14:48.389983+0000","last_became_active":"2023-09-23T21:14:16.668530+0000","last_became_peered":"2023-09-23T21:14:16.668530+0000","last_unstale":"2023-09-23T21:14:48.389983+0000","last_undegraded":"2023-09-23T21:14:48.389983+0000","last_fullsized":"2023-09-23T21:14:48.389983+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2547,"log_dups_size":0,"ondisk_log_size":2547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:17.360743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166920192,"num_objects":2547,"num_object_clones":0,"num_object_copies":7641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2547,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":141,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689614+0000","last_change":"2023-09-23T21:14:16.684996+0000","last_active":"2023-09-23T21:14:43.689614+0000","last_peered":"2023-09-23T21:14:43.689614+0000","last_clean":"2023-09-23T21:14:43.689614+0000","last_became_active":"2023-09-23T21:14:16.670959+0000","last_became_peered":"2023-09-23T21:14:16.670959+0000","last_unstale":"2023-09-23T21:14:43.689614+0000","last_undegraded":"2023-09-23T21:14:43.689614+0000","last_fullsized":"2023-09-23T21:14:43.689614+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:06.274493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":335,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708662+0000","last_change":"2023-09-23T21:14:26.695768+0000","last_active":"2023-09-23T21:14:43.708662+0000","last_peered":"2023-09-23T21:14:43.708662+0000","last_clean":"2023-09-23T21:14:43.708662+0000","last_became_active":"2023-09-23T21:14:26.695340+0000","last_became_peered":"2023-09-23T21:14:26.695340+0000","last_unstale":"2023-09-23T21:14:43.708662+0000","last_undegraded":"2023-09-23T21:14:43.708662+0000","last_fullsized":"2023-09-23T21:14:43.708662+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:57.536572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.687219+0000","last_change":"2023-09-23T21:14:26.685854+0000","last_active":"2023-09-23T21:14:44.687219+0000","last_peered":"2023-09-23T21:14:44.687219+0000","last_clean":"2023-09-23T21:14:44.687219+0000","last_became_active":"2023-09-23T21:14:26.685542+0000","last_became_peered":"2023-09-23T21:14:26.685542+0000","last_unstale":"2023-09-23T21:14:44.687219+0000","last_undegraded":"2023-09-23T21:14:44.687219+0000","last_fullsized":"2023-09-23T21:14:44.687219+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:42.426172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"500'2308","reported_seq":2335,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.983132+0000","last_change":"2023-09-23T21:14:12.650976+0000","last_active":"2023-09-23T21:14:44.983132+0000","last_peered":"2023-09-23T21:14:44.983132+0000","last_clean":"2023-09-23T21:14:44.983132+0000","last_became_active":"2023-09-23T21:14:12.650707+0000","last_became_peered":"2023-09-23T21:14:12.650707+0000","last_unstale":"2023-09-23T21:14:44.983132+0000","last_undegraded":"2023-09-23T21:14:44.983132+0000","last_fullsized":"2023-09-23T21:14:44.983132+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":2308,"log_dups_size":0,"ondisk_log_size":2308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:51.274874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151257088,"num_objects":2308,"num_object_clones":0,"num_object_copies":6924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2308,"num_write_kb":147712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":406,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.709032+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:14:43.709032+0000","last_peered":"2023-09-23T21:14:43.709032+0000","last_clean":"2023-09-23T21:14:43.709032+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:14:43.709032+0000","last_undegraded":"2023-09-23T21:14:43.709032+0000","last_fullsized":"2023-09-23T21:14:43.709032+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":37,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689703+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:14:43.689703+0000","last_peered":"2023-09-23T21:14:43.689703+0000","last_clean":"2023-09-23T21:14:43.689703+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:14:43.689703+0000","last_undegraded":"2023-09-23T21:14:43.689703+0000","last_fullsized":"2023-09-23T21:14:43.689703+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":185,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.692549+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:14:43.692549+0000","last_peered":"2023-09-23T21:14:43.692549+0000","last_clean":"2023-09-23T21:14:43.692549+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:14:43.692549+0000","last_undegraded":"2023-09-23T21:14:43.692549+0000","last_fullsized":"2023-09-23T21:14:43.692549+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":117,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.688575+0000","last_change":"2023-09-23T21:14:26.690902+0000","last_active":"2023-09-23T21:14:43.688575+0000","last_peered":"2023-09-23T21:14:43.688575+0000","last_clean":"2023-09-23T21:14:43.688575+0000","last_became_active":"2023-09-23T21:14:26.690536+0000","last_became_peered":"2023-09-23T21:14:26.690536+0000","last_unstale":"2023-09-23T21:14:43.688575+0000","last_undegraded":"2023-09-23T21:14:43.688575+0000","last_fullsized":"2023-09-23T21:14:43.688575+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:45.638503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":553,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684550+0000","last_change":"2023-09-23T21:14:26.671535+0000","last_active":"2023-09-23T21:14:43.684550+0000","last_peered":"2023-09-23T21:14:43.684550+0000","last_clean":"2023-09-23T21:14:43.684550+0000","last_became_active":"2023-09-23T21:14:26.671174+0000","last_became_peered":"2023-09-23T21:14:26.671174+0000","last_unstale":"2023-09-23T21:14:43.684550+0000","last_undegraded":"2023-09-23T21:14:43.684550+0000","last_fullsized":"2023-09-23T21:14:43.684550+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:17.199253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":176,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689618+0000","last_change":"2023-09-23T21:14:26.697769+0000","last_active":"2023-09-23T21:14:43.689618+0000","last_peered":"2023-09-23T21:14:43.689618+0000","last_clean":"2023-09-23T21:14:43.689618+0000","last_became_active":"2023-09-23T21:14:26.689518+0000","last_became_peered":"2023-09-23T21:14:26.689518+0000","last_unstale":"2023-09-23T21:14:43.689618+0000","last_undegraded":"2023-09-23T21:14:43.689618+0000","last_fullsized":"2023-09-23T21:14:43.689618+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:23.331952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":178,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.687957+0000","last_change":"2023-09-23T21:14:26.673173+0000","last_active":"2023-09-23T21:14:44.687957+0000","last_peered":"2023-09-23T21:14:44.687957+0000","last_clean":"2023-09-23T21:14:44.687957+0000","last_became_active":"2023-09-23T21:14:26.672854+0000","last_became_peered":"2023-09-23T21:14:26.672854+0000","last_unstale":"2023-09-23T21:14:44.687957+0000","last_undegraded":"2023-09-23T21:14:44.687957+0000","last_fullsized":"2023-09-23T21:14:44.687957+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:47.837126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":381,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.708622+0000","last_change":"2023-09-23T20:55:55.463136+0000","last_active":"2023-09-23T21:14:43.708622+0000","last_peered":"2023-09-23T21:14:43.708622+0000","last_clean":"2023-09-23T21:14:43.708622+0000","last_became_active":"2023-09-23T20:55:55.462804+0000","last_became_peered":"2023-09-23T20:55:55.462804+0000","last_unstale":"2023-09-23T21:14:43.708622+0000","last_undegraded":"2023-09-23T21:14:43.708622+0000","last_fullsized":"2023-09-23T21:14:43.708622+0000","mapping_epoch":245,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:01.572372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":170,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.687639+0000","last_change":"2023-09-23T21:14:16.662395+0000","last_active":"2023-09-23T21:14:44.687639+0000","last_peered":"2023-09-23T21:14:44.687639+0000","last_clean":"2023-09-23T21:14:44.687639+0000","last_became_active":"2023-09-23T21:14:16.661975+0000","last_became_peered":"2023-09-23T21:14:16.661975+0000","last_unstale":"2023-09-23T21:14:44.687639+0000","last_undegraded":"2023-09-23T21:14:44.687639+0000","last_fullsized":"2023-09-23T21:14:44.687639+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:35.824206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":144,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-09-23T21:14:44.687900+0000","last_change":"2023-09-23T21:04:41.419317+0000","last_active":"2023-09-23T21:14:44.687900+0000","last_peered":"2023-09-23T21:14:44.687900+0000","last_clean":"2023-09-23T21:14:44.687900+0000","last_became_active":"2023-09-23T21:04:41.411856+0000","last_became_peered":"2023-09-23T21:04:41.411856+0000","last_unstale":"2023-09-23T21:14:44.687900+0000","last_undegraded":"2023-09-23T21:14:44.687900+0000","last_fullsized":"2023-09-23T21:14:44.687900+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:59.511706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":161,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.680511+0000","last_change":"2023-09-23T21:14:16.665420+0000","last_active":"2023-09-23T21:14:43.680511+0000","last_peered":"2023-09-23T21:14:43.680511+0000","last_clean":"2023-09-23T21:14:43.680511+0000","last_became_active":"2023-09-23T21:14:16.664834+0000","last_became_peered":"2023-09-23T21:14:16.664834+0000","last_unstale":"2023-09-23T21:14:43.680511+0000","last_undegraded":"2023-09-23T21:14:43.680511+0000","last_fullsized":"2023-09-23T21:14:43.680511+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:48.454144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":425,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.684716+0000","last_change":"2023-09-23T21:14:16.676748+0000","last_active":"2023-09-23T21:14:43.684716+0000","last_peered":"2023-09-23T21:14:43.684716+0000","last_clean":"2023-09-23T21:14:43.684716+0000","last_became_active":"2023-09-23T21:14:16.670039+0000","last_became_peered":"2023-09-23T21:14:16.670039+0000","last_unstale":"2023-09-23T21:14:43.684716+0000","last_undegraded":"2023-09-23T21:14:43.684716+0000","last_fullsized":"2023-09-23T21:14:43.684716+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:20.025680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":499,"state":"active+clean","last_fresh":"2023-09-23T21:14:43.689118+0000","last_change":"2023-09-23T21:14:26.675246+0000","last_active":"2023-09-23T21:14:43.689118+0000","last_peered":"2023-09-23T21:14:43.689118+0000","last_clean":"2023-09-23T21:14:43.689118+0000","last_became_active":"2023-09-23T21:14:26.674914+0000","last_became_peered":"2023-09-23T21:14:26.674914+0000","last_unstale":"2023-09-23T21:14:43.689118+0000","last_undegraded":"2023-09-23T21:14:43.689118+0000","last_fullsized":"2023-09-23T21:14:43.689118+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:31.814154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2585001984,"num_objects":39444,"num_object_clones":0,"num_object_copies":118332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":952,"num_objects_misplaced":113,"num_objects_unfound":0,"num_objects_dirty":39444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39444,"num_write_kb":2524416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3949,"num_bytes_recovered":258801664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7982088192,"data_stored":7982088192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39444,"ondisk_log_size":39444,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":28072,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201833,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1831988,"kb_used_data":1135788,"kb_used_omap":0,"kb_used_meta":696192,"kb_avail":91913164,"statfs":{"total":95995035648,"available":94119079936,"internally_reserved":0,"allocated":1163046912,"data_stored":1161699608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":712900608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":8.2390000000000008,"5min":4.1269999999999998,"15min":2.0529999999999999},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.20699999999999999},"max":{"1min":141.33199999999999,"5min":141.33199999999999,"15min":141.33199999999999},"last":2.6269999999999998},{"interface":"front","average":{"1min":8.3190000000000008,"5min":4.0259999999999998,"15min":2.0489999999999999},"min":{"1min":0.25900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":141.31700000000001,"5min":141.31700000000001,"15min":141.31700000000001},"last":3.234}]},{"osd":1,"last update":"Sat Sep 23 21:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":3.4609999999999999,"15min":3.1789999999999998},"min":{"1min":0.26100000000000001,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":10.407999999999999,"5min":76.007999999999996,"15min":114.16500000000001},"last":3.2069999999999999},{"interface":"front","average":{"1min":1.327,"5min":4.79,"15min":3.7050000000000001},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8650000000000002,"5min":117.637,"15min":117.637},"last":2.6899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":8.4359999999999999,"5min":4.6699999999999999,"15min":4.157},"min":{"1min":0.33500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":135.529,"5min":135.529,"15min":183.44200000000001},"last":3.137},{"interface":"front","average":{"1min":8.6349999999999998,"5min":4.7489999999999997,"15min":4.0899999999999999},"min":{"1min":0.32500000000000001,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":135.56700000000001,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.625}]},{"osd":3,"last update":"Sat Sep 23 21:14:00 2023","interfaces":[{"interface":"back","average":{"1min":8.3520000000000003,"5min":6.6180000000000003,"15min":3.6560000000000001},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":135.66200000000001,"5min":135.66200000000001,"15min":135.66200000000001},"last":2.7189999999999999},{"interface":"front","average":{"1min":9.3469999999999995,"5min":7.2199999999999998,"15min":3.7490000000000001},"min":{"1min":0.375,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":135.47,"5min":135.47,"15min":135.47},"last":0.65700000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.60499999999999998,"15min":0.79800000000000004},"min":{"1min":0.29599999999999999,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":4.3230000000000004,"5min":4.3230000000000004,"15min":15.942},"last":3.177},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.65700000000000003,"15min":0.73899999999999999},"min":{"1min":0.34000000000000002,"5min":0.28000000000000003,"15min":0.26500000000000001},"max":{"1min":3.7400000000000002,"5min":6.8920000000000003,"15min":16.725999999999999},"last":0.27700000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.76200000000000001,"15min":0.95199999999999996},"min":{"1min":0.30199999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6699999999999999,"5min":4.6699999999999999,"15min":10.891},"last":2.6629999999999998},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.748,"15min":0.86099999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":10.005000000000001,"5min":10.005000000000001,"15min":13.867000000000001},"last":3.2559999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":2.0430000000000001,"15min":1.6299999999999999},"min":{"1min":0.34899999999999998,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":5.5369999999999999,"5min":143.18700000000001,"15min":143.18700000000001},"last":3.2829999999999999},{"interface":"front","average":{"1min":1.3120000000000001,"5min":2.1379999999999999,"15min":1.6220000000000001},"min":{"1min":0.32100000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.507,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.68200000000000005}]}]},{"osd":6,"up_from":47,"seq":201863463453,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1659880,"kb_used_data":1007008,"kb_used_omap":0,"kb_used_meta":652864,"kb_avail":92085272,"statfs":{"total":95995035648,"available":94295318528,"internally_reserved":0,"allocated":1031176192,"data_stored":1029841377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":668532736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":7.6959999999999997,"5min":4.7009999999999996,"15min":2.9900000000000002},"min":{"1min":0.22500000000000001,"5min":0.214,"15min":0.20999999999999999},"max":{"1min":121.762,"5min":149.61600000000001,"15min":149.61600000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":8.9320000000000004,"5min":4.6200000000000001,"15min":2.9980000000000002},"min":{"1min":0.23100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":121.73099999999999,"5min":149.303,"15min":149.303},"last":5.7939999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:14:22 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":4.1820000000000004,"15min":3.379},"min":{"1min":0.30599999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":13.214,"5min":129.37899999999999,"15min":129.37899999999999},"last":5.8319999999999999},{"interface":"front","average":{"1min":2.0110000000000001,"5min":4.1950000000000003,"15min":3.3239999999999998},"min":{"1min":0.40600000000000003,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":13.292,"5min":129.40100000000001,"15min":129.40100000000001},"last":5.9960000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":8.4960000000000004,"5min":6.9829999999999997,"15min":5.657},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26800000000000002},"max":{"1min":122.797,"5min":133.46600000000001,"15min":199.23400000000001},"last":1.899},{"interface":"front","average":{"1min":9.2300000000000004,"5min":6.7919999999999998,"15min":5.5720000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26200000000000001},"max":{"1min":122.809,"5min":133.33000000000001,"15min":199.08000000000001},"last":5.9050000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":8.8710000000000004,"5min":6.8470000000000004,"15min":4.4189999999999996},"min":{"1min":0.38600000000000001,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":126.649,"5min":137.875,"15min":178.40000000000001},"last":7.149},{"interface":"front","average":{"1min":9.1059999999999999,"5min":6.5460000000000003,"15min":4.4139999999999997},"min":{"1min":0.318,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":125.324,"5min":137.60900000000001,"15min":178.26599999999999},"last":1.9279999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.71399999999999997,"15min":0.90500000000000003},"min":{"1min":0.311,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.774,"5min":9.0890000000000004,"15min":13.1},"last":5.7629999999999999},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.623,"15min":0.751},"min":{"1min":0.29399999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.8250000000000002,"5min":6.4400000000000004,"15min":6.4400000000000004},"last":11.949999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:13:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.77400000000000002,"15min":0.91200000000000003},"min":{"1min":0.32300000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.8390000000000004,"5min":9.0519999999999996,"15min":11.281000000000001},"last":6.2389999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.65700000000000003,"15min":0.88900000000000001},"min":{"1min":0.36099999999999999,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":4.7270000000000003,"5min":6.0190000000000001,"15min":8.6229999999999993},"last":6.1920000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:14:22 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56000000000000005,"15min":0.88},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70699999999999996,"5min":5.0590000000000002,"15min":7.8079999999999998},"last":1.954},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.64500000000000002,"15min":0.92900000000000005},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.97199999999999998,"5min":4.7549999999999999,"15min":7.1849999999999996},"last":7.9119999999999999}]}]},{"osd":1,"up_from":0,"seq":1185410973900,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991640,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781530,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2226060,"kb_used_data":1327620,"kb_used_omap":7,"kb_used_meta":898424,"kb_avail":91519092,"statfs":{"total":95995035648,"available":93715550208,"internally_reserved":0,"allocated":1359482880,"data_stored":1357110599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":919986257},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:13:51 2023","interfaces":[{"interface":"back","average":{"1min":4.9379999999999997,"5min":3.6920000000000002,"15min":2.1389999999999998},"min":{"1min":0.29199999999999998,"5min":0.19800000000000001,"15min":0.17699999999999999},"max":{"1min":25.297000000000001,"5min":103.021,"15min":103.021},"last":0.58399999999999996},{"interface":"front","average":{"1min":5.3840000000000003,"5min":4.4100000000000001,"15min":2.8359999999999999},"min":{"1min":0.31,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":18.879999999999999,"5min":103.154,"15min":103.154},"last":0.35599999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:14:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8599999999999999,"5min":5.3150000000000004,"15min":4.3330000000000002},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.188},"max":{"1min":12.500999999999999,"5min":143.482,"15min":212.12},"last":0.69399999999999995},{"interface":"front","average":{"1min":3.976,"5min":5.9260000000000002,"15min":4.3230000000000004},"min":{"1min":0.28399999999999997,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":25.155000000000001,"5min":143.61000000000001,"15min":213.02799999999999},"last":0.67500000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:14:01 2023","interfaces":[{"interface":"back","average":{"1min":7.0469999999999997,"5min":6.3949999999999996,"15min":4.7460000000000004},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.216},"max":{"1min":81.653999999999996,"5min":143.50800000000001,"15min":143.50800000000001},"last":4.4989999999999997},{"interface":"front","average":{"1min":9.3849999999999998,"5min":7.0709999999999997,"15min":4.7939999999999996},"min":{"1min":0.33000000000000002,"5min":0.249,"15min":0.23899999999999999},"max":{"1min":82.241,"5min":143.27799999999999,"15min":143.27799999999999},"last":4.5270000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:13:51 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":4.3899999999999997,"15min":2.2080000000000002},"min":{"1min":0.34799999999999998,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":25.231000000000002,"5min":225.80799999999999,"15min":225.80799999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":5.3090000000000002,"5min":5.3239999999999998,"15min":2.879},"min":{"1min":0.49099999999999999,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":12.590999999999999,"5min":225.79300000000001,"15min":225.79300000000001},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:13:51 2023","interfaces":[{"interface":"back","average":{"1min":5.0270000000000001,"5min":3.7829999999999999,"15min":2.431},"min":{"1min":0.46600000000000003,"5min":0.35999999999999999,"15min":0.29999999999999999},"max":{"1min":12.555,"5min":98.682000000000002,"15min":98.682000000000002},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.2509999999999999,"5min":4.4619999999999997,"15min":2.3279999999999998},"min":{"1min":0.43099999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":10.93,"5min":143.34700000000001,"15min":143.34700000000001},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:13:51 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":3.2719999999999998,"15min":2.669},"min":{"1min":0.42499999999999999,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":10.885999999999999,"5min":143.41399999999999,"15min":175.72499999999999},"last":0.443},{"interface":"front","average":{"1min":3.484,"5min":1.95,"15min":2.1389999999999998},"min":{"1min":0.53500000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":25.204999999999998,"5min":25.204999999999998,"15min":176.20400000000001},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:13:51 2023","interfaces":[{"interface":"back","average":{"1min":3.8029999999999999,"5min":2.0699999999999998,"15min":1.6100000000000001},"min":{"1min":0.45300000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":25.260000000000002,"5min":25.260000000000002,"15min":25.260000000000002},"last":0.81399999999999995},{"interface":"front","average":{"1min":5.0510000000000002,"5min":2.879,"15min":2.1419999999999999},"min":{"1min":0.52000000000000002,"5min":0.41399999999999998,"15min":0.34499999999999997},"max":{"1min":12.613,"5min":12.613,"15min":16.875},"last":0.40600000000000003}]}]},{"osd":3,"up_from":148,"seq":635655160124,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1699168,"kb_used_data":898072,"kb_used_omap":2,"kb_used_meta":801085,"kb_avail":92045984,"statfs":{"total":95995035648,"available":94255087616,"internally_reserved":0,"allocated":919625728,"data_stored":916999949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":820311093},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:13:47 2023","interfaces":[{"interface":"back","average":{"1min":5.3099999999999996,"5min":3.1989999999999998,"15min":2.1829999999999998},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":25.722999999999999,"5min":25.722999999999999,"15min":44.963000000000001},"last":0.245},{"interface":"front","average":{"1min":4.7919999999999998,"5min":3.1120000000000001,"15min":1.798},"min":{"1min":0.217,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":18.277999999999999,"5min":18.277999999999999,"15min":19.738},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:14:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4750000000000001,"5min":12.419,"15min":6.9500000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.20999999999999999},"max":{"1min":12.497,"5min":237.072,"15min":237.072},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.0720000000000001,"5min":12.172000000000001,"15min":6.9180000000000001},"min":{"1min":0.26000000000000001,"5min":0.20999999999999999,"15min":0.182},"max":{"1min":10.723000000000001,"5min":234.21899999999999,"15min":234.21899999999999},"last":0.317}]},{"osd":2,"last update":"Sat Sep 23 21:13:47 2023","interfaces":[{"interface":"back","average":{"1min":4.3600000000000003,"5min":5.0640000000000001,"15min":3.3159999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":13.666,"5min":93.272000000000006,"15min":93.272000000000006},"last":0.82599999999999996},{"interface":"front","average":{"1min":4.556,"5min":4.6719999999999997,"15min":2.8679999999999999},"min":{"1min":0.32600000000000001,"5min":0.23899999999999999,"15min":0.222},"max":{"1min":12.127000000000001,"5min":95.233999999999995,"15min":95.233999999999995},"last":0.442}]},{"osd":4,"last update":"Sat Sep 23 21:13:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":2.3500000000000001,"15min":2.7410000000000001},"min":{"1min":0.28299999999999997,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":10.698,"5min":15.763,"15min":111.36},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.8069999999999999,"5min":2.0920000000000001,"15min":2.5209999999999999},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.27300000000000002},"max":{"1min":13.644,"5min":13.644,"15min":112.473},"last":0.86499999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:13:47 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":2.2570000000000001,"15min":1.7190000000000001},"min":{"1min":0.31,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.7289999999999992,"5min":16.855,"15min":36.488999999999997},"last":8.0830000000000002},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.4790000000000001,"15min":2.0899999999999999},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":10.605,"5min":15.789999999999999,"15min":36.295000000000002},"last":0.73399999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:13:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":6.4539999999999997,"15min":3.3460000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.312},"max":{"1min":10.673999999999999,"5min":304.22399999999999,"15min":304.22399999999999},"last":0.501},{"interface":"front","average":{"1min":2.4249999999999998,"5min":6.0640000000000001,"15min":3.0880000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":13.609,"5min":304.15800000000002,"15min":304.15800000000002},"last":1.998}]},{"osd":7,"last update":"Sat Sep 23 21:13:47 2023","interfaces":[{"interface":"back","average":{"1min":2.9820000000000002,"5min":2.4359999999999999,"15min":2.0339999999999998},"min":{"1min":0.34200000000000003,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.7010000000000005,"5min":16.899000000000001,"15min":74.019000000000005},"last":0.84699999999999998},{"interface":"front","average":{"1min":2.6269999999999998,"5min":2.738,"15min":2.472},"min":{"1min":0.372,"5min":0.372,"15min":0.32900000000000001},"max":{"1min":10.648999999999999,"5min":15.816000000000001,"15min":73.364999999999995},"last":0.53300000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986743,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2117552,"kb_used_data":1465960,"kb_used_omap":0,"kb_used_meta":651584,"kb_avail":91627600,"statfs":{"total":95995035648,"available":93826662400,"internally_reserved":0,"allocated":1501143040,"data_stored":1499799541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":667222016},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:13:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2330000000000005,"5min":4.6760000000000002,"15min":2.371},"min":{"1min":0.219,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":109.23099999999999,"5min":226.751,"15min":226.751},"last":1.655},{"interface":"front","average":{"1min":7.7290000000000001,"5min":4.758,"15min":2.351},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.223},"max":{"1min":109.16800000000001,"5min":226.43299999999999,"15min":226.43299999999999},"last":2.601}]},{"osd":1,"last update":"Sat Sep 23 21:14:18 2023","interfaces":[{"interface":"back","average":{"1min":6.5570000000000004,"5min":3.9470000000000001,"15min":4.9889999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":119.55800000000001,"5min":119.55800000000001,"15min":127.411},"last":2.5579999999999998},{"interface":"front","average":{"1min":5.9690000000000003,"5min":3.343,"15min":4.1859999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":116.78100000000001,"5min":116.78100000000001,"15min":118.075},"last":1.8560000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:13:49 2023","interfaces":[{"interface":"back","average":{"1min":12.233000000000001,"5min":7.5890000000000004,"15min":5.2329999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":155.51300000000001,"5min":210.50299999999999,"15min":210.50299999999999},"last":1.7869999999999999},{"interface":"front","average":{"1min":12.597,"5min":8.0700000000000003,"15min":5.194},"min":{"1min":0.314,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":155.77099999999999,"5min":210.59,"15min":210.59},"last":1.8180000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:14:18 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":8.6440000000000001,"15min":3.7210000000000001},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":111.22199999999999,"5min":214.64099999999999,"15min":214.64099999999999},"last":1.758},{"interface":"front","average":{"1min":7.2439999999999998,"5min":9.1850000000000005,"15min":3.972},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":111.39400000000001,"5min":214.54900000000001,"15min":214.54900000000001},"last":1.8839999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":0.96999999999999997,"15min":0.83499999999999996},"min":{"1min":0.27800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":34.497999999999998,"5min":34.497999999999998,"15min":34.497999999999998},"last":1.8340000000000001},{"interface":"front","average":{"1min":2.4500000000000002,"5min":0.95399999999999996,"15min":0.85699999999999998},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":34.883000000000003,"5min":34.883000000000003,"15min":34.883000000000003},"last":8.2309999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.1220000000000001,"15min":0.99199999999999999},"min":{"1min":0.28799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.0049999999999999,"5min":36.557000000000002,"15min":36.557000000000002},"last":2.6320000000000001},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.0609999999999999,"15min":1.002},"min":{"1min":0.34200000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.6040000000000001,"5min":38.468000000000004,"15min":38.468000000000004},"last":2.2999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:14:34 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.69599999999999995,"15min":1.508},"min":{"1min":0.35199999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.8399999999999999,"5min":6.8399999999999999,"15min":145.721},"last":1.911},{"interface":"front","average":{"1min":0.75,"5min":0.65400000000000003,"15min":1.633},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.30499999999999999},"max":{"1min":3.577,"5min":6.6580000000000004,"15min":148.45699999999999},"last":0.377}]}]},{"osd":5,"up_from":39,"seq":167503725107,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2298276,"kb_used_data":1615644,"kb_used_omap":0,"kb_used_meta":682624,"kb_avail":91446876,"statfs":{"total":95995035648,"available":93641601024,"internally_reserved":0,"allocated":1654419456,"data_stored":1653084011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":699006976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:13:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.9820000000000002,"15min":1.599},"min":{"1min":0.29099999999999998,"5min":0.217,"15min":0.217},"max":{"1min":8.6660000000000004,"5min":82.280000000000001,"15min":82.280000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.1829999999999998,"5min":2.9940000000000002,"15min":1.7070000000000001},"min":{"1min":0.23400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":16.977,"5min":85.236000000000004,"15min":85.236000000000004},"last":2.109}]},{"osd":1,"last update":"Sat Sep 23 21:14:33 2023","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":2.9830000000000001,"15min":4.3869999999999996},"min":{"1min":0.26300000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.1950000000000003,"5min":74.920000000000002,"15min":233.77099999999999},"last":1.641},{"interface":"front","average":{"1min":1.3049999999999999,"5min":2.7040000000000002,"15min":4.2350000000000003},"min":{"1min":0.32100000000000001,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":5.1799999999999997,"5min":74.978999999999999,"15min":233.75200000000001},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:13:50 2023","interfaces":[{"interface":"back","average":{"1min":2.992,"5min":3.3879999999999999,"15min":3.6680000000000001},"min":{"1min":0.309,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":10.454000000000001,"5min":78.917000000000002,"15min":231.34700000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.9809999999999999,"5min":3.2330000000000001,"15min":3.7610000000000001},"min":{"1min":0.45300000000000001,"5min":0.28499999999999998,"15min":0.27300000000000002},"max":{"1min":10.621,"5min":78.888999999999996,"15min":231.381},"last":2.1480000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:13:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9020000000000001,"5min":7.5910000000000002,"15min":4.1449999999999996},"min":{"1min":0.34599999999999997,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":11.597,"5min":263.887,"15min":263.887},"last":1.5840000000000001},{"interface":"front","average":{"1min":3.3210000000000002,"5min":7.3869999999999996,"15min":4.0030000000000001},"min":{"1min":0.41699999999999998,"5min":0.30099999999999999,"15min":0.29699999999999999},"max":{"1min":10.519,"5min":263.70100000000002,"15min":263.70100000000002},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:14:33 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.61799999999999999,"15min":0.64000000000000001},"min":{"1min":0.28699999999999998,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":1.3759999999999999,"5min":7.5259999999999998,"15min":12.739000000000001},"last":3.5019999999999998},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.57399999999999995,"15min":0.76000000000000001},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.0099999999999998,"5min":5.0099999999999998,"15min":10.792},"last":2.0800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:13:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.5640000000000001,"15min":1.343},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":2.0939999999999999,"5min":83.451999999999998,"15min":83.451999999999998},"last":2.133},{"interface":"front","average":{"1min":0.73799999999999999,"5min":1.6459999999999999,"15min":1.393},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":4.4139999999999997,"5min":83.484999999999999,"15min":83.484999999999999},"last":12.725}]},{"osd":7,"last update":"Sat Sep 23 21:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.752,"15min":0.90900000000000003},"min":{"1min":0.32800000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9009999999999998,"5min":3.96,"15min":9.6329999999999991},"last":2.2149999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.66900000000000004,"15min":0.66700000000000004},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":1.288,"5min":3.0790000000000002,"15min":4.29},"last":0.45700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":283443200,"data_stored":283443200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1364590592,"data_stored":1364590592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":927334400,"data_stored":927334400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1512112128,"data_stored":1512112128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1690828800,"data_stored":1690828800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1029046272,"data_stored":1029046272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1174732800,"data_stored":1174732800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:14:52.216 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:14:52.216 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_8 pg_num 26 2023-09-23T21:14:52.516 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:14:52.516 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:14:52.516 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:14:52.517 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:14:52.517 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[219709]: pgmap v1600: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 954/127893 objects degraded (0.746%); 113/127893 objects misplaced (0.088%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[219709]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[219709]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[219709]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[219709]: osdmap e502: 8 total, 8 up, 6 in 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[219709]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 954/127893 objects degraded (0.746%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:52.547 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[211526]: pgmap v1600: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 954/127893 objects degraded (0.746%); 113/127893 objects misplaced (0.088%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:14:52.548 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[211526]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.548 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[211526]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.548 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[211526]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.548 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[211526]: osdmap e502: 8 total, 8 up, 6 in 2023-09-23T21:14:52.548 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[211526]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.548 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 954/127893 objects degraded (0.746%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:52.591 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:14:52.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:52.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:14:52.669 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 45994 45978 87.0564 87.0625 0.0146306 0.0114816 2023-09-23T21:14:52.705 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:52 smithi123 ceph-mon[193903]: pgmap v1600: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 954/127893 objects degraded (0.746%); 113/127893 objects misplaced (0.088%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T21:14:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:52 smithi123 ceph-mon[193903]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:52 smithi123 ceph-mon[193903]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:52 smithi123 ceph-mon[193903]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:52 smithi123 ceph-mon[193903]: osdmap e502: 8 total, 8 up, 6 in 2023-09-23T21:14:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:52 smithi123 ceph-mon[193903]: failed to encode map e502 with expected crc 2023-09-23T21:14:52.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 954/127893 objects degraded (0.746%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:14:52.777 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:52.906 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:52.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:52.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:53.016 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:53.043 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.087 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:53.192 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:53.256 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.373 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:53.378 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2352896964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:53.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:53.396 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:14:53.480 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:53.532 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:53.598 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:53.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:53.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:53.617 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:14:53.635 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2352896964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:53.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2352896964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:14:53.669 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 47080 47064 86.4922 67.875 0.0115464 0.011557 2023-09-23T21:14:53.705 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:53.749 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:54.326 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:54.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:54.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:54.431 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:54.501 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:54.517 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:54.596 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:54.612 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:54.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 62 MiB/s wr, 995 op/s; 954/127893 objects degraded (0.746%); 113/127893 objects misplaced (0.088%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T21:14:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2352896964' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: osdmap e503: 8 total, 8 up, 6 in 2023-09-23T21:14:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[211526]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 62 MiB/s wr, 995 op/s; 954/127893 objects degraded (0.746%); 113/127893 objects misplaced (0.088%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2352896964' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: osdmap e503: 8 total, 8 up, 6 in 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:54 smithi121 ceph-mon[219709]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.671 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 48443 48427 86.4509 85.1875 0.00761725 0.0115631 2023-09-23T21:14:54.726 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:14:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 62 MiB/s wr, 995 op/s; 954/127893 objects degraded (0.746%); 113/127893 objects misplaced (0.088%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T21:14:54.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2352896964' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: osdmap e503: 8 total, 8 up, 6 in 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:54 smithi123 ceph-mon[193903]: failed to encode map e503 with expected crc 2023-09-23T21:14:54.795 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:54.803 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:54.922 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:14:54.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:54.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:14:55.037 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:55.098 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:55.129 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:14:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:14:54] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.43.0" 2023-09-23T21:14:55.209 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:55.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:55.323 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:55.389 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:55.501 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:55.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:55.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:55.616 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:14:55.650 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:55.671 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 49653 49637 86.1501 75.625 0.0128274 0.0116018 2023-09-23T21:14:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:55 smithi123 ceph-mon[193903]: pgmap v1604: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 363/138153 objects misplaced (0.263%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T21:14:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:55 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:14:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:55 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:14:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[211526]: pgmap v1604: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 363/138153 objects misplaced (0.263%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[219709]: pgmap v1604: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 363/138153 objects misplaced (0.263%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T21:14:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:14:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:14:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:56.429 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:14:56.671 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 50806 50790 85.7692 72.0625 0.0150883 0.0116538 2023-09-23T21:14:56.866 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:56 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 954/127893 objects degraded (0.746%), 2 pgs degraded) 2023-09-23T21:14:56.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:56 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 954/127893 objects degraded (0.746%), 2 pgs degraded) 2023-09-23T21:14:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:56 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 954/127893 objects degraded (0.746%), 2 pgs degraded) 2023-09-23T21:14:57.160 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:14:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:57.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:14:57.264 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:57.293 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:57.664 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:57.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:57.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:57.671 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 51038 51022 83.8941 14.5 0.00235829 0.0116345 2023-09-23T21:14:57.768 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:57.814 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:57.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: failed to encode map e504 with expected crc 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: failed to encode map e504 with expected crc 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: osdmap e504: 8 total, 8 up, 6 in 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: failed to encode map e504 with expected crc 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: failed to encode map e504 with expected crc 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: failed to encode map e504 with expected crc 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: failed to encode map e504 with expected crc 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: pgmap v1606: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 58 MiB/s wr, 925 op/s; 363/144549 objects misplaced (0.251%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:57.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:58.015 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:14:58.031 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:14:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:58.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:14:58.036 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.036 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.036 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: osdmap e504: 8 total, 8 up, 6 in 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: pgmap v1606: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 58 MiB/s wr, 925 op/s; 363/144549 objects misplaced (0.251%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.037 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: osdmap e504: 8 total, 8 up, 6 in 2023-09-23T21:14:58.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: failed to encode map e504 with expected crc 2023-09-23T21:14:58.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: pgmap v1606: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 87 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 58 MiB/s wr, 925 op/s; 363/144549 objects misplaced (0.251%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T21:14:58.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:14:58.038 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:14:58.137 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:14:58.202 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:58.454 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:14:58.488 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:14:58.520 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:14:58.552 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:14:58.582 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:14:58.616 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:14:58.632 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:14:58.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:58.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:14:58.652 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:14:58.671 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 52321 52305 83.7989 80.1875 0.0201656 0.0119284 2023-09-23T21:14:58.688 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:14:58.691 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.692 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.692 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.692 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.692 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.692 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.692 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.692 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:14:58.715 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 13 pg_num to 26 2023-09-23T21:14:58.721 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[211526]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.721 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[211526]: osdmap e505: 8 total, 8 up, 6 in 2023-09-23T21:14:58.721 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[211526]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.721 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[211526]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.721 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/737872093' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:14:58.721 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1271426191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[219709]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[219709]: osdmap e505: 8 total, 8 up, 6 in 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[219709]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[219709]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/737872093' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:14:58.722 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1271426191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:58.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:14:58.756 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:14:58.756 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:14:58.802 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:58.904 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:14:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:58.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:14:58.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:58 smithi123 ceph-mon[193903]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:58 smithi123 ceph-mon[193903]: osdmap e505: 8 total, 8 up, 6 in 2023-09-23T21:14:58.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:58 smithi123 ceph-mon[193903]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:58 smithi123 ceph-mon[193903]: failed to encode map e505 with expected crc 2023-09-23T21:14:58.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/737872093' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:14:58.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T21:14:58.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1271426191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:14:59.022 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:59.102 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:59.206 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:59.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:59.323 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:14:59.351 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:59.394 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:14:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:59.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:14:59.500 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:14:59.539 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:59.636 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:14:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:14:59.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:14:59.671 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:14:59.672746+0000 min lat: 0.00124323 max lat: 1.4251 avg lat: 0.0119958 2023-09-23T21:14:59.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:14:59.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 53364 53348 83.3335 65.1875 0.00894306 0.0119958 2023-09-23T21:14:59.752 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:14:59.828 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:14:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: failed to encode map e506 with expected crc 2023-09-23T21:14:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: osdmap e506: 8 total, 8 up, 6 in 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: failed to encode map e506 with expected crc 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: failed to encode map e506 with expected crc 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: failed to encode map e506 with expected crc 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: failed to encode map e506 with expected crc 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: failed to encode map e506 with expected crc 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: failed to encode map e506 with expected crc 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: pgmap v1609: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s; 11 MiB/s, 180 objects/s recovering 2023-09-23T21:14:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:14:59.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:14:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:15:00.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.150 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: osdmap e506: 8 total, 8 up, 6 in 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: pgmap v1609: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s; 11 MiB/s, 180 objects/s recovering 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:15:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: osdmap e506: 8 total, 8 up, 6 in 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: failed to encode map e506 with expected crc 2023-09-23T21:15:00.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: pgmap v1609: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s; 11 MiB/s, 180 objects/s recovering 2023-09-23T21:15:00.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:15:00.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:14:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T21:15:00.562 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:00.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:00.665 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:00.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 53447 53431 81.4277 5.1875 0.215288 0.0120125 2023-09-23T21:15:00.697 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:00.778 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:00.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:00.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[211526]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[211526]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[211526]: osdmap e507: 8 total, 8 up, 6 in 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[211526]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[211526]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[219709]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.876 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[219709]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.877 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:15:00.877 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[219709]: osdmap e507: 8 total, 8 up, 6 in 2023-09-23T21:15:00.877 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[219709]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.877 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:00 smithi121 ceph-mon[219709]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.893 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:00.948 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:00 smithi123 ceph-mon[193903]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:00 smithi123 ceph-mon[193903]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:00 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T21:15:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:00 smithi123 ceph-mon[193903]: osdmap e507: 8 total, 8 up, 6 in 2023-09-23T21:15:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:00 smithi123 ceph-mon[193903]: failed to encode map e507 with expected crc 2023-09-23T21:15:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:00 smithi123 ceph-mon[193903]: failed to encode map e507 with expected crc 2023-09-23T21:15:01.049 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:01.155 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:01.181 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.222 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:01.329 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:01.355 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:15:01.398 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.519 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:01.624 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:01.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 54609 54593 81.218 72.625 0.00926316 0.012307 2023-09-23T21:15:01.693 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.795 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:01.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:01.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:01.906 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: osdmap e508: 8 total, 8 up, 6 in 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[211526]: pgmap v1612: 99 pgs: 3 peering, 8 unknown, 88 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: osdmap e508: 8 total, 8 up, 6 in 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:01 smithi121 ceph-mon[219709]: pgmap v1612: 99 pgs: 3 peering, 8 unknown, 88 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:15:01.966 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:01.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: osdmap e508: 8 total, 8 up, 6 in 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: failed to encode map e508 with expected crc 2023-09-23T21:15:01.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:01 smithi123 ceph-mon[193903]: pgmap v1612: 99 pgs: 3 peering, 8 unknown, 88 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 5.7 MiB/s, 90 objects/s recovering 2023-09-23T21:15:02.094 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:02.133 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:02.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:02.237 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:02.280 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:02.347 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:02.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:02.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:02.451 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:02.529 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:02.631 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:02.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:02.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 56198 56182 81.6385 99.3125 0.00798506 0.0122439 2023-09-23T21:15:02.741 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:02.766 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:15:02.786 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:02.849 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:02.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:02.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3763413648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:02.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3763413648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:02.929 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:15:02.954 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:03.031 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3763413648' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:03.034 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:03.153 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:03.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:03.266 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:03.292 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:03.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 57438 57422 81.5443 77.5 0.00821877 0.012258 2023-09-23T21:15:04.043 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:04.045 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:04.045 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.045 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3763413648' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:04.045 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: osdmap e509: 8 total, 8 up, 6 in 2023-09-23T21:15:04.045 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.046 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.046 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.046 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: 13.0 scrub starts 2023-09-23T21:15:04.046 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[219709]: pgmap v1614: 99 pgs: 3 peering, 8 unknown, 88 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 27 MiB/s wr, 430 op/s 2023-09-23T21:15:04.046 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:04.046 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3763413648' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:04.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: osdmap e509: 8 total, 8 up, 6 in 2023-09-23T21:15:04.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: 13.0 scrub starts 2023-09-23T21:15:04.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:03 smithi121 ceph-mon[211526]: pgmap v1614: 99 pgs: 3 peering, 8 unknown, 88 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 27 MiB/s wr, 430 op/s 2023-09-23T21:15:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:04.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:04.149 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:04.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:04.174 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.175 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3763413648' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:04.175 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: osdmap e509: 8 total, 8 up, 6 in 2023-09-23T21:15:04.175 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.175 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.175 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: failed to encode map e509 with expected crc 2023-09-23T21:15:04.175 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: 13.0 scrub starts 2023-09-23T21:15:04.175 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:03 smithi123 ceph-mon[193903]: pgmap v1614: 99 pgs: 3 peering, 8 unknown, 88 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 27 MiB/s wr, 430 op/s 2023-09-23T21:15:04.177 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:04.221 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:04.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:04.325 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:04.361 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:04.581 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:04.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:04.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 58625 58609 81.3807 74.1875 0.0102158 0.012283 2023-09-23T21:15:04.685 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:04.721 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:04.748 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:04.881 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:04.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:04.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:04.999 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:05.041 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:15:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:15:04] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.43.0" 2023-09-23T21:15:05.042 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:04 smithi121 ceph-mon[211526]: 13.0 scrub ok 2023-09-23T21:15:05.042 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:04 smithi121 ceph-mon[211526]: 13.8 scrub starts 2023-09-23T21:15:05.042 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:04 smithi121 ceph-mon[219709]: 13.0 scrub ok 2023-09-23T21:15:05.042 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:04 smithi121 ceph-mon[219709]: 13.8 scrub starts 2023-09-23T21:15:05.044 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:05.136 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:05.137 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:05.167 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:05.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:05.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:04 smithi123 ceph-mon[193903]: 13.0 scrub ok 2023-09-23T21:15:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:04 smithi123 ceph-mon[193903]: 13.8 scrub starts 2023-09-23T21:15:05.274 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:05.349 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:05.490 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:05.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:05.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:05.603 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:05.672 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:05.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 59919 59903 81.3696 80.875 0.00858261 0.0122836 2023-09-23T21:15:05.704 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 3, 7] out_osds: [0, 1] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:15:05.704 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:15:05.705 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [4, 5, 6, 2, 3, 7] 2023-09-23T21:15:05.705 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T21:15:05.783 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:05.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:05.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:05.897 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:05.948 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:05 smithi123 ceph-mon[193903]: 13.7 scrub starts 2023-09-23T21:15:05.948 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:05 smithi123 ceph-mon[193903]: 13.8 scrub ok 2023-09-23T21:15:05.948 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:05 smithi123 ceph-mon[193903]: 13.18 scrub starts 2023-09-23T21:15:05.948 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:05 smithi123 ceph-mon[193903]: 13.2 scrub starts 2023-09-23T21:15:05.949 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:05 smithi123 ceph-mon[193903]: 13.17 scrub starts 2023-09-23T21:15:05.949 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:05 smithi123 ceph-mon[193903]: 13.5 scrub starts 2023-09-23T21:15:05.949 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:05 smithi123 ceph-mon[193903]: pgmap v1615: 99 pgs: 2 active+clean+scrubbing, 2 peering, 2 unknown, 93 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 36 MiB/s wr, 569 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T21:15:05.953 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:05.999 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:06.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:06.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:06.108 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:06.134 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[219709]: 13.7 scrub starts 2023-09-23T21:15:06.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[219709]: 13.8 scrub ok 2023-09-23T21:15:06.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[219709]: 13.18 scrub starts 2023-09-23T21:15:06.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[219709]: 13.2 scrub starts 2023-09-23T21:15:06.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[219709]: 13.17 scrub starts 2023-09-23T21:15:06.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[219709]: 13.5 scrub starts 2023-09-23T21:15:06.135 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[219709]: pgmap v1615: 99 pgs: 2 active+clean+scrubbing, 2 peering, 2 unknown, 93 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 36 MiB/s wr, 569 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T21:15:06.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[211526]: 13.7 scrub starts 2023-09-23T21:15:06.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[211526]: 13.8 scrub ok 2023-09-23T21:15:06.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[211526]: 13.18 scrub starts 2023-09-23T21:15:06.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[211526]: 13.2 scrub starts 2023-09-23T21:15:06.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[211526]: 13.17 scrub starts 2023-09-23T21:15:06.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[211526]: 13.5 scrub starts 2023-09-23T21:15:06.137 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:05 smithi121 ceph-mon[211526]: pgmap v1615: 99 pgs: 2 active+clean+scrubbing, 2 peering, 2 unknown, 93 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 36 MiB/s wr, 569 op/s; 21 KiB/s, 0 objects/s recovering 2023-09-23T21:15:06.139 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:06.591 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:15:06.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 60817 60801 80.8324 56.125 0.0138425 0.0123673 2023-09-23T21:15:06.758 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:06.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:06.863 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:06.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:06 smithi121 ceph-mon[211526]: 13.18 scrub ok 2023-09-23T21:15:06.911 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:06 smithi121 ceph-mon[211526]: 13.5 scrub ok 2023-09-23T21:15:06.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:06 smithi121 ceph-mon[219709]: 13.18 scrub ok 2023-09-23T21:15:06.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:06 smithi121 ceph-mon[219709]: 13.5 scrub ok 2023-09-23T21:15:06.925 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:06.945 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:15:07.010 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:07.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:07.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:07.121 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:07.166 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:06 smithi123 ceph-mon[193903]: 13.18 scrub ok 2023-09-23T21:15:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:06 smithi123 ceph-mon[193903]: 13.5 scrub ok 2023-09-23T21:15:07.346 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:07.376 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:07.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:07.486 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:07.500 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:15:07.536 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:15:07.541 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:07.549 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:07.569 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:15:07.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:15:07.629 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:15:07.645 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:07.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:07.666 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:15:07.672 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 62238 62222 80.9985 88.8125 0.00437285 0.0123405 2023-09-23T21:15:07.701 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:15:07.736 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:15:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.737 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.737 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.737 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.737 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:15:07.756 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:07.797 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:07.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[211526]: 13.3 scrub starts 2023-09-23T21:15:07.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[211526]: pgmap v1616: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 768 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T21:15:07.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:15:07.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:15:07.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1384576490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[219709]: 13.3 scrub starts 2023-09-23T21:15:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[219709]: pgmap v1616: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 768 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T21:15:07.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:15:07.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:15:07.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1384576490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:07 smithi123 ceph-mon[193903]: 13.3 scrub starts 2023-09-23T21:15:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:07 smithi123 ceph-mon[193903]: pgmap v1616: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 768 op/s; 17 KiB/s, 0 objects/s recovering 2023-09-23T21:15:07.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:07 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:15:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:07 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T21:15:07.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1384576490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:08.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 62781 62765 80.0381 33.9375 0.00269666 0.0122952 2023-09-23T21:15:08.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: 13.1 scrub starts 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: 13.3 scrub ok 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: osdmap e510: 8 total, 8 up, 6 in 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[211526]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: 13.1 scrub starts 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: 13.3 scrub ok 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: osdmap e510: 8 total, 8 up, 6 in 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: failed to encode map e510 with expected crc 2023-09-23T21:15:08.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:08 smithi121 ceph-mon[219709]: failed to encode map e510 with expected crc 2023-09-23T21:15:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: 13.1 scrub starts 2023-09-23T21:15:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: 13.3 scrub ok 2023-09-23T21:15:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: failed to encode map e510 with expected crc 2023-09-23T21:15:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: failed to encode map e510 with expected crc 2023-09-23T21:15:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T21:15:09.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: osdmap e510: 8 total, 8 up, 6 in 2023-09-23T21:15:09.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: failed to encode map e510 with expected crc 2023-09-23T21:15:09.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: failed to encode map e510 with expected crc 2023-09-23T21:15:09.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:08 smithi123 ceph-mon[193903]: failed to encode map e510 with expected crc 2023-09-23T21:15:09.362 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:09.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:09.466 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:09.496 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:09.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 62781 62765 78.4376 0 - 0.0122952 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: 13.4 scrub starts 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: 13.4 scrub ok 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: 13.6 scrub starts 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: osdmap e511: 8 total, 8 up, 6 in 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: 13.6 scrub ok 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: pgmap v1619: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 27 MiB/s wr, 439 op/s 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: 13.4 scrub starts 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: 13.4 scrub ok 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: 13.6 scrub starts 2023-09-23T21:15:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: osdmap e511: 8 total, 8 up, 6 in 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: failed to encode map e511 with expected crc 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: 13.6 scrub ok 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: pgmap v1619: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 27 MiB/s wr, 439 op/s 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:15:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:15:10.103 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:10.185 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:10.207 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: 13.4 scrub starts 2023-09-23T21:15:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: 13.4 scrub ok 2023-09-23T21:15:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: 13.6 scrub starts 2023-09-23T21:15:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: failed to encode map e511 with expected crc 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: osdmap e511: 8 total, 8 up, 6 in 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: failed to encode map e511 with expected crc 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: failed to encode map e511 with expected crc 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: failed to encode map e511 with expected crc 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: failed to encode map e511 with expected crc 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: 13.6 scrub ok 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: pgmap v1619: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 27 MiB/s wr, 439 op/s 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:15:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T21:15:10.237 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:10.293 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:10.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:10.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:10.429 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:10.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 62781 62765 76.8998 0 - 0.0122952 2023-09-23T21:15:10.705 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:10.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:10.811 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:10.812 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.3. 2023-09-23T21:15:10.868 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:10.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 conmon[211495]: 2023-09-23T21:15:10.766+0000 7fdc50c66700 -1 mon.a@0(leader).osd e512 definitely_dead 0 2023-09-23T21:15:10.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: 13.9 scrub starts 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: 13.13 scrub starts 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: failed to encode map e512 with expected crc 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: failed to encode map e512 with expected crc 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: osdmap e512: 8 total, 8 up, 6 in 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: failed to encode map e512 with expected crc 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: 13.9 scrub ok 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1859161354' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T21:15:10.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[211526]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T21:15:10.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: 13.9 scrub starts 2023-09-23T21:15:10.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: 13.13 scrub starts 2023-09-23T21:15:10.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: failed to encode map e512 with expected crc 2023-09-23T21:15:10.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: failed to encode map e512 with expected crc 2023-09-23T21:15:10.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:15:10.910 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: osdmap e512: 8 total, 8 up, 6 in 2023-09-23T21:15:10.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: failed to encode map e512 with expected crc 2023-09-23T21:15:10.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: 13.9 scrub ok 2023-09-23T21:15:10.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1859161354' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T21:15:10.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:10 smithi121 ceph-mon[219709]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T21:15:10.913 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:10.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:11.022 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:11.054 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:11.109 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:11.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:11.212 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: 13.9 scrub starts 2023-09-23T21:15:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: 13.13 scrub starts 2023-09-23T21:15:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: failed to encode map e512 with expected crc 2023-09-23T21:15:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: failed to encode map e512 with expected crc 2023-09-23T21:15:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T21:15:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: osdmap e512: 8 total, 8 up, 6 in 2023-09-23T21:15:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: failed to encode map e512 with expected crc 2023-09-23T21:15:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: 13.9 scrub ok 2023-09-23T21:15:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1859161354' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T21:15:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:10 smithi123 ceph-mon[193903]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T21:15:11.238 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:11.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 62781 62765 75.4212 0 - 0.0122952 2023-09-23T21:15:11.684 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:15:11.709 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:11.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:11.832 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:11.860 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:11.970 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1859161354' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: osdmap e513: 8 total, 8 up, 5 in 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.996 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1104808503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: pgmap v1622: 99 pgs: 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 77 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 1917/188523 objects degraded (1.017%) 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: osdmap e514: 8 total, 8 up, 5 in 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:11.997 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1859161354' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: osdmap e513: 8 total, 8 up, 5 in 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.998 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e513 with expected crc 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1104808503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: pgmap v1622: 99 pgs: 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 77 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 1917/188523 objects degraded (1.017%) 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: osdmap e514: 8 total, 8 up, 5 in 2023-09-23T21:15:11.999 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:11 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:12.176 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:12.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1859161354' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T21:15:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: osdmap e513: 8 total, 8 up, 5 in 2023-09-23T21:15:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e513 with expected crc 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1104808503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: pgmap v1622: 99 pgs: 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 77 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 1917/188523 objects degraded (1.017%) 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:15:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:12.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: osdmap e514: 8 total, 8 up, 5 in 2023-09-23T21:15:12.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:11 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:12.282 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:12.309 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:12.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 62851 62835 74.0809 1.09375 0.00209757 0.013359 2023-09-23T21:15:12.824 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:12.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:12.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:12.927 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:12.961 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:13.033 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:13.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:13.127 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:13.141 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:13.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:15:13.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 1917/188523 objects degraded (1.017%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: osdmap e515: 8 total, 8 up, 5 in 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[211526]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 1917/188523 objects degraded (1.017%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: osdmap e515: 8 total, 8 up, 5 in 2023-09-23T21:15:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:12 smithi121 ceph-mon[219709]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.193 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:13.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e514 with expected crc 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 1917/188523 objects degraded (1.017%), 1 pg degraded (PG_DEGRADED) 2023-09-23T21:15:13.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:13.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:13.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: osdmap e515: 8 total, 8 up, 5 in 2023-09-23T21:15:13.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:12 smithi123 ceph-mon[193903]: failed to encode map e515 with expected crc 2023-09-23T21:15:13.263 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:13.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:13.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:13.370 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:13.416 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:13.512 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:15:13.512 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:15:13.513 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:15:13.513 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:15:13.513 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:15:13.543 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:13.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:13.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:13.645 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:13.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 63649 63633 73.6326 49.875 0.0208932 0.0135766 2023-09-23T21:15:13.722 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:13.840 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:13.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:13.954 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:14.022 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:14.091 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:14.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:14.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:14.198 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:14.231 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[211526]: failed to encode map e515 with expected crc 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[211526]: failed to encode map e515 with expected crc 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[211526]: pgmap v1625: 99 pgs: 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 77 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 1917/188523 objects degraded (1.017%) 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[219709]: failed to encode map e515 with expected crc 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[219709]: failed to encode map e515 with expected crc 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:14 smithi121 ceph-mon[219709]: pgmap v1625: 99 pgs: 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 77 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 1917/188523 objects degraded (1.017%) 2023-09-23T21:15:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:14 smithi123 ceph-mon[193903]: failed to encode map e515 with expected crc 2023-09-23T21:15:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:14 smithi123 ceph-mon[193903]: failed to encode map e515 with expected crc 2023-09-23T21:15:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:14 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:14 smithi123 ceph-mon[193903]: pgmap v1625: 99 pgs: 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 77 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 1917/188523 objects degraded (1.017%) 2023-09-23T21:15:14.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 64393 64377 73.1393 46.5 0.0304089 0.0136671 2023-09-23T21:15:14.890 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:14.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:14.978 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:15:14.979 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:15:14.994 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:15.034 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:15:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:15:14] "GET /metrics HTTP/1.1" 200 36672 "" "Prometheus/2.43.0" 2023-09-23T21:15:15.059 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:15.161 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:15.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:15.269 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:15.295 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:15.569 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:15.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:15.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:15.673 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:15.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 65306 65290 72.8522 57.0625 0.00632156 0.0137224 2023-09-23T21:15:15.750 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:15.868 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:15.872 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:15.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:15.963 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:15.986 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:16.046 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:16.175 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:16.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:16.284 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:15:16.293 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:16.318 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:15:16.331 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:15:16.355 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:15:16.378 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:16.393 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:15:16.431 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:15:16.455 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:15:16.485 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.486 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:15:16.487 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:16.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:16.601 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:16.621 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:16 smithi121 ceph-mon[211526]: pgmap v1626: 99 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 68 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 21571/189597 objects degraded (11.377%) 2023-09-23T21:15:16.621 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2654755694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:16.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:16 smithi121 ceph-mon[219709]: pgmap v1626: 99 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 68 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 21571/189597 objects degraded (11.377%) 2023-09-23T21:15:16.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2654755694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:16.659 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:16.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:16 smithi123 ceph-mon[193903]: pgmap v1626: 99 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 1 unknown, 19 peering, 68 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 21571/189597 objects degraded (11.377%) 2023-09-23T21:15:16.659 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2654755694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:16.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 66252 66236 72.6114 59.125 0.0186129 0.0137662 2023-09-23T21:15:16.790 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:16.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:16.820 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:15:16.820 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:15:16.821 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:15:16.821 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:15:16.821 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:15:16.821 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:15:16.897 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:16.994 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:17.121 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:17.236 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:17.264 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:17.646 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:17.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:17.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 15 67283 67268 72.4714 64.5 0.0209149 0.0137936 2023-09-23T21:15:17.750 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:17.840 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:17.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:17 smithi121 ceph-mon[211526]: pgmap v1627: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 25544/193005 objects degraded (13.235%); 1616/193005 objects misplaced (0.837%) 2023-09-23T21:15:17.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:17 smithi121 ceph-mon[219709]: pgmap v1627: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 25544/193005 objects degraded (13.235%); 1616/193005 objects misplaced (0.837%) 2023-09-23T21:15:17.978 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:17.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:17 smithi123 ceph-mon[193903]: pgmap v1627: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 25544/193005 objects degraded (13.235%); 1616/193005 objects misplaced (0.837%) 2023-09-23T21:15:17.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:17.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:18.091 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:18.117 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:18.181 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:18.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:18.294 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:18.326 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:18.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 68143 68127 72.153 53.6875 0.0171329 0.0138545 2023-09-23T21:15:18.964 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:18 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 25544/193005 objects degraded (13.235%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:18.964 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:18 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 6 pgs peering) 2023-09-23T21:15:18.965 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:18 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 25544/193005 objects degraded (13.235%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:18.965 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:18 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 6 pgs peering) 2023-09-23T21:15:18.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:18 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 25544/193005 objects degraded (13.235%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:18.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:18 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 6 pgs peering) 2023-09-23T21:15:19.181 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:19.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:19.285 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:19.343 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:19.358 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:19.453 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:19.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:19.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:19.574 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:19.638 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:19.673 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:15:19.674515+0000 min lat: 0.00124323 max lat: 4.30095 avg lat: 0.0139179 2023-09-23T21:15:19.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:15:19.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 68982 68966 71.8244 52.4375 0.010394 0.0139179 2023-09-23T21:15:19.785 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:19.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:19.900 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:19.944 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:19 smithi121 ceph-mon[211526]: pgmap v1628: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 2.5 MiB/s wr, 40 op/s; 25544/201141 objects degraded (12.700%); 1616/201141 objects misplaced (0.803%); 2.7 MiB/s, 44 objects/s recovering 2023-09-23T21:15:19.947 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:19 smithi121 ceph-mon[219709]: pgmap v1628: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 2.5 MiB/s wr, 40 op/s; 25544/201141 objects degraded (12.700%); 1616/201141 objects misplaced (0.803%); 2.7 MiB/s, 44 objects/s recovering 2023-09-23T21:15:19.959 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:19.959 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:19.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:19 smithi123 ceph-mon[193903]: pgmap v1628: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 2.5 MiB/s wr, 40 op/s; 25544/201141 objects degraded (12.700%); 1616/201141 objects misplaced (0.803%); 2.7 MiB/s, 44 objects/s recovering 2023-09-23T21:15:19.998 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:20.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:20.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:20.224 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:20.298 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:20.426 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:20.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:20.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:20.580 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:20.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 69933 69917 71.6213 59.4375 0.00476231 0.0139583 2023-09-23T21:15:20.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1990974076' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:20.707 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:15:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:20.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:15:21.017 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1990974076' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:21.017 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:21.018 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1990974076' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:21.018 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:15:21.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 70777 70761 71.3169 52.75 0.0134794 0.0140172 2023-09-23T21:15:21.858 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:21.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:21.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:21.962 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:21 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:21 smithi123 ceph-mon[193903]: failed to encode map e516 with expected crc 2023-09-23T21:15:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:21 smithi123 ceph-mon[193903]: osdmap e516: 8 total, 8 up, 5 in 2023-09-23T21:15:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:21 smithi123 ceph-mon[193903]: failed to encode map e516 with expected crc 2023-09-23T21:15:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:21 smithi123 ceph-mon[193903]: failed to encode map e516 with expected crc 2023-09-23T21:15:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:21 smithi123 ceph-mon[193903]: pgmap v1630: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 16 MiB/s wr, 254 op/s; 25544/206766 objects degraded (12.354%); 1240/206766 objects misplaced (0.600%); 5.2 MiB/s, 84 objects/s recovering 2023-09-23T21:15:22.001 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:22.042 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:22.147 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:22.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:22.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:22.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[219709]: failed to encode map e516 with expected crc 2023-09-23T21:15:22.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[219709]: osdmap e516: 8 total, 8 up, 5 in 2023-09-23T21:15:22.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[219709]: failed to encode map e516 with expected crc 2023-09-23T21:15:22.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[219709]: failed to encode map e516 with expected crc 2023-09-23T21:15:22.149 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[219709]: pgmap v1630: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 16 MiB/s wr, 254 op/s; 25544/206766 objects degraded (12.354%); 1240/206766 objects misplaced (0.600%); 5.2 MiB/s, 84 objects/s recovering 2023-09-23T21:15:22.150 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:22.150 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:15:22.150 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[211526]: failed to encode map e516 with expected crc 2023-09-23T21:15:22.150 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[211526]: osdmap e516: 8 total, 8 up, 5 in 2023-09-23T21:15:22.150 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[211526]: failed to encode map e516 with expected crc 2023-09-23T21:15:22.150 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[211526]: failed to encode map e516 with expected crc 2023-09-23T21:15:22.150 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:21 smithi121 ceph-mon[211526]: pgmap v1630: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 16 MiB/s wr, 254 op/s; 25544/206766 objects degraded (12.354%); 1240/206766 objects misplaced (0.600%); 5.2 MiB/s, 84 objects/s recovering 2023-09-23T21:15:22.180 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:15:22.187 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:22.487 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:22.560 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:22.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:22.628 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:22.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 71638 71622 71.039 53.8125 0.0157021 0.0140716 2023-09-23T21:15:22.799 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:22.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:22.904 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:22.941 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:23.006 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:23.008 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:23.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:23.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:23.069 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:15:23.069 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:15:23.069 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:15:23.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:23.154 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:23.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 72616 72600 70.8841 61.125 0.0101279 0.0141028 2023-09-23T21:15:23.694 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:23.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:23.798 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:23.860 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:23.974 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:23.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:24.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:24.168 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:24.300 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:24.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:24.415 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:24.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:24 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:24.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:24 smithi123 ceph-mon[193903]: pgmap v1631: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 MiB/s wr, 219 op/s; 25544/206766 objects degraded (12.354%); 1240/206766 objects misplaced (0.600%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T21:15:24.483 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:24.518 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:15:24.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:24 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:24.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:24 smithi121 ceph-mon[211526]: pgmap v1631: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 MiB/s wr, 219 op/s; 25544/206766 objects degraded (12.354%); 1240/206766 objects misplaced (0.600%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T21:15:24.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:24 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:24.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:24 smithi121 ceph-mon[219709]: pgmap v1631: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 14 MiB/s wr, 219 op/s; 25544/206766 objects degraded (12.354%); 1240/206766 objects misplaced (0.600%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T21:15:24.556 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:24.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:24.662 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:24.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 73461 73445 70.606 52.8125 0.010704 0.0141586 2023-09-23T21:15:24.692 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:24.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:15:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:15:24] "GET /metrics HTTP/1.1" 200 36672 "" "Prometheus/2.43.0" 2023-09-23T21:15:25.096 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:25.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:25.205 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:25.278 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:25.376 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:25.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:25.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:25.483 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:25.532 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:25.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:25 smithi121 ceph-mon[211526]: pgmap v1632: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 53 MiB/s wr, 842 op/s; 25544/214869 objects degraded (11.888%); 1240/214869 objects misplaced (0.577%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:15:25.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:25 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 25544/214869 objects degraded (11.888%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:25.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:25 smithi121 ceph-mon[219709]: pgmap v1632: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 53 MiB/s wr, 842 op/s; 25544/214869 objects degraded (11.888%); 1240/214869 objects misplaced (0.577%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:15:25.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:25 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 25544/214869 objects degraded (11.888%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:25.665 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:25.674 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 74291 74275 70.3222 51.875 0.0238135 0.0142155 2023-09-23T21:15:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:25.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:25.679 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:25 smithi123 ceph-mon[193903]: pgmap v1632: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 53 MiB/s wr, 842 op/s; 25544/214869 objects degraded (11.888%); 1240/214869 objects misplaced (0.577%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T21:15:25.679 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:25 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 25544/214869 objects degraded (11.888%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:25.779 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:25.815 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:26.067 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:26.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:26.172 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:26.256 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:26.286 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:15:26.308 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:15:26.332 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:15:26.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:15:26.362 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:26.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:26.393 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:15:26.424 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:15:26.454 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:15:26.470 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:26.485 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:26.491 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:15:26.497 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.497 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.497 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.497 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.498 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.498 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.498 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.498 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:15:26.521 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:26.620 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:15:26.620 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.47003.0:75173 13.a e516/498)", 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.619243+0000", 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.00068529799999999998, 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00071312100000000005, 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619243+0000", 2023-09-23T21:15:26.621 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619243+0000", 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.622 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619242+0000", 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619247+0000", 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4700000000000004e-06 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619258+0000", 2023-09-23T21:15:26.623 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1782e-05 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619281+0000", 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2986999999999999e-05 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.624 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.47003.0:75179 13.a e516/498)", 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.619511+0000", 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.00041759300000000001, 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00050403199999999996, 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "queued for pg", 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.625 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619511+0000", 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619511+0000", 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619510+0000", 2023-09-23T21:15:26.626 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619514+0000", 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3189999999999996e-06 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.627 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619526+0000", 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1802e-05 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619533+0000", 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.596e-06 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.628 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPushReply(13.10 516/511 [PushReplyOp(13:0d6fd58d:::benchmark_data_smithi121_621349_object36383:head)])", 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.619642+0000", 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.00028668500000000001, 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00043067799999999999, 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:15:26.629 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619642+0000", 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619642+0000", 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.630 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619641+0000", 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619646+0000", 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.904e-06 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.631 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619657+0000", 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0733e-05 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619663+0000", 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2480000000000001e-06 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619711+0000", 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8269999999999997e-05 2023-09-23T21:15:26.632 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47003.0:75183 13.f e516/482 13:fa92d6bf:::benchmark_data_smithi121_621349_object75182:head v 516'4672, mlcod=516'4670)", 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.619875+0000", 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 5.2982999999999997e-05, 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00025530699999999997, 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.633 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619875+0000", 2023-09-23T21:15:26.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.634 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.634 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.634 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619875+0000", 2023-09-23T21:15:26.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.635 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.635 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619875+0000", 2023-09-23T21:15:26.635 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:15:26.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619930+0000", 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5661000000000001e-05 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619939+0000", 2023-09-23T21:15:26.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7790000000000001e-06 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619945+0000", 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4509999999999997e-06 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619983+0000", 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8010999999999999e-05 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.620006+0000", 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2597999999999999e-05 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:75173 13.a 13:5b96e75b:::benchmark_data_smithi121_621349_object75172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.607936+0000", 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.011991784, 2023-09-23T21:15:26.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012257786999999999, 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 75173 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.607936+0000", 2023-09-23T21:15:26.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.607936+0000", 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.607936+0000", 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.607953+0000", 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7084000000000001e-05 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.607963+0000", 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0387000000000001e-05 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.607968+0000", 2023-09-23T21:15:26.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3800000000000002e-06 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.607994+0000", 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5729e-05 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.608098+0000", 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010374099999999999 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.608186+0000", 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8375999999999995e-05 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.610127+0000", 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001940331 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615420+0000", 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052934610000000002 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:75179 13.a 13:5da1c274:::benchmark_data_smithi121_621349_object75178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-09-23T21:15:26.645 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.615634+0000", 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0042940670000000004, 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046663240000000003, 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 75179 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615634+0000", 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615634+0000", 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615633+0000", 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615652+0000", 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9123e-05 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615664+0000", 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.183e-05 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615670+0000", 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3789999999999997e-06 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615705+0000", 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5732000000000003e-05 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:15:26.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615827+0000", 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121855 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.615922+0000", 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5311999999999998e-05 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.619938+0000", 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004015991 2023-09-23T21:15:26.650 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:75181 13.a 13:5a9b95aa:::benchmark_data_smithi121_621349_object75180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.617630+0000", 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0022978069999999998, 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002749605, 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:15:26.651 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 75181 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617630+0000", 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617630+0000", 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617629+0000", 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617655+0000", 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6196000000000001e-05 2023-09-23T21:15:26.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617671+0000", 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5458e-05 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617682+0000", 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1021e-05 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617817+0000", 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000135556 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617920+0000", 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000102722 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.618063+0000", 2023-09-23T21:15:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000142881 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:75182 13.b 13:d03945f8:::benchmark_data_smithi121_621349_object75181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:26.617693+0000", 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0022356500000000001, 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027599170000000002, 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:15:26.656 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 75182 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617693+0000", 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:15:26.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617693+0000", 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617692+0000", 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617709+0000", 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7118000000000001e-05 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617716+0000", 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7449999999999998e-06 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617720+0000", 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1219999999999997e-06 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617771+0000", 2023-09-23T21:15:26.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0961999999999998e-05 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.617903+0000", 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000132006 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.618052+0000", 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000149192 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:15:26.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:26.620279+0000", 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0022266550000000001 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 8 2023-09-23T21:15:26.661 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:15:26.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2920880108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:26.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2920880108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:26.675 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 75234 75218 70.1509 58.9375 0.025262 0.0142501 2023-09-23T21:15:26.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2920880108' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:27.473 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:27.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:27.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:27.577 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:27.609 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:27.675 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 76176 76160 69.985 58.875 0.00853447 0.0142838 2023-09-23T21:15:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:27 smithi121 ceph-mon[211526]: pgmap v1633: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 57 MiB/s wr, 904 op/s; 25544/220140 objects degraded (11.604%); 848/220140 objects misplaced (0.385%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:15:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:27.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:27.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:27 smithi121 ceph-mon[219709]: pgmap v1633: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 57 MiB/s wr, 904 op/s; 25544/220140 objects degraded (11.604%); 848/220140 objects misplaced (0.385%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:15:27.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:27.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:27.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:27 smithi123 ceph-mon[193903]: pgmap v1633: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 57 MiB/s wr, 904 op/s; 25544/220140 objects degraded (11.604%); 848/220140 objects misplaced (0.385%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:15:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:28.304 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:28.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:28.320 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:15:28.409 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:28.442 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:28.675 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 77055 77039 69.7669 54.9375 0.00891817 0.0143289 2023-09-23T21:15:28.702 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:28.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:28.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:28.806 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:28.833 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:29.228 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:29.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:29.331 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:29.369 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:29.408 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:29.437 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:29.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:29.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:29.544 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:29.634 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:29.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 77774 77758 69.4122 44.9375 0.00644547 0.0144012 2023-09-23T21:15:29.759 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:29.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:29.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:29.945 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:30.073 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:30.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:30.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:30.123 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:15:30.124 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:15:30.124 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.2657904389710002 2023-09-23T21:15:30.124 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 0.2657904389710002 2023-09-23T21:15:30.183 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:30.213 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:15:30.249 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:30.373 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:30.377 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:15:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:30.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:30.482 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:30.506 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:30 smithi121 ceph-mon[219709]: pgmap v1634: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 56 MiB/s wr, 895 op/s; 25544/228009 objects degraded (11.203%); 848/228009 objects misplaced (0.372%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:15:30.506 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1727181117' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:30.507 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:30 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:30.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:30 smithi121 ceph-mon[211526]: pgmap v1634: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 56 MiB/s wr, 895 op/s; 25544/228009 objects degraded (11.203%); 848/228009 objects misplaced (0.372%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:15:30.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1727181117' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:30.507 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:30 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:30.509 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:30.622 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:30.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:30.675 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 78544 78528 69.1124 48.125 0.0217844 0.0144647 2023-09-23T21:15:30.726 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:30.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:30 smithi123 ceph-mon[193903]: pgmap v1634: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 56 MiB/s wr, 895 op/s; 25544/228009 objects degraded (11.203%); 848/228009 objects misplaced (0.372%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T21:15:30.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1727181117' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:30.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:30 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:30.793 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:30.941 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:30.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:31.049 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:31.078 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:31.392 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: osdmap e517: 8 total, 8 up, 5 in 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.393 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[219709]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.650 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:31.650 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:31.650 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: osdmap e517: 8 total, 8 up, 5 in 2023-09-23T21:15:31.650 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.650 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.650 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.651 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.651 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.651 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:31 smithi121 ceph-mon[211526]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 79310 79294 68.8174 47.875 0.029095 0.0145255 2023-09-23T21:15:31.683 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:31.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:31.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:31.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:31.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: osdmap e517: 8 total, 8 up, 5 in 2023-09-23T21:15:31.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:31 smithi123 ceph-mon[193903]: failed to encode map e517 with expected crc 2023-09-23T21:15:31.775 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:31.787 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:31.824 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:31.831 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:31.934 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:31.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:31.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:32.039 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:32.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:32.244 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:15:32.244 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:15:32.244 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:15:32.244 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:15:32.244 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:15:32.389 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:32.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:32.493 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:32 smithi121 ceph-mon[219709]: pgmap v1636: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 879 op/s; 25544/233166 objects degraded (10.955%); 420/233166 objects misplaced (0.180%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:32.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 25544/233166 objects degraded (10.955%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:32 smithi121 ceph-mon[211526]: pgmap v1636: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 879 op/s; 25544/233166 objects degraded (10.955%); 420/233166 objects misplaced (0.180%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:32.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 25544/233166 objects degraded (10.955%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:32.526 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:32.640 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:32.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:32.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:32.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 80235 80219 68.6666 57.8125 0.0159298 0.0145587 2023-09-23T21:15:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:32 smithi123 ceph-mon[193903]: pgmap v1636: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 879 op/s; 25544/233166 objects degraded (10.955%); 420/233166 objects misplaced (0.180%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 25544/233166 objects degraded (10.955%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:32.744 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:32.842 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:32.896 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:32.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:33.001 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:33.028 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:33.528 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:15:33.528 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:33.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:33.631 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:33.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 80975 80959 68.3636 46.25 0.0273338 0.0146228 2023-09-23T21:15:33.678 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:15:33.704 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:33.793 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:33.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:33.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:33.902 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:33.995 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:34.136 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:34.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:34.255 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:34.287 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:34.404 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:34 smithi123 ceph-mon[193903]: pgmap v1637: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 879 op/s; 25544/233166 objects degraded (10.955%); 420/233166 objects misplaced (0.180%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:34.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:34 smithi121 ceph-mon[211526]: pgmap v1637: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 879 op/s; 25544/233166 objects degraded (10.955%); 420/233166 objects misplaced (0.180%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:34.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:34 smithi121 ceph-mon[219709]: pgmap v1637: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 55 MiB/s wr, 879 op/s; 25544/233166 objects degraded (10.955%); 420/233166 objects misplaced (0.180%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:34.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 81753 81737 68.1004 48.625 0.0234347 0.014679 2023-09-23T21:15:34.709 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:34.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:34.816 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:34.887 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:35.006 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:15:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:15:34] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.43.0" 2023-09-23T21:15:35.006 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:35.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:35.058 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:35.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:35.176 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:35.461 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:35.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:35.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:15:35.534 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:15:35.555 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:15:35.568 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:35.583 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:15:35.613 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:15:35.638 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:35.646 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:15:35.671 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:35.675 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:15:35.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 82624 82608 67.9206 54.4375 0.0149008 0.0147097 2023-09-23T21:15:35.699 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:15:35.701 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.701 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.702 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.702 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.702 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.702 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.702 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.702 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:15:35.747 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:35.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:35.854 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:35.915 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:36.026 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:36.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:36.138 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:36.192 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:36.250 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:36.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:36.359 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:36.389 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:36.436 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.6 to 0.26579 (440a) 2023-09-23T21:15:36.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: pgmap v1638: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s; 25544/240180 objects degraded (10.635%); 420/240180 objects misplaced (0.175%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:36.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: osdmap e518: 8 total, 8 up, 5 in 2023-09-23T21:15:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1731490451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4188593351' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]: dispatch 2023-09-23T21:15:36.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:36 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]: dispatch 2023-09-23T21:15:36.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: pgmap v1638: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s; 25544/240180 objects degraded (10.635%); 420/240180 objects misplaced (0.175%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: osdmap e518: 8 total, 8 up, 5 in 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1731490451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4188593351' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]: dispatch 2023-09-23T21:15:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]: dispatch 2023-09-23T21:15:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: pgmap v1638: 99 pgs: 1 active+recovering+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s; 25544/240180 objects degraded (10.635%); 420/240180 objects misplaced (0.175%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:15:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: osdmap e518: 8 total, 8 up, 5 in 2023-09-23T21:15:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: failed to encode map e518 with expected crc 2023-09-23T21:15:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1731490451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:36.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4188593351' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]: dispatch 2023-09-23T21:15:36.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:36 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]: dispatch 2023-09-23T21:15:36.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 83107 83091 67.4306 30.1875 0.00384081 0.0147909 2023-09-23T21:15:37.427 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.427 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.427 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.427 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]': finished 2023-09-23T21:15:37.427 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: osdmap e519: 8 total, 8 up, 5 in 2023-09-23T21:15:37.427 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.427 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 25544/240180 objects degraded (10.635%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:37.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 83107 83091 66.5661 0 - 0.0147909 2023-09-23T21:15:37.704 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:37.708 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.708 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.708 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.708 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]': finished 2023-09-23T21:15:37.708 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: osdmap e519: 8 total, 8 up, 5 in 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 25544/240180 objects degraded (10.635%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: osdmap e520: 8 total, 8 up, 5 in 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.709 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[219709]: pgmap v1642: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 41 MiB/s wr, 657 op/s; 23673/245001 objects degraded (9.662%); 1822/245001 objects misplaced (0.744%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.2657904389710002}]': finished 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: osdmap e519: 8 total, 8 up, 5 in 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e519 with expected crc 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 25544/240180 objects degraded (10.635%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.710 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: osdmap e520: 8 total, 8 up, 5 in 2023-09-23T21:15:37.711 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.711 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.711 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.711 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.711 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:37 smithi121 ceph-mon[211526]: pgmap v1642: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 41 MiB/s wr, 657 op/s; 23673/245001 objects degraded (9.662%); 1822/245001 objects misplaced (0.744%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:15:37.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:37.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: osdmap e520: 8 total, 8 up, 5 in 2023-09-23T21:15:37.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: failed to encode map e520 with expected crc 2023-09-23T21:15:37.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:37 smithi123 ceph-mon[193903]: pgmap v1642: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 41 MiB/s wr, 657 op/s; 23673/245001 objects degraded (9.662%); 1822/245001 objects misplaced (0.744%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T21:15:37.812 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:37.907 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:38.018 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:38.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:38.140 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:38.239 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:38.371 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:38.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:38.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:38.483 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:38.560 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:38.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 83638 83622 66.1436 16.5938 0.0104814 0.0151144 2023-09-23T21:15:38.677 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:38.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:38.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:38.729 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:15:38.729 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:15:38.752 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: osdmap e521: 8 total, 8 up, 5 in 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[211526]: 13.f starting backfill to osd.5 from (0'0,0'0] MAX to 518'5159 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.753 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: osdmap e521: 8 total, 8 up, 5 in 2023-09-23T21:15:38.754 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.754 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.754 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.754 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.754 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:38 smithi121 ceph-mon[219709]: 13.f starting backfill to osd.5 from (0'0,0'0] MAX to 518'5159 2023-09-23T21:15:38.798 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:38.871 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.871 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.871 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: osdmap e521: 8 total, 8 up, 5 in 2023-09-23T21:15:38.871 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: failed to encode map e521 with expected crc 2023-09-23T21:15:38.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:38 smithi123 ceph-mon[193903]: 13.f starting backfill to osd.5 from (0'0,0'0] MAX to 518'5159 2023-09-23T21:15:38.872 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:39.015 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:39.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:39.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:39.127 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:39.164 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:39.221 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:39.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:39.325 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:39.356 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:39.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:15:39.677511+0000 min lat: 0.00124323 max lat: 4.30095 avg lat: 0.0151497 2023-09-23T21:15:39.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:15:39.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 84493 84477 65.9847 53.4375 0.0161671 0.0151497 2023-09-23T21:15:39.978 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:15:39.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:39 smithi123 ceph-mon[193903]: pgmap v1644: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 37776/249927 objects degraded (15.115%); 7008/249927 objects misplaced (2.804%); 7.6 MiB/s, 123 objects/s recovering 2023-09-23T21:15:40.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:39 smithi121 ceph-mon[211526]: pgmap v1644: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 37776/249927 objects degraded (15.115%); 7008/249927 objects misplaced (2.804%); 7.6 MiB/s, 123 objects/s recovering 2023-09-23T21:15:40.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:39 smithi121 ceph-mon[219709]: pgmap v1644: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 51 MiB/s wr, 812 op/s; 37776/249927 objects degraded (15.115%); 7008/249927 objects misplaced (2.804%); 7.6 MiB/s, 123 objects/s recovering 2023-09-23T21:15:40.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 85257 85241 65.7596 47.75 0.0141991 0.015202 2023-09-23T21:15:40.811 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:40.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:40.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:15:40.919 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:41.001 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.153 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:41.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:41.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:41.213 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:41.266 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:41.326 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.392 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:41.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:41.400 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:41.497 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:41.535 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.666 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:41.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:41.676 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 86103 86087 65.6024 52.875 0.0231307 0.0152381 2023-09-23T21:15:41.770 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:41.803 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:41.917 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:41.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:42.023 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:42.106 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:42.236 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:42.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:42.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:42.358 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:42.390 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:42 smithi123 ceph-mon[193903]: pgmap v1645: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 44 MiB/s wr, 706 op/s; 48505/252666 objects degraded (19.197%); 8972/252666 objects misplaced (3.551%); 8.6 MiB/s, 139 objects/s recovering 2023-09-23T21:15:42.390 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/584770660' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:42.390 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:42 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:42.390 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48505/252666 objects degraded (19.197%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:42.395 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:42.401 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:15:42.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[219709]: pgmap v1645: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 44 MiB/s wr, 706 op/s; 48505/252666 objects degraded (19.197%); 8972/252666 objects misplaced (3.551%); 8.6 MiB/s, 139 objects/s recovering 2023-09-23T21:15:42.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/584770660' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48505/252666 objects degraded (19.197%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[211526]: pgmap v1645: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 44 MiB/s wr, 706 op/s; 48505/252666 objects degraded (19.197%); 8972/252666 objects misplaced (3.551%); 8.6 MiB/s, 139 objects/s recovering 2023-09-23T21:15:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/584770660' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:15:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48505/252666 objects degraded (19.197%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:42.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 86889 86873 65.4039 49.125 0.0364825 0.0152841 2023-09-23T21:15:43.324 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:15:43.324 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:15:43.325 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_8 2023-09-23T21:15:43.325 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:15:43.404 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:43.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:43.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:43.511 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:43.546 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:43.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[211526]: failed to encode map e522 with expected crc 2023-09-23T21:15:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[211526]: osdmap e522: 8 total, 8 up, 5 in 2023-09-23T21:15:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[211526]: failed to encode map e522 with expected crc 2023-09-23T21:15:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[219709]: failed to encode map e522 with expected crc 2023-09-23T21:15:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[219709]: osdmap e522: 8 total, 8 up, 5 in 2023-09-23T21:15:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[219709]: failed to encode map e522 with expected crc 2023-09-23T21:15:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:43.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 87754 87738 65.2688 54.0625 0.0100658 0.0153156 2023-09-23T21:15:43.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:43 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:15:43.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:43 smithi123 ceph-mon[193903]: failed to encode map e522 with expected crc 2023-09-23T21:15:43.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:43 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:15:43.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:43 smithi123 ceph-mon[193903]: osdmap e522: 8 total, 8 up, 5 in 2023-09-23T21:15:43.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:43 smithi123 ceph-mon[193903]: failed to encode map e522 with expected crc 2023-09-23T21:15:43.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:44.137 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:44.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:44.244 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:44.276 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:44.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:44 smithi121 ceph-mon[211526]: pgmap v1647: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 24 MiB/s wr, 385 op/s; 48505/252666 objects degraded (19.197%); 8972/252666 objects misplaced (3.551%); 3.2 MiB/s, 53 objects/s recovering 2023-09-23T21:15:44.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:44 smithi121 ceph-mon[219709]: pgmap v1647: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 24 MiB/s wr, 385 op/s; 48505/252666 objects degraded (19.197%); 8972/252666 objects misplaced (3.551%); 3.2 MiB/s, 53 objects/s recovering 2023-09-23T21:15:44.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 88500 88484 65.0494 46.625 0.00863496 0.0153686 2023-09-23T21:15:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:44 smithi123 ceph-mon[193903]: pgmap v1647: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 24 MiB/s wr, 385 op/s; 48505/252666 objects degraded (19.197%); 8972/252666 objects misplaced (3.551%); 3.2 MiB/s, 53 objects/s recovering 2023-09-23T21:15:44.854 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:44.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:44.958 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:45.011 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:45.058 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:45.141 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:15:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:15:44] "GET /metrics HTTP/1.1" 200 36714 "" "Prometheus/2.43.0" 2023-09-23T21:15:45.184 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:45.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:45.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:45.289 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:45.368 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:45.437 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:15:45.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:15:45.485 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:15:45.491 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:45.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:45.510 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:15:45.545 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:15:45.580 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:15:45.607 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:45.615 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:15:45.648 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:45.664 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:15:45.665 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.665 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.666 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.666 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.666 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.666 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.666 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.666 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:15:45.677 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 89333 89317 64.8984 52.0625 0.0244623 0.0154038 2023-09-23T21:15:46.063 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:46.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:46.164 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:15:46.169 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:46.240 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:46.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:46 smithi121 ceph-mon[211526]: pgmap v1648: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 42 MiB/s wr, 664 op/s; 48505/260958 objects degraded (18.587%); 9253/260958 objects misplaced (3.546%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:15:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1362732396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:46 smithi121 ceph-mon[219709]: pgmap v1648: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 42 MiB/s wr, 664 op/s; 48505/260958 objects degraded (18.587%); 9253/260958 objects misplaced (3.546%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:15:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1362732396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:46.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 90189 90173 64.7653 53.5 0.0418233 0.0154328 2023-09-23T21:15:46.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:46 smithi123 ceph-mon[193903]: pgmap v1648: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 42 MiB/s wr, 664 op/s; 48505/260958 objects degraded (18.587%); 9253/260958 objects misplaced (3.546%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:15:46.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1362732396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:46.882 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:15:46.882 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:15:46.883 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:15:46.883 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:15:47.465 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:47.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:47.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:47.569 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:15:47.631 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:47.673 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:15:47.675 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:47 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48505/260958 objects degraded (18.587%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:47.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:15:47.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 90917 90901 64.5463 45.5 0.00684299 0.015477 2023-09-23T21:15:47.777 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:15:47.806 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:47 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48505/260958 objects degraded (18.587%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:47.806 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:47 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48505/260958 objects degraded (18.587%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:47.810 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:48.027 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:15:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:48.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:15:48.059 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:15:48.136 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:48.210 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:48.319 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:48.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:48.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:48.430 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:48.460 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:48.471 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:48 smithi123 ceph-mon[193903]: pgmap v1649: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 44 MiB/s wr, 696 op/s; 48505/265110 objects degraded (18.296%); 9062/265110 objects misplaced (3.418%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:15:48.585 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:48 smithi121 ceph-mon[211526]: pgmap v1649: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 44 MiB/s wr, 696 op/s; 48505/265110 objects degraded (18.296%); 9062/265110 objects misplaced (3.418%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:15:48.585 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:48 smithi121 ceph-mon[219709]: pgmap v1649: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 44 MiB/s wr, 696 op/s; 48505/265110 objects degraded (18.296%); 9062/265110 objects misplaced (3.418%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T21:15:48.635 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:48.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:48.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 91712 91696 64.3793 49.6875 0.0198224 0.0155279 2023-09-23T21:15:48.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:48.823 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:48.837 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:48.956 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:48.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:49.063 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:49.095 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:15:49.097 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:15:49.119 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:49.236 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:49.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:49.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:49.349 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:49.385 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:49.549 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:49 smithi121 ceph-mon[211526]: from='client.37143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:49.549 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:49 smithi121 ceph-mon[211526]: pgmap v1650: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 764 op/s; 48505/272880 objects degraded (17.775%); 9303/272880 objects misplaced (3.409%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:15:49.549 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:49 smithi121 ceph-mon[219709]: from='client.37143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:49.549 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:49 smithi121 ceph-mon[219709]: pgmap v1650: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 764 op/s; 48505/272880 objects degraded (17.775%); 9303/272880 objects misplaced (3.409%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:15:49.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.01 2023-09-23T21:15:49.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 92491 2023-09-23T21:15:49.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 64.2227 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 36.32 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 200.562 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 1027 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 581.137 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 3209 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0155669 2023-09-23T21:15:49.681 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.0704349 2023-09-23T21:15:49.682 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 4.30095 2023-09-23T21:15:49.682 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00124323 2023-09-23T21:15:49.687 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:15:49.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:49 smithi123 ceph-mon[193903]: from='client.37143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:15:49.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:49 smithi123 ceph-mon[193903]: pgmap v1650: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 764 op/s; 48505/272880 objects degraded (17.775%); 9303/272880 objects misplaced (3.409%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T21:15:50.327 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:15:51.304 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1649,"stamp":"2023-09-23T21:15:47.314757+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5790373792,"num_objects":88370,"num_object_clones":0,"num_object_copies":265110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":48505,"num_objects_misplaced":9062,"num_objects_unfound":0,"num_objects_dirty":88370,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":73481,"num_write_kb":4685965,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6711,"num_bytes_recovered":435820068,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115224,"ondisk_log_size":115224,"up":297,"acting":275,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":345,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":19338880,"kb_used_data":15488216,"kb_used_omap":7,"kb_used_meta":3850616,"kb_avail":449386880,"statfs":{"total":479975178240,"available":460172165120,"internally_reserved":0,"allocated":15859933184,"data_stored":15851933294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":3943030865},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":439287808,"num_objects":6703,"num_object_clones":0,"num_object_copies":20109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24832,"num_objects_misplaced":7240,"num_objects_unfound":0,"num_objects_dirty":6703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6703,"num_write_kb":428992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35001602,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.622843"},"pg_stats":[{"pgid":"13.14","version":"522'4562","reported_seq":6729,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:45.933618+0000","last_change":"2023-09-23T21:15:37.953666+0000","last_active":"2023-09-23T21:15:45.933618+0000","last_peered":"2023-09-23T21:15:45.933618+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:15:37.947742+0000","last_became_peered":"2023-09-23T21:15:37.947742+0000","last_unstale":"2023-09-23T21:15:45.933618+0000","last_undegraded":"2023-09-23T21:15:37.724073+0000","last_fullsized":"2023-09-23T21:15:37.696156+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4562,"log_dups_size":0,"ondisk_log_size":4562,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186843136,"num_objects":2851,"num_object_clones":0,"num_object_copies":8553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2613,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2851,"num_write_kb":182464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":238},{"shards":"2,5,6","objects":2613}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":123,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696719+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:15:37.696719+0000","last_peered":"2023-09-23T21:15:37.696719+0000","last_clean":"2023-09-23T21:15:37.696719+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:15:37.696719+0000","last_undegraded":"2023-09-23T21:15:37.696719+0000","last_fullsized":"2023-09-23T21:15:37.696719+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":152,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.702312+0000","last_change":"2023-09-23T21:15:36.786608+0000","last_active":"2023-09-23T21:15:37.702312+0000","last_peered":"2023-09-23T21:15:37.702312+0000","last_clean":"2023-09-23T21:15:37.702312+0000","last_became_active":"2023-09-23T21:15:36.786251+0000","last_became_peered":"2023-09-23T21:15:36.786251+0000","last_unstale":"2023-09-23T21:15:37.702312+0000","last_undegraded":"2023-09-23T21:15:37.702312+0000","last_fullsized":"2023-09-23T21:15:37.702312+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:15.804266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.17","version":"522'4338","reported_seq":4407,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.320059+0000","last_change":"2023-09-23T21:15:37.925492+0000","last_active":"2023-09-23T21:15:43.320059+0000","last_peered":"2023-09-23T21:15:43.320059+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:15:37.917083+0000","last_became_peered":"2023-09-23T21:15:37.917083+0000","last_unstale":"2023-09-23T21:15:43.320059+0000","last_undegraded":"2023-09-23T21:15:37.769758+0000","last_fullsized":"2023-09-23T21:15:37.736401+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4338,"log_dups_size":0,"ondisk_log_size":4338,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176619520,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2584,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2695,"num_write_kb":172480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":2584},{"shards":"4,7","objects":111}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":309,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.700025+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:15:37.700025+0000","last_peered":"2023-09-23T21:15:37.700025+0000","last_clean":"2023-09-23T21:15:37.700025+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:15:37.700025+0000","last_undegraded":"2023-09-23T21:15:37.700025+0000","last_fullsized":"2023-09-23T21:15:37.700025+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":169,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704992+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:15:37.704992+0000","last_peered":"2023-09-23T21:15:37.704992+0000","last_clean":"2023-09-23T21:15:37.704992+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:15:37.704992+0000","last_undegraded":"2023-09-23T21:15:37.704992+0000","last_fullsized":"2023-09-23T21:15:37.704992+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"522'4383","reported_seq":4450,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.305002+0000","last_change":"2023-09-23T21:15:12.803518+0000","last_active":"2023-09-23T21:15:43.305002+0000","last_peered":"2023-09-23T21:15:43.305002+0000","last_clean":"2023-09-23T21:15:09.799088+0000","last_became_active":"2023-09-23T21:15:12.799547+0000","last_became_peered":"2023-09-23T21:15:12.799547+0000","last_unstale":"2023-09-23T21:15:43.305002+0000","last_undegraded":"2023-09-23T21:15:12.697902+0000","last_fullsized":"2023-09-23T21:15:12.679711+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4383,"log_dups_size":0,"ondisk_log_size":4383,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178388992,"num_objects":2722,"num_object_clones":0,"num_object_copies":8166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1956,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2722,"num_write_kb":174208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1956},{"shards":"4,5","objects":766}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":181,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.724372+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:15:37.724372+0000","last_peered":"2023-09-23T21:15:37.724372+0000","last_clean":"2023-09-23T21:15:37.724372+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:15:37.724372+0000","last_undegraded":"2023-09-23T21:15:37.724372+0000","last_fullsized":"2023-09-23T21:15:37.724372+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":416,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.770980+0000","last_change":"2023-09-23T21:15:36.720709+0000","last_active":"2023-09-23T21:15:37.770980+0000","last_peered":"2023-09-23T21:15:37.770980+0000","last_clean":"2023-09-23T21:15:37.770980+0000","last_became_active":"2023-09-23T21:15:36.720461+0000","last_became_peered":"2023-09-23T21:15:36.720461+0000","last_unstale":"2023-09-23T21:15:37.770980+0000","last_undegraded":"2023-09-23T21:15:37.770980+0000","last_fullsized":"2023-09-23T21:15:37.770980+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:34.395890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.19","version":"522'4330","reported_seq":4407,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.240406+0000","last_change":"2023-09-23T21:15:38.015633+0000","last_active":"2023-09-23T21:15:43.240406+0000","last_peered":"2023-09-23T21:15:43.240406+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:15:38.010899+0000","last_became_peered":"2023-09-23T21:15:38.010899+0000","last_unstale":"2023-09-23T21:15:43.240406+0000","last_undegraded":"2023-09-23T21:15:37.722607+0000","last_fullsized":"2023-09-23T21:15:37.705389+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4330,"log_dups_size":0,"ondisk_log_size":4330,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176586752,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2695,"num_write_kb":172448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":151},{"shards":"2,5,6","objects":2544}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":106,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.707351+0000","last_change":"2023-09-23T21:15:36.715191+0000","last_active":"2023-09-23T21:15:37.707351+0000","last_peered":"2023-09-23T21:15:37.707351+0000","last_clean":"2023-09-23T21:15:37.707351+0000","last_became_active":"2023-09-23T21:15:36.714882+0000","last_became_peered":"2023-09-23T21:15:36.714882+0000","last_unstale":"2023-09-23T21:15:37.707351+0000","last_undegraded":"2023-09-23T21:15:37.707351+0000","last_fullsized":"2023-09-23T21:15:37.707351+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:36.489500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.055349+0000","last_change":"2023-09-23T21:15:36.718536+0000","last_active":"2023-09-23T21:15:41.055349+0000","last_peered":"2023-09-23T21:15:41.055349+0000","last_clean":"2023-09-23T21:15:41.055349+0000","last_became_active":"2023-09-23T21:15:36.717389+0000","last_became_peered":"2023-09-23T21:15:36.717389+0000","last_unstale":"2023-09-23T21:15:41.055349+0000","last_undegraded":"2023-09-23T21:15:41.055349+0000","last_fullsized":"2023-09-23T21:15:41.055349+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:49.029938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.18","version":"522'4433","reported_seq":4548,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:45.999591+0000","last_change":"2023-09-23T21:15:37.899454+0000","last_active":"2023-09-23T21:15:45.999591+0000","last_peered":"2023-09-23T21:15:45.999591+0000","last_clean":"2023-09-23T21:15:36.291044+0000","last_became_active":"2023-09-23T21:15:37.895682+0000","last_became_peered":"2023-09-23T21:15:37.895682+0000","last_unstale":"2023-09-23T21:15:45.999591+0000","last_undegraded":"2023-09-23T21:15:37.723119+0000","last_fullsized":"2023-09-23T21:15:37.695587+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":4433,"log_dups_size":0,"ondisk_log_size":4433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:16.086598+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181796864,"num_objects":2774,"num_object_clones":0,"num_object_copies":8322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":920,"num_write_kb":58880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":198},{"shards":"2,5,6","objects":2576}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":175,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.706965+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:15:37.706965+0000","last_peered":"2023-09-23T21:15:37.706965+0000","last_clean":"2023-09-23T21:15:37.706965+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:15:37.706965+0000","last_undegraded":"2023-09-23T21:15:37.706965+0000","last_fullsized":"2023-09-23T21:15:37.706965+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":134,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704514+0000","last_change":"2023-09-23T21:15:36.718310+0000","last_active":"2023-09-23T21:15:37.704514+0000","last_peered":"2023-09-23T21:15:37.704514+0000","last_clean":"2023-09-23T21:15:37.704514+0000","last_became_active":"2023-09-23T21:15:36.717872+0000","last_became_peered":"2023-09-23T21:15:36.717872+0000","last_unstale":"2023-09-23T21:15:37.704514+0000","last_undegraded":"2023-09-23T21:15:37.704514+0000","last_fullsized":"2023-09-23T21:15:37.704514+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:58.556574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":109,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.705577+0000","last_change":"2023-09-23T21:15:36.816209+0000","last_active":"2023-09-23T21:15:37.705577+0000","last_peered":"2023-09-23T21:15:37.705577+0000","last_clean":"2023-09-23T21:15:37.705577+0000","last_became_active":"2023-09-23T21:15:36.724436+0000","last_became_peered":"2023-09-23T21:15:36.724436+0000","last_unstale":"2023-09-23T21:15:37.705577+0000","last_undegraded":"2023-09-23T21:15:37.705577+0000","last_fullsized":"2023-09-23T21:15:37.705577+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:57.112425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":270,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.057138+0000","last_change":"2023-09-23T21:15:36.717042+0000","last_active":"2023-09-23T21:15:41.057138+0000","last_peered":"2023-09-23T21:15:41.057138+0000","last_clean":"2023-09-23T21:15:41.057138+0000","last_became_active":"2023-09-23T21:15:36.716735+0000","last_became_peered":"2023-09-23T21:15:36.716735+0000","last_unstale":"2023-09-23T21:15:41.057138+0000","last_undegraded":"2023-09-23T21:15:41.057138+0000","last_fullsized":"2023-09-23T21:15:41.057138+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:33.432298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":129,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.703727+0000","last_change":"2023-09-23T21:15:36.735118+0000","last_active":"2023-09-23T21:15:37.703727+0000","last_peered":"2023-09-23T21:15:37.703727+0000","last_clean":"2023-09-23T21:15:37.703727+0000","last_became_active":"2023-09-23T21:15:36.699810+0000","last_became_peered":"2023-09-23T21:15:36.699810+0000","last_unstale":"2023-09-23T21:15:37.703727+0000","last_undegraded":"2023-09-23T21:15:37.703727+0000","last_fullsized":"2023-09-23T21:15:37.703727+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:45.321201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":450,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696789+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:15:37.696789+0000","last_peered":"2023-09-23T21:15:37.696789+0000","last_clean":"2023-09-23T21:15:37.696789+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:15:37.696789+0000","last_undegraded":"2023-09-23T21:15:37.696789+0000","last_fullsized":"2023-09-23T21:15:37.696789+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":105,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.707353+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:15:37.707353+0000","last_peered":"2023-09-23T21:15:37.707353+0000","last_clean":"2023-09-23T21:15:37.707353+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:15:37.707353+0000","last_undegraded":"2023-09-23T21:15:37.707353+0000","last_fullsized":"2023-09-23T21:15:37.707353+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":173,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.703566+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:15:37.703566+0000","last_peered":"2023-09-23T21:15:37.703566+0000","last_clean":"2023-09-23T21:15:37.703566+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:15:37.703566+0000","last_undegraded":"2023-09-23T21:15:37.703566+0000","last_fullsized":"2023-09-23T21:15:37.703566+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":101,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704787+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:15:37.704787+0000","last_peered":"2023-09-23T21:15:37.704787+0000","last_clean":"2023-09-23T21:15:37.704787+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:15:37.704787+0000","last_undegraded":"2023-09-23T21:15:37.704787+0000","last_fullsized":"2023-09-23T21:15:37.704787+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.708341+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:15:37.708341+0000","last_peered":"2023-09-23T21:15:37.708341+0000","last_clean":"2023-09-23T21:15:37.708341+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:15:37.708341+0000","last_undegraded":"2023-09-23T21:15:37.708341+0000","last_fullsized":"2023-09-23T21:15:37.708341+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704605+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:15:37.704605+0000","last_peered":"2023-09-23T21:15:37.704605+0000","last_clean":"2023-09-23T21:15:37.704605+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:15:37.704605+0000","last_undegraded":"2023-09-23T21:15:37.704605+0000","last_fullsized":"2023-09-23T21:15:37.704605+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":123,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.702531+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:15:37.702531+0000","last_peered":"2023-09-23T21:15:37.702531+0000","last_clean":"2023-09-23T21:15:37.702531+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:15:37.702531+0000","last_undegraded":"2023-09-23T21:15:37.702531+0000","last_fullsized":"2023-09-23T21:15:37.702531+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":558,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696099+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:15:37.696099+0000","last_peered":"2023-09-23T21:15:37.696099+0000","last_clean":"2023-09-23T21:15:37.696099+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:15:37.696099+0000","last_undegraded":"2023-09-23T21:15:37.696099+0000","last_fullsized":"2023-09-23T21:15:37.696099+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":173,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.771150+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:15:37.771150+0000","last_peered":"2023-09-23T21:15:37.771150+0000","last_clean":"2023-09-23T21:15:37.771150+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:15:37.771150+0000","last_undegraded":"2023-09-23T21:15:37.771150+0000","last_fullsized":"2023-09-23T21:15:37.771150+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"522'4396","reported_seq":4465,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:45.976476+0000","last_change":"2023-09-23T21:15:38.014276+0000","last_active":"2023-09-23T21:15:45.976476+0000","last_peered":"2023-09-23T21:15:45.976476+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:15:38.009853+0000","last_became_peered":"2023-09-23T21:15:38.009853+0000","last_unstale":"2023-09-23T21:15:45.976476+0000","last_undegraded":"2023-09-23T21:15:37.722932+0000","last_fullsized":"2023-09-23T21:15:37.695966+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4396,"log_dups_size":0,"ondisk_log_size":4396,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180846592,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2759,"num_write_kb":176608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":195},{"shards":"2,4,6","objects":2564}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":103,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.771244+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:15:37.771244+0000","last_peered":"2023-09-23T21:15:37.771244+0000","last_clean":"2023-09-23T21:15:37.771244+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:15:37.771244+0000","last_undegraded":"2023-09-23T21:15:37.771244+0000","last_fullsized":"2023-09-23T21:15:37.771244+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":165,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.700044+0000","last_change":"2023-09-23T21:15:36.714867+0000","last_active":"2023-09-23T21:15:37.700044+0000","last_peered":"2023-09-23T21:15:37.700044+0000","last_clean":"2023-09-23T21:15:37.700044+0000","last_became_active":"2023-09-23T21:15:36.714569+0000","last_became_peered":"2023-09-23T21:15:36.714569+0000","last_unstale":"2023-09-23T21:15:37.700044+0000","last_undegraded":"2023-09-23T21:15:37.700044+0000","last_fullsized":"2023-09-23T21:15:37.700044+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:05.675620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"522'4420","reported_seq":6496,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.391597+0000","last_change":"2023-09-23T21:15:37.924162+0000","last_active":"2023-09-23T21:15:43.391597+0000","last_peered":"2023-09-23T21:15:43.391597+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:15:37.916339+0000","last_became_peered":"2023-09-23T21:15:37.916339+0000","last_unstale":"2023-09-23T21:15:43.391597+0000","last_undegraded":"2023-09-23T21:15:37.720532+0000","last_fullsized":"2023-09-23T21:15:37.701179+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":4420,"log_dups_size":0,"ondisk_log_size":4420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:56:07.285781+0000","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181600256,"num_objects":2771,"num_object_clones":0,"num_object_copies":8313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":995,"num_write_kb":63680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2623},{"shards":"2,7","objects":148}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":117,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.705076+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:15:37.705076+0000","last_peered":"2023-09-23T21:15:37.705076+0000","last_clean":"2023-09-23T21:15:37.705076+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:15:37.705076+0000","last_undegraded":"2023-09-23T21:15:37.705076+0000","last_fullsized":"2023-09-23T21:15:37.705076+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":343,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704416+0000","last_change":"2023-09-23T21:15:36.710858+0000","last_active":"2023-09-23T21:15:37.704416+0000","last_peered":"2023-09-23T21:15:37.704416+0000","last_clean":"2023-09-23T21:15:37.704416+0000","last_became_active":"2023-09-23T21:15:36.710564+0000","last_became_peered":"2023-09-23T21:15:36.710564+0000","last_unstale":"2023-09-23T21:15:37.704416+0000","last_undegraded":"2023-09-23T21:15:37.704416+0000","last_fullsized":"2023-09-23T21:15:37.704416+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:34.074726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.3","version":"522'4489","reported_seq":4605,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.429818+0000","last_change":"2023-09-23T21:15:12.859496+0000","last_active":"2023-09-23T21:15:43.429818+0000","last_peered":"2023-09-23T21:15:43.429818+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:15:12.855788+0000","last_became_peered":"2023-09-23T21:15:12.855788+0000","last_unstale":"2023-09-23T21:15:43.429818+0000","last_undegraded":"2023-09-23T21:15:12.705784+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":4489,"log_dups_size":0,"ondisk_log_size":4489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:08.789183+0000","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185729024,"num_objects":2834,"num_object_clones":0,"num_object_copies":8502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2022,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":861,"num_write_kb":55104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":2022},{"shards":"4,5","objects":812}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":103,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.708351+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:15:37.708351+0000","last_peered":"2023-09-23T21:15:37.708351+0000","last_clean":"2023-09-23T21:15:37.708351+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:15:37.708351+0000","last_undegraded":"2023-09-23T21:15:37.708351+0000","last_fullsized":"2023-09-23T21:15:37.708351+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":405,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704867+0000","last_change":"2023-09-23T21:15:36.719723+0000","last_active":"2023-09-23T21:15:37.704867+0000","last_peered":"2023-09-23T21:15:37.704867+0000","last_clean":"2023-09-23T21:15:37.704867+0000","last_became_active":"2023-09-23T21:15:36.719022+0000","last_became_peered":"2023-09-23T21:15:36.719022+0000","last_unstale":"2023-09-23T21:15:37.704867+0000","last_undegraded":"2023-09-23T21:15:37.704867+0000","last_fullsized":"2023-09-23T21:15:37.704867+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:40.754123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"522'4407","reported_seq":4499,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.406010+0000","last_change":"2023-09-23T21:15:36.820548+0000","last_active":"2023-09-23T21:15:43.406010+0000","last_peered":"2023-09-23T21:15:43.406010+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:15:12.940714+0000","last_became_peered":"2023-09-23T21:15:12.940714+0000","last_unstale":"2023-09-23T21:15:43.406010+0000","last_undegraded":"2023-09-23T21:15:36.819776+0000","last_fullsized":"2023-09-23T21:15:12.681398+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4407,"log_dups_size":0,"ondisk_log_size":4407,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179961856,"num_objects":2746,"num_object_clones":0,"num_object_copies":8238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1907,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2746,"num_write_kb":175744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1907},{"shards":"4,5","objects":839}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":307,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.057231+0000","last_change":"2023-09-23T21:15:36.717524+0000","last_active":"2023-09-23T21:15:41.057231+0000","last_peered":"2023-09-23T21:15:41.057231+0000","last_clean":"2023-09-23T21:15:41.057231+0000","last_became_active":"2023-09-23T21:15:36.716684+0000","last_became_peered":"2023-09-23T21:15:36.716684+0000","last_unstale":"2023-09-23T21:15:41.057231+0000","last_undegraded":"2023-09-23T21:15:41.057231+0000","last_fullsized":"2023-09-23T21:15:41.057231+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:19.720726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.706960+0000","last_change":"2023-09-23T21:15:36.718920+0000","last_active":"2023-09-23T21:15:37.706960+0000","last_peered":"2023-09-23T21:15:37.706960+0000","last_clean":"2023-09-23T21:15:37.706960+0000","last_became_active":"2023-09-23T21:15:36.718644+0000","last_became_peered":"2023-09-23T21:15:36.718644+0000","last_unstale":"2023-09-23T21:15:37.706960+0000","last_undegraded":"2023-09-23T21:15:37.706960+0000","last_fullsized":"2023-09-23T21:15:37.706960+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:51.768643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.5","version":"522'4396","reported_seq":4567,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.437016+0000","last_change":"2023-09-23T21:15:36.812668+0000","last_active":"2023-09-23T21:15:43.437016+0000","last_peered":"2023-09-23T21:15:43.437016+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:15:36.809179+0000","last_became_peered":"2023-09-23T21:15:36.809179+0000","last_unstale":"2023-09-23T21:15:43.437016+0000","last_undegraded":"2023-09-23T21:15:36.711291+0000","last_fullsized":"2023-09-23T21:15:36.684980+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":4396,"log_dups_size":0,"ondisk_log_size":4396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:11.989879+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176357376,"num_objects":2691,"num_object_clones":0,"num_object_copies":8073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2543,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":840,"num_write_kb":53760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1943},{"shards":"4,5","objects":148},{"shards":"4,5,6","objects":600}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":189,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.707224+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:15:37.707224+0000","last_peered":"2023-09-23T21:15:37.707224+0000","last_clean":"2023-09-23T21:15:37.707224+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:15:37.707224+0000","last_undegraded":"2023-09-23T21:15:37.707224+0000","last_fullsized":"2023-09-23T21:15:37.707224+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":338,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.703373+0000","last_change":"2023-09-23T21:15:36.700128+0000","last_active":"2023-09-23T21:15:37.703373+0000","last_peered":"2023-09-23T21:15:37.703373+0000","last_clean":"2023-09-23T21:15:37.703373+0000","last_became_active":"2023-09-23T21:15:36.699374+0000","last_became_peered":"2023-09-23T21:15:36.699374+0000","last_unstale":"2023-09-23T21:15:37.703373+0000","last_undegraded":"2023-09-23T21:15:37.703373+0000","last_fullsized":"2023-09-23T21:15:37.703373+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:40.893590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.4","version":"522'4485","reported_seq":6744,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:46.031005+0000","last_change":"2023-09-23T21:15:37.918328+0000","last_active":"2023-09-23T21:15:46.031005+0000","last_peered":"2023-09-23T21:15:46.031005+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:15:37.913417+0000","last_became_peered":"2023-09-23T21:15:37.913417+0000","last_unstale":"2023-09-23T21:15:46.031005+0000","last_undegraded":"2023-09-23T21:15:37.722000+0000","last_fullsized":"2023-09-23T21:15:37.696311+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":4485,"log_dups_size":0,"ondisk_log_size":4485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:53.357576+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184352768,"num_objects":2813,"num_object_clones":0,"num_object_copies":8439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2638,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":785,"num_write_kb":50240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":175},{"shards":"2,5,6","objects":2638}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":466,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.700450+0000","last_change":"2023-09-23T21:15:36.716276+0000","last_active":"2023-09-23T21:15:37.700450+0000","last_peered":"2023-09-23T21:15:37.700450+0000","last_clean":"2023-09-23T21:15:37.700450+0000","last_became_active":"2023-09-23T21:15:36.715995+0000","last_became_peered":"2023-09-23T21:15:36.715995+0000","last_unstale":"2023-09-23T21:15:37.700450+0000","last_undegraded":"2023-09-23T21:15:37.700450+0000","last_fullsized":"2023-09-23T21:15:37.700450+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:02.365152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":554,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696545+0000","last_change":"2023-09-23T21:15:36.785937+0000","last_active":"2023-09-23T21:15:37.696545+0000","last_peered":"2023-09-23T21:15:37.696545+0000","last_clean":"2023-09-23T21:15:37.696545+0000","last_became_active":"2023-09-23T21:15:36.785603+0000","last_became_peered":"2023-09-23T21:15:36.785603+0000","last_unstale":"2023-09-23T21:15:37.696545+0000","last_undegraded":"2023-09-23T21:15:37.696545+0000","last_fullsized":"2023-09-23T21:15:37.696545+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:51.148832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"522'4348","reported_seq":4423,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.259688+0000","last_change":"2023-09-23T21:15:37.925459+0000","last_active":"2023-09-23T21:15:43.259688+0000","last_peered":"2023-09-23T21:15:43.259688+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:15:37.908666+0000","last_became_peered":"2023-09-23T21:15:37.908666+0000","last_unstale":"2023-09-23T21:15:43.259688+0000","last_undegraded":"2023-09-23T21:15:37.734531+0000","last_fullsized":"2023-09-23T21:15:37.717463+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4348,"log_dups_size":0,"ondisk_log_size":4348,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2705,"num_write_kb":173120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":2535},{"shards":"4,7","objects":170}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":159,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.700186+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:15:37.700186+0000","last_peered":"2023-09-23T21:15:37.700186+0000","last_clean":"2023-09-23T21:15:37.700186+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:15:37.700186+0000","last_undegraded":"2023-09-23T21:15:37.700186+0000","last_fullsized":"2023-09-23T21:15:37.700186+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.702197+0000","last_change":"2023-09-23T21:15:36.720164+0000","last_active":"2023-09-23T21:15:37.702197+0000","last_peered":"2023-09-23T21:15:37.702197+0000","last_clean":"2023-09-23T21:15:37.702197+0000","last_became_active":"2023-09-23T21:15:36.719910+0000","last_became_peered":"2023-09-23T21:15:36.719910+0000","last_unstale":"2023-09-23T21:15:37.702197+0000","last_undegraded":"2023-09-23T21:15:37.702197+0000","last_fullsized":"2023-09-23T21:15:37.702197+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:10.098103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.6","version":"522'4346","reported_seq":4522,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.403708+0000","last_change":"2023-09-23T21:15:36.787083+0000","last_active":"2023-09-23T21:15:43.403708+0000","last_peered":"2023-09-23T21:15:43.403708+0000","last_clean":"2023-09-23T21:15:09.799249+0000","last_became_active":"2023-09-23T21:15:12.799540+0000","last_became_peered":"2023-09-23T21:15:12.799540+0000","last_unstale":"2023-09-23T21:15:43.403708+0000","last_undegraded":"2023-09-23T21:15:36.785930+0000","last_fullsized":"2023-09-23T21:15:12.679693+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":4346,"log_dups_size":0,"ondisk_log_size":4346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:59.312702+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175505408,"num_objects":2678,"num_object_clones":0,"num_object_copies":8034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":733,"num_write_kb":46912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1938},{"shards":"4,5","objects":740}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":173,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.722262+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:15:37.722262+0000","last_peered":"2023-09-23T21:15:37.722262+0000","last_clean":"2023-09-23T21:15:37.722262+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:15:37.722262+0000","last_undegraded":"2023-09-23T21:15:37.722262+0000","last_fullsized":"2023-09-23T21:15:37.722262+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":261,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.770815+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:15:37.770815+0000","last_peered":"2023-09-23T21:15:37.770815+0000","last_clean":"2023-09-23T21:15:37.770815+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:15:37.770815+0000","last_undegraded":"2023-09-23T21:15:37.770815+0000","last_fullsized":"2023-09-23T21:15:37.770815+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"522'5790","reported_seq":6696,"reported_epoch":522,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:15:46.042640+0000","last_change":"2023-09-23T21:15:36.828415+0000","last_active":"2023-09-23T21:15:46.042640+0000","last_peered":"2023-09-23T21:15:46.042640+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:15:12.862954+0000","last_became_peered":"2023-09-23T21:15:12.862954+0000","last_unstale":"2023-09-23T21:15:46.042640+0000","last_undegraded":"2023-09-23T21:15:46.042640+0000","last_fullsized":"2023-09-23T21:15:12.676181+0000","mapping_epoch":514,"log_start":"492'1083","ondisk_log_start":"492'1083","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4707,"log_dups_size":0,"ondisk_log_size":4707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:55.849205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379453440,"num_objects":5790,"num_object_clones":0,"num_object_copies":17370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3595,"num_objects_unfound":0,"num_objects_dirty":5790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5790,"num_write_kb":370560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":424,"num_bytes_recovered":27787264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":3595},{"shards":"2,5","objects":2195}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":446,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.703643+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:15:37.703643+0000","last_peered":"2023-09-23T21:15:37.703643+0000","last_clean":"2023-09-23T21:15:37.703643+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:15:37.703643+0000","last_undegraded":"2023-09-23T21:15:37.703643+0000","last_fullsized":"2023-09-23T21:15:37.703643+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":197,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.701984+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:15:37.701984+0000","last_peered":"2023-09-23T21:15:37.701984+0000","last_clean":"2023-09-23T21:15:37.701984+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:15:37.701984+0000","last_undegraded":"2023-09-23T21:15:37.701984+0000","last_fullsized":"2023-09-23T21:15:37.701984+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.703735+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:15:37.703735+0000","last_peered":"2023-09-23T21:15:37.703735+0000","last_clean":"2023-09-23T21:15:37.703735+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:15:37.703735+0000","last_undegraded":"2023-09-23T21:15:37.703735+0000","last_fullsized":"2023-09-23T21:15:37.703735+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"522'5529","reported_seq":7627,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.432622+0000","last_change":"2023-09-23T21:15:36.805204+0000","last_active":"2023-09-23T21:15:43.432622+0000","last_peered":"2023-09-23T21:15:43.432622+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:15:12.880410+0000","last_became_peered":"2023-09-23T21:15:12.880410+0000","last_unstale":"2023-09-23T21:15:43.432622+0000","last_undegraded":"2023-09-23T21:15:36.753137+0000","last_fullsized":"2023-09-23T21:15:12.707776+0000","mapping_epoch":514,"log_start":"493'1364","ondisk_log_start":"493'1364","created":482,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4165,"log_dups_size":0,"ondisk_log_size":4165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:41.662804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362348544,"num_objects":5529,"num_object_clones":0,"num_object_copies":16587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3951,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5529,"num_write_kb":353856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1008,"num_bytes_recovered":66060288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3951},{"shards":"4,7","objects":1578}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":371,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704718+0000","last_change":"2023-09-23T21:15:36.719951+0000","last_active":"2023-09-23T21:15:37.704718+0000","last_peered":"2023-09-23T21:15:37.704718+0000","last_clean":"2023-09-23T21:15:37.704718+0000","last_became_active":"2023-09-23T21:15:36.719432+0000","last_became_peered":"2023-09-23T21:15:36.719432+0000","last_unstale":"2023-09-23T21:15:37.704718+0000","last_undegraded":"2023-09-23T21:15:37.704718+0000","last_fullsized":"2023-09-23T21:15:37.704718+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:59.778469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.057168+0000","last_change":"2023-09-23T21:15:36.718996+0000","last_active":"2023-09-23T21:15:41.057168+0000","last_peered":"2023-09-23T21:15:41.057168+0000","last_clean":"2023-09-23T21:15:41.057168+0000","last_became_active":"2023-09-23T21:15:36.717461+0000","last_became_peered":"2023-09-23T21:15:36.717461+0000","last_unstale":"2023-09-23T21:15:41.057168+0000","last_undegraded":"2023-09-23T21:15:41.057168+0000","last_fullsized":"2023-09-23T21:15:41.057168+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:51.584158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":226,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.722887+0000","last_change":"2023-09-23T21:15:36.718291+0000","last_active":"2023-09-23T21:15:37.722887+0000","last_peered":"2023-09-23T21:15:37.722887+0000","last_clean":"2023-09-23T21:15:37.722887+0000","last_became_active":"2023-09-23T21:15:36.717181+0000","last_became_peered":"2023-09-23T21:15:36.717181+0000","last_unstale":"2023-09-23T21:15:37.722887+0000","last_undegraded":"2023-09-23T21:15:37.722887+0000","last_fullsized":"2023-09-23T21:15:37.722887+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:57.775200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"522'5378","reported_seq":5426,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:15:43.365491+0000","last_change":"2023-09-23T21:14:12.651062+0000","last_active":"2023-09-23T21:15:43.365491+0000","last_peered":"2023-09-23T21:15:43.365491+0000","last_clean":"2023-09-23T21:15:43.365491+0000","last_became_active":"2023-09-23T21:14:12.650805+0000","last_became_peered":"2023-09-23T21:14:12.650805+0000","last_unstale":"2023-09-23T21:15:43.365491+0000","last_undegraded":"2023-09-23T21:15:43.365491+0000","last_fullsized":"2023-09-23T21:15:43.365491+0000","mapping_epoch":482,"log_start":"489'833","ondisk_log_start":"489'833","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4545,"log_dups_size":0,"ondisk_log_size":4545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:43.223080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452608,"num_objects":5378,"num_object_clones":0,"num_object_copies":16134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5378,"num_write_kb":344192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":530,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.695936+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:15:37.695936+0000","last_peered":"2023-09-23T21:15:37.695936+0000","last_clean":"2023-09-23T21:15:37.695936+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:15:37.695936+0000","last_undegraded":"2023-09-23T21:15:37.695936+0000","last_fullsized":"2023-09-23T21:15:37.695936+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":258,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.695659+0000","last_change":"2023-09-23T21:15:36.718723+0000","last_active":"2023-09-23T21:15:37.695659+0000","last_peered":"2023-09-23T21:15:37.695659+0000","last_clean":"2023-09-23T21:15:37.695659+0000","last_became_active":"2023-09-23T21:15:36.716966+0000","last_became_peered":"2023-09-23T21:15:36.716966+0000","last_unstale":"2023-09-23T21:15:37.695659+0000","last_undegraded":"2023-09-23T21:15:37.695659+0000","last_fullsized":"2023-09-23T21:15:37.695659+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:10.887236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":175,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.770898+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:15:37.770898+0000","last_peered":"2023-09-23T21:15:37.770898+0000","last_clean":"2023-09-23T21:15:37.770898+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:15:37.770898+0000","last_undegraded":"2023-09-23T21:15:37.770898+0000","last_fullsized":"2023-09-23T21:15:37.770898+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"522'5578","reported_seq":7656,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:15:43.446686+0000","last_change":"2023-09-23T21:14:56.664518+0000","last_active":"2023-09-23T21:15:43.446686+0000","last_peered":"2023-09-23T21:15:43.446686+0000","last_clean":"2023-09-23T21:15:43.446686+0000","last_became_active":"2023-09-23T21:14:51.384818+0000","last_became_peered":"2023-09-23T21:14:51.384818+0000","last_unstale":"2023-09-23T21:15:43.446686+0000","last_undegraded":"2023-09-23T21:15:43.446686+0000","last_fullsized":"2023-09-23T21:15:43.446686+0000","mapping_epoch":501,"log_start":"493'1544","ondisk_log_start":"493'1544","created":482,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4034,"log_dups_size":0,"ondisk_log_size":4034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:05.711518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365559808,"num_objects":5578,"num_object_clones":0,"num_object_copies":16734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5578,"num_write_kb":356992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1005,"num_bytes_recovered":65863680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":350,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.057213+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:15:41.057213+0000","last_peered":"2023-09-23T21:15:41.057213+0000","last_clean":"2023-09-23T21:15:41.057213+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:15:41.057213+0000","last_undegraded":"2023-09-23T21:15:41.057213+0000","last_fullsized":"2023-09-23T21:15:41.057213+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":856,"reported_epoch":521,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:37.889694+0000","last_change":"2023-09-23T21:15:37.889694+0000","last_active":"2023-09-23T21:15:37.889694+0000","last_peered":"2023-09-23T21:15:37.889694+0000","last_clean":"2023-09-23T21:15:35.435956+0000","last_became_active":"2023-09-23T21:15:37.879389+0000","last_became_peered":"2023-09-23T21:15:37.879389+0000","last_unstale":"2023-09-23T21:15:37.889694+0000","last_undegraded":"2023-09-23T21:15:37.700827+0000","last_fullsized":"2023-09-23T21:15:37.700230+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:15.483465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.703329+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:15:37.703329+0000","last_peered":"2023-09-23T21:15:37.703329+0000","last_clean":"2023-09-23T21:15:37.703329+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:15:37.703329+0000","last_undegraded":"2023-09-23T21:15:37.703329+0000","last_fullsized":"2023-09-23T21:15:37.703329+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":337,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.699298+0000","last_change":"2023-09-23T21:15:36.805725+0000","last_active":"2023-09-23T21:15:37.699298+0000","last_peered":"2023-09-23T21:15:37.699298+0000","last_clean":"2023-09-23T21:15:37.699298+0000","last_became_active":"2023-09-23T21:15:36.778557+0000","last_became_peered":"2023-09-23T21:15:36.778557+0000","last_unstale":"2023-09-23T21:15:37.699298+0000","last_undegraded":"2023-09-23T21:15:37.699298+0000","last_fullsized":"2023-09-23T21:15:37.699298+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:24.564634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"522'5467","reported_seq":5525,"reported_epoch":522,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:15:43.360300+0000","last_change":"2023-09-23T21:15:37.912383+0000","last_active":"2023-09-23T21:15:43.360300+0000","last_peered":"2023-09-23T21:15:43.360300+0000","last_clean":"2023-09-23T21:15:36.326799+0000","last_became_active":"2023-09-23T21:15:37.894353+0000","last_became_peered":"2023-09-23T21:15:37.894353+0000","last_unstale":"2023-09-23T21:15:43.360300+0000","last_undegraded":"2023-09-23T21:15:43.360300+0000","last_fullsized":"2023-09-23T21:15:43.360300+0000","mapping_epoch":520,"log_start":"489'900","ondisk_log_start":"489'900","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4567,"log_dups_size":0,"ondisk_log_size":4567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:03.607087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358285312,"num_objects":5467,"num_object_clones":0,"num_object_copies":16401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5467,"num_objects_unfound":0,"num_objects_dirty":5467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5467,"num_write_kb":349888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":5467}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":139,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.722162+0000","last_change":"2023-09-23T21:15:36.718478+0000","last_active":"2023-09-23T21:15:37.722162+0000","last_peered":"2023-09-23T21:15:37.722162+0000","last_clean":"2023-09-23T21:15:37.722162+0000","last_became_active":"2023-09-23T21:15:36.717508+0000","last_became_peered":"2023-09-23T21:15:36.717508+0000","last_unstale":"2023-09-23T21:15:37.722162+0000","last_undegraded":"2023-09-23T21:15:37.722162+0000","last_fullsized":"2023-09-23T21:15:37.722162+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:23.183805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":18,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704514+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:15:37.704514+0000","last_peered":"2023-09-23T21:15:37.704514+0000","last_clean":"2023-09-23T21:15:37.704514+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:15:37.704514+0000","last_undegraded":"2023-09-23T21:15:37.704514+0000","last_fullsized":"2023-09-23T21:15:37.704514+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":430,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.697851+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:15:37.697851+0000","last_peered":"2023-09-23T21:15:37.697851+0000","last_clean":"2023-09-23T21:15:37.697851+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:15:37.697851+0000","last_undegraded":"2023-09-23T21:15:37.697851+0000","last_fullsized":"2023-09-23T21:15:37.697851+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"522'5539","reported_seq":7625,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:15:46.029830+0000","last_change":"2023-09-23T21:14:43.693596+0000","last_active":"2023-09-23T21:15:46.029830+0000","last_peered":"2023-09-23T21:15:46.029830+0000","last_clean":"2023-09-23T21:15:46.029830+0000","last_became_active":"2023-09-23T21:14:43.690785+0000","last_became_peered":"2023-09-23T21:14:43.690785+0000","last_unstale":"2023-09-23T21:15:46.029830+0000","last_undegraded":"2023-09-23T21:15:46.029830+0000","last_fullsized":"2023-09-23T21:15:46.029830+0000","mapping_epoch":498,"log_start":"489'841","ondisk_log_start":"489'841","created":482,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4698,"log_dups_size":0,"ondisk_log_size":4698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:00.973146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363003904,"num_objects":5539,"num_object_clones":0,"num_object_copies":16617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5539,"num_write_kb":354496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1010,"num_bytes_recovered":66191360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":119,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.705699+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:15:37.705699+0000","last_peered":"2023-09-23T21:15:37.705699+0000","last_clean":"2023-09-23T21:15:37.705699+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:15:37.705699+0000","last_undegraded":"2023-09-23T21:15:37.705699+0000","last_fullsized":"2023-09-23T21:15:37.705699+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":325,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.722984+0000","last_change":"2023-09-23T21:15:36.718380+0000","last_active":"2023-09-23T21:15:37.722984+0000","last_peered":"2023-09-23T21:15:37.722984+0000","last_clean":"2023-09-23T21:15:37.722984+0000","last_became_active":"2023-09-23T21:15:36.717363+0000","last_became_peered":"2023-09-23T21:15:36.717363+0000","last_unstale":"2023-09-23T21:15:37.722984+0000","last_undegraded":"2023-09-23T21:15:37.722984+0000","last_fullsized":"2023-09-23T21:15:37.722984+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:11.961939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":227,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.704883+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:15:37.704883+0000","last_peered":"2023-09-23T21:15:37.704883+0000","last_clean":"2023-09-23T21:15:37.704883+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:15:37.704883+0000","last_undegraded":"2023-09-23T21:15:37.704883+0000","last_fullsized":"2023-09-23T21:15:37.704883+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"522'4353","reported_seq":4518,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.381552+0000","last_change":"2023-09-23T21:15:38.027486+0000","last_active":"2023-09-23T21:15:43.381552+0000","last_peered":"2023-09-23T21:15:43.381552+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:15:38.015308+0000","last_became_peered":"2023-09-23T21:15:38.015308+0000","last_unstale":"2023-09-23T21:15:43.381552+0000","last_undegraded":"2023-09-23T21:15:37.751992+0000","last_fullsized":"2023-09-23T21:15:37.734941+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":4353,"log_dups_size":0,"ondisk_log_size":4353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:13.280295+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178913280,"num_objects":2730,"num_object_clones":0,"num_object_copies":8190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":791,"num_write_kb":50624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":155},{"shards":"2,5,6","objects":2575}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":177,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.771341+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:15:37.771341+0000","last_peered":"2023-09-23T21:15:37.771341+0000","last_clean":"2023-09-23T21:15:37.771341+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:15:37.771341+0000","last_undegraded":"2023-09-23T21:15:37.771341+0000","last_fullsized":"2023-09-23T21:15:37.771341+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":365,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.695802+0000","last_change":"2023-09-23T21:15:36.715325+0000","last_active":"2023-09-23T21:15:37.695802+0000","last_peered":"2023-09-23T21:15:37.695802+0000","last_clean":"2023-09-23T21:15:37.695802+0000","last_became_active":"2023-09-23T21:15:36.714995+0000","last_became_peered":"2023-09-23T21:15:36.714995+0000","last_unstale":"2023-09-23T21:15:37.695802+0000","last_undegraded":"2023-09-23T21:15:37.695802+0000","last_fullsized":"2023-09-23T21:15:37.695802+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:21.699827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":192,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.722058+0000","last_change":"2023-09-23T21:15:36.718660+0000","last_active":"2023-09-23T21:15:37.722058+0000","last_peered":"2023-09-23T21:15:37.722058+0000","last_clean":"2023-09-23T21:15:37.722058+0000","last_became_active":"2023-09-23T21:15:36.717919+0000","last_became_peered":"2023-09-23T21:15:36.717919+0000","last_unstale":"2023-09-23T21:15:37.722058+0000","last_undegraded":"2023-09-23T21:15:37.722058+0000","last_fullsized":"2023-09-23T21:15:37.722058+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:59.596117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"522'4521","reported_seq":4634,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:46.029211+0000","last_change":"2023-09-23T21:15:37.899429+0000","last_active":"2023-09-23T21:15:46.029211+0000","last_peered":"2023-09-23T21:15:46.029211+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:15:37.895550+0000","last_became_peered":"2023-09-23T21:15:37.895550+0000","last_unstale":"2023-09-23T21:15:46.029211+0000","last_undegraded":"2023-09-23T21:15:37.722673+0000","last_fullsized":"2023-09-23T21:15:37.695773+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":4521,"log_dups_size":0,"ondisk_log_size":4521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:11.453135+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597952,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1033,"num_write_kb":66112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":206},{"shards":"2,5,6","objects":2626}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":428,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696789+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:15:37.696789+0000","last_peered":"2023-09-23T21:15:37.696789+0000","last_clean":"2023-09-23T21:15:37.696789+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:15:37.696789+0000","last_undegraded":"2023-09-23T21:15:37.696789+0000","last_fullsized":"2023-09-23T21:15:37.696789+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":59,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696032+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:15:37.696032+0000","last_peered":"2023-09-23T21:15:37.696032+0000","last_clean":"2023-09-23T21:15:37.696032+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:15:37.696032+0000","last_undegraded":"2023-09-23T21:15:37.696032+0000","last_fullsized":"2023-09-23T21:15:37.696032+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":207,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.753202+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:15:37.753202+0000","last_peered":"2023-09-23T21:15:37.753202+0000","last_clean":"2023-09-23T21:15:37.753202+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:15:37.753202+0000","last_undegraded":"2023-09-23T21:15:37.753202+0000","last_fullsized":"2023-09-23T21:15:37.753202+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"522'4455","reported_seq":4541,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.434750+0000","last_change":"2023-09-23T21:15:36.812663+0000","last_active":"2023-09-23T21:15:43.434750+0000","last_peered":"2023-09-23T21:15:43.434750+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:15:36.808663+0000","last_became_peered":"2023-09-23T21:15:36.808663+0000","last_unstale":"2023-09-23T21:15:43.434750+0000","last_undegraded":"2023-09-23T21:15:36.712242+0000","last_fullsized":"2023-09-23T21:15:36.687952+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4455,"log_dups_size":0,"ondisk_log_size":4455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182321152,"num_objects":2782,"num_object_clones":0,"num_object_copies":8346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2782,"num_write_kb":178048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":2007},{"shards":"4,5","objects":116},{"shards":"4,5,6","objects":659}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":149,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.707430+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:15:37.707430+0000","last_peered":"2023-09-23T21:15:37.707430+0000","last_clean":"2023-09-23T21:15:37.707430+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:15:37.707430+0000","last_undegraded":"2023-09-23T21:15:37.707430+0000","last_fullsized":"2023-09-23T21:15:37.707430+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":584,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.723067+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:15:37.723067+0000","last_peered":"2023-09-23T21:15:37.723067+0000","last_clean":"2023-09-23T21:15:37.723067+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:15:37.723067+0000","last_undegraded":"2023-09-23T21:15:37.723067+0000","last_fullsized":"2023-09-23T21:15:37.723067+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"522'4385","reported_seq":4450,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.417378+0000","last_change":"2023-09-23T21:15:12.946273+0000","last_active":"2023-09-23T21:15:43.417378+0000","last_peered":"2023-09-23T21:15:43.417378+0000","last_clean":"2023-09-23T21:15:07.989292+0000","last_became_active":"2023-09-23T21:15:12.941733+0000","last_became_peered":"2023-09-23T21:15:12.941733+0000","last_unstale":"2023-09-23T21:15:43.417378+0000","last_undegraded":"2023-09-23T21:15:12.704994+0000","last_fullsized":"2023-09-23T21:15:12.681408+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4385,"log_dups_size":0,"ondisk_log_size":4385,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1970,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2724,"num_write_kb":174336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1970},{"shards":"4,5","objects":754}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":212,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.695266+0000","last_change":"2023-09-23T21:15:36.731913+0000","last_active":"2023-09-23T21:15:37.695266+0000","last_peered":"2023-09-23T21:15:37.695266+0000","last_clean":"2023-09-23T21:15:37.695266+0000","last_became_active":"2023-09-23T21:15:36.715596+0000","last_became_peered":"2023-09-23T21:15:36.715596+0000","last_unstale":"2023-09-23T21:15:37.695266+0000","last_undegraded":"2023-09-23T21:15:37.695266+0000","last_fullsized":"2023-09-23T21:15:37.695266+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:14.247487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696699+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:15:37.696699+0000","last_peered":"2023-09-23T21:15:37.696699+0000","last_clean":"2023-09-23T21:15:37.696699+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:15:37.696699+0000","last_undegraded":"2023-09-23T21:15:37.696699+0000","last_fullsized":"2023-09-23T21:15:37.696699+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"522'4440","reported_seq":4504,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:43.402464+0000","last_change":"2023-09-23T21:15:12.860448+0000","last_active":"2023-09-23T21:15:43.402464+0000","last_peered":"2023-09-23T21:15:43.402464+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:15:43.402464+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4440,"log_dups_size":0,"ondisk_log_size":4440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180649984,"num_objects":2757,"num_object_clones":0,"num_object_copies":8271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1941,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2757,"num_write_kb":176416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1941},{"shards":"4,5","objects":816}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.705495+0000","last_change":"2023-09-23T21:15:36.818381+0000","last_active":"2023-09-23T21:15:37.705495+0000","last_peered":"2023-09-23T21:15:37.705495+0000","last_clean":"2023-09-23T21:15:37.705495+0000","last_became_active":"2023-09-23T21:15:36.724491+0000","last_became_peered":"2023-09-23T21:15:36.724491+0000","last_unstale":"2023-09-23T21:15:37.705495+0000","last_undegraded":"2023-09-23T21:15:37.705495+0000","last_fullsized":"2023-09-23T21:15:37.705495+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:08.475242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":166,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696323+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:15:37.696323+0000","last_peered":"2023-09-23T21:15:37.696323+0000","last_clean":"2023-09-23T21:15:37.696323+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:15:37.696323+0000","last_undegraded":"2023-09-23T21:15:37.696323+0000","last_fullsized":"2023-09-23T21:15:37.696323+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"522'4384","reported_seq":10218,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T21:15:46.014336+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:15:46.014336+0000","last_peered":"2023-09-23T21:15:46.014336+0000","last_clean":"2023-09-23T21:15:46.014336+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:15:46.014336+0000","last_undegraded":"2023-09-23T21:15:46.014336+0000","last_fullsized":"2023-09-23T21:15:46.014336+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4384,"log_dups_size":0,"ondisk_log_size":4384,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178323456,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2721,"num_write_kb":174144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":14,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:41.060458+0000","last_change":"2023-09-23T21:15:36.709281+0000","last_active":"2023-09-23T21:15:41.060458+0000","last_peered":"2023-09-23T21:15:41.060458+0000","last_clean":"2023-09-23T21:15:41.060458+0000","last_became_active":"2023-09-23T21:15:36.709011+0000","last_became_peered":"2023-09-23T21:15:36.709011+0000","last_unstale":"2023-09-23T21:15:41.060458+0000","last_undegraded":"2023-09-23T21:15:41.060458+0000","last_fullsized":"2023-09-23T21:15:41.060458+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:56.451393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":372,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.696022+0000","last_change":"2023-09-23T21:15:36.718816+0000","last_active":"2023-09-23T21:15:37.696022+0000","last_peered":"2023-09-23T21:15:37.696022+0000","last_clean":"2023-09-23T21:15:37.696022+0000","last_became_active":"2023-09-23T21:15:36.718449+0000","last_became_peered":"2023-09-23T21:15:36.718449+0000","last_unstale":"2023-09-23T21:15:37.696022+0000","last_undegraded":"2023-09-23T21:15:37.696022+0000","last_fullsized":"2023-09-23T21:15:37.696022+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:36.076184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"522'4421","reported_seq":4873,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:15:46.054860+0000","last_change":"2023-09-23T21:15:36.707049+0000","last_active":"2023-09-23T21:15:46.054860+0000","last_peered":"2023-09-23T21:15:46.054860+0000","last_clean":"2023-09-23T21:15:08.796333+0000","last_became_active":"2023-09-23T21:15:11.758753+0000","last_became_peered":"2023-09-23T21:15:11.758753+0000","last_unstale":"2023-09-23T21:15:46.054860+0000","last_undegraded":"2023-09-23T21:15:36.705557+0000","last_fullsized":"2023-09-23T21:15:11.693080+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4421,"log_dups_size":0,"ondisk_log_size":4421,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182484992,"num_objects":2785,"num_object_clones":0,"num_object_copies":8355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1731,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2785,"num_write_kb":178208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":12386304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":1731},{"shards":"2,6","objects":1054}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":462,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.697853+0000","last_change":"2023-09-23T21:15:36.737658+0000","last_active":"2023-09-23T21:15:37.697853+0000","last_peered":"2023-09-23T21:15:37.697853+0000","last_clean":"2023-09-23T21:15:37.697853+0000","last_became_active":"2023-09-23T21:15:36.716750+0000","last_became_peered":"2023-09-23T21:15:36.716750+0000","last_unstale":"2023-09-23T21:15:37.697853+0000","last_undegraded":"2023-09-23T21:15:37.697853+0000","last_fullsized":"2023-09-23T21:15:37.697853+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:30.186342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-09-23T21:15:37.695139+0000","last_change":"2023-09-23T21:15:36.716348+0000","last_active":"2023-09-23T21:15:37.695139+0000","last_peered":"2023-09-23T21:15:37.695139+0000","last_clean":"2023-09-23T21:15:37.695139+0000","last_became_active":"2023-09-23T21:15:36.716039+0000","last_became_peered":"2023-09-23T21:15:36.716039+0000","last_unstale":"2023-09-23T21:15:37.695139+0000","last_undegraded":"2023-09-23T21:15:37.695139+0000","last_fullsized":"2023-09-23T21:15:37.695139+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:59.988277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":26,"stat_sum":{"num_bytes":5789777920,"num_objects":88346,"num_object_clones":0,"num_object_copies":265038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":48503,"num_objects_misplaced":9062,"num_objects_unfound":0,"num_objects_dirty":88346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73181,"num_write_kb":4683520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6569,"num_bytes_recovered":430473216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17495556096,"data_stored":17495556096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115008,"ondisk_log_size":115008,"up":78,"acting":57,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":33540,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201844,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3267208,"kb_used_data":2555840,"kb_used_omap":0,"kb_used_meta":711360,"kb_avail":90477944,"statfs":{"total":95995035648,"available":92649414656,"internally_reserved":0,"allocated":2617180160,"data_stored":2615796352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":728432640},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":4.1479999999999997,"15min":2.133},"min":{"1min":0.32100000000000001,"5min":0.214,"15min":0.20699999999999999},"max":{"1min":8.5030000000000001,"5min":141.33199999999999,"15min":141.33199999999999},"last":1.23},{"interface":"front","average":{"1min":1.6539999999999999,"5min":4.125,"15min":2.1309999999999998},"min":{"1min":0.31900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.0210000000000008,"5min":141.31700000000001,"15min":141.31700000000001},"last":0.58799999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:15:13 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":3.6629999999999998,"15min":3.2770000000000001},"min":{"1min":0.27700000000000002,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":8.9450000000000003,"5min":76.007999999999996,"15min":114.16500000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.9099999999999999,"5min":4.9989999999999997,"15min":3.802},"min":{"1min":0.29499999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.5980000000000008,"5min":117.637,"15min":117.637},"last":1.2649999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:15:05 2023","interfaces":[{"interface":"back","average":{"1min":3.012,"5min":4.9340000000000002,"15min":4.3310000000000004},"min":{"1min":0.27600000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":31.128,"5min":135.529,"15min":183.44200000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":7.6539999999999999,"5min":5.9390000000000001,"15min":4.5700000000000003},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":128.45699999999999,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.78600000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:15:05 2023","interfaces":[{"interface":"back","average":{"1min":12.387,"5min":8.5830000000000002,"15min":4.3769999999999998},"min":{"1min":0.39000000000000001,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":121.669,"5min":135.66200000000001,"15min":135.66200000000001},"last":1.29},{"interface":"front","average":{"1min":12.444000000000001,"5min":9.0999999999999996,"15min":4.4489999999999998},"min":{"1min":0.30399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":121.879,"5min":135.47,"15min":135.47},"last":0.82099999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:15:34 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":1.089,"15min":0.96399999999999997},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":10.989000000000001,"5min":10.989000000000001,"15min":15.942},"last":2.794},{"interface":"front","average":{"1min":2.6459999999999999,"5min":1.0880000000000001,"15min":0.88300000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":8.5600000000000005,"5min":8.5600000000000005,"15min":16.725999999999999},"last":2.9950000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:15:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":1.2230000000000001,"15min":1.1200000000000001},"min":{"1min":0.33500000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.231,"5min":11.231,"15min":11.231},"last":7.4450000000000003},{"interface":"front","average":{"1min":3.2610000000000001,"5min":1.29,"15min":1.044},"min":{"1min":0.33000000000000002,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":9.0440000000000005,"5min":10.005000000000001,"15min":13.867000000000001},"last":6.0049999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:15:34 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":2.6219999999999999,"15min":1.825},"min":{"1min":0.39500000000000002,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":11.038,"5min":143.18700000000001,"15min":143.18700000000001},"last":1.669},{"interface":"front","average":{"1min":2.0819999999999999,"5min":2.4289999999999998,"15min":1.724},"min":{"1min":0.38600000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.1920000000000002,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.89300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463465,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3204480,"kb_used_data":2509688,"kb_used_omap":0,"kb_used_meta":694784,"kb_avail":90540672,"statfs":{"total":95995035648,"available":92713648128,"internally_reserved":0,"allocated":2569920512,"data_stored":2568545187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":711458816},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":4.6849999999999996,"15min":2.6619999999999999},"min":{"1min":0.248,"5min":0.214,"15min":0.20999999999999999},"max":{"1min":9.3290000000000006,"5min":149.61600000000001,"15min":149.61600000000001},"last":1.169},{"interface":"front","average":{"1min":1.6160000000000001,"5min":4.6550000000000002,"15min":2.7029999999999998},"min":{"1min":0.26300000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.387,"5min":149.303,"15min":149.303},"last":0.499}]},{"osd":1,"last update":"Sat Sep 23 21:15:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":4.3150000000000004,"15min":3.4489999999999998},"min":{"1min":0.313,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.8319999999999999,"5min":129.37899999999999,"15min":129.37899999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.4339999999999999,"5min":4.3419999999999996,"15min":3.3820000000000001},"min":{"1min":0.27300000000000002,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":5.9960000000000004,"5min":129.40100000000001,"15min":129.40100000000001},"last":2.4049999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":5.9009999999999998,"15min":5.6950000000000003},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.26800000000000002},"max":{"1min":5.1719999999999997,"5min":133.46600000000001,"15min":199.23400000000001},"last":1.212},{"interface":"front","average":{"1min":1.661,"5min":5.7850000000000001,"15min":5.6310000000000002},"min":{"1min":0.309,"5min":0.28999999999999998,"15min":0.26200000000000001},"max":{"1min":5.9050000000000002,"5min":133.33000000000001,"15min":199.08000000000001},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":6.8120000000000003,"15min":4.4770000000000003},"min":{"1min":0.36699999999999999,"5min":0.30399999999999999,"15min":0.28100000000000003},"max":{"1min":7.149,"5min":137.875,"15min":178.40000000000001},"last":2.371},{"interface":"front","average":{"1min":1.27,"5min":6.5069999999999997,"15min":4.4370000000000003},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":5.048,"5min":137.60900000000001,"15min":178.26599999999999},"last":2.3860000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":0.76100000000000001,"15min":0.97499999999999998},"min":{"1min":0.34300000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.274,"5min":6.274,"15min":13.1},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.806,"5min":0.78800000000000003,"15min":0.84099999999999997},"min":{"1min":0.38900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.949999999999999,"5min":11.949999999999999,"15min":11.949999999999999},"last":2.4399999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":0.91700000000000004,"15min":1.014},"min":{"1min":0.34100000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":12.355,"5min":12.355,"15min":12.355},"last":2.4209999999999998},{"interface":"front","average":{"1min":2.2490000000000001,"5min":0.92400000000000004,"15min":1.006},"min":{"1min":0.38600000000000001,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":11.042,"5min":11.042,"15min":11.042},"last":4.3890000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:15:24 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":0.70899999999999996,"15min":0.94099999999999995},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.3559999999999999,"5min":4.3559999999999999,"15min":7.8079999999999998},"last":1.419},{"interface":"front","average":{"1min":2.2690000000000001,"5min":0.97599999999999998,"15min":1.034},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.9119999999999999,"5min":7.9119999999999999,"15min":7.9119999999999999},"last":2.931}]}]},{"osd":1,"up_from":0,"seq":1185410973911,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991651,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781542,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4355640,"kb_used_data":3362864,"kb_used_omap":7,"kb_used_meta":992760,"kb_avail":89389512,"statfs":{"total":95995035648,"available":91534860288,"internally_reserved":0,"allocated":3443572736,"data_stored":3441087473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1016586321},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:55 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":3.4239999999999999,"15min":2.149},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.189},"max":{"1min":1.7509999999999999,"5min":103.021,"15min":103.021},"last":0.248},{"interface":"front","average":{"1min":1.1240000000000001,"5min":4.0590000000000002,"15min":2.879},"min":{"1min":0.25900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.552,"5min":103.154,"15min":103.154},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:15:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":5.2480000000000002,"15min":4.3369999999999997},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.188},"max":{"1min":10.766,"5min":143.482,"15min":212.12},"last":0.309},{"interface":"front","average":{"1min":0.55400000000000005,"5min":5.7830000000000004,"15min":4.3369999999999997},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":1.6200000000000001,"5min":143.61000000000001,"15min":213.02799999999999},"last":1.474}]},{"osd":3,"last update":"Sat Sep 23 21:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":5.202,"15min":4.7460000000000004},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.6630000000000003,"5min":143.50800000000001,"15min":143.50800000000001},"last":2.0350000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":5.8970000000000002,"15min":4.8179999999999996},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":4.5270000000000001,"5min":143.27799999999999,"15min":143.27799999999999},"last":1.512}]},{"osd":4,"last update":"Sat Sep 23 21:14:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":4.3010000000000002,"15min":2.226},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.6360000000000001,"5min":225.80799999999999,"15min":225.80799999999999},"last":2.016},{"interface":"front","average":{"1min":1.3540000000000001,"5min":5.1280000000000001,"15min":2.9279999999999999},"min":{"1min":0.35499999999999998,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":11.43,"5min":225.79300000000001,"15min":225.79300000000001},"last":2.0910000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:14:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":3.6429999999999998,"15min":2.504},"min":{"1min":0.31,"5min":0.31,"15min":0.29999999999999999},"max":{"1min":11.467000000000001,"5min":98.682000000000002,"15min":98.682000000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.5189999999999999,"5min":4.5090000000000003,"15min":2.3759999999999999},"min":{"1min":0.42499999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.7089999999999996,"5min":143.34700000000001,"15min":143.34700000000001},"last":1.9990000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:14:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":3.2770000000000001,"15min":2.702},"min":{"1min":0.40200000000000002,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":5.5919999999999996,"5min":143.41399999999999,"15min":175.72499999999999},"last":5.798},{"interface":"front","average":{"1min":0.90200000000000002,"5min":1.839,"15min":2.1640000000000001},"min":{"1min":0.39700000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.008,"5min":25.204999999999998,"15min":176.20400000000001},"last":1.833}]},{"osd":7,"last update":"Sat Sep 23 21:14:55 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.8919999999999999,"15min":1.633},"min":{"1min":0.47599999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.3079999999999998,"5min":25.260000000000002,"15min":25.260000000000002},"last":6.1859999999999999},{"interface":"front","average":{"1min":1.3899999999999999,"5min":2.5910000000000002,"15min":2.1920000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":11.492000000000001,"5min":12.613,"15min":16.875},"last":0.75800000000000001}]}]},{"osd":3,"up_from":0,"seq":635655160136,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986754,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3930632,"kb_used_data":3227392,"kb_used_omap":0,"kb_used_meta":703232,"kb_avail":89814520,"statfs":{"total":95995035648,"available":91970068480,"internally_reserved":0,"allocated":3304849408,"data_stored":3303465863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":720109568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":4.6310000000000002,"15min":2.4260000000000002},"min":{"1min":0.26400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":4.782,"5min":226.751,"15min":226.751},"last":8.2149999999999999},{"interface":"front","average":{"1min":1.173,"5min":4.5739999999999998,"15min":2.4049999999999998},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.223},"max":{"1min":5.8540000000000001,"5min":226.43299999999999,"15min":226.43299999999999},"last":0.60899999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:15:20 2023","interfaces":[{"interface":"back","average":{"1min":3.9399999999999999,"5min":4.5739999999999998,"15min":5.226},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":35.179000000000002,"5min":119.55800000000001,"15min":127.411},"last":0.63800000000000001},{"interface":"front","average":{"1min":3.819,"5min":3.8860000000000001,"15min":4.4119999999999999},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":34.883000000000003,"5min":116.78100000000001,"15min":118.075},"last":8.2550000000000008}]},{"osd":2,"last update":"Sat Sep 23 21:14:51 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":6.0659999999999998,"15min":5.2489999999999997},"min":{"1min":0.34799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.1040000000000001,"5min":210.50299999999999,"15min":210.50299999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.724,"5min":6.5529999999999999,"15min":5.2699999999999996},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":7.5750000000000002,"5min":210.59,"15min":210.59},"last":8.3030000000000008}]},{"osd":3,"last update":"Sat Sep 23 21:15:20 2023","interfaces":[{"interface":"back","average":{"1min":5.5110000000000001,"5min":7.6760000000000002,"15min":4.0430000000000001},"min":{"1min":0.36199999999999999,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":74.75,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":6.0599999999999996,"5min":8.343,"15min":4.3390000000000004},"min":{"1min":0.38400000000000001,"5min":0.32700000000000001,"15min":0.25600000000000001},"max":{"1min":74.814999999999998,"5min":214.54900000000001,"15min":214.54900000000001},"last":8.2799999999999994}]},{"osd":5,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.175,"15min":0.90300000000000002},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":5.4390000000000001,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.48099999999999998},{"interface":"front","average":{"1min":3.6360000000000001,"5min":1.583,"15min":1.0669999999999999},"min":{"1min":0.307,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":11.566000000000001,"5min":34.883000000000003,"15min":34.883000000000003},"last":8.3279999999999994}]},{"osd":6,"last update":"Sat Sep 23 21:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":0.74399999999999999,"15min":1.0489999999999999},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.0919999999999996,"5min":7.0919999999999996,"15min":36.557000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.76700000000000002,"15min":1.0349999999999999},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.3289999999999997,"5min":6.6040000000000001,"15min":38.468000000000004},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.103,"15min":1.6419999999999999},"min":{"1min":0.505,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":6.2999999999999998,"5min":6.8399999999999999,"15min":145.721},"last":8.3510000000000009},{"interface":"front","average":{"1min":2.718,"5min":1.099,"15min":1.782},"min":{"1min":0.377,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":10.677,"5min":10.677,"15min":148.45699999999999},"last":1.1990000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725118,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4580920,"kb_used_data":3832432,"kb_used_omap":0,"kb_used_meta":748480,"kb_avail":89164232,"statfs":{"total":95995035648,"available":91304173568,"internally_reserved":0,"allocated":3924410368,"data_stored":3923038419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":766443520},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":2.7730000000000001,"15min":1.627},"min":{"1min":0.22600000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.7380000000000004,"5min":82.280000000000001,"15min":82.280000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.3180000000000001,"5min":2.8769999999999998,"15min":1.7689999999999999},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":4.7699999999999996,"5min":85.236000000000004,"15min":85.236000000000004},"last":2.6949999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1309999999999998,"5min":3.2229999999999999,"15min":4.4790000000000001},"min":{"1min":0.41099999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.548,"5min":74.920000000000002,"15min":233.77099999999999},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.077,"5min":2.7290000000000001,"15min":4.2560000000000002},"min":{"1min":0.30199999999999999,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":7.5030000000000001,"5min":74.978999999999999,"15min":233.75200000000001},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":3.206,"15min":3.6829999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.1070000000000002,"5min":78.917000000000002,"15min":231.34700000000001},"last":1.6839999999999999},{"interface":"front","average":{"1min":1.407,"5min":3.2160000000000002,"15min":3.8260000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27300000000000002},"max":{"1min":4.8380000000000001,"5min":78.888999999999996,"15min":231.381},"last":2.6549999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:14:50 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":7.7320000000000002,"15min":3.96},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.26800000000000002},"max":{"1min":9.3780000000000001,"5min":263.887,"15min":263.887},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.7929999999999999,"5min":7.3029999999999999,"15min":3.8090000000000002},"min":{"1min":0.40799999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.638,"5min":263.70100000000002,"15min":263.70100000000002},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.052,"15min":0.78900000000000003},"min":{"1min":0.50900000000000001,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":8.6969999999999992,"5min":8.6969999999999992,"15min":12.739000000000001},"last":3.415},{"interface":"front","average":{"1min":1.2909999999999999,"5min":0.752,"15min":0.81699999999999995},"min":{"1min":0.39700000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.3399999999999999,"5min":5.0099999999999998,"15min":10.792},"last":2.9220000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:14:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.71,"15min":1.413},"min":{"1min":0.33400000000000002,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":4.0430000000000001,"5min":83.451999999999998,"15min":83.451999999999998},"last":2.7709999999999999},{"interface":"front","average":{"1min":2.8959999999999999,"5min":2.0710000000000002,"15min":1.5549999999999999},"min":{"1min":0.33900000000000002,"5min":0.29399999999999998,"15min":0.27400000000000002},"max":{"1min":12.725,"5min":83.484999999999999,"15min":83.484999999999999},"last":2.9060000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:15:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.1279999999999999,"15min":1.03},"min":{"1min":0.55900000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.0250000000000004,"5min":7.0250000000000004,"15min":9.6329999999999991},"last":2.7400000000000002},{"interface":"front","average":{"1min":1.7110000000000001,"5min":0.91600000000000004,"15min":0.748},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":7.5670000000000002,"5min":7.5670000000000002,"15min":7.5670000000000002},"last":0.38100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4259840,"data_stored":4259840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3445948416,"data_stored":3445948416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1551433728,"data_stored":1551433728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3337814016,"data_stored":3337814016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3958702080,"data_stored":3958702080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2564030464,"data_stored":2564030464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2633367552,"data_stored":2633367552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:15:51.310 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:15:51.311 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_8 pg_num 36 2023-09-23T21:15:51.393 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:51.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:51.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:51.501 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:51.579 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:51.717 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:51.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:51.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:51.833 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:51.897 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:52.010 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:52.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:52.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:52.126 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:52.202 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:52.339 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:52.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:52.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:52.454 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:15:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:52 smithi123 ceph-mon[193903]: pgmap v1651: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 799 op/s; 48505/274845 objects degraded (17.648%); 9115/274845 objects misplaced (3.316%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48505/274845 objects degraded (17.648%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4250533558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:52.527 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:52.627 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:52 smithi121 ceph-mon[219709]: pgmap v1651: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 799 op/s; 48505/274845 objects degraded (17.648%); 9115/274845 objects misplaced (3.316%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:52.628 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48505/274845 objects degraded (17.648%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:52.628 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4250533558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:52.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:52 smithi121 ceph-mon[211526]: pgmap v1651: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 799 op/s; 48505/274845 objects degraded (17.648%); 9115/274845 objects misplaced (3.316%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:52.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48505/274845 objects degraded (17.648%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:52.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4250533558' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:15:52.632 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:52.643 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:15:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:52.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:15:52.730 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:15:52.756 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:52.791 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:52.834 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:53.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:53.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4250533558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:53.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[211526]: osdmap e523: 8 total, 8 up, 5 in 2023-09-23T21:15:53.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[211526]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[211526]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.375 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[211526]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.649 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:53.652 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:53.652 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4250533558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:53.652 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[219709]: osdmap e523: 8 total, 8 up, 5 in 2023-09-23T21:15:53.652 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[219709]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.652 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[219709]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.652 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:53 smithi121 ceph-mon[219709]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:53.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:53.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:53 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:15:53.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4250533558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:15:53.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:53 smithi123 ceph-mon[193903]: osdmap e523: 8 total, 8 up, 5 in 2023-09-23T21:15:53.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:53 smithi123 ceph-mon[193903]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:53 smithi123 ceph-mon[193903]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:53 smithi123 ceph-mon[193903]: failed to encode map e523 with expected crc 2023-09-23T21:15:53.754 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:53.842 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:53.963 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:53.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:53.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:54.080 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:54.131 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:54.597 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:54.601 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:54 smithi121 ceph-mon[211526]: failed to encode map e523 with expected crc 2023-09-23T21:15:54.601 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:54 smithi121 ceph-mon[211526]: pgmap v1653: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 799 op/s; 48505/274845 objects degraded (17.648%); 9115/274845 objects misplaced (3.316%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:54.601 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:54 smithi121 ceph-mon[219709]: failed to encode map e523 with expected crc 2023-09-23T21:15:54.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:54 smithi121 ceph-mon[219709]: pgmap v1653: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 799 op/s; 48505/274845 objects degraded (17.648%); 9115/274845 objects misplaced (3.316%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:54.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:54.633 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:15:54.633 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:15:54.633 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:15:54.633 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:15:54.633 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:15:54.704 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:54 smithi123 ceph-mon[193903]: failed to encode map e523 with expected crc 2023-09-23T21:15:54.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:54 smithi123 ceph-mon[193903]: pgmap v1653: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 48 MiB/s wr, 799 op/s; 48505/274845 objects degraded (17.648%); 9115/274845 objects misplaced (3.316%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:54.740 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:54.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:15:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:15:54] "GET /metrics HTTP/1.1" 200 36714 "" "Prometheus/2.43.0" 2023-09-23T21:15:55.233 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:55.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:55.338 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:55.368 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:56.083 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:56.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:56.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:56.116 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:15:56.155 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:15:56.188 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:56.245 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:56.425 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:56.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:56.532 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:15:56.621 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:56.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:56 smithi121 ceph-mon[211526]: pgmap v1654: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 35 MiB/s wr, 776 op/s; 48505/271185 objects degraded (17.886%); 9021/271185 objects misplaced (3.327%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:56.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:56 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:15:56.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:56 smithi121 ceph-mon[219709]: pgmap v1654: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 35 MiB/s wr, 776 op/s; 48505/271185 objects degraded (17.886%); 9021/271185 objects misplaced (3.327%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:56.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:56 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:15:56.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:15:56.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:56 smithi123 ceph-mon[193903]: pgmap v1654: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 35 MiB/s wr, 776 op/s; 48505/271185 objects degraded (17.886%); 9021/271185 objects misplaced (3.327%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T21:15:56.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:56 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:15:56.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:56.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:15:56.847 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:56.878 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:57.392 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:57 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48505/271185 objects degraded (17.886%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:57.656 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:57.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:57 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48505/271185 objects degraded (17.886%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:57.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:57 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48505/271185 objects degraded (17.886%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:15:57.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:57.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:57.718 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:15:57.744 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:15:57.764 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:57.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:15:57.797 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:15:57.812 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:57.839 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:15:57.876 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:57.877 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:15:57.919 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:15:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:57.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:15:57.926 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:15:57.927 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:15:57.981 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:15:57.992 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:15:57.992 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:15:57.992 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:15:57.992 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:15:57.992 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:15:57.992 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:15:57.993 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:15:57.993 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:15:58.025 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:15:58.070 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:58.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[211526]: pgmap v1655: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 26 MiB/s wr, 787 op/s; 48505/266703 objects degraded (18.187%); 8795/266703 objects misplaced (3.298%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T21:15:58.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:58.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:58.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2152473012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:58.623 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3246419531' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T21:15:58.623 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T21:15:58.623 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[219709]: pgmap v1655: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 26 MiB/s wr, 787 op/s; 48505/266703 objects degraded (18.187%); 8795/266703 objects misplaced (3.298%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T21:15:58.623 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:58.624 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:58.624 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2152473012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:58.624 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3246419531' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T21:15:58.624 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T21:15:58.711 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 13 pg_num to 36 2023-09-23T21:15:58.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:58 smithi123 ceph-mon[193903]: pgmap v1655: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 26 MiB/s wr, 787 op/s; 48505/266703 objects degraded (18.187%); 8795/266703 objects misplaced (3.298%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T21:15:58.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:15:58.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:15:58.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2152473012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:15:58.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3246419531' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T21:15:58.740 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T21:15:58.752 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:15:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:58.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:15:58.856 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:15:58.914 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:59.096 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:15:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:59.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:15:59.214 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:15:59.246 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:59.514 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:15:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:15:59.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:15:59.618 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:15:59.649 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:15:59.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]': finished 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[211526]: osdmap e524: 8 total, 8 up, 5 in 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[211526]: failed to encode map e524 with expected crc 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[211526]: failed to encode map e524 with expected crc 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[211526]: pgmap v1657: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 127 B/s rd, 5.1 MiB/s wr, 834 op/s; 48505/258753 objects degraded (18.746%); 8554/258753 objects misplaced (3.306%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]': finished 2023-09-23T21:15:59.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[219709]: osdmap e524: 8 total, 8 up, 5 in 2023-09-23T21:15:59.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[219709]: failed to encode map e524 with expected crc 2023-09-23T21:15:59.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[219709]: failed to encode map e524 with expected crc 2023-09-23T21:15:59.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[219709]: pgmap v1657: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 127 B/s rd, 5.1 MiB/s wr, 834 op/s; 48505/258753 objects degraded (18.746%); 8554/258753 objects misplaced (3.306%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T21:15:59.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T21:15:59.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:15:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T21:15:59.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:59 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "36"}]': finished 2023-09-23T21:15:59.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:59 smithi123 ceph-mon[193903]: osdmap e524: 8 total, 8 up, 5 in 2023-09-23T21:15:59.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:59 smithi123 ceph-mon[193903]: failed to encode map e524 with expected crc 2023-09-23T21:15:59.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:59 smithi123 ceph-mon[193903]: failed to encode map e524 with expected crc 2023-09-23T21:15:59.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:59 smithi123 ceph-mon[193903]: pgmap v1657: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 21 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 127 B/s rd, 5.1 MiB/s wr, 834 op/s; 48505/258753 objects degraded (18.746%); 8554/258753 objects misplaced (3.306%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T21:15:59.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:59 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T21:15:59.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:15:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T21:16:00.446 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:00.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:00.549 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:00.579 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:00.630 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:00.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:00.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:00.730 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[211526]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.730 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[211526]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[211526]: osdmap e525: 8 total, 8 up, 5 in 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[211526]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[211526]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[219709]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[219709]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T21:16:00.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[219709]: osdmap e525: 8 total, 8 up, 5 in 2023-09-23T21:16:00.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[219709]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.732 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:00 smithi121 ceph-mon[219709]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.734 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:00.819 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:00.915 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:00.921 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:00 smithi123 ceph-mon[193903]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:00 smithi123 ceph-mon[193903]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:00 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T21:16:00.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:00 smithi123 ceph-mon[193903]: osdmap e525: 8 total, 8 up, 5 in 2023-09-23T21:16:00.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:00 smithi123 ceph-mon[193903]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:00 smithi123 ceph-mon[193903]: failed to encode map e525 with expected crc 2023-09-23T21:16:00.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:00.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:01.024 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:01.053 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.209 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:01.314 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:01.344 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.492 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:01.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:01.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:01.624 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:01.824 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:01.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: osdmap e526: 8 total, 8 up, 5 in 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: 13.f continuing backfill to osd.5 from (493'1700,524'6261] MIN to 524'6261 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: pgmap v1660: 109 pgs: 2 active+recovery_wait+degraded, 2 peering, 7 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 862 op/s; 47078/255648 objects degraded (18.415%); 6863/255648 objects misplaced (2.685%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48505/258753 objects degraded (18.746%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: osdmap e527: 8 total, 8 up, 5 in 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:01.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: osdmap e526: 8 total, 8 up, 5 in 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e526 with expected crc 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: 13.f continuing backfill to osd.5 from (493'1700,524'6261] MIN to 524'6261 2023-09-23T21:16:01.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: pgmap v1660: 109 pgs: 2 active+recovery_wait+degraded, 2 peering, 7 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 862 op/s; 47078/255648 objects degraded (18.415%); 6863/255648 objects misplaced (2.685%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:16:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48505/258753 objects degraded (18.746%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: osdmap e527: 8 total, 8 up, 5 in 2023-09-23T21:16:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:01.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:01 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:01.929 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:01.974 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.026 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:02.033 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.033 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.033 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.033 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: osdmap e526: 8 total, 8 up, 5 in 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e526 with expected crc 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: 13.f continuing backfill to osd.5 from (493'1700,524'6261] MIN to 524'6261 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: pgmap v1660: 109 pgs: 2 active+recovery_wait+degraded, 2 peering, 7 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 862 op/s; 47078/255648 objects degraded (18.415%); 6863/255648 objects misplaced (2.685%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48505/258753 objects degraded (18.746%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:02.034 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.035 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: osdmap e527: 8 total, 8 up, 5 in 2023-09-23T21:16:02.035 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.035 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:01 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.133 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:02.163 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.510 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:02.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:02.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:02.648 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.717 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:02.819 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:02.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e528 with expected crc 2023-09-23T21:16:02.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e528 with expected crc 2023-09-23T21:16:02.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: osdmap e528: 8 total, 8 up, 5 in 2023-09-23T21:16:02.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[211526]: failed to encode map e528 with expected crc 2023-09-23T21:16:02.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e527 with expected crc 2023-09-23T21:16:02.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e528 with expected crc 2023-09-23T21:16:02.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e528 with expected crc 2023-09-23T21:16:02.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: osdmap e528: 8 total, 8 up, 5 in 2023-09-23T21:16:02.825 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:02 smithi121 ceph-mon[219709]: failed to encode map e528 with expected crc 2023-09-23T21:16:02.879 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:02.956 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:16:02.972 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:02.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:02.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:03.082 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:03.094 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:03.094 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:03.114 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:03.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:03.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:03.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:03.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e527 with expected crc 2023-09-23T21:16:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e528 with expected crc 2023-09-23T21:16:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e528 with expected crc 2023-09-23T21:16:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: osdmap e528: 8 total, 8 up, 5 in 2023-09-23T21:16:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:02 smithi123 ceph-mon[193903]: failed to encode map e528 with expected crc 2023-09-23T21:16:03.368 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:03.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:03.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:03.473 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:03.502 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:03.546 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:03.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:03.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:03.652 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:03.679 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:03.722 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:16:03.737 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:03.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:03.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:03.849 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:03.921 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: failed to encode map e528 with expected crc 2023-09-23T21:16:03.921 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: 13.1f continuing backfill to osd.5 from (493'1700,524'6261] MIN to 524'6261 2023-09-23T21:16:03.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3396796457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:03.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: pgmap v1663: 109 pgs: 4 active+recovery_wait+degraded, 7 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 287 op/s; 49871/255297 objects degraded (19.535%); 6863/255297 objects misplaced (2.688%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:16:03.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:03.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: failed to encode map e529 with expected crc 2023-09-23T21:16:03.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3396796457' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:03.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: osdmap e529: 8 total, 8 up, 5 in 2023-09-23T21:16:03.922 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:03 smithi123 ceph-mon[193903]: failed to encode map e529 with expected crc 2023-09-23T21:16:03.922 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:03.971 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:16:04.039 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:04.151 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: failed to encode map e528 with expected crc 2023-09-23T21:16:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: 13.1f continuing backfill to osd.5 from (493'1700,524'6261] MIN to 524'6261 2023-09-23T21:16:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3396796457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: pgmap v1663: 109 pgs: 4 active+recovery_wait+degraded, 7 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 287 op/s; 49871/255297 objects degraded (19.535%); 6863/255297 objects misplaced (2.688%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:16:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: failed to encode map e529 with expected crc 2023-09-23T21:16:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3396796457' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: osdmap e529: 8 total, 8 up, 5 in 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[211526]: failed to encode map e529 with expected crc 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: failed to encode map e528 with expected crc 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: 13.1f continuing backfill to osd.5 from (493'1700,524'6261] MIN to 524'6261 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3396796457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: pgmap v1663: 109 pgs: 4 active+recovery_wait+degraded, 7 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 19 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 287 op/s; 49871/255297 objects degraded (19.535%); 6863/255297 objects misplaced (2.688%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: failed to encode map e529 with expected crc 2023-09-23T21:16:04.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3396796457' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: osdmap e529: 8 total, 8 up, 5 in 2023-09-23T21:16:04.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:03 smithi121 ceph-mon[219709]: failed to encode map e529 with expected crc 2023-09-23T21:16:04.307 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.450 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:04.557 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:04.626 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.734 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:04.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:04.798 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:04.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:04.882 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:04.945 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:04.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:04 smithi123 ceph-mon[193903]: failed to encode map e529 with expected crc 2023-09-23T21:16:05.044 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:05.050 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:16:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:16:04] "GET /metrics HTTP/1.1" 200 36711 "" "Prometheus/2.43.0" 2023-09-23T21:16:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:05.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:05.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:04 smithi121 ceph-mon[219709]: failed to encode map e529 with expected crc 2023-09-23T21:16:05.052 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:04 smithi121 ceph-mon[211526]: failed to encode map e529 with expected crc 2023-09-23T21:16:05.152 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:05.191 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:05.241 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:05.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:05.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:05.345 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:05.426 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:05.755 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:05 smithi121 ceph-mon[211526]: pgmap v1665: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 478 op/s; 47840/252156 objects degraded (18.972%); 6761/252156 objects misplaced (2.681%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:16:05.755 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:05 smithi121 ceph-mon[219709]: pgmap v1665: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 478 op/s; 47840/252156 objects degraded (18.972%); 6761/252156 objects misplaced (2.681%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:16:06.011 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:05 smithi123 ceph-mon[193903]: pgmap v1665: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 478 op/s; 47840/252156 objects degraded (18.972%); 6761/252156 objects misplaced (2.681%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T21:16:06.041 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:06.146 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:06.177 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:06.428 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:16:06.428 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:16:06.428 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T21:16:06.428 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:16:06.428 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:16:06.429 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T21:16:06.616 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:06.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:06.721 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:06.789 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:06.871 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:06.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:06.947 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:16:06.975 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:07.027 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 47840/252156 objects degraded (18.972%), 27 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:07.031 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:07.114 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 47840/252156 objects degraded (18.972%), 27 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:07.115 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 47840/252156 objects degraded (18.972%), 27 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:07.153 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:07.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:07.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:07.263 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:07.295 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:07.350 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:07.406 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:07.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:07.507 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:16:07.513 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:07.540 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:16:07.559 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:07.567 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:16:07.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:16:07.631 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:16:07.667 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:16:07.702 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:16:07.734 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.736 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:16:07.776 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:07.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:07.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:07.881 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:07.915 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[219709]: pgmap v1666: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 206 op/s; 47840/251934 objects degraded (18.989%); 6090/251934 objects misplaced (2.417%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3681331373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[211526]: pgmap v1666: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 206 op/s; 47840/251934 objects degraded (18.989%); 6090/251934 objects misplaced (2.417%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T21:16:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3681331373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:16:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:16:08.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.483 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:08.528 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:08.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:08.635 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:08.696 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.696 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:08 smithi123 ceph-mon[193903]: pgmap v1666: 109 pgs: 1 peering, 1 active+recovery_wait+remapped, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 206 op/s; 47840/251934 objects degraded (18.989%); 6090/251934 objects misplaced (2.417%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T21:16:08.696 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.696 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3681331373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:08.696 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:08 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:16:08.696 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:08 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:16:08.696 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:08.720 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:08.824 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:08.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:08.932 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:08.964 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:09.726 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:09.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:09.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:09.830 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:09.863 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:10.069 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:10.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:10.174 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:10.201 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:10.332 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:10 smithi121 ceph-mon[219709]: pgmap v1667: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 649 op/s; 49106/240786 objects degraded (20.394%); 5824/240786 objects misplaced (2.419%); 12 MiB/s, 190 objects/s recovering 2023-09-23T21:16:10.601 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:10.603 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:10 smithi121 ceph-mon[211526]: pgmap v1667: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 649 op/s; 49106/240786 objects degraded (20.394%); 5824/240786 objects misplaced (2.419%); 12 MiB/s, 190 objects/s recovering 2023-09-23T21:16:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:10.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:10.705 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:10.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:10 smithi123 ceph-mon[193903]: pgmap v1667: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 649 op/s; 49106/240786 objects degraded (20.394%); 5824/240786 objects misplaced (2.419%); 12 MiB/s, 190 objects/s recovering 2023-09-23T21:16:10.739 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:10.850 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:10.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:10.955 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:10.988 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:11.081 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:11.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:11.100 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:16:11.100 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:16:11.100 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:16:11.186 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:11.273 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:16:11.281 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:11.391 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:11.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:11.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:11.503 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:11.570 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:11.675 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:11.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:11.779 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:11.816 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:12.184 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:12.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:12.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:12.275 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:12.290 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:12.365 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:12.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:12 smithi121 ceph-mon[219709]: pgmap v1668: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 588 op/s; 49106/240438 objects degraded (20.424%); 5316/240438 objects misplaced (2.211%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:16:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 49106/240438 objects degraded (20.424%), 28 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:12.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3138048522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:12 smithi121 ceph-mon[211526]: pgmap v1668: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 588 op/s; 49106/240438 objects degraded (20.424%); 5316/240438 objects misplaced (2.211%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:16:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 49106/240438 objects degraded (20.424%), 28 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:12.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3138048522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:12.445 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:12.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:12.548 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:12.580 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:12.707 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:16:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:12 smithi123 ceph-mon[193903]: pgmap v1668: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 588 op/s; 49106/240438 objects degraded (20.424%); 5316/240438 objects misplaced (2.211%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:16:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 49106/240438 objects degraded (20.424%), 28 pgs degraded (PG_DEGRADED) 2023-09-23T21:16:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3138048522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:12.738 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:12.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:12.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:12.844 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:12.875 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:13.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:13.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:13.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:13.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: failed to encode map e530 with expected crc 2023-09-23T21:16:13.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: failed to encode map e530 with expected crc 2023-09-23T21:16:13.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3138048522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:13.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: osdmap e530: 8 total, 8 up, 5 in 2023-09-23T21:16:13.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:13 smithi123 ceph-mon[193903]: pgmap v1670: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 536 op/s; 48927/236706 objects degraded (20.670%); 5316/236706 objects misplaced (2.246%); 13 MiB/s, 214 objects/s recovering 2023-09-23T21:16:13.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: failed to encode map e530 with expected crc 2023-09-23T21:16:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: failed to encode map e530 with expected crc 2023-09-23T21:16:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3138048522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: osdmap e530: 8 total, 8 up, 5 in 2023-09-23T21:16:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[219709]: pgmap v1670: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 536 op/s; 48927/236706 objects degraded (20.670%); 5316/236706 objects misplaced (2.246%); 13 MiB/s, 214 objects/s recovering 2023-09-23T21:16:13.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:13.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:13.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:13.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: failed to encode map e530 with expected crc 2023-09-23T21:16:13.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: failed to encode map e530 with expected crc 2023-09-23T21:16:13.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3138048522' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:13.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: osdmap e530: 8 total, 8 up, 5 in 2023-09-23T21:16:13.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:13 smithi121 ceph-mon[211526]: pgmap v1670: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.8 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 536 op/s; 48927/236706 objects degraded (20.670%); 5316/236706 objects misplaced (2.246%); 13 MiB/s, 214 objects/s recovering 2023-09-23T21:16:14.082 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:14.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:14.189 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:14.223 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:14.324 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:14.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:14.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:14.429 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:14.508 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:14.553 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:16:14.573 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:16:14.593 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:16:14.612 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:16:14.654 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:16:14.656 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:14.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:14.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:14.677 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:16:14.697 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:16:14.722 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:16:14.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.730 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.731 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.731 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:16:14.762 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:14.786 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/209964141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:14.789 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:14.850 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:14.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:14.954 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:14.981 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:16:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:16:14] "GET /metrics HTTP/1.1" 200 36717 "" "Prometheus/2.43.0" 2023-09-23T21:16:14.982 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/209964141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:14.982 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/209964141' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:14.985 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.035 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:15.139 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:15.166 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.207 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:15.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:15.311 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:15.342 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.577 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:15.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:15.681 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:15.711 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:15 smithi123 ceph-mon[193903]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:15 smithi123 ceph-mon[193903]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:15 smithi123 ceph-mon[193903]: osdmap e531: 8 total, 8 up, 5 in 2023-09-23T21:16:15.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:15 smithi123 ceph-mon[193903]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:15 smithi123 ceph-mon[193903]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:15 smithi123 ceph-mon[193903]: pgmap v1672: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 962 op/s; 48285/228825 objects degraded (21.101%); 5064/228825 objects misplaced (2.213%); 16 MiB/s, 253 objects/s recovering 2023-09-23T21:16:15.760 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:15.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:15.865 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:15.892 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[211526]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.892 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[211526]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.893 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[211526]: osdmap e531: 8 total, 8 up, 5 in 2023-09-23T21:16:15.893 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[211526]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.893 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[211526]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.893 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[211526]: pgmap v1672: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 962 op/s; 48285/228825 objects degraded (21.101%); 5064/228825 objects misplaced (2.213%); 16 MiB/s, 253 objects/s recovering 2023-09-23T21:16:15.893 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[219709]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.893 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[219709]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.893 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[219709]: osdmap e531: 8 total, 8 up, 5 in 2023-09-23T21:16:15.894 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[219709]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.894 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[219709]: failed to encode map e531 with expected crc 2023-09-23T21:16:15.894 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:15 smithi121 ceph-mon[219709]: pgmap v1672: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 962 op/s; 48285/228825 objects degraded (21.101%); 5064/228825 objects misplaced (2.213%); 16 MiB/s, 253 objects/s recovering 2023-09-23T21:16:15.895 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:15.946 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:15.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:15.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:16.053 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:16.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:16:16.144 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:16.299 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:16.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:16.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:16.315 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:16.315 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:16.416 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:16.445 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:16.499 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:16.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:16.567 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:16.603 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:16.636 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:16.703 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:16.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:16.807 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:16.869 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: osdmap e532: 8 total, 8 up, 5 in 2023-09-23T21:16:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 48285/228825 objects degraded (21.101%), 28 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:16.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: osdmap e532: 8 total, 8 up, 5 in 2023-09-23T21:16:16.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.988 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 48285/228825 objects degraded (21.101%), 28 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: osdmap e532: 8 total, 8 up, 5 in 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: failed to encode map e532 with expected crc 2023-09-23T21:16:16.989 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 48285/228825 objects degraded (21.101%), 28 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:17.119 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:17.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:17.185 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:16:17.186 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:16:17.186 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T21:16:17.186 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:16:17.223 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:17.309 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:17.418 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:17.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:17.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:17.525 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:17.552 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:17.643 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:16:17.859 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:17.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:17 smithi121 ceph-mon[219709]: pgmap v1674: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 660 op/s; 48202/228552 objects degraded (21.090%); 4753/228552 objects misplaced (2.080%); 12 MiB/s, 202 objects/s recovering 2023-09-23T21:16:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:17.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:17.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:17 smithi121 ceph-mon[211526]: pgmap v1674: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 660 op/s; 48202/228552 objects degraded (21.090%); 4753/228552 objects misplaced (2.080%); 12 MiB/s, 202 objects/s recovering 2023-09-23T21:16:17.966 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:18.006 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:18.080 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:18.082 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:18.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:18.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:18.187 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:18.200 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:18.218 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:17 smithi123 ceph-mon[193903]: pgmap v1674: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovery_wait, 1 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 660 op/s; 48202/228552 objects degraded (21.090%); 4753/228552 objects misplaced (2.080%); 12 MiB/s, 202 objects/s recovering 2023-09-23T21:16:18.316 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:18.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:18.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:18.421 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:18.500 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:18.731 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:16:18.732 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:18.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:18.850 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:18.871 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:16:18.871 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/343411214' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:18.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:18.891 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:19.095 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:19.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:19.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:19.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/343411214' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:19.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:19.202 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:19.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/343411214' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:19.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:19.232 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:19.431 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:19.936 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:19.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:20.040 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:20.071 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:20.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:19 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:20.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:19 smithi123 ceph-mon[193903]: failed to encode map e533 with expected crc 2023-09-23T21:16:20.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:19 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:20.075 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:19 smithi123 ceph-mon[193903]: osdmap e533: 8 total, 8 up, 5 in 2023-09-23T21:16:20.075 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:19 smithi123 ceph-mon[193903]: failed to encode map e533 with expected crc 2023-09-23T21:16:20.075 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:19 smithi123 ceph-mon[193903]: pgmap v1676: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 961 op/s; 44519/219390 objects degraded (20.292%); 6831/219390 objects misplaced (3.114%); 24 MiB/s, 405 objects/s recovering 2023-09-23T21:16:20.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[219709]: failed to encode map e533 with expected crc 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[219709]: osdmap e533: 8 total, 8 up, 5 in 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[219709]: failed to encode map e533 with expected crc 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[219709]: pgmap v1676: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 961 op/s; 44519/219390 objects degraded (20.292%); 6831/219390 objects misplaced (3.114%); 24 MiB/s, 405 objects/s recovering 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[211526]: failed to encode map e533 with expected crc 2023-09-23T21:16:20.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:20.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[211526]: osdmap e533: 8 total, 8 up, 5 in 2023-09-23T21:16:20.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[211526]: failed to encode map e533 with expected crc 2023-09-23T21:16:20.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:19 smithi121 ceph-mon[211526]: pgmap v1676: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 961 op/s; 44519/219390 objects degraded (20.292%); 6831/219390 objects misplaced (3.114%); 24 MiB/s, 405 objects/s recovering 2023-09-23T21:16:20.474 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:20.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:20.581 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:20.611 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:20.696 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:20.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:20.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:20.810 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:20.844 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:21.020 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:21.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:21.125 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:21.209 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:21.271 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:21.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:21.384 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:21.451 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:21.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:16:21.498 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:16:21.528 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:16:21.553 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:16:21.576 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:16:21.589 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:21.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:21.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:21.609 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:16:21.645 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:16:21.670 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:16:21.671 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.671 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.671 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.672 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.672 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.672 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.672 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.672 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:16:21.695 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:21.739 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:21.903 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:21.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:22.008 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:16:22.010 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:22.010 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:16:22.063 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:22.117 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:22.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:22.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:22.225 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:22.258 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:22.532 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:22 smithi121 ceph-mon[211526]: pgmap v1677: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 492 op/s; 44519/219057 objects degraded (20.323%); 6831/219057 objects misplaced (3.118%); 16 MiB/s, 270 objects/s recovering 2023-09-23T21:16:22.532 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:22 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1109302609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:22.532 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 44519/219057 objects degraded (20.323%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:22.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:22 smithi121 ceph-mon[219709]: pgmap v1677: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 492 op/s; 44519/219057 objects degraded (20.323%); 6831/219057 objects misplaced (3.118%); 16 MiB/s, 270 objects/s recovering 2023-09-23T21:16:22.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:22 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1109302609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:22.532 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 44519/219057 objects degraded (20.323%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:22.729 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:16:22.729 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:16:22.729 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:16:22.729 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:16:22.729 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:16:22.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:22 smithi123 ceph-mon[193903]: pgmap v1677: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 492 op/s; 44519/219057 objects degraded (20.323%); 6831/219057 objects misplaced (3.118%); 16 MiB/s, 270 objects/s recovering 2023-09-23T21:16:22.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:22 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1109302609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:22.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 44519/219057 objects degraded (20.323%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:22.813 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:16:22.865 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:22.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:22.969 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:23.057 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:23.144 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1677,"stamp":"2023-09-23T21:16:21.324468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4784265144,"num_objects":73019,"num_object_clones":0,"num_object_copies":219057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":44519,"num_objects_misplaced":6831,"num_objects_unfound":0,"num_objects_dirty":73019,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":97236,"num_write_kb":4954830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12009,"num_bytes_recovered":777197092,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":180534,"ondisk_log_size":180534,"up":327,"acting":312,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":346,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":17695988,"kb_used_data":13668492,"kb_used_omap":7,"kb_used_meta":4027128,"kb_avail":451029772,"statfs":{"total":479975178240,"available":461854486528,"internally_reserved":0,"allocated":13996535808,"data_stored":13988327223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4123779153},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-213385216,"num_objects":-3256,"num_object_clones":0,"num_object_copies":-9768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3766,"num_objects_misplaced":1767,"num_objects_unfound":0,"num_objects_dirty":-3256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3256,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1792,"num_bytes_recovered":111935488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.617846"},"pg_stats":[{"pgid":"13.23","version":"532'5068","reported_seq":5552,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.486112+0000","last_change":"2023-09-23T21:16:00.853306+0000","last_active":"2023-09-23T21:16:18.486112+0000","last_peered":"2023-09-23T21:16:18.486112+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:00.848945+0000","last_became_peered":"2023-09-23T21:16:00.848945+0000","last_unstale":"2023-09-23T21:16:18.486112+0000","last_undegraded":"2023-09-23T21:16:00.815130+0000","last_fullsized":"2023-09-23T21:16:18.486112+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5068,"log_dups_size":0,"ondisk_log_size":5068,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77234176,"num_objects":1179,"num_object_clones":0,"num_object_copies":3537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":825,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":807,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":175,"num_bytes_recovered":11468800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":701},{"shards":"4,5,7","objects":478}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.22","version":"532'4996","reported_seq":5465,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.454382+0000","last_change":"2023-09-23T21:16:00.842228+0000","last_active":"2023-09-23T21:16:18.454382+0000","last_peered":"2023-09-23T21:16:18.454382+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:00.839299+0000","last_became_peered":"2023-09-23T21:16:00.839299+0000","last_unstale":"2023-09-23T21:16:18.454382+0000","last_undegraded":"2023-09-23T21:16:00.787945+0000","last_fullsized":"2023-09-23T21:16:18.454382+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4996,"log_dups_size":0,"ondisk_log_size":4996,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74907648,"num_objects":1143,"num_object_clones":0,"num_object_copies":3429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":777,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1757,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":185,"num_bytes_recovered":12124160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":666},{"shards":"4,5,6","objects":477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.21","version":"532'4908","reported_seq":5359,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:17.795107+0000","last_change":"2023-09-23T21:16:00.817069+0000","last_active":"2023-09-23T21:16:17.795107+0000","last_peered":"2023-09-23T21:16:17.795107+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:00.815146+0000","last_became_peered":"2023-09-23T21:16:00.815146+0000","last_unstale":"2023-09-23T21:16:17.795107+0000","last_undegraded":"2023-09-23T21:16:00.800077+0000","last_fullsized":"2023-09-23T21:16:17.795107+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4908,"log_dups_size":0,"ondisk_log_size":4908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74039296,"num_objects":1130,"num_object_clones":0,"num_object_copies":3390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1063,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1727,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":182,"num_bytes_recovered":11927552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":948},{"shards":"2,4,7","objects":182}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.20","version":"532'4991","reported_seq":7429,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.429851+0000","last_change":"2023-09-23T21:16:00.913587+0000","last_active":"2023-09-23T21:16:18.429851+0000","last_peered":"2023-09-23T21:16:18.429851+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:00.909085+0000","last_became_peered":"2023-09-23T21:16:00.909085+0000","last_unstale":"2023-09-23T21:16:18.429851+0000","last_undegraded":"2023-09-23T21:16:00.886179+0000","last_fullsized":"2023-09-23T21:16:18.429851+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":4991,"log_dups_size":0,"ondisk_log_size":4991,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76808192,"num_objects":1172,"num_object_clones":0,"num_object_copies":3516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1144,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":868,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":173,"num_bytes_recovered":11206656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,4,7","objects":149},{"shards":"2,6,7","objects":1023}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.14","version":"532'5245","reported_seq":7422,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.004930+0000","last_change":"2023-09-23T21:15:37.953666+0000","last_active":"2023-09-23T21:16:18.004930+0000","last_peered":"2023-09-23T21:16:18.004930+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:15:37.947742+0000","last_became_peered":"2023-09-23T21:15:37.947742+0000","last_unstale":"2023-09-23T21:16:18.004930+0000","last_undegraded":"2023-09-23T21:15:37.724073+0000","last_fullsized":"2023-09-23T21:15:37.696156+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5245,"log_dups_size":0,"ondisk_log_size":5245,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152829952,"num_objects":2332,"num_object_clones":0,"num_object_copies":6996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2613,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3534,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":-281},{"shards":"2,5,6","objects":2012}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":133,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.725138+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:16:14.725138+0000","last_peered":"2023-09-23T21:16:14.725138+0000","last_clean":"2023-09-23T21:16:14.725138+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:16:14.725138+0000","last_undegraded":"2023-09-23T21:16:14.725138+0000","last_fullsized":"2023-09-23T21:16:14.725138+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":162,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.717919+0000","last_change":"2023-09-23T21:15:36.786608+0000","last_active":"2023-09-23T21:16:14.717919+0000","last_peered":"2023-09-23T21:16:14.717919+0000","last_clean":"2023-09-23T21:16:14.717919+0000","last_became_active":"2023-09-23T21:15:36.786251+0000","last_became_peered":"2023-09-23T21:15:36.786251+0000","last_unstale":"2023-09-23T21:16:14.717919+0000","last_undegraded":"2023-09-23T21:16:14.717919+0000","last_fullsized":"2023-09-23T21:16:14.717919+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:15.804266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.17","version":"532'5142","reported_seq":5222,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.048251+0000","last_change":"2023-09-23T21:15:37.925492+0000","last_active":"2023-09-23T21:16:18.048251+0000","last_peered":"2023-09-23T21:16:18.048251+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:15:37.917083+0000","last_became_peered":"2023-09-23T21:15:37.917083+0000","last_unstale":"2023-09-23T21:16:18.048251+0000","last_undegraded":"2023-09-23T21:15:37.769758+0000","last_fullsized":"2023-09-23T21:15:37.736401+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5142,"log_dups_size":0,"ondisk_log_size":5142,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145686528,"num_objects":2223,"num_object_clones":0,"num_object_copies":6669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2584,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3499,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":1946},{"shards":"4,7","objects":-361}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":319,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.711853+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:16:14.711853+0000","last_peered":"2023-09-23T21:16:14.711853+0000","last_clean":"2023-09-23T21:16:14.711853+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:16:14.711853+0000","last_undegraded":"2023-09-23T21:16:14.711853+0000","last_fullsized":"2023-09-23T21:16:14.711853+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":179,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883149+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:16:14.883149+0000","last_peered":"2023-09-23T21:16:14.883149+0000","last_clean":"2023-09-23T21:16:14.883149+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:16:14.883149+0000","last_undegraded":"2023-09-23T21:16:14.883149+0000","last_fullsized":"2023-09-23T21:16:14.883149+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"532'5132","reported_seq":5806,"reported_epoch":532,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.484393+0000","last_change":"2023-09-23T21:16:14.481799+0000","last_active":"2023-09-23T21:16:18.484393+0000","last_peered":"2023-09-23T21:16:18.484393+0000","last_clean":"2023-09-23T21:15:09.799088+0000","last_became_active":"2023-09-23T21:15:12.799547+0000","last_became_peered":"2023-09-23T21:15:12.799547+0000","last_unstale":"2023-09-23T21:16:18.484393+0000","last_undegraded":"2023-09-23T21:15:12.697902+0000","last_fullsized":"2023-09-23T21:15:12.679711+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5132,"log_dups_size":0,"ondisk_log_size":5132,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149749760,"num_objects":2285,"num_object_clones":0,"num_object_copies":6855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":529,"num_objects_misplaced":1129,"num_objects_unfound":0,"num_objects_dirty":2285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3471,"num_write_kb":184192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1129},{"shards":"4,5","objects":627}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":191,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.726428+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:16:14.726428+0000","last_peered":"2023-09-23T21:16:14.726428+0000","last_clean":"2023-09-23T21:16:14.726428+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:16:14.726428+0000","last_undegraded":"2023-09-23T21:16:14.726428+0000","last_fullsized":"2023-09-23T21:16:14.726428+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":426,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883961+0000","last_change":"2023-09-23T21:15:36.720709+0000","last_active":"2023-09-23T21:16:14.883961+0000","last_peered":"2023-09-23T21:16:14.883961+0000","last_clean":"2023-09-23T21:16:14.883961+0000","last_became_active":"2023-09-23T21:15:36.720461+0000","last_became_peered":"2023-09-23T21:15:36.720461+0000","last_unstale":"2023-09-23T21:16:14.883961+0000","last_undegraded":"2023-09-23T21:16:14.883961+0000","last_fullsized":"2023-09-23T21:16:14.883961+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:34.395890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.19","version":"532'5086","reported_seq":5173,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.324160+0000","last_change":"2023-09-23T21:15:38.015633+0000","last_active":"2023-09-23T21:16:18.324160+0000","last_peered":"2023-09-23T21:16:18.324160+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:15:38.010899+0000","last_became_peered":"2023-09-23T21:15:38.010899+0000","last_unstale":"2023-09-23T21:16:18.324160+0000","last_undegraded":"2023-09-23T21:15:37.722607+0000","last_fullsized":"2023-09-23T21:15:37.705389+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142770176,"num_objects":2179,"num_object_clones":0,"num_object_copies":6537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3451,"num_write_kb":180128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":-365},{"shards":"2,5,6","objects":1908}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.720832+0000","last_change":"2023-09-23T21:15:36.715191+0000","last_active":"2023-09-23T21:16:14.720832+0000","last_peered":"2023-09-23T21:16:14.720832+0000","last_clean":"2023-09-23T21:16:14.720832+0000","last_became_active":"2023-09-23T21:15:36.714882+0000","last_became_peered":"2023-09-23T21:15:36.714882+0000","last_unstale":"2023-09-23T21:16:14.720832+0000","last_undegraded":"2023-09-23T21:16:14.720832+0000","last_fullsized":"2023-09-23T21:16:14.720832+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:36.489500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.724553+0000","last_change":"2023-09-23T21:15:36.718536+0000","last_active":"2023-09-23T21:16:14.724553+0000","last_peered":"2023-09-23T21:16:14.724553+0000","last_clean":"2023-09-23T21:16:14.724553+0000","last_became_active":"2023-09-23T21:15:36.717389+0000","last_became_peered":"2023-09-23T21:15:36.717389+0000","last_unstale":"2023-09-23T21:16:14.724553+0000","last_undegraded":"2023-09-23T21:16:14.724553+0000","last_fullsized":"2023-09-23T21:16:14.724553+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:49.029938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.18","version":"532'5118","reported_seq":5243,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.045887+0000","last_change":"2023-09-23T21:15:37.899454+0000","last_active":"2023-09-23T21:16:18.045887+0000","last_peered":"2023-09-23T21:16:18.045887+0000","last_clean":"2023-09-23T21:15:36.291044+0000","last_became_active":"2023-09-23T21:15:37.895682+0000","last_became_peered":"2023-09-23T21:15:37.895682+0000","last_unstale":"2023-09-23T21:16:18.045887+0000","last_undegraded":"2023-09-23T21:15:37.723119+0000","last_fullsized":"2023-09-23T21:15:37.695587+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":5118,"log_dups_size":0,"ondisk_log_size":5118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:16.086598+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148832256,"num_objects":2271,"num_object_clones":0,"num_object_copies":6813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1605,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":-305},{"shards":"2,5,6","objects":1982}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":185,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.721746+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:16:14.721746+0000","last_peered":"2023-09-23T21:16:14.721746+0000","last_clean":"2023-09-23T21:16:14.721746+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:16:14.721746+0000","last_undegraded":"2023-09-23T21:16:14.721746+0000","last_fullsized":"2023-09-23T21:16:14.721746+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":144,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882552+0000","last_change":"2023-09-23T21:15:36.718310+0000","last_active":"2023-09-23T21:16:14.882552+0000","last_peered":"2023-09-23T21:16:14.882552+0000","last_clean":"2023-09-23T21:16:14.882552+0000","last_became_active":"2023-09-23T21:15:36.717872+0000","last_became_peered":"2023-09-23T21:15:36.717872+0000","last_unstale":"2023-09-23T21:16:14.882552+0000","last_undegraded":"2023-09-23T21:16:14.882552+0000","last_fullsized":"2023-09-23T21:16:14.882552+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:58.556574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1b","version":"532'6845","reported_seq":9153,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.046280+0000","last_change":"2023-09-23T21:16:00.814154+0000","last_active":"2023-09-23T21:16:18.046280+0000","last_peered":"2023-09-23T21:16:18.046280+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:00.808957+0000","last_became_peered":"2023-09-23T21:16:00.808957+0000","last_unstale":"2023-09-23T21:16:18.046280+0000","last_undegraded":"2023-09-23T21:16:00.770362+0000","last_fullsized":"2023-09-23T21:16:18.046280+0000","mapping_epoch":525,"log_start":"493'1783","ondisk_log_start":"493'1783","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5062,"log_dups_size":0,"ondisk_log_size":5062,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154042368,"num_objects":2351,"num_object_clones":0,"num_object_copies":7053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1415,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3610,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":621,"num_bytes_recovered":40697856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1415},{"shards":"2,4,5","objects":936}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":119,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883493+0000","last_change":"2023-09-23T21:15:36.816209+0000","last_active":"2023-09-23T21:16:14.883493+0000","last_peered":"2023-09-23T21:16:14.883493+0000","last_clean":"2023-09-23T21:16:14.883493+0000","last_became_active":"2023-09-23T21:15:36.724436+0000","last_became_peered":"2023-09-23T21:15:36.724436+0000","last_unstale":"2023-09-23T21:16:14.883493+0000","last_undegraded":"2023-09-23T21:16:14.883493+0000","last_fullsized":"2023-09-23T21:16:14.883493+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:57.112425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":280,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.725060+0000","last_change":"2023-09-23T21:15:36.717042+0000","last_active":"2023-09-23T21:16:14.725060+0000","last_peered":"2023-09-23T21:16:14.725060+0000","last_clean":"2023-09-23T21:16:14.725060+0000","last_became_active":"2023-09-23T21:15:36.716735+0000","last_became_peered":"2023-09-23T21:15:36.716735+0000","last_unstale":"2023-09-23T21:16:14.725060+0000","last_undegraded":"2023-09-23T21:16:14.725060+0000","last_fullsized":"2023-09-23T21:16:14.725060+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:33.432298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.1a","version":"532'6591","reported_seq":8697,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:18.010609+0000","last_change":"2023-09-23T21:16:00.812483+0000","last_active":"2023-09-23T21:16:18.010609+0000","last_peered":"2023-09-23T21:16:18.010609+0000","last_clean":"2023-09-23T21:16:18.010609+0000","last_became_active":"2023-09-23T21:16:00.806979+0000","last_became_peered":"2023-09-23T21:16:00.806979+0000","last_unstale":"2023-09-23T21:16:18.010609+0000","last_undegraded":"2023-09-23T21:16:18.010609+0000","last_fullsized":"2023-09-23T21:16:18.010609+0000","mapping_epoch":525,"log_start":"493'1441","ondisk_log_start":"493'1441","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5150,"log_dups_size":0,"ondisk_log_size":5150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3479,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":139,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883411+0000","last_change":"2023-09-23T21:15:36.735118+0000","last_active":"2023-09-23T21:16:14.883411+0000","last_peered":"2023-09-23T21:16:14.883411+0000","last_clean":"2023-09-23T21:16:14.883411+0000","last_became_active":"2023-09-23T21:15:36.699810+0000","last_became_peered":"2023-09-23T21:15:36.699810+0000","last_unstale":"2023-09-23T21:16:14.883411+0000","last_undegraded":"2023-09-23T21:16:14.883411+0000","last_fullsized":"2023-09-23T21:16:14.883411+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:45.321201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":460,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.710880+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:16:14.710880+0000","last_peered":"2023-09-23T21:16:14.710880+0000","last_clean":"2023-09-23T21:16:14.710880+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:16:14.710880+0000","last_undegraded":"2023-09-23T21:16:14.710880+0000","last_fullsized":"2023-09-23T21:16:14.710880+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1d","version":"532'6715","reported_seq":9565,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.464978+0000","last_change":"2023-09-23T21:16:03.792593+0000","last_active":"2023-09-23T21:16:18.464978+0000","last_peered":"2023-09-23T21:16:18.464978+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:16:03.773235+0000","last_became_peered":"2023-09-23T21:16:03.773235+0000","last_unstale":"2023-09-23T21:16:18.464978+0000","last_undegraded":"2023-09-23T21:16:03.727693+0000","last_fullsized":"2023-09-23T21:16:18.464978+0000","mapping_epoch":528,"log_start":"497'2164","ondisk_log_start":"497'2164","created":525,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4551,"log_dups_size":0,"ondisk_log_size":4551,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151814156,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1607,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3538,"num_write_kb":187329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":865,"num_bytes_recovered":56688640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":1367},{"shards":"4,5,7","objects":950}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":115,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883393+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:16:14.883393+0000","last_peered":"2023-09-23T21:16:14.883393+0000","last_clean":"2023-09-23T21:16:14.883393+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:16:14.883393+0000","last_undegraded":"2023-09-23T21:16:14.883393+0000","last_fullsized":"2023-09-23T21:16:14.883393+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":183,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883241+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:16:14.883241+0000","last_peered":"2023-09-23T21:16:14.883241+0000","last_clean":"2023-09-23T21:16:14.883241+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:16:14.883241+0000","last_undegraded":"2023-09-23T21:16:14.883241+0000","last_fullsized":"2023-09-23T21:16:14.883241+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1c","version":"532'6736","reported_seq":8835,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:18.412287+0000","last_change":"2023-09-23T21:16:00.824659+0000","last_active":"2023-09-23T21:16:18.412287+0000","last_peered":"2023-09-23T21:16:18.412287+0000","last_clean":"2023-09-23T21:16:18.412287+0000","last_became_active":"2023-09-23T21:16:00.820619+0000","last_became_peered":"2023-09-23T21:16:00.820619+0000","last_unstale":"2023-09-23T21:16:18.412287+0000","last_undegraded":"2023-09-23T21:16:18.412287+0000","last_fullsized":"2023-09-23T21:16:18.412287+0000","mapping_epoch":525,"log_start":"500'2344","ondisk_log_start":"500'2344","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4392,"log_dups_size":0,"ondisk_log_size":4392,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152469504,"num_objects":2327,"num_object_clones":0,"num_object_copies":6981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3552,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882754+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:16:14.882754+0000","last_peered":"2023-09-23T21:16:14.882754+0000","last_clean":"2023-09-23T21:16:14.882754+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:16:14.882754+0000","last_undegraded":"2023-09-23T21:16:14.882754+0000","last_fullsized":"2023-09-23T21:16:14.882754+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883636+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:16:14.883636+0000","last_peered":"2023-09-23T21:16:14.883636+0000","last_clean":"2023-09-23T21:16:14.883636+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:16:14.883636+0000","last_undegraded":"2023-09-23T21:16:14.883636+0000","last_fullsized":"2023-09-23T21:16:14.883636+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1f","version":"532'6651","reported_seq":6731,"reported_epoch":532,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T21:16:18.384270+0000","last_change":"2023-09-23T21:16:02.754390+0000","last_active":"2023-09-23T21:16:18.384270+0000","last_peered":"2023-09-23T21:16:18.384270+0000","last_clean":"2023-09-23T21:15:36.326799+0000","last_became_active":"2023-09-23T21:16:02.737320+0000","last_became_peered":"2023-09-23T21:16:02.737320+0000","last_unstale":"2023-09-23T21:16:18.384270+0000","last_undegraded":"2023-09-23T21:16:18.384270+0000","last_fullsized":"2023-09-23T21:16:18.384270+0000","mapping_epoch":527,"log_start":"493'1700","ondisk_log_start":"493'1700","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4951,"log_dups_size":0,"ondisk_log_size":4951,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148668416,"num_objects":2269,"num_object_clones":0,"num_object_copies":6807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2269,"num_objects_unfound":0,"num_objects_dirty":2269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3521,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2269}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882910+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:16:14.882910+0000","last_peered":"2023-09-23T21:16:14.882910+0000","last_clean":"2023-09-23T21:16:14.882910+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:16:14.882910+0000","last_undegraded":"2023-09-23T21:16:14.882910+0000","last_fullsized":"2023-09-23T21:16:14.882910+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":133,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883068+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:16:14.883068+0000","last_peered":"2023-09-23T21:16:14.883068+0000","last_clean":"2023-09-23T21:16:14.883068+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:16:14.883068+0000","last_undegraded":"2023-09-23T21:16:14.883068+0000","last_fullsized":"2023-09-23T21:16:14.883068+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1e","version":"532'6547","reported_seq":6614,"reported_epoch":532,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:16:18.420695+0000","last_change":"2023-09-23T21:16:00.911470+0000","last_active":"2023-09-23T21:16:18.420695+0000","last_peered":"2023-09-23T21:16:18.420695+0000","last_clean":"2023-09-23T21:15:59.749850+0000","last_became_active":"2023-09-23T21:16:00.844392+0000","last_became_peered":"2023-09-23T21:16:00.844392+0000","last_unstale":"2023-09-23T21:16:18.420695+0000","last_undegraded":"2023-09-23T21:16:18.420695+0000","last_fullsized":"2023-09-23T21:16:18.420695+0000","mapping_epoch":525,"log_start":"493'1633","ondisk_log_start":"493'1633","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4914,"log_dups_size":0,"ondisk_log_size":4914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148537344,"num_objects":2267,"num_object_clones":0,"num_object_copies":6801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3456,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":2267}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":569,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:15.719298+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:16:15.719298+0000","last_peered":"2023-09-23T21:16:15.719298+0000","last_clean":"2023-09-23T21:16:15.719298+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:16:15.719298+0000","last_undegraded":"2023-09-23T21:16:15.719298+0000","last_fullsized":"2023-09-23T21:16:15.719298+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":183,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.886121+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:16:14.886121+0000","last_peered":"2023-09-23T21:16:14.886121+0000","last_clean":"2023-09-23T21:16:14.886121+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:16:14.886121+0000","last_undegraded":"2023-09-23T21:16:14.886121+0000","last_fullsized":"2023-09-23T21:16:14.886121+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"532'4906","reported_seq":5356,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.055077+0000","last_change":"2023-09-23T21:16:01.726069+0000","last_active":"2023-09-23T21:16:18.055077+0000","last_peered":"2023-09-23T21:16:18.055077+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:01.722838+0000","last_became_peered":"2023-09-23T21:16:01.722838+0000","last_unstale":"2023-09-23T21:16:18.055077+0000","last_undegraded":"2023-09-23T21:16:01.697703+0000","last_fullsized":"2023-09-23T21:16:18.055077+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4906,"log_dups_size":0,"ondisk_log_size":4906,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74170368,"num_objects":1131,"num_object_clones":0,"num_object_copies":3393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1138,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1724,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":11862016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1020},{"shards":"2,4,7","objects":111}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":113,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.886208+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:16:14.886208+0000","last_peered":"2023-09-23T21:16:14.886208+0000","last_clean":"2023-09-23T21:16:14.886208+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:16:14.886208+0000","last_undegraded":"2023-09-23T21:16:14.886208+0000","last_fullsized":"2023-09-23T21:16:14.886208+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":175,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.713099+0000","last_change":"2023-09-23T21:15:36.714867+0000","last_active":"2023-09-23T21:16:14.713099+0000","last_peered":"2023-09-23T21:16:14.713099+0000","last_clean":"2023-09-23T21:16:14.713099+0000","last_became_active":"2023-09-23T21:15:36.714569+0000","last_became_peered":"2023-09-23T21:15:36.714569+0000","last_unstale":"2023-09-23T21:16:14.713099+0000","last_undegraded":"2023-09-23T21:16:14.713099+0000","last_fullsized":"2023-09-23T21:16:14.713099+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:05.675620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"532'5000","reported_seq":7454,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.366838+0000","last_change":"2023-09-23T21:16:01.731454+0000","last_active":"2023-09-23T21:16:18.366838+0000","last_peered":"2023-09-23T21:16:18.366838+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:01.727726+0000","last_became_peered":"2023-09-23T21:16:01.727726+0000","last_unstale":"2023-09-23T21:16:18.366838+0000","last_undegraded":"2023-09-23T21:16:01.712365+0000","last_fullsized":"2023-09-23T21:16:18.366838+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5000,"log_dups_size":0,"ondisk_log_size":5000,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1126,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":877,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":180,"num_bytes_recovered":11730944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,4,7","objects":157},{"shards":"2,6,7","objects":1006}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":127,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883251+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:16:14.883251+0000","last_peered":"2023-09-23T21:16:14.883251+0000","last_clean":"2023-09-23T21:16:14.883251+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:16:14.883251+0000","last_undegraded":"2023-09-23T21:16:14.883251+0000","last_fullsized":"2023-09-23T21:16:14.883251+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":353,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882449+0000","last_change":"2023-09-23T21:15:36.710858+0000","last_active":"2023-09-23T21:16:14.882449+0000","last_peered":"2023-09-23T21:16:14.882449+0000","last_clean":"2023-09-23T21:16:14.882449+0000","last_became_active":"2023-09-23T21:15:36.710564+0000","last_became_peered":"2023-09-23T21:15:36.710564+0000","last_unstale":"2023-09-23T21:16:14.882449+0000","last_undegraded":"2023-09-23T21:16:14.882449+0000","last_fullsized":"2023-09-23T21:16:14.882449+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:34.074726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.3","version":"532'5081","reported_seq":5592,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.462142+0000","last_change":"2023-09-23T21:16:01.735614+0000","last_active":"2023-09-23T21:16:18.462142+0000","last_peered":"2023-09-23T21:16:18.462142+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:01.731847+0000","last_became_peered":"2023-09-23T21:16:01.731847+0000","last_unstale":"2023-09-23T21:16:18.462142+0000","last_undegraded":"2023-09-23T21:16:01.699708+0000","last_fullsized":"2023-09-23T21:16:18.462142+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5081,"log_dups_size":0,"ondisk_log_size":5081,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76382208,"num_objects":1165,"num_object_clones":0,"num_object_copies":3495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":835,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":820,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":187,"num_bytes_recovered":12255232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":692},{"shards":"4,5,7","objects":473}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":113,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883646+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:16:14.883646+0000","last_peered":"2023-09-23T21:16:14.883646+0000","last_clean":"2023-09-23T21:16:14.883646+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:16:14.883646+0000","last_undegraded":"2023-09-23T21:16:14.883646+0000","last_fullsized":"2023-09-23T21:16:14.883646+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":415,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883686+0000","last_change":"2023-09-23T21:15:36.719723+0000","last_active":"2023-09-23T21:16:14.883686+0000","last_peered":"2023-09-23T21:16:14.883686+0000","last_clean":"2023-09-23T21:16:14.883686+0000","last_became_active":"2023-09-23T21:15:36.719022+0000","last_became_peered":"2023-09-23T21:15:36.719022+0000","last_unstale":"2023-09-23T21:16:14.883686+0000","last_undegraded":"2023-09-23T21:16:14.883686+0000","last_fullsized":"2023-09-23T21:16:14.883686+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:40.754123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"532'4990","reported_seq":5450,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.455204+0000","last_change":"2023-09-23T21:16:01.750035+0000","last_active":"2023-09-23T21:16:18.455204+0000","last_peered":"2023-09-23T21:16:18.455204+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:01.742399+0000","last_became_peered":"2023-09-23T21:16:01.742399+0000","last_unstale":"2023-09-23T21:16:18.455204+0000","last_undegraded":"2023-09-23T21:16:01.706463+0000","last_fullsized":"2023-09-23T21:16:18.455204+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4990,"log_dups_size":0,"ondisk_log_size":4990,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75300864,"num_objects":1149,"num_object_clones":0,"num_object_copies":3447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":778,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1751,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":179,"num_bytes_recovered":11730944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":645},{"shards":"4,5,6","objects":504}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":317,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.724647+0000","last_change":"2023-09-23T21:15:36.717524+0000","last_active":"2023-09-23T21:16:14.724647+0000","last_peered":"2023-09-23T21:16:14.724647+0000","last_clean":"2023-09-23T21:16:14.724647+0000","last_became_active":"2023-09-23T21:15:36.716684+0000","last_became_peered":"2023-09-23T21:15:36.716684+0000","last_unstale":"2023-09-23T21:16:14.724647+0000","last_undegraded":"2023-09-23T21:16:14.724647+0000","last_fullsized":"2023-09-23T21:16:14.724647+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:19.720726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882979+0000","last_change":"2023-09-23T21:15:36.718920+0000","last_active":"2023-09-23T21:16:14.882979+0000","last_peered":"2023-09-23T21:16:14.882979+0000","last_clean":"2023-09-23T21:16:14.882979+0000","last_became_active":"2023-09-23T21:15:36.718644+0000","last_became_peered":"2023-09-23T21:15:36.718644+0000","last_unstale":"2023-09-23T21:16:14.882979+0000","last_undegraded":"2023-09-23T21:16:14.882979+0000","last_fullsized":"2023-09-23T21:16:14.882979+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:51.768643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.5","version":"532'5149","reported_seq":5330,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.390872+0000","last_change":"2023-09-23T21:15:36.812668+0000","last_active":"2023-09-23T21:16:18.390872+0000","last_peered":"2023-09-23T21:16:18.390872+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:15:36.809179+0000","last_became_peered":"2023-09-23T21:15:36.809179+0000","last_unstale":"2023-09-23T21:16:18.390872+0000","last_undegraded":"2023-09-23T21:15:36.711291+0000","last_fullsized":"2023-09-23T21:15:36.684980+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":5149,"log_dups_size":0,"ondisk_log_size":5149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:11.989879+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148766720,"num_objects":2270,"num_object_clones":0,"num_object_copies":6810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2543,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1593,"num_write_kb":64384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1356},{"shards":"4,5","objects":-273},{"shards":"4,5,6","objects":600}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":199,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883262+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:16:14.883262+0000","last_peered":"2023-09-23T21:16:14.883262+0000","last_clean":"2023-09-23T21:16:14.883262+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:16:14.883262+0000","last_undegraded":"2023-09-23T21:16:14.883262+0000","last_fullsized":"2023-09-23T21:16:14.883262+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":348,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882992+0000","last_change":"2023-09-23T21:15:36.700128+0000","last_active":"2023-09-23T21:16:14.882992+0000","last_peered":"2023-09-23T21:16:14.882992+0000","last_clean":"2023-09-23T21:16:14.882992+0000","last_became_active":"2023-09-23T21:15:36.699374+0000","last_became_peered":"2023-09-23T21:15:36.699374+0000","last_unstale":"2023-09-23T21:16:14.882992+0000","last_undegraded":"2023-09-23T21:16:14.882992+0000","last_fullsized":"2023-09-23T21:16:14.882992+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:40.893590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.4","version":"532'5212","reported_seq":7481,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:17.940065+0000","last_change":"2023-09-23T21:15:37.918328+0000","last_active":"2023-09-23T21:16:17.940065+0000","last_peered":"2023-09-23T21:16:17.940065+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:15:37.913417+0000","last_became_peered":"2023-09-23T21:15:37.913417+0000","last_unstale":"2023-09-23T21:16:17.940065+0000","last_undegraded":"2023-09-23T21:15:37.722000+0000","last_fullsized":"2023-09-23T21:15:37.696311+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":5212,"log_dups_size":0,"ondisk_log_size":5212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:53.357576+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145489920,"num_objects":2220,"num_object_clones":0,"num_object_copies":6660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2638,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1512,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":-418},{"shards":"2,5,6","objects":1978}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":476,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.711907+0000","last_change":"2023-09-23T21:15:36.716276+0000","last_active":"2023-09-23T21:16:14.711907+0000","last_peered":"2023-09-23T21:16:14.711907+0000","last_clean":"2023-09-23T21:16:14.711907+0000","last_became_active":"2023-09-23T21:15:36.715995+0000","last_became_peered":"2023-09-23T21:15:36.715995+0000","last_unstale":"2023-09-23T21:16:14.711907+0000","last_undegraded":"2023-09-23T21:16:14.711907+0000","last_fullsized":"2023-09-23T21:16:14.711907+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:02.365152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":564,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.727778+0000","last_change":"2023-09-23T21:15:36.785937+0000","last_active":"2023-09-23T21:16:14.727778+0000","last_peered":"2023-09-23T21:16:14.727778+0000","last_clean":"2023-09-23T21:16:14.727778+0000","last_became_active":"2023-09-23T21:15:36.785603+0000","last_became_peered":"2023-09-23T21:15:36.785603+0000","last_unstale":"2023-09-23T21:16:14.727778+0000","last_undegraded":"2023-09-23T21:16:14.727778+0000","last_fullsized":"2023-09-23T21:16:14.727778+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:51.148832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"532'5089","reported_seq":5174,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.380433+0000","last_change":"2023-09-23T21:15:37.925459+0000","last_active":"2023-09-23T21:16:18.380433+0000","last_peered":"2023-09-23T21:16:18.380433+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:15:37.908666+0000","last_became_peered":"2023-09-23T21:15:37.908666+0000","last_unstale":"2023-09-23T21:16:18.380433+0000","last_undegraded":"2023-09-23T21:15:37.734531+0000","last_fullsized":"2023-09-23T21:15:37.717463+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5089,"log_dups_size":0,"ondisk_log_size":5089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148635648,"num_objects":2268,"num_object_clones":0,"num_object_copies":6804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3446,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":1946},{"shards":"4,7","objects":-267}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":169,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.712081+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:16:14.712081+0000","last_peered":"2023-09-23T21:16:14.712081+0000","last_clean":"2023-09-23T21:16:14.712081+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:16:14.712081+0000","last_undegraded":"2023-09-23T21:16:14.712081+0000","last_fullsized":"2023-09-23T21:16:14.712081+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":22,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882671+0000","last_change":"2023-09-23T21:15:36.720164+0000","last_active":"2023-09-23T21:16:14.882671+0000","last_peered":"2023-09-23T21:16:14.882671+0000","last_clean":"2023-09-23T21:16:14.882671+0000","last_became_active":"2023-09-23T21:15:36.719910+0000","last_became_peered":"2023-09-23T21:15:36.719910+0000","last_unstale":"2023-09-23T21:16:14.882671+0000","last_undegraded":"2023-09-23T21:16:14.882671+0000","last_fullsized":"2023-09-23T21:16:14.882671+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:10.098103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.6","version":"532'5063","reported_seq":5249,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.387080+0000","last_change":"2023-09-23T21:15:36.787083+0000","last_active":"2023-09-23T21:16:18.387080+0000","last_peered":"2023-09-23T21:16:18.387080+0000","last_clean":"2023-09-23T21:15:09.799249+0000","last_became_active":"2023-09-23T21:15:12.799540+0000","last_became_peered":"2023-09-23T21:15:12.799540+0000","last_unstale":"2023-09-23T21:16:18.387080+0000","last_undegraded":"2023-09-23T21:15:36.785930+0000","last_fullsized":"2023-09-23T21:15:12.679693+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:59.312702+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149356544,"num_objects":2279,"num_object_clones":0,"num_object_copies":6837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1450,"num_write_kb":57088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1387},{"shards":"4,5","objects":341}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":183,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.884192+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:16:14.884192+0000","last_peered":"2023-09-23T21:16:14.884192+0000","last_clean":"2023-09-23T21:16:14.884192+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:16:14.884192+0000","last_undegraded":"2023-09-23T21:16:14.884192+0000","last_fullsized":"2023-09-23T21:16:14.884192+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":271,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883824+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:16:14.883824+0000","last_peered":"2023-09-23T21:16:14.883824+0000","last_clean":"2023-09-23T21:16:14.883824+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:16:14.883824+0000","last_undegraded":"2023-09-23T21:16:14.883824+0000","last_fullsized":"2023-09-23T21:16:14.883824+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"532'6844","reported_seq":9187,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.024299+0000","last_change":"2023-09-23T21:16:01.728998+0000","last_active":"2023-09-23T21:16:18.024299+0000","last_peered":"2023-09-23T21:16:18.024299+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:01.723118+0000","last_became_peered":"2023-09-23T21:16:01.723118+0000","last_unstale":"2023-09-23T21:16:18.024299+0000","last_undegraded":"2023-09-23T21:16:01.697539+0000","last_fullsized":"2023-09-23T21:16:18.024299+0000","mapping_epoch":526,"log_start":"493'1783","ondisk_log_start":"493'1783","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5061,"log_dups_size":0,"ondisk_log_size":5061,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154107904,"num_objects":2351,"num_object_clones":0,"num_object_copies":7053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1345,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3608,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":638,"num_bytes_recovered":41811968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1345},{"shards":"2,4,5","objects":1006}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":456,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883324+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:16:14.883324+0000","last_peered":"2023-09-23T21:16:14.883324+0000","last_clean":"2023-09-23T21:16:14.883324+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:16:14.883324+0000","last_undegraded":"2023-09-23T21:16:14.883324+0000","last_fullsized":"2023-09-23T21:16:14.883324+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":207,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.713815+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:16:14.713815+0000","last_peered":"2023-09-23T21:16:14.713815+0000","last_clean":"2023-09-23T21:16:14.713815+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:16:14.713815+0000","last_undegraded":"2023-09-23T21:16:14.713815+0000","last_fullsized":"2023-09-23T21:16:14.713815+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.713759+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:16:14.713759+0000","last_peered":"2023-09-23T21:16:14.713759+0000","last_clean":"2023-09-23T21:16:14.713759+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:16:14.713759+0000","last_undegraded":"2023-09-23T21:16:14.713759+0000","last_fullsized":"2023-09-23T21:16:14.713759+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"532'6714","reported_seq":9554,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:18.448607+0000","last_change":"2023-09-23T21:16:01.774573+0000","last_active":"2023-09-23T21:16:18.448607+0000","last_peered":"2023-09-23T21:16:18.448607+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:16:01.764537+0000","last_became_peered":"2023-09-23T21:16:01.764537+0000","last_unstale":"2023-09-23T21:16:18.448607+0000","last_undegraded":"2023-09-23T21:16:01.707917+0000","last_fullsized":"2023-09-23T21:16:18.448607+0000","mapping_epoch":526,"log_start":"497'2164","ondisk_log_start":"497'2164","created":482,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151879692,"num_objects":2318,"num_object_clones":0,"num_object_copies":6954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3537,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":864,"num_bytes_recovered":56623104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":1364},{"shards":"4,5,7","objects":954}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":381,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882739+0000","last_change":"2023-09-23T21:15:36.719951+0000","last_active":"2023-09-23T21:16:14.882739+0000","last_peered":"2023-09-23T21:16:14.882739+0000","last_clean":"2023-09-23T21:16:14.882739+0000","last_became_active":"2023-09-23T21:15:36.719432+0000","last_became_peered":"2023-09-23T21:15:36.719432+0000","last_unstale":"2023-09-23T21:16:14.882739+0000","last_undegraded":"2023-09-23T21:16:14.882739+0000","last_fullsized":"2023-09-23T21:16:14.882739+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:59.778469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":23,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.726037+0000","last_change":"2023-09-23T21:15:36.718996+0000","last_active":"2023-09-23T21:16:14.726037+0000","last_peered":"2023-09-23T21:16:14.726037+0000","last_clean":"2023-09-23T21:16:14.726037+0000","last_became_active":"2023-09-23T21:15:36.717461+0000","last_became_peered":"2023-09-23T21:15:36.717461+0000","last_unstale":"2023-09-23T21:16:14.726037+0000","last_undegraded":"2023-09-23T21:16:14.726037+0000","last_fullsized":"2023-09-23T21:16:14.726037+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:51.584158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":236,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.729786+0000","last_change":"2023-09-23T21:15:36.718291+0000","last_active":"2023-09-23T21:16:14.729786+0000","last_peered":"2023-09-23T21:16:14.729786+0000","last_clean":"2023-09-23T21:16:14.729786+0000","last_became_active":"2023-09-23T21:15:36.717181+0000","last_became_peered":"2023-09-23T21:15:36.717181+0000","last_unstale":"2023-09-23T21:16:14.729786+0000","last_undegraded":"2023-09-23T21:16:14.729786+0000","last_fullsized":"2023-09-23T21:16:14.729786+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:57.775200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"532'6509","reported_seq":6577,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:17.949433+0000","last_change":"2023-09-23T21:16:00.841315+0000","last_active":"2023-09-23T21:16:17.949433+0000","last_peered":"2023-09-23T21:16:17.949433+0000","last_clean":"2023-09-23T21:16:17.949433+0000","last_became_active":"2023-09-23T21:16:00.829607+0000","last_became_peered":"2023-09-23T21:16:00.829607+0000","last_unstale":"2023-09-23T21:16:17.949433+0000","last_undegraded":"2023-09-23T21:16:17.949433+0000","last_fullsized":"2023-09-23T21:16:17.949433+0000","mapping_epoch":482,"log_start":"493'1633","ondisk_log_start":"493'1633","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4876,"log_dups_size":0,"ondisk_log_size":4876,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151027712,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3417,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":540,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.725280+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:16:14.725280+0000","last_peered":"2023-09-23T21:16:14.725280+0000","last_clean":"2023-09-23T21:16:14.725280+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:16:14.725280+0000","last_undegraded":"2023-09-23T21:16:14.725280+0000","last_fullsized":"2023-09-23T21:16:14.725280+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":268,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.724640+0000","last_change":"2023-09-23T21:15:36.718723+0000","last_active":"2023-09-23T21:16:14.724640+0000","last_peered":"2023-09-23T21:16:14.724640+0000","last_clean":"2023-09-23T21:16:14.724640+0000","last_became_active":"2023-09-23T21:15:36.716966+0000","last_became_peered":"2023-09-23T21:15:36.716966+0000","last_unstale":"2023-09-23T21:16:14.724640+0000","last_undegraded":"2023-09-23T21:16:14.724640+0000","last_fullsized":"2023-09-23T21:16:14.724640+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:10.887236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":185,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883893+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:16:14.883893+0000","last_peered":"2023-09-23T21:16:14.883893+0000","last_clean":"2023-09-23T21:16:14.883893+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:16:14.883893+0000","last_undegraded":"2023-09-23T21:16:14.883893+0000","last_fullsized":"2023-09-23T21:16:14.883893+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"532'6751","reported_seq":8851,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:18.356788+0000","last_change":"2023-09-23T21:16:00.748968+0000","last_active":"2023-09-23T21:16:18.356788+0000","last_peered":"2023-09-23T21:16:18.356788+0000","last_clean":"2023-09-23T21:16:18.356788+0000","last_became_active":"2023-09-23T21:16:00.745123+0000","last_became_peered":"2023-09-23T21:16:00.745123+0000","last_unstale":"2023-09-23T21:16:18.356788+0000","last_undegraded":"2023-09-23T21:16:18.356788+0000","last_fullsized":"2023-09-23T21:16:18.356788+0000","mapping_epoch":501,"log_start":"500'2344","ondisk_log_start":"500'2344","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4407,"log_dups_size":0,"ondisk_log_size":4407,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151486464,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3566,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":502,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":360,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.726997+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:16:14.726997+0000","last_peered":"2023-09-23T21:16:14.726997+0000","last_clean":"2023-09-23T21:16:14.726997+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:16:14.726997+0000","last_undegraded":"2023-09-23T21:16:14.726997+0000","last_fullsized":"2023-09-23T21:16:14.726997+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":866,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:14.883611+0000","last_change":"2023-09-23T21:15:37.889694+0000","last_active":"2023-09-23T21:16:14.883611+0000","last_peered":"2023-09-23T21:16:14.883611+0000","last_clean":"2023-09-23T21:15:35.435956+0000","last_became_active":"2023-09-23T21:15:37.879389+0000","last_became_peered":"2023-09-23T21:15:37.879389+0000","last_unstale":"2023-09-23T21:16:14.883611+0000","last_undegraded":"2023-09-23T21:15:37.700827+0000","last_fullsized":"2023-09-23T21:15:37.700230+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:15.483465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":104,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.713359+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:16:14.713359+0000","last_peered":"2023-09-23T21:16:14.713359+0000","last_clean":"2023-09-23T21:16:14.713359+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:16:14.713359+0000","last_undegraded":"2023-09-23T21:16:14.713359+0000","last_fullsized":"2023-09-23T21:16:14.713359+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":347,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.714216+0000","last_change":"2023-09-23T21:15:36.805725+0000","last_active":"2023-09-23T21:16:14.714216+0000","last_peered":"2023-09-23T21:16:14.714216+0000","last_clean":"2023-09-23T21:16:14.714216+0000","last_became_active":"2023-09-23T21:15:36.778557+0000","last_became_peered":"2023-09-23T21:15:36.778557+0000","last_unstale":"2023-09-23T21:16:14.714216+0000","last_undegraded":"2023-09-23T21:16:14.714216+0000","last_fullsized":"2023-09-23T21:16:14.714216+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:24.564634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"532'6643","reported_seq":6720,"reported_epoch":532,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:16:17.976946+0000","last_change":"2023-09-23T21:16:00.842289+0000","last_active":"2023-09-23T21:16:17.976946+0000","last_peered":"2023-09-23T21:16:17.976946+0000","last_clean":"2023-09-23T21:15:36.326799+0000","last_became_active":"2023-09-23T21:16:00.822378+0000","last_became_peered":"2023-09-23T21:16:00.822378+0000","last_unstale":"2023-09-23T21:16:17.976946+0000","last_undegraded":"2023-09-23T21:16:17.976946+0000","last_fullsized":"2023-09-23T21:16:17.976946+0000","mapping_epoch":520,"log_start":"493'1700","ondisk_log_start":"493'1700","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149192704,"num_objects":2276,"num_object_clones":0,"num_object_copies":6828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2276,"num_objects_unfound":0,"num_objects_dirty":2276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3512,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2276}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":149,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883773+0000","last_change":"2023-09-23T21:15:36.718478+0000","last_active":"2023-09-23T21:16:14.883773+0000","last_peered":"2023-09-23T21:16:14.883773+0000","last_clean":"2023-09-23T21:16:14.883773+0000","last_became_active":"2023-09-23T21:15:36.717508+0000","last_became_peered":"2023-09-23T21:15:36.717508+0000","last_unstale":"2023-09-23T21:16:14.883773+0000","last_undegraded":"2023-09-23T21:16:14.883773+0000","last_fullsized":"2023-09-23T21:16:14.883773+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:23.183805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":28,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882549+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:16:14.882549+0000","last_peered":"2023-09-23T21:16:14.882549+0000","last_clean":"2023-09-23T21:16:14.882549+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:16:14.882549+0000","last_undegraded":"2023-09-23T21:16:14.882549+0000","last_fullsized":"2023-09-23T21:16:14.882549+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":440,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.729516+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:16:14.729516+0000","last_peered":"2023-09-23T21:16:14.729516+0000","last_clean":"2023-09-23T21:16:14.729516+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:16:14.729516+0000","last_undegraded":"2023-09-23T21:16:14.729516+0000","last_fullsized":"2023-09-23T21:16:14.729516+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"532'6572","reported_seq":8678,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:18.025307+0000","last_change":"2023-09-23T21:16:00.763134+0000","last_active":"2023-09-23T21:16:18.025307+0000","last_peered":"2023-09-23T21:16:18.025307+0000","last_clean":"2023-09-23T21:16:18.025307+0000","last_became_active":"2023-09-23T21:16:00.758875+0000","last_became_peered":"2023-09-23T21:16:00.758875+0000","last_unstale":"2023-09-23T21:16:18.025307+0000","last_undegraded":"2023-09-23T21:16:18.025307+0000","last_fullsized":"2023-09-23T21:16:18.025307+0000","mapping_epoch":498,"log_start":"493'1441","ondisk_log_start":"493'1441","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5131,"log_dups_size":0,"ondisk_log_size":5131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147652608,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3460,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":129,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883546+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:16:14.883546+0000","last_peered":"2023-09-23T21:16:14.883546+0000","last_clean":"2023-09-23T21:16:14.883546+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:16:14.883546+0000","last_undegraded":"2023-09-23T21:16:14.883546+0000","last_fullsized":"2023-09-23T21:16:14.883546+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":335,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.729846+0000","last_change":"2023-09-23T21:15:36.718380+0000","last_active":"2023-09-23T21:16:14.729846+0000","last_peered":"2023-09-23T21:16:14.729846+0000","last_clean":"2023-09-23T21:16:14.729846+0000","last_became_active":"2023-09-23T21:15:36.717363+0000","last_became_peered":"2023-09-23T21:15:36.717363+0000","last_unstale":"2023-09-23T21:16:14.729846+0000","last_undegraded":"2023-09-23T21:16:14.729846+0000","last_fullsized":"2023-09-23T21:16:14.729846+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:11.961939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":237,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.718941+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:16:14.718941+0000","last_peered":"2023-09-23T21:16:14.718941+0000","last_clean":"2023-09-23T21:16:14.718941+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:16:14.718941+0000","last_undegraded":"2023-09-23T21:16:14.718941+0000","last_fullsized":"2023-09-23T21:16:14.718941+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"532'5128","reported_seq":5303,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.449234+0000","last_change":"2023-09-23T21:15:38.027486+0000","last_active":"2023-09-23T21:16:18.449234+0000","last_peered":"2023-09-23T21:16:18.449234+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:15:38.015308+0000","last_became_peered":"2023-09-23T21:15:38.015308+0000","last_unstale":"2023-09-23T21:16:18.449234+0000","last_undegraded":"2023-09-23T21:15:37.751992+0000","last_fullsized":"2023-09-23T21:15:37.734941+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":5128,"log_dups_size":0,"ondisk_log_size":5128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:13.280295+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149094400,"num_objects":2275,"num_object_clones":0,"num_object_copies":6825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1566,"num_write_kb":60864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":-300},{"shards":"2,5,6","objects":1960}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":187,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.886307+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:16:14.886307+0000","last_peered":"2023-09-23T21:16:14.886307+0000","last_clean":"2023-09-23T21:16:14.886307+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:16:14.886307+0000","last_undegraded":"2023-09-23T21:16:14.886307+0000","last_fullsized":"2023-09-23T21:16:14.886307+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":375,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.710825+0000","last_change":"2023-09-23T21:15:36.715325+0000","last_active":"2023-09-23T21:16:14.710825+0000","last_peered":"2023-09-23T21:16:14.710825+0000","last_clean":"2023-09-23T21:16:14.710825+0000","last_became_active":"2023-09-23T21:15:36.714995+0000","last_became_peered":"2023-09-23T21:15:36.714995+0000","last_unstale":"2023-09-23T21:16:14.710825+0000","last_undegraded":"2023-09-23T21:16:14.710825+0000","last_fullsized":"2023-09-23T21:16:14.710825+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:21.699827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":202,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.882666+0000","last_change":"2023-09-23T21:15:36.718660+0000","last_active":"2023-09-23T21:16:14.882666+0000","last_peered":"2023-09-23T21:16:14.882666+0000","last_clean":"2023-09-23T21:16:14.882666+0000","last_became_active":"2023-09-23T21:15:36.717919+0000","last_became_peered":"2023-09-23T21:15:36.717919+0000","last_unstale":"2023-09-23T21:16:14.882666+0000","last_undegraded":"2023-09-23T21:16:14.882666+0000","last_fullsized":"2023-09-23T21:16:14.882666+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:59.596117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"532'5199","reported_seq":5322,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:17.977100+0000","last_change":"2023-09-23T21:15:37.899429+0000","last_active":"2023-09-23T21:16:17.977100+0000","last_peered":"2023-09-23T21:16:17.977100+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:15:37.895550+0000","last_became_peered":"2023-09-23T21:15:37.895550+0000","last_unstale":"2023-09-23T21:16:17.977100+0000","last_undegraded":"2023-09-23T21:15:37.722673+0000","last_fullsized":"2023-09-23T21:15:37.695773+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:11.453135+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153616384,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1711,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":-282},{"shards":"2,5,6","objects":2043}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":439,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:15.720494+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:16:15.720494+0000","last_peered":"2023-09-23T21:16:15.720494+0000","last_clean":"2023-09-23T21:16:15.720494+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:16:15.720494+0000","last_undegraded":"2023-09-23T21:16:15.720494+0000","last_fullsized":"2023-09-23T21:16:15.720494+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":69,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.711090+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:16:14.711090+0000","last_peered":"2023-09-23T21:16:14.711090+0000","last_clean":"2023-09-23T21:16:14.711090+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:16:14.711090+0000","last_undegraded":"2023-09-23T21:16:14.711090+0000","last_fullsized":"2023-09-23T21:16:14.711090+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":217,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.718290+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:16:14.718290+0000","last_peered":"2023-09-23T21:16:14.718290+0000","last_clean":"2023-09-23T21:16:14.718290+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:16:14.718290+0000","last_undegraded":"2023-09-23T21:16:14.718290+0000","last_fullsized":"2023-09-23T21:16:14.718290+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"532'5244","reported_seq":5340,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.482944+0000","last_change":"2023-09-23T21:15:36.812663+0000","last_active":"2023-09-23T21:16:18.482944+0000","last_peered":"2023-09-23T21:16:18.482944+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:15:36.808663+0000","last_became_peered":"2023-09-23T21:15:36.808663+0000","last_unstale":"2023-09-23T21:16:18.482944+0000","last_undegraded":"2023-09-23T21:15:36.712242+0000","last_fullsized":"2023-09-23T21:15:36.687952+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5244,"log_dups_size":0,"ondisk_log_size":5244,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149618688,"num_objects":2283,"num_object_clones":0,"num_object_copies":6849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3571,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1363},{"shards":"4,5","objects":-383},{"shards":"4,5,6","objects":659}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":159,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883479+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:16:14.883479+0000","last_peered":"2023-09-23T21:16:14.883479+0000","last_clean":"2023-09-23T21:16:14.883479+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:16:14.883479+0000","last_undegraded":"2023-09-23T21:16:14.883479+0000","last_fullsized":"2023-09-23T21:16:14.883479+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":594,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.729869+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:16:14.729869+0000","last_peered":"2023-09-23T21:16:14.729869+0000","last_clean":"2023-09-23T21:16:14.729869+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:16:14.729869+0000","last_undegraded":"2023-09-23T21:16:14.729869+0000","last_fullsized":"2023-09-23T21:16:14.729869+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"532'5139","reported_seq":5775,"reported_epoch":532,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.457911+0000","last_change":"2023-09-23T21:16:14.478236+0000","last_active":"2023-09-23T21:16:18.457911+0000","last_peered":"2023-09-23T21:16:18.457911+0000","last_clean":"2023-09-23T21:15:07.989292+0000","last_became_active":"2023-09-23T21:15:12.941733+0000","last_became_peered":"2023-09-23T21:15:12.941733+0000","last_unstale":"2023-09-23T21:16:18.457911+0000","last_undegraded":"2023-09-23T21:15:12.704994+0000","last_fullsized":"2023-09-23T21:15:12.681408+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5139,"log_dups_size":0,"ondisk_log_size":5139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147980288,"num_objects":2258,"num_object_clones":0,"num_object_copies":6774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":533,"num_objects_misplaced":1157,"num_objects_unfound":0,"num_objects_dirty":2258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3478,"num_write_kb":183552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":280,"num_bytes_recovered":18350080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":1157},{"shards":"4,5","objects":568}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":222,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.711804+0000","last_change":"2023-09-23T21:15:36.731913+0000","last_active":"2023-09-23T21:16:14.711804+0000","last_peered":"2023-09-23T21:16:14.711804+0000","last_clean":"2023-09-23T21:16:14.711804+0000","last_became_active":"2023-09-23T21:15:36.715596+0000","last_became_peered":"2023-09-23T21:15:36.715596+0000","last_unstale":"2023-09-23T21:16:14.711804+0000","last_undegraded":"2023-09-23T21:16:14.711804+0000","last_fullsized":"2023-09-23T21:16:14.711804+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:14.247487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":189,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.726224+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:16:14.726224+0000","last_peered":"2023-09-23T21:16:14.726224+0000","last_clean":"2023-09-23T21:16:14.726224+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:16:14.726224+0000","last_undegraded":"2023-09-23T21:16:14.726224+0000","last_fullsized":"2023-09-23T21:16:14.726224+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"532'5228","reported_seq":5302,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:16:18.421802+0000","last_change":"2023-09-23T21:15:12.860448+0000","last_active":"2023-09-23T21:16:18.421802+0000","last_peered":"2023-09-23T21:16:18.421802+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:16:18.421802+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5228,"log_dups_size":0,"ondisk_log_size":5228,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146833408,"num_objects":2241,"num_object_clones":0,"num_object_copies":6723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1941,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3545,"num_write_kb":185120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":1289},{"shards":"4,5","objects":300}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":26,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.883409+0000","last_change":"2023-09-23T21:15:36.818381+0000","last_active":"2023-09-23T21:16:14.883409+0000","last_peered":"2023-09-23T21:16:14.883409+0000","last_clean":"2023-09-23T21:16:14.883409+0000","last_became_active":"2023-09-23T21:15:36.724491+0000","last_became_peered":"2023-09-23T21:15:36.724491+0000","last_unstale":"2023-09-23T21:16:14.883409+0000","last_undegraded":"2023-09-23T21:16:14.883409+0000","last_fullsized":"2023-09-23T21:16:14.883409+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:08.475242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.724871+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:16:14.724871+0000","last_peered":"2023-09-23T21:16:14.724871+0000","last_clean":"2023-09-23T21:16:14.724871+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:16:14.724871+0000","last_undegraded":"2023-09-23T21:16:14.724871+0000","last_fullsized":"2023-09-23T21:16:14.724871+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"532'5039","reported_seq":10883,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-09-23T21:16:17.793177+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:16:17.793177+0000","last_peered":"2023-09-23T21:16:17.793177+0000","last_clean":"2023-09-23T21:16:17.793177+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:16:17.793177+0000","last_undegraded":"2023-09-23T21:16:17.793177+0000","last_fullsized":"2023-09-23T21:16:17.793177+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5039,"log_dups_size":0,"ondisk_log_size":5039,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147193856,"num_objects":2246,"num_object_clones":0,"num_object_copies":6738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3376,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":24,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.729531+0000","last_change":"2023-09-23T21:15:36.709281+0000","last_active":"2023-09-23T21:16:14.729531+0000","last_peered":"2023-09-23T21:16:14.729531+0000","last_clean":"2023-09-23T21:16:14.729531+0000","last_became_active":"2023-09-23T21:15:36.709011+0000","last_became_peered":"2023-09-23T21:15:36.709011+0000","last_unstale":"2023-09-23T21:16:14.729531+0000","last_undegraded":"2023-09-23T21:16:14.729531+0000","last_fullsized":"2023-09-23T21:16:14.729531+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:56.451393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":382,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.725735+0000","last_change":"2023-09-23T21:15:36.718816+0000","last_active":"2023-09-23T21:16:14.725735+0000","last_peered":"2023-09-23T21:16:14.725735+0000","last_clean":"2023-09-23T21:16:14.725735+0000","last_became_active":"2023-09-23T21:15:36.718449+0000","last_became_peered":"2023-09-23T21:15:36.718449+0000","last_unstale":"2023-09-23T21:16:14.725735+0000","last_undegraded":"2023-09-23T21:16:14.725735+0000","last_fullsized":"2023-09-23T21:16:14.725735+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:36.076184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"533'5177","reported_seq":9114,"reported_epoch":533,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:16:21.040905+0000","last_change":"2023-09-23T21:16:15.767599+0000","last_active":"2023-09-23T21:16:21.040905+0000","last_peered":"2023-09-23T21:16:21.040905+0000","last_clean":"2023-09-23T21:15:08.796333+0000","last_became_active":"2023-09-23T21:16:15.742634+0000","last_became_peered":"2023-09-23T21:16:15.742634+0000","last_unstale":"2023-09-23T21:16:21.040905+0000","last_undegraded":"2023-09-23T21:16:21.040905+0000","last_fullsized":"2023-09-23T21:16:21.040905+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5177,"log_dups_size":0,"ondisk_log_size":5177,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144867328,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3541,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":["6","2","4","5"],"object_location_counts":[{"shards":"2,5,6","objects":2211}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":472,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.728057+0000","last_change":"2023-09-23T21:15:36.737658+0000","last_active":"2023-09-23T21:16:14.728057+0000","last_peered":"2023-09-23T21:16:14.728057+0000","last_clean":"2023-09-23T21:16:14.728057+0000","last_became_active":"2023-09-23T21:15:36.716750+0000","last_became_peered":"2023-09-23T21:15:36.716750+0000","last_unstale":"2023-09-23T21:16:14.728057+0000","last_undegraded":"2023-09-23T21:16:14.728057+0000","last_fullsized":"2023-09-23T21:16:14.728057+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:30.186342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":189,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-09-23T21:16:14.710127+0000","last_change":"2023-09-23T21:15:36.716348+0000","last_active":"2023-09-23T21:16:14.710127+0000","last_peered":"2023-09-23T21:16:14.710127+0000","last_clean":"2023-09-23T21:16:14.710127+0000","last_became_active":"2023-09-23T21:15:36.716039+0000","last_became_peered":"2023-09-23T21:15:36.716039+0000","last_unstale":"2023-09-23T21:16:14.710127+0000","last_undegraded":"2023-09-23T21:16:14.710127+0000","last_fullsized":"2023-09-23T21:16:14.710127+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:59.988277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":36,"stat_sum":{"num_bytes":4783669272,"num_objects":72995,"num_object_clones":0,"num_object_copies":218985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":44517,"num_objects_misplaced":6831,"num_objects_unfound":0,"num_objects_dirty":72995,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":96936,"num_write_kb":4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11867,"num_bytes_recovered":771850240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15494426624,"data_stored":15494414408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":180318,"ondisk_log_size":180318,"up":108,"acting":94,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":33540,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201851,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2973892,"kb_used_data":2233596,"kb_used_omap":0,"kb_used_meta":740288,"kb_avail":90771260,"statfs":{"total":95995035648,"available":92949770240,"internally_reserved":0,"allocated":2287202304,"data_stored":2285777363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":758054912},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":3.5129999999999999,"15min":2.1920000000000002},"min":{"1min":0.218,"5min":0.214,"15min":0.20699999999999999},"max":{"1min":5.7949999999999999,"5min":141.33199999999999,"15min":141.33199999999999},"last":7.726},{"interface":"front","average":{"1min":1.9079999999999999,"5min":3.6230000000000002,"15min":2.2280000000000002},"min":{"1min":0.27800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.8330000000000002,"5min":141.31700000000001,"15min":141.31700000000001},"last":0.307}]},{"osd":1,"last update":"Sat Sep 23 21:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":3.0030000000000001,"15min":3.3889999999999998},"min":{"1min":0.317,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.8700000000000001,"5min":76.007999999999996,"15min":114.16500000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.294,"5min":4.077,"15min":3.8610000000000002},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.681,"5min":117.637,"15min":117.637},"last":7.7699999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":5.0549999999999997,"15min":4.4119999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.7999999999999998,"5min":135.529,"15min":183.44200000000001},"last":0.246},{"interface":"front","average":{"1min":1.6679999999999999,"5min":6.016,"15min":4.6349999999999998},"min":{"1min":0.318,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":5.7859999999999996,"5min":135.56700000000001,"15min":183.46199999999999},"last":2.8250000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":7.9820000000000002,"15min":4.2640000000000002},"min":{"1min":0.30499999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":5.7279999999999998,"5min":135.66200000000001,"15min":135.66200000000001},"last":7.7960000000000003},{"interface":"front","average":{"1min":1.897,"5min":8.4979999999999993,"15min":4.375},"min":{"1min":0.318,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":5.8600000000000003,"5min":135.47,"15min":135.47},"last":2.903}]},{"osd":4,"last update":"Sat Sep 23 21:15:34 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":1.089,"15min":0.96399999999999997},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.26500000000000001},"max":{"1min":10.989000000000001,"5min":10.989000000000001,"15min":15.942},"last":1.1990000000000001},{"interface":"front","average":{"1min":2.6459999999999999,"5min":1.0880000000000001,"15min":0.88300000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":8.5600000000000005,"5min":8.5600000000000005,"15min":16.725999999999999},"last":2.9260000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:15:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":1.2230000000000001,"15min":1.1200000000000001},"min":{"1min":0.33500000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.231,"5min":11.231,"15min":11.231},"last":7.8200000000000003},{"interface":"front","average":{"1min":3.2610000000000001,"5min":1.29,"15min":1.044},"min":{"1min":0.33000000000000002,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":9.0440000000000005,"5min":10.005000000000001,"15min":13.867000000000001},"last":2.758}]},{"osd":6,"last update":"Sat Sep 23 21:15:34 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":2.6219999999999999,"15min":1.825},"min":{"1min":0.39500000000000002,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":11.038,"5min":143.18700000000001,"15min":143.18700000000001},"last":1.228},{"interface":"front","average":{"1min":2.0819999999999999,"5min":2.4289999999999998,"15min":1.724},"min":{"1min":0.38600000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.1920000000000002,"5min":142.70400000000001,"15min":142.70400000000001},"last":2.8719999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463472,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3128624,"kb_used_data":2418728,"kb_used_omap":0,"kb_used_meta":709888,"kb_avail":90616528,"statfs":{"total":95995035648,"available":92791324672,"internally_reserved":0,"allocated":2476777472,"data_stored":2475376416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":726925312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:52 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":4.5259999999999998,"15min":2.6720000000000002},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":3.1059999999999999,"5min":149.61600000000001,"15min":149.61600000000001},"last":0.372},{"interface":"front","average":{"1min":1.661,"5min":4.7279999999999998,"15min":2.7450000000000001},"min":{"1min":0.30499999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.2089999999999996,"5min":149.303,"15min":149.303},"last":0.503}]},{"osd":1,"last update":"Sat Sep 23 21:15:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":4.3150000000000004,"15min":3.4489999999999998},"min":{"1min":0.313,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.8319999999999999,"5min":129.37899999999999,"15min":129.37899999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.4339999999999999,"5min":4.3419999999999996,"15min":3.3820000000000001},"min":{"1min":0.27300000000000002,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":5.9960000000000004,"5min":129.40100000000001,"15min":129.40100000000001},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:15:52 2023","interfaces":[{"interface":"back","average":{"1min":4.6689999999999996,"5min":6.6550000000000002,"15min":5.9489999999999998},"min":{"1min":0.32000000000000001,"5min":0.27900000000000003,"15min":0.26800000000000002},"max":{"1min":58.877000000000002,"5min":133.46600000000001,"15min":199.23400000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":5.077,"5min":6.6509999999999998,"15min":5.9050000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":59.154000000000003,"5min":133.33000000000001,"15min":199.08000000000001},"last":0.42699999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:15:52 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":6.9989999999999997,"15min":4.069},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.4489999999999998,"5min":137.875,"15min":178.40000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.7490000000000001,"5min":6.6929999999999996,"15min":4.0490000000000004},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":6.8860000000000001,"5min":137.60900000000001,"15min":178.26599999999999},"last":0.48599999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:15:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":1.246,"15min":1.117},"min":{"1min":0.34999999999999998,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":7.4589999999999996,"5min":7.4589999999999996,"15min":13.1},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.1479999999999999,"15min":0.95699999999999996},"min":{"1min":0.38900000000000001,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":8.7680000000000007,"5min":11.949999999999999,"15min":11.949999999999999},"last":5.5899999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:15:52 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":1.417,"15min":1.1599999999999999},"min":{"1min":0.35899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":9.3900000000000006,"5min":12.355,"15min":12.355},"last":0.77700000000000002},{"interface":"front","average":{"1min":3.4860000000000002,"5min":1.506,"15min":1.1950000000000001},"min":{"1min":0.48799999999999999,"5min":0.31900000000000001,"15min":0.28499999999999998},"max":{"1min":8.8160000000000007,"5min":11.042,"15min":11.042},"last":1.889}]},{"osd":7,"last update":"Sat Sep 23 21:15:24 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":0.70899999999999996,"15min":0.94099999999999995},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.3559999999999999,"5min":4.3559999999999999,"15min":7.8079999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.2690000000000001,"5min":0.97599999999999998,"15min":1.034},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.9119999999999999,"5min":7.9119999999999999,"15min":7.9119999999999999},"last":0.58799999999999997}]}]},{"osd":1,"up_from":0,"seq":1185410973918,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991658,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781549,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3848968,"kb_used_data":2806080,"kb_used_omap":7,"kb_used_meta":1042808,"kb_avail":89896184,"statfs":{"total":95995035648,"available":92053692416,"internally_reserved":0,"allocated":2873425920,"data_stored":2870881092,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1067835473},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":3.2639999999999998,"15min":2.1360000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.189},"max":{"1min":3.2040000000000002,"5min":103.021,"15min":103.021},"last":0.17699999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":3.7909999999999999,"15min":2.8759999999999999},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.2959999999999998,"5min":103.154,"15min":103.154},"last":0.252}]},{"osd":1,"last update":"Sat Sep 23 21:15:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":5.2480000000000002,"15min":4.3369999999999997},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.188},"max":{"1min":10.766,"5min":143.482,"15min":212.12},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":5.7830000000000004,"15min":4.3369999999999997},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":1.6200000000000001,"5min":143.61000000000001,"15min":213.02799999999999},"last":0.312}]},{"osd":3,"last update":"Sat Sep 23 21:16:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.8300000000000001,"15min":4.3220000000000001},"min":{"1min":0.27100000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.8260000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.76400000000000001,"5min":3.4820000000000002,"15min":4.4189999999999996},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.2850000000000001,"5min":82.241,"15min":143.27799999999999},"last":0.433}]},{"osd":4,"last update":"Sat Sep 23 21:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":1.79,"15min":2.3100000000000001},"min":{"1min":0.32600000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.6890000000000001,"5min":25.231000000000002,"15min":225.80799999999999},"last":1.518},{"interface":"front","average":{"1min":1.851,"5min":2.5139999999999998,"15min":3.0089999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":5.7889999999999997,"5min":12.590999999999999,"15min":225.79300000000001},"last":1.5369999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":2.4329999999999998,"15min":2.5449999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0129999999999999,"5min":12.555,"15min":98.682000000000002},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.7689999999999999,"15min":2.4609999999999999},"min":{"1min":0.34399999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.667999999999999,"5min":12.667999999999999,"15min":143.34700000000001},"last":1.2210000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:15:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.819,"15min":2.819},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.308},"max":{"1min":5.798,"5min":10.885999999999999,"15min":175.72499999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.865,"15min":2.2160000000000002},"min":{"1min":0.36399999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.1559999999999997,"5min":25.204999999999998,"15min":176.20400000000001},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:15:59 2023","interfaces":[{"interface":"back","average":{"1min":2.9020000000000001,"5min":2.2290000000000001,"15min":1.7809999999999999},"min":{"1min":0.45300000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.7469999999999999,"5min":25.260000000000002,"15min":25.260000000000002},"last":0.371},{"interface":"front","average":{"1min":1.9410000000000001,"5min":2.552,"15min":2.2679999999999998},"min":{"1min":0.46999999999999997,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":6.2140000000000004,"5min":12.613,"15min":16.875},"last":0.45900000000000002}]}]},{"osd":3,"up_from":0,"seq":635655160143,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986761,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3577412,"kb_used_data":2832828,"kb_used_omap":0,"kb_used_meta":744384,"kb_avail":90167740,"statfs":{"total":95995035648,"available":92331765760,"internally_reserved":0,"allocated":2900815872,"data_stored":2899391194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":762249216},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":5.1189999999999998,"15min":2.597},"min":{"1min":0.28100000000000003,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":8.2149999999999999,"5min":226.751,"15min":226.751},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.52,"5min":4.7519999999999998,"15min":2.4809999999999999},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.223},"max":{"1min":7.173,"5min":226.43299999999999,"15min":226.43299999999999},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:15:20 2023","interfaces":[{"interface":"back","average":{"1min":3.9399999999999999,"5min":4.5739999999999998,"15min":5.226},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":35.179000000000002,"5min":119.55800000000001,"15min":127.411},"last":0.34599999999999997},{"interface":"front","average":{"1min":3.819,"5min":3.8860000000000001,"15min":4.4119999999999999},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":34.883000000000003,"5min":116.78100000000001,"15min":118.075},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:15:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":6.1840000000000002,"15min":5.3150000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.673,"5min":210.50299999999999,"15min":210.50299999999999},"last":90.950000000000003},{"interface":"front","average":{"1min":3.113,"5min":6.9580000000000002,"15min":5.4260000000000002},"min":{"1min":0.44,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":8.3030000000000008,"5min":210.59,"15min":210.59},"last":90.846000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:15:20 2023","interfaces":[{"interface":"back","average":{"1min":5.5110000000000001,"5min":7.6760000000000002,"15min":4.0430000000000001},"min":{"1min":0.36199999999999999,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":74.75,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.44},{"interface":"front","average":{"1min":6.0599999999999996,"5min":8.343,"15min":4.3390000000000004},"min":{"1min":0.38400000000000001,"5min":0.32700000000000001,"15min":0.25600000000000001},"max":{"1min":74.814999999999998,"5min":214.54900000000001,"15min":214.54900000000001},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.175,"15min":0.90300000000000002},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":5.4390000000000001,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":3.6360000000000001,"5min":1.583,"15min":1.0669999999999999},"min":{"1min":0.307,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":11.566000000000001,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.32800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:15:51 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":1.2,"15min":1.202},"min":{"1min":0.35799999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.446999999999999,"5min":10.446999999999999,"15min":36.557000000000002},"last":0.313},{"interface":"front","average":{"1min":3.121,"5min":1.2849999999999999,"15min":1.206},"min":{"1min":0.51900000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4160000000000004,"5min":6.6040000000000001,"15min":38.468000000000004},"last":0.502}]},{"osd":7,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.103,"15min":1.6419999999999999},"min":{"1min":0.505,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":6.2999999999999998,"5min":6.8399999999999999,"15min":145.721},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.718,"5min":1.099,"15min":1.782},"min":{"1min":0.377,"5min":0.33500000000000002,"15min":0.30499999999999999},"max":{"1min":10.677,"5min":10.677,"15min":148.45699999999999},"last":0.51600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725125,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4167092,"kb_used_data":3377260,"kb_used_omap":0,"kb_used_meta":789760,"kb_avail":89578060,"statfs":{"total":95995035648,"available":91727933440,"internally_reserved":0,"allocated":3458314240,"data_stored":3456901158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":808714240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:15:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":2.7650000000000001,"15min":1.6459999999999999},"min":{"1min":0.26300000000000001,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":3.048,"5min":82.280000000000001,"15min":82.280000000000001},"last":3.105},{"interface":"front","average":{"1min":1.6859999999999999,"5min":3.073,"15min":1.8560000000000001},"min":{"1min":0.36399999999999999,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":7.234,"5min":85.236000000000004,"15min":85.236000000000004},"last":0.54000000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1309999999999998,"5min":3.2229999999999999,"15min":4.4790000000000001},"min":{"1min":0.41099999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.548,"5min":74.920000000000002,"15min":233.77099999999999},"last":2.5920000000000001},{"interface":"front","average":{"1min":1.077,"5min":2.7290000000000001,"15min":4.2560000000000002},"min":{"1min":0.30199999999999999,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":7.5030000000000001,"5min":74.978999999999999,"15min":233.75200000000001},"last":3.0600000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:15:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":3.1970000000000001,"15min":3.706},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":7.6379999999999999,"5min":78.917000000000002,"15min":231.34700000000001},"last":3.3260000000000001},{"interface":"front","average":{"1min":1.613,"5min":3.335,"15min":3.9020000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.1859999999999999,"5min":78.888999999999996,"15min":231.381},"last":0.57299999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:15:51 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":3.137,"15min":4.0449999999999999},"min":{"1min":0.41199999999999998,"5min":0.317,"15min":0.26800000000000002},"max":{"1min":6.798,"5min":41.981000000000002,"15min":263.887},"last":2.625},{"interface":"front","average":{"1min":1.1879999999999999,"5min":2.6240000000000001,"15min":3.8380000000000001},"min":{"1min":0.39700000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.6100000000000003,"5min":42.113,"15min":263.70100000000002},"last":3.1880000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:15:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.052,"15min":0.78900000000000003},"min":{"1min":0.50900000000000001,"5min":0.27200000000000002,"15min":0.25},"max":{"1min":8.6969999999999992,"5min":8.6969999999999992,"15min":12.739000000000001},"last":3.9769999999999999},{"interface":"front","average":{"1min":1.2909999999999999,"5min":0.752,"15min":0.81699999999999995},"min":{"1min":0.39700000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.3399999999999999,"5min":5.0099999999999998,"15min":10.792},"last":0.78300000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:15:51 2023","interfaces":[{"interface":"back","average":{"1min":2.556,"5min":1.198,"15min":1.552},"min":{"1min":0.42199999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":8.9130000000000003,"5min":8.9130000000000003,"15min":83.451999999999998},"last":4.024},{"interface":"front","average":{"1min":3.6819999999999999,"5min":1.774,"15min":1.772},"min":{"1min":0.38200000000000001,"5min":0.30399999999999999,"15min":0.27400000000000002},"max":{"1min":12.042,"5min":12.725,"15min":83.484999999999999},"last":2.6509999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:15:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.1279999999999999,"15min":1.03},"min":{"1min":0.55900000000000005,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.0250000000000004,"5min":7.0250000000000004,"15min":9.6329999999999991},"last":0.627},{"interface":"front","average":{"1min":1.7110000000000001,"5min":0.91600000000000004,"15min":0.748},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":7.5670000000000002,"5min":7.5670000000000002,"15min":7.5670000000000002},"last":5.6710000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2864513024,"data_stored":2864513024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1551433728,"data_stored":1551433728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2881097728,"data_stored":2881093656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3452375040,"data_stored":3452370968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2470903808,"data_stored":2470903808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2274103296,"data_stored":2274099224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:16:23.152 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-09-23T21:16:23.166 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:23.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:23.235 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:23.272 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:23.314 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:23.383 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:23.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:23.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:23.391 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:23 smithi123 ceph-mon[193903]: from='client.47177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:16:23.491 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:23.578 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:23.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:23 smithi121 ceph-mon[211526]: from='client.47177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:16:23.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:23 smithi121 ceph-mon[219709]: from='client.47177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:16:23.686 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:23.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:23.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:23.800 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:23.833 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:23.935 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:23.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:24.039 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:24.065 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:24.108 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:24.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:24.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:24.211 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:24.294 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:24.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:24 smithi121 ceph-mon[211526]: pgmap v1678: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 540 op/s; 44198/215838 objects degraded (20.477%); 6831/215838 objects misplaced (3.165%); 16 MiB/s, 264 objects/s recovering 2023-09-23T21:16:24.420 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:24.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:24.526 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:24.585 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:24 smithi123 ceph-mon[193903]: pgmap v1678: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 540 op/s; 44198/215838 objects degraded (20.477%); 6831/215838 objects misplaced (3.165%); 16 MiB/s, 264 objects/s recovering 2023-09-23T21:16:24.585 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:24.595 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:24.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:24 smithi121 ceph-mon[219709]: pgmap v1678: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 540 op/s; 44198/215838 objects degraded (20.477%); 6831/215838 objects misplaced (3.165%); 16 MiB/s, 264 objects/s recovering 2023-09-23T21:16:24.779 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:24.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:24.800 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:16:24.897 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:25.010 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:25.065 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:16:25.100 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:16:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:16:24] "GET /metrics HTTP/1.1" 200 36717 "" "Prometheus/2.43.0" 2023-09-23T21:16:25.139 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:25.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:25.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:25.261 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:25.363 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:25.507 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:25.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:25.554 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:25.613 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:25.646 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:26.014 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:26.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:26.123 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:26.160 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:26.198 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:26.213 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:26.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:26.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:26.317 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:26.412 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:26.426 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:16:26.541 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:26.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:26.606 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:26 smithi121 ceph-mon[219709]: pgmap v1679: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 687 op/s; 43671/208983 objects degraded (20.897%); 5885/208983 objects misplaced (2.816%); 21 MiB/s, 352 objects/s recovering 2023-09-23T21:16:26.606 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3051023259' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:26.606 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:26.607 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:26 smithi121 ceph-mon[211526]: pgmap v1679: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 687 op/s; 43671/208983 objects degraded (20.897%); 5885/208983 objects misplaced (2.816%); 21 MiB/s, 352 objects/s recovering 2023-09-23T21:16:26.607 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3051023259' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:26.607 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:26.646 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:26.684 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:26.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:26 smithi123 ceph-mon[193903]: pgmap v1679: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 687 op/s; 43671/208983 objects degraded (20.897%); 5885/208983 objects misplaced (2.816%); 21 MiB/s, 352 objects/s recovering 2023-09-23T21:16:26.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3051023259' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:26.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:27.522 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[219709]: failed to encode map e534 with expected crc 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[219709]: failed to encode map e534 with expected crc 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[219709]: osdmap e534: 8 total, 8 up, 5 in 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 43671/208983 objects degraded (20.897%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/382327153' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:27.523 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[211526]: failed to encode map e534 with expected crc 2023-09-23T21:16:27.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[211526]: failed to encode map e534 with expected crc 2023-09-23T21:16:27.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:27.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[211526]: osdmap e534: 8 total, 8 up, 5 in 2023-09-23T21:16:27.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 43671/208983 objects degraded (20.897%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:27.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/382327153' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T21:16:27.730 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 1 pgp_num to 1 2023-09-23T21:16:27.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:27 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:27.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:27 smithi123 ceph-mon[193903]: failed to encode map e534 with expected crc 2023-09-23T21:16:27.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:27 smithi123 ceph-mon[193903]: failed to encode map e534 with expected crc 2023-09-23T21:16:27.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:27.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:27 smithi123 ceph-mon[193903]: osdmap e534: 8 total, 8 up, 5 in 2023-09-23T21:16:27.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:27 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 43671/208983 objects degraded (20.897%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:27.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/382327153' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T21:16:27.851 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:27.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:27.956 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:27.984 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:28.029 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:28.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:28.132 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:28.166 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:28.575 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:28.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:28.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[211526]: pgmap v1681: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 422 op/s; 43671/208668 objects degraded (20.928%); 5885/208668 objects misplaced (2.820%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[211526]: failed to encode map e535 with expected crc 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/382327153' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[211526]: osdmap e535: 8 total, 8 up, 5 in 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[211526]: failed to encode map e535 with expected crc 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[219709]: pgmap v1681: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 422 op/s; 43671/208668 objects degraded (20.928%); 5885/208668 objects misplaced (2.820%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[219709]: failed to encode map e535 with expected crc 2023-09-23T21:16:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/382327153' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T21:16:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[219709]: osdmap e535: 8 total, 8 up, 5 in 2023-09-23T21:16:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:28 smithi121 ceph-mon[219709]: failed to encode map e535 with expected crc 2023-09-23T21:16:28.679 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:28.709 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:28.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:28 smithi123 ceph-mon[193903]: pgmap v1681: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 422 op/s; 43671/208668 objects degraded (20.928%); 5885/208668 objects misplaced (2.820%); 12 MiB/s, 188 objects/s recovering 2023-09-23T21:16:28.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:28.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:28.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:28 smithi123 ceph-mon[193903]: failed to encode map e535 with expected crc 2023-09-23T21:16:28.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/382327153' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T21:16:28.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:28 smithi123 ceph-mon[193903]: osdmap e535: 8 total, 8 up, 5 in 2023-09-23T21:16:28.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:28 smithi123 ceph-mon[193903]: failed to encode map e535 with expected crc 2023-09-23T21:16:28.808 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:28.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:28.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:28.912 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:28.952 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:29.103 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:29.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:29.175 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:16:29.196 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:16:29.209 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:29.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:16:29.239 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:29.245 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:16:29.273 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:16:29.293 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:29.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:29.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:29.308 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:16:29.336 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:16:29.378 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:16:29.379 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.379 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.379 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.379 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.379 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.379 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.379 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.380 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:16:29.397 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:29.432 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:29.488 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:29.488 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:29.592 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:29.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:29.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:29.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3682560696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3682560696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:29.696 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:29.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3682560696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:29.745 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:29.792 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:29.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:29.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:29.896 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:29.994 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.102 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:30.220 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:30.248 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.291 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:30.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:30.394 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:30.427 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:16:30.482 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:30.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:30 smithi121 ceph-mon[219709]: pgmap v1683: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 833 op/s; 42836/199047 objects degraded (21.521%); 4750/199047 objects misplaced (2.386%); 26 MiB/s, 419 objects/s recovering 2023-09-23T21:16:30.488 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:30 smithi121 ceph-mon[211526]: pgmap v1683: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 833 op/s; 42836/199047 objects degraded (21.521%); 4750/199047 objects misplaced (2.386%); 26 MiB/s, 419 objects/s recovering 2023-09-23T21:16:30.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:30.592 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:30.622 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.686 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:30.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:30.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:30 smithi123 ceph-mon[193903]: pgmap v1683: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 833 op/s; 42836/199047 objects degraded (21.521%); 4750/199047 objects misplaced (2.386%); 26 MiB/s, 419 objects/s recovering 2023-09-23T21:16:30.792 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:30.869 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:30.883 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:30.938 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:16:30.998 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:31.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:31.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:31.121 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:31.220 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:31.357 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:31.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:31.472 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:31.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:31 smithi123 ceph-mon[193903]: pgmap v1684: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 712 op/s; 42836/198735 objects degraded (21.554%); 4750/198735 objects misplaced (2.390%); 24 MiB/s, 379 objects/s recovering 2023-09-23T21:16:31.504 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:31.562 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:31.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:31.645 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:31.667 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:31.691 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:31 smithi121 ceph-mon[211526]: pgmap v1684: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 712 op/s; 42836/198735 objects degraded (21.554%); 4750/198735 objects misplaced (2.390%); 24 MiB/s, 379 objects/s recovering 2023-09-23T21:16:31.692 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:31 smithi121 ceph-mon[219709]: pgmap v1684: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 712 op/s; 42836/198735 objects degraded (21.554%); 4750/198735 objects misplaced (2.390%); 24 MiB/s, 379 objects/s recovering 2023-09-23T21:16:31.692 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:31.736 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:31.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:31.840 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:31.870 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:32.152 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:32.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:32.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:32.231 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:16:32.258 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:32.309 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:32.425 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:32.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:32.530 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:32.571 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:32.633 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:32.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:32.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:32.700 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:32.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 42836/198735 objects degraded (21.554%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:32.736 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:32.773 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 42836/198735 objects degraded (21.554%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:32.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 42836/198735 objects degraded (21.554%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:32.778 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:32.846 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:32.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:32.949 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:32.998 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:33.056 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:33.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:33.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:33.160 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:33.197 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:33.302 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:33.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:33.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:33.406 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:33.431 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:33.456 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:16:33.477 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:33.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:33.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:33.581 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:33.607 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:33.659 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:33.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/690889021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:33.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:33 smithi121 ceph-mon[219709]: pgmap v1685: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 42525/195414 objects degraded (21.761%); 4750/195414 objects misplaced (2.431%); 16 MiB/s, 259 objects/s recovering 2023-09-23T21:16:33.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/690889021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:33.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:33 smithi121 ceph-mon[211526]: pgmap v1685: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 42525/195414 objects degraded (21.761%); 4750/195414 objects misplaced (2.431%); 16 MiB/s, 259 objects/s recovering 2023-09-23T21:16:33.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:33.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:33.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/690889021' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:16:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:33 smithi123 ceph-mon[193903]: pgmap v1685: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 42525/195414 objects degraded (21.761%); 4750/195414 objects misplaced (2.431%); 16 MiB/s, 259 objects/s recovering 2023-09-23T21:16:33.765 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:33.779 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:16:33.779 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:16:33.838 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:34.042 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:34.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:34.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:34.166 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:34.269 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:34.509 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:34.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:34.521 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:34 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:34.521 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:34 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/690889021' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:34.521 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:34 smithi123 ceph-mon[193903]: osdmap e536: 8 total, 8 up, 5 in 2023-09-23T21:16:34.521 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:34 smithi123 ceph-mon[193903]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.521 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:34 smithi123 ceph-mon[193903]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.521 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:34 smithi123 ceph-mon[193903]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.621 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:34.653 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:34.730 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:34.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/690889021' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[211526]: osdmap e536: 8 total, 8 up, 5 in 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[211526]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[211526]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[211526]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/690889021' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:16:34.735 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[219709]: osdmap e536: 8 total, 8 up, 5 in 2023-09-23T21:16:34.736 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[219709]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.736 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[219709]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.736 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:34 smithi121 ceph-mon[219709]: failed to encode map e536 with expected crc 2023-09-23T21:16:34.756 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:34.835 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:34.871 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:34.948 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:34.949 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:35.004 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:16:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:16:34] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.43.0" 2023-09-23T21:16:35.214 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:35.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:35.222 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:16:35.248 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:16:35.275 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:16:35.299 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:16:35.318 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:35.339 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:16:35.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:16:35.399 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:16:35.420 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:35.426 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:16:35.548 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:35.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:35.662 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:35.694 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4280654714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:35 smithi123 ceph-mon[193903]: pgmap v1687: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 837 op/s; 41435/188553 objects degraded (21.975%); 3919/188553 objects misplaced (2.078%); 25 MiB/s, 471 objects/s recovering 2023-09-23T21:16:35.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4280654714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:36.714 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:35 smithi121 ceph-mon[219709]: pgmap v1687: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 837 op/s; 41435/188553 objects degraded (21.975%); 3919/188553 objects misplaced (2.078%); 25 MiB/s, 471 objects/s recovering 2023-09-23T21:16:36.715 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4280654714' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:36.715 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:35 smithi121 ceph-mon[211526]: pgmap v1687: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 837 op/s; 41435/188553 objects degraded (21.975%); 3919/188553 objects misplaced (2.078%); 25 MiB/s, 471 objects/s recovering 2023-09-23T21:16:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:36.716 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:36.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:36.717 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: osdmap e537: 8 total, 8 up, 5 in 2023-09-23T21:16:36.717 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.718 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.718 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.718 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.718 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.718 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.719 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.719 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[211526]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: osdmap e537: 8 total, 8 up, 5 in 2023-09-23T21:16:36.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.720 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.720 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.720 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.720 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.720 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.720 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:36 smithi121 ceph-mon[219709]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.720 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:16:36.722 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:16:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: osdmap e537: 8 total, 8 up, 5 in 2023-09-23T21:16:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:36 smithi123 ceph-mon[193903]: failed to encode map e537 with expected crc 2023-09-23T21:16:36.818 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:36.856 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:36.916 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:36.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:37.021 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:37.058 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:37.108 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:37.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:37.214 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:37.232 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:37.242 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:37.338 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:37.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:37.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:37.360 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:37.442 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:37.468 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[211526]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[211526]: osdmap e538: 8 total, 8 up, 5 in 2023-09-23T21:16:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[211526]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[211526]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[211526]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41435/188553 objects degraded (21.975%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:37.580 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[211526]: pgmap v1690: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 582 op/s; 41435/188256 objects degraded (22.010%); 3919/188256 objects misplaced (2.082%); 15 MiB/s, 334 objects/s recovering 2023-09-23T21:16:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[219709]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[219709]: osdmap e538: 8 total, 8 up, 5 in 2023-09-23T21:16:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[219709]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[219709]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[219709]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41435/188553 objects degraded (21.975%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:37.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:37 smithi121 ceph-mon[219709]: pgmap v1690: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 582 op/s; 41435/188256 objects degraded (22.010%); 3919/188256 objects misplaced (2.082%); 15 MiB/s, 334 objects/s recovering 2023-09-23T21:16:37.616 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:37.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:37.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:37 smithi123 ceph-mon[193903]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:37 smithi123 ceph-mon[193903]: osdmap e538: 8 total, 8 up, 5 in 2023-09-23T21:16:37.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:37 smithi123 ceph-mon[193903]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:37 smithi123 ceph-mon[193903]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:37 smithi123 ceph-mon[193903]: failed to encode map e538 with expected crc 2023-09-23T21:16:37.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41435/188553 objects degraded (21.975%), 27 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:37.714 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:37 smithi123 ceph-mon[193903]: pgmap v1690: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 582 op/s; 41435/188256 objects degraded (22.010%); 3919/188256 objects misplaced (2.082%); 15 MiB/s, 334 objects/s recovering 2023-09-23T21:16:37.722 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:37.758 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:37.811 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:37.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:37.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:37.916 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:37.991 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.046 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:38.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:38.149 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:38.179 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.226 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:38.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:38.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:38.330 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:38.360 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[211526]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[211526]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[211526]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[211526]: osdmap e539: 8 total, 8 up, 5 in 2023-09-23T21:16:38.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[211526]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4009929988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:16:38.493 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:16:38.495 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:16:38.566 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:38.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:38.670 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:38.698 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[219709]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[219709]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[219709]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[219709]: osdmap e539: 8 total, 8 up, 5 in 2023-09-23T21:16:38.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[219709]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4009929988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:16:38.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:38 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:16:38.793 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:38.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:38.902 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:38.917 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:16:38.929 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:38.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:38 smithi123 ceph-mon[193903]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:38 smithi123 ceph-mon[193903]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:38 smithi123 ceph-mon[193903]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:38 smithi123 ceph-mon[193903]: osdmap e539: 8 total, 8 up, 5 in 2023-09-23T21:16:38.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:38 smithi123 ceph-mon[193903]: failed to encode map e539 with expected crc 2023-09-23T21:16:38.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4009929988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:16:38.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:38 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:16:39.286 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:39.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:39.390 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:39.430 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:16:39.460 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:39.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[211526]: failed to encode map e540 with expected crc 2023-09-23T21:16:39.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:16:39.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[211526]: osdmap e540: 8 total, 8 up, 5 in 2023-09-23T21:16:39.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[211526]: pgmap v1693: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 773 op/s; 39158/179271 objects degraded (21.843%); 4349/179271 objects misplaced (2.426%); 17 MiB/s, 410 objects/s recovering 2023-09-23T21:16:39.543 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[219709]: failed to encode map e540 with expected crc 2023-09-23T21:16:39.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:16:39.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[219709]: osdmap e540: 8 total, 8 up, 5 in 2023-09-23T21:16:39.544 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:39 smithi121 ceph-mon[219709]: pgmap v1693: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 773 op/s; 39158/179271 objects degraded (21.843%); 4349/179271 objects misplaced (2.426%); 17 MiB/s, 410 objects/s recovering 2023-09-23T21:16:39.626 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:39.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:39.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:39 smithi123 ceph-mon[193903]: failed to encode map e540 with expected crc 2023-09-23T21:16:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:39 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:16:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:39 smithi123 ceph-mon[193903]: osdmap e540: 8 total, 8 up, 5 in 2023-09-23T21:16:39.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:39 smithi123 ceph-mon[193903]: pgmap v1693: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 773 op/s; 39158/179271 objects degraded (21.843%); 4349/179271 objects misplaced (2.426%); 17 MiB/s, 410 objects/s recovering 2023-09-23T21:16:39.735 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:39.755 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:39.839 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:39.916 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:39.996 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:40.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:40.109 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:40.122 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:16:40.122 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:16:40.122 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:16:40.122 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:16:40.122 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:16:40.202 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:40.387 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:40.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:40.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:40.498 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:40.530 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:16:40.541 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:40.555 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2777661503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:40.555 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:40.555 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2777661503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:40.555 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2777661503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:40.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:41.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:41.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:41.534 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: osdmap e541: 8 total, 8 up, 5 in 2023-09-23T21:16:41.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.535 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[219709]: pgmap v1695: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 636 op/s; 39158/178995 objects degraded (21.877%); 4349/178995 objects misplaced (2.430%); 14 MiB/s, 338 objects/s recovering 2023-09-23T21:16:41.670 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:16:41.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:41.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: osdmap e541: 8 total, 8 up, 5 in 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:41 smithi121 ceph-mon[211526]: pgmap v1695: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 636 op/s; 39158/178995 objects degraded (21.877%); 4349/178995 objects misplaced (2.430%); 14 MiB/s, 338 objects/s recovering 2023-09-23T21:16:41.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:41.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: osdmap e541: 8 total, 8 up, 5 in 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: failed to encode map e541 with expected crc 2023-09-23T21:16:41.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:41 smithi123 ceph-mon[193903]: pgmap v1695: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 636 op/s; 39158/178995 objects degraded (21.877%); 4349/178995 objects misplaced (2.430%); 14 MiB/s, 338 objects/s recovering 2023-09-23T21:16:42.354 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:42.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:42.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:42.438 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:42.464 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:42.503 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:42.784 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39158/178995 objects degraded (21.877%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:42.784 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:42.784 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:42.784 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39158/178995 objects degraded (21.877%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:42.785 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:42.785 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:42.796 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39158/178995 objects degraded (21.877%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:42.797 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:16:42.797 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:42.801 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:42.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:42.907 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:42.944 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:43.010 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:43.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:43.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:43.050 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:16:43.071 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:16:43.105 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:16:43.115 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:43.129 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:16:43.169 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:16:43.204 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:43.209 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:16:43.234 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:16:43.261 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:16:43.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.263 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.263 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.263 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.263 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.263 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:16:43.307 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:43.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:43.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:43.415 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:43.479 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:43.547 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3059238292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:43.547 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3171781457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:16:43.547 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:43 smithi121 ceph-mon[211526]: pgmap v1696: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 712 op/s; 38807/175425 objects degraded (22.122%); 4349/175425 objects misplaced (2.479%); 15 MiB/s, 331 objects/s recovering 2023-09-23T21:16:43.548 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:16:43.554 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:43.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:43.566 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3059238292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:43.566 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3171781457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:16:43.566 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:43 smithi123 ceph-mon[193903]: pgmap v1696: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 712 op/s; 38807/175425 objects degraded (22.122%); 4349/175425 objects misplaced (2.479%); 15 MiB/s, 331 objects/s recovering 2023-09-23T21:16:43.666 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:43.695 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:43.737 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:43.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:43.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:43.841 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:43.871 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3059238292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:43.871 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3171781457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:16:43.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:43 smithi121 ceph-mon[219709]: pgmap v1696: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 712 op/s; 38807/175425 objects degraded (22.122%); 4349/175425 objects misplaced (2.479%); 15 MiB/s, 331 objects/s recovering 2023-09-23T21:16:43.875 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:44.344 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:16:44.736 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:44.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:44.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[211526]: failed to encode map e542 with expected crc 2023-09-23T21:16:44.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[211526]: failed to encode map e542 with expected crc 2023-09-23T21:16:44.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3171781457' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:16:44.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[211526]: osdmap e542: 8 total, 8 up, 5 in 2023-09-23T21:16:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[219709]: failed to encode map e542 with expected crc 2023-09-23T21:16:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[219709]: failed to encode map e542 with expected crc 2023-09-23T21:16:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3171781457' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:16:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:44 smithi121 ceph-mon[219709]: osdmap e542: 8 total, 8 up, 5 in 2023-09-23T21:16:44.831 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:16:44.840 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:44.896 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:44 smithi123 ceph-mon[193903]: failed to encode map e542 with expected crc 2023-09-23T21:16:44.897 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:44 smithi123 ceph-mon[193903]: failed to encode map e542 with expected crc 2023-09-23T21:16:44.897 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3171781457' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:16:44.897 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:44 smithi123 ceph-mon[193903]: osdmap e542: 8 total, 8 up, 5 in 2023-09-23T21:16:44.905 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:44.935 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:45.068 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:45.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:45.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:45.129 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:16:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:16:44] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.43.0" 2023-09-23T21:16:45.189 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:45.270 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:45.457 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:45.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:45.570 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:45.572 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:45 smithi123 ceph-mon[193903]: pgmap v1698: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 732 op/s; 38228/168315 objects degraded (22.712%); 3818/168315 objects misplaced (2.268%); 18 MiB/s, 361 objects/s recovering 2023-09-23T21:16:45.623 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:45.673 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:45.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:45.776 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:45.804 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:45.804 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:45.875 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:45.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:45 smithi121 ceph-mon[211526]: pgmap v1698: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 732 op/s; 38228/168315 objects degraded (22.712%); 3818/168315 objects misplaced (2.268%); 18 MiB/s, 361 objects/s recovering 2023-09-23T21:16:45.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:45 smithi121 ceph-mon[219709]: pgmap v1698: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 732 op/s; 38228/168315 objects degraded (22.712%); 3818/168315 objects misplaced (2.268%); 18 MiB/s, 361 objects/s recovering 2023-09-23T21:16:46.008 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:46.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:46.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:46.115 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:46.121 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:46.213 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:46.354 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:46.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:46.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:46.468 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:46.505 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:46.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:16:46.653 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:46.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:46.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:46.754 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:46 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 38228/168315 objects degraded (22.712%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:46.760 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:46.786 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:47.028 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:46 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 38228/168315 objects degraded (22.712%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:47.202 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:46 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 38228/168315 objects degraded (22.712%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:47.223 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:16:47.240 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:47.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:47.348 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:47.379 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:47.680 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:47.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:47.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:47.730 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:47.780 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:16:47.785 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:47.873 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:47.991 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:48.004 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:48.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:48.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:48.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1840088016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:48.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:48.054 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:47 smithi121 ceph-mon[211526]: pgmap v1699: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 469 op/s; 38228/168009 objects degraded (22.754%); 3818/168009 objects misplaced (2.272%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:16:48.055 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1840088016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:48.055 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:48.055 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:47 smithi121 ceph-mon[219709]: pgmap v1699: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 469 op/s; 38228/168009 objects degraded (22.754%); 3818/168009 objects misplaced (2.272%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:16:48.108 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:16:48.200 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1840088016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:48.200 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:16:48.200 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:47 smithi123 ceph-mon[193903]: pgmap v1699: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 469 op/s; 38228/168009 objects degraded (22.754%); 3818/168009 objects misplaced (2.272%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:16:48.205 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:48.374 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:16:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:48.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:16:48.477 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:48.513 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:48.771 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:48.772 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[211526]: failed to encode map e543 with expected crc 2023-09-23T21:16:48.772 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:48.772 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[211526]: osdmap e543: 8 total, 8 up, 5 in 2023-09-23T21:16:48.772 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[211526]: failed to encode map e543 with expected crc 2023-09-23T21:16:48.971 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:48.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:48.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:48 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:48 smithi123 ceph-mon[193903]: failed to encode map e543 with expected crc 2023-09-23T21:16:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:48 smithi123 ceph-mon[193903]: osdmap e543: 8 total, 8 up, 5 in 2023-09-23T21:16:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:48 smithi123 ceph-mon[193903]: failed to encode map e543 with expected crc 2023-09-23T21:16:49.076 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:49.132 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:16:49.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[219709]: failed to encode map e543 with expected crc 2023-09-23T21:16:49.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:16:49.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[219709]: osdmap e543: 8 total, 8 up, 5 in 2023-09-23T21:16:49.133 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:48 smithi121 ceph-mon[219709]: failed to encode map e543 with expected crc 2023-09-23T21:16:49.169 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:49.296 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:49.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:49.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:49.433 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:49.476 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:49.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:49.581 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:49.608 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:49.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:49 smithi121 ceph-mon[211526]: pgmap v1701: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 838 op/s; 37470/158856 objects degraded (23.587%); 3305/158856 objects misplaced (2.081%); 18 MiB/s, 288 objects/s recovering 2023-09-23T21:16:49.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4140705953' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:16:49.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:49 smithi121 ceph-mon[219709]: pgmap v1701: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 838 op/s; 37470/158856 objects degraded (23.587%); 3305/158856 objects misplaced (2.081%); 18 MiB/s, 288 objects/s recovering 2023-09-23T21:16:49.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4140705953' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:16:49.791 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:16:49.855 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:49.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:49.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:49.960 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:49.986 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.080 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:50.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:50.139 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:49 smithi123 ceph-mon[193903]: pgmap v1701: 109 pgs: 1 active+recovery_wait+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 838 op/s; 37470/158856 objects degraded (23.587%); 3305/158856 objects misplaced (2.081%); 18 MiB/s, 288 objects/s recovering 2023-09-23T21:16:50.140 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4140705953' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:16:50.184 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:50.212 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.554 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:50.556 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:16:50.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:50.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:50.658 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:50.694 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.725 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:16:50.726 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:16:50.755 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:50.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:50.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:50.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[211526]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[211526]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4140705953' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:16:50.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[211526]: osdmap e544: 8 total, 8 up, 5 in 2023-09-23T21:16:50.790 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[211526]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[219709]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[219709]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4140705953' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:16:50.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[219709]: osdmap e544: 8 total, 8 up, 5 in 2023-09-23T21:16:50.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:50 smithi121 ceph-mon[219709]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.859 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:50.950 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:50 smithi123 ceph-mon[193903]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.950 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:50 smithi123 ceph-mon[193903]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.950 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4140705953' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:16:50.951 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:50 smithi123 ceph-mon[193903]: osdmap e544: 8 total, 8 up, 5 in 2023-09-23T21:16:50.951 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:50 smithi123 ceph-mon[193903]: failed to encode map e544 with expected crc 2023-09-23T21:16:50.956 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:50.983 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:50.988 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:16:51.010 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:16:51.032 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:16:51.055 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:16:51.061 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:51.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:51.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:51.083 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:16:51.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:16:51.133 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:16:51.154 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.156 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:16:51.159 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:16:51.169 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:51.235 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:51.605 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:51.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:51.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:51.710 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:51.739 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:51.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: osdmap e545: 8 total, 8 up, 5 in 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3944680528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: pgmap v1704: 109 pgs: 2 peering, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 543 op/s; 37471/158520 objects degraded (23.638%); 18/158520 objects misplaced (0.011%); 11 MiB/s, 181 objects/s recovering 2023-09-23T21:16:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 37470/158856 objects degraded (23.587%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: osdmap e545: 8 total, 8 up, 5 in 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: failed to encode map e545 with expected crc 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3944680528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: pgmap v1704: 109 pgs: 2 peering, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 543 op/s; 37471/158520 objects degraded (23.638%); 18/158520 objects misplaced (0.011%); 11 MiB/s, 181 objects/s recovering 2023-09-23T21:16:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 37470/158856 objects degraded (23.587%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: failed to encode map e545 with expected crc 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: osdmap e545: 8 total, 8 up, 5 in 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: failed to encode map e545 with expected crc 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: failed to encode map e545 with expected crc 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: failed to encode map e545 with expected crc 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: failed to encode map e545 with expected crc 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: failed to encode map e545 with expected crc 2023-09-23T21:16:52.216 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3944680528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:16:52.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: pgmap v1704: 109 pgs: 2 peering, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 543 op/s; 37471/158520 objects degraded (23.638%); 18/158520 objects misplaced (0.011%); 11 MiB/s, 181 objects/s recovering 2023-09-23T21:16:52.217 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 37470/158856 objects degraded (23.587%), 25 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:52.237 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:16:52.759 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:52.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:52.864 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:52.889 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:52.914 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:53.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[219709]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[219709]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[219709]: osdmap e546: 8 total, 8 up, 5 in 2023-09-23T21:16:53.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[219709]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[219709]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.101 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[219709]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[211526]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[211526]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[211526]: osdmap e546: 8 total, 8 up, 5 in 2023-09-23T21:16:53.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[211526]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[211526]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.102 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:52 smithi121 ceph-mon[211526]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.149 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:52 smithi123 ceph-mon[193903]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:52 smithi123 ceph-mon[193903]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:52 smithi123 ceph-mon[193903]: osdmap e546: 8 total, 8 up, 5 in 2023-09-23T21:16:53.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:52 smithi123 ceph-mon[193903]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:52 smithi123 ceph-mon[193903]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.150 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:52 smithi123 ceph-mon[193903]: failed to encode map e546 with expected crc 2023-09-23T21:16:53.311 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:53.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:53.414 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:53.484 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:53.511 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:53.722 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:53.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:53.836 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:53.922 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:54.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:53 smithi121 ceph-mon[219709]: pgmap v1706: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 682 op/s; 31550/156624 objects degraded (20.144%); 65/156624 objects misplaced (0.042%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:16:54.011 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:53 smithi121 ceph-mon[211526]: pgmap v1706: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 682 op/s; 31550/156624 objects degraded (20.144%); 65/156624 objects misplaced (0.042%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:16:54.042 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:54.046 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:53 smithi123 ceph-mon[193903]: pgmap v1706: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 682 op/s; 31550/156624 objects degraded (20.144%); 65/156624 objects misplaced (0.042%); 17 MiB/s, 266 objects/s recovering 2023-09-23T21:16:54.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:54.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:54.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:54.173 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:54.215 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:54.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:54.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:54.318 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:54.380 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:54.722 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:54.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:54.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:54.823 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:16:54.829 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:54.863 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:54.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1744418322' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:16:55.126 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:16:55.155 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:16:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:16:54] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.43.0" 2023-09-23T21:16:55.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1744418322' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:16:55.156 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1744418322' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:16:55.235 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:55.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:55.339 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:55.406 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:55.450 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:55.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:55.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:55.526 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:16:55.526 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:16:55.526 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:16:55.526 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:16:55.526 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:16:55.554 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:16:55.581 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:55.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1744418322' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: osdmap e547: 8 total, 8 up, 5 in 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[211526]: pgmap v1708: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 360 op/s; 25722/152856 objects degraded (16.828%); 47/152856 objects misplaced (0.031%); 11 MiB/s, 258 objects/s recovering 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1744418322' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:16:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: osdmap e547: 8 total, 8 up, 5 in 2023-09-23T21:16:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: failed to encode map e547 with expected crc 2023-09-23T21:16:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:55 smithi121 ceph-mon[219709]: pgmap v1708: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 360 op/s; 25722/152856 objects degraded (16.828%); 47/152856 objects misplaced (0.031%); 11 MiB/s, 258 objects/s recovering 2023-09-23T21:16:55.952 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:16:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:55.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:16:56.056 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:16:56.076 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:16:56.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: failed to encode map e547 with expected crc 2023-09-23T21:16:56.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1744418322' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:16:56.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: osdmap e547: 8 total, 8 up, 5 in 2023-09-23T21:16:56.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: failed to encode map e547 with expected crc 2023-09-23T21:16:56.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: failed to encode map e547 with expected crc 2023-09-23T21:16:56.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: failed to encode map e547 with expected crc 2023-09-23T21:16:56.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: failed to encode map e547 with expected crc 2023-09-23T21:16:56.080 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:55 smithi123 ceph-mon[193903]: pgmap v1708: 109 pgs: 1 active+recovering+undersized+remapped, 2 peering, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 360 op/s; 25722/152856 objects degraded (16.828%); 47/152856 objects misplaced (0.031%); 11 MiB/s, 258 objects/s recovering 2023-09-23T21:16:56.082 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:56.126 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:16:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:56.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:16:56.230 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:56.290 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:56.336 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:56.572 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:16:56.601 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:56.603 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:16:56.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:56.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:56.716 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:16:56.818 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 25722/152856 objects degraded (16.828%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:56 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3693366851' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:56 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 25722/152856 objects degraded (16.828%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:56 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3693366851' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:56.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:56 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:56.946 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:16:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:56.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:16:56.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 25722/152856 objects degraded (16.828%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:16:56.965 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:56 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3693366851' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:56.966 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:56 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:16:57.056 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:16:57.160 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:57.280 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:16:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:57.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:16:57.349 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:16:57.391 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:57.425 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:57.729 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:16:57.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:57 smithi123 ceph-mon[193903]: pgmap v1709: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 25722/151914 objects degraded (16.932%); 47/151914 objects misplaced (0.031%); 9.6 MiB/s, 219 objects/s recovering 2023-09-23T21:16:57.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:57.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:57 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:57.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:57 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:57.933 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:57 smithi123 ceph-mon[193903]: osdmap e548: 8 total, 8 up, 5 in 2023-09-23T21:16:58.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[219709]: pgmap v1709: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 25722/151914 objects degraded (16.932%); 47/151914 objects misplaced (0.031%); 9.6 MiB/s, 219 objects/s recovering 2023-09-23T21:16:58.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:58.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:58.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:58.100 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[219709]: osdmap e548: 8 total, 8 up, 5 in 2023-09-23T21:16:58.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[211526]: pgmap v1709: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 25722/151914 objects degraded (16.932%); 47/151914 objects misplaced (0.031%); 9.6 MiB/s, 219 objects/s recovering 2023-09-23T21:16:58.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:16:58.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:16:58.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:16:58.101 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:57 smithi121 ceph-mon[211526]: osdmap e548: 8 total, 8 up, 5 in 2023-09-23T21:16:58.889 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:16:58.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:58.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:16:58.993 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:16:59.020 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:59.085 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:58 smithi123 ceph-mon[193903]: failed to encode map e548 with expected crc 2023-09-23T21:16:59.085 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:58 smithi123 ceph-mon[193903]: failed to encode map e548 with expected crc 2023-09-23T21:16:59.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:58 smithi121 ceph-mon[211526]: failed to encode map e548 with expected crc 2023-09-23T21:16:59.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:58 smithi121 ceph-mon[211526]: failed to encode map e548 with expected crc 2023-09-23T21:16:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:58 smithi121 ceph-mon[219709]: failed to encode map e548 with expected crc 2023-09-23T21:16:59.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:58 smithi121 ceph-mon[219709]: failed to encode map e548 with expected crc 2023-09-23T21:16:59.478 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:16:59.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:59.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:16:59.582 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:16:59.619 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:59.704 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:16:59.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:16:59.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:16:59.808 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:16:59.843 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:16:59.874 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:16:59.874 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:147730 12.1 12:806729b5:::benchmark_data_smithi121_563784_object57327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:58.648450+0000", 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 241.22196976800001, 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0452919920000001, 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:16:59.875 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 147730 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648450+0000", 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648450+0000", 2023-09-23T21:16:59.876 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648449+0000", 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648452+0000", 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4420000000000001e-06 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.877 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648457+0000", 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5079999999999996e-06 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648462+0000", 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4680000000000003e-06 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648498+0000", 2023-09-23T21:16:59.878 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6494e-05 2023-09-23T21:16:59.879 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.879 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.879 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.879 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648608+0000", 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010921699999999999 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648701+0000", 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3184999999999996e-05 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.659355+0000", 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010654083 2023-09-23T21:16:59.880 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.661711+0000", 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023556670000000001 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.668680+0000", 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0069692670000000003 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.881 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.882 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676424+0000", 2023-09-23T21:16:59.882 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077446140000000004 2023-09-23T21:16:59.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.883 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.883 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676434+0000", 2023-09-23T21:16:59.883 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3880000000000006e-06 2023-09-23T21:16:59.883 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.883 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.884 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.884 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.693662+0000", 2023-09-23T21:16:59.884 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0172280970000001 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.693742+0000", 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9950999999999994e-05 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:147732 12.1 12:883b207e:::benchmark_data_smithi121_563784_object57329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-09-23T21:16:59.885 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:58.648473+0000", 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 241.22194705999999, 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0457270919999999, 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 147732 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.886 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648473+0000", 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648473+0000", 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648472+0000", 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:59.887 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648474+0000", 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0200000000000001e-06 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648480+0000", 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7660000000000004e-06 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.888 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648483+0000", 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.428e-06 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.652857+0000", 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043731270000000001 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.652967+0000", 2023-09-23T21:16:59.889 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011064 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.653071+0000", 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010343700000000001 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.663197+0000", 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010125881999999999 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.890 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.663414+0000", 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000217906 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.674405+0000", 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010990541 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676477+0000", 2023-09-23T21:16:59.891 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020718540000000001 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676484+0000", 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4919999999999996e-06 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694141+0000", 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0176565399999999 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.892 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.893 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.893 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694200+0000", 2023-09-23T21:16:59.894 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8783000000000003e-05 2023-09-23T21:16:59.894 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.894 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.894 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:147747 12.1 12:8d405dd7:::benchmark_data_smithi121_563784_object57344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:58.843469+0000", 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 241.02695086700001, 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88383067199999998, 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.895 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 147747 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843469+0000", 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.896 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843469+0000", 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843467+0000", 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:16:59.897 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843480+0000", 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3183000000000001e-05 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843513+0000", 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3658999999999997e-05 2023-09-23T21:16:59.898 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843526+0000", 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2847000000000001e-05 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843632+0000", 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010543 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.899 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843650+0000", 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8427000000000002e-05 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694222+0000", 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85057223400000004 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.900 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694338+0000", 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011524800000000001 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694455+0000", 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117815 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.901 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.705071+0000", 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010615591000000001 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.705272+0000", 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00020080400000000001 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.902 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.727207+0000", 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021935316 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.727258+0000", 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0495999999999999e-05 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.903 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.727299+0000", 2023-09-23T21:16:59.904 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1851999999999998e-05 2023-09-23T21:16:59.904 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.904 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.904 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.904 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.904 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29430 13.4 13:2f637a96:::benchmark_data_smithi121_621349_object29429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.422763573, 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90678336900000001, 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.905 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29430 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.906 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447655+0000", 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.907 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447720+0000", 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4391000000000001e-05 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447728+0000", 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4300000000000006e-06 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.908 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447738+0000", 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0118999999999999e-05 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447838+0000", 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9207000000000001e-05 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.909 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447846+0000", 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2439999999999993e-06 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338557+0000", 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.890711217 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.910 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338700+0000", 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143361 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338864+0000", 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016315700000000001 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.911 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342396+0000", 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035323749999999999 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354238+0000", 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011842363 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.912 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354340+0000", 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101962 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354396+0000", 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6174000000000001e-05 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.913 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354439+0000", 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2994999999999997e-05 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.914 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29438 13.4 13:2ddaff29:::benchmark_data_smithi121_621349_object29437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.418817222, 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90300633600000002, 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29438 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.915 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:16:59.916 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451601+0000", 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451653+0000", 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2207999999999999e-05 2023-09-23T21:16:59.917 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451664+0000", 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0692e-05 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451669+0000", 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2660000000000001e-06 2023-09-23T21:16:59.918 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.452098+0000", 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00042897200000000001 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.452110+0000", 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.202e-05 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.919 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.339881+0000", 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88777067300000001 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.339995+0000", 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114695 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.920 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340148+0000", 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015272199999999999 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343455+0000", 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003307454 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.921 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354283+0000", 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010827448999999999 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354522+0000", 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00023920500000000001 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.922 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354571+0000", 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8539999999999999e-05 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354609+0000", 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8046999999999997e-05 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.923 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29448 13.4 13:22d32e47:::benchmark_data_smithi121_621349_object29447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.414074005, 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89842967699999998, 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.924 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29448 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.925 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456345+0000", 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456397+0000", 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2176999999999999e-05 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456411+0000", 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4219000000000001e-05 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.927 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456417+0000", 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6210000000000001e-06 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456475+0000", 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8056000000000001e-05 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.928 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456485+0000", 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0071000000000001e-05 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340736+0000", 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88425111199999995 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340816+0000", 2023-09-23T21:16:59.929 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0597000000000001e-05 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340916+0000", 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9607999999999999e-05 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343463+0000", 2023-09-23T21:16:59.930 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002547278 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354482+0000", 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011018857 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354722+0000", 2023-09-23T21:16:59.931 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024004300000000001 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354754+0000", 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1299e-05 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354775+0000", 2023-09-23T21:16:59.932 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1764e-05 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29452 13.4 13:23edd8ec:::benchmark_data_smithi121_621349_object29451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.41250062, 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89701267699999998, 2023-09-23T21:16:59.933 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29452 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.934 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.935 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457953+0000", 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4428000000000003e-05 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.936 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457961+0000", 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6809999999999998e-06 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457965+0000", 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6999999999999999e-06 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.458018+0000", 2023-09-23T21:16:59.937 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2782999999999999e-05 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.458027+0000", 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3030000000000005e-06 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.938 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.939 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341620+0000", 2023-09-23T21:16:59.939 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88359242400000004 2023-09-23T21:16:59.939 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.939 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341748+0000", 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012769800000000001 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341830+0000", 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2670999999999995e-05 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.940 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345316+0000", 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034856290000000001 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354632+0000", 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0093166240000000008 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.941 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354861+0000", 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022811500000000001 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354897+0000", 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6327e-05 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.942 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354932+0000", 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4768999999999999e-05 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29475 13.4 13:21b7c357:::benchmark_data_smithi121_621349_object29474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.943 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.397988252, 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88263625400000001, 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29475 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.944 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472430+0000", 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472465+0000", 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4814000000000002e-05 2023-09-23T21:16:59.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472485+0000", 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0435e-05 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472493+0000", 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5240000000000003e-06 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.947 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472551+0000", 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8548999999999998e-05 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472562+0000", 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0855e-05 2023-09-23T21:16:59.948 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342433+0000", 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86987049900000002 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342547+0000", 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114554 2023-09-23T21:16:59.949 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342679+0000", 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000131984 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347274+0000", 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045946049999999999 2023-09-23T21:16:59.950 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354817+0000", 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0075430949999999997 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354987+0000", 2023-09-23T21:16:59.951 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017053899999999999 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355035+0000", 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7305999999999997e-05 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.952 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355068+0000", 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2978000000000002e-05 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29478 13.4 13:2842156f:::benchmark_data_smithi121_621349_object29477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.39721073699999, 2023-09-23T21:16:59.953 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88206223699999997, 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29478 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.954 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.955 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:16:59.955 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.955 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.955 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.955 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473208+0000", 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.956 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473278+0000", 2023-09-23T21:16:59.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0062999999999996e-05 2023-09-23T21:16:59.957 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.957 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.957 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.957 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473296+0000", 2023-09-23T21:16:59.957 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7852000000000001e-05 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473303+0000", 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3479999999999998e-06 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.958 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473375+0000", 2023-09-23T21:16:59.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1407000000000003e-05 2023-09-23T21:16:59.959 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.959 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.959 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.959 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473396+0000", 2023-09-23T21:16:59.959 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1634999999999999e-05 2023-09-23T21:16:59.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.960 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.960 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343498+0000", 2023-09-23T21:16:59.960 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.87010185600000001 2023-09-23T21:16:59.960 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.960 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343568+0000", 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0224999999999999e-05 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343651+0000", 2023-09-23T21:16:59.961 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2723000000000001e-05 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347284+0000", 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003633168 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.962 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355131+0000", 2023-09-23T21:16:59.963 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078466939999999995 2023-09-23T21:16:59.963 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.963 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.963 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.963 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355175+0000", 2023-09-23T21:16:59.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4586999999999998e-05 2023-09-23T21:16:59.964 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.964 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.964 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.964 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355226+0000", 2023-09-23T21:16:59.964 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0695000000000002e-05 2023-09-23T21:16:59.965 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.965 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.965 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.965 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355271+0000", 2023-09-23T21:16:59.965 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4932000000000003e-05 2023-09-23T21:16:59.965 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.965 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.966 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.966 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.966 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.966 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29505 13.4 13:2e31c810:::benchmark_data_smithi121_621349_object29504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.966 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:16:59.967 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.38027230200001, 2023-09-23T21:16:59.967 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.865266541, 2023-09-23T21:16:59.967 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.967 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.967 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.967 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29505 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:16:59.968 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490146+0000", 2023-09-23T21:16:59.969 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490797+0000", 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00065088600000000004 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490813+0000", 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6189999999999999e-05 2023-09-23T21:16:59.970 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490820+0000", 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6730000000000003e-06 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490881+0000", 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0927999999999997e-05 2023-09-23T21:16:59.971 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490890+0000", 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4599999999999992e-06 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344328+0000", 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85343784700000003 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.972 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344457+0000", 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128563 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344619+0000", 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016246899999999999 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.973 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349443+0000", 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048236540000000001 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355307+0000", 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058641479999999996 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.974 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355337+0000", 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9658999999999998e-05 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355380+0000", 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3547999999999997e-05 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.975 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355414+0000", 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3769999999999997e-05 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29527 13.4 13:29cd74c5:::benchmark_data_smithi121_621349_object29526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:16:59.976 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.36540618800001, 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85053144899999999, 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29527 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.977 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.978 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505012+0000", 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505051+0000", 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8639000000000001e-05 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.979 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505066+0000", 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5812999999999999e-05 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505073+0000", 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4250000000000003e-06 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.980 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505137+0000", 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4097999999999995e-05 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505144+0000", 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2559999999999996e-06 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.981 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345347+0000", 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84020272100000004 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345451+0000", 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103905 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345554+0000", 2023-09-23T21:16:59.982 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103387 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349453+0000", 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003898641 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355452+0000", 2023-09-23T21:16:59.983 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059990449999999997 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355477+0000", 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5531999999999999e-05 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355516+0000", 2023-09-23T21:16:59.984 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8699000000000003e-05 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355545+0000", 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8785999999999998e-05 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.985 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29536 13.4 13:2bfcf8e1:::benchmark_data_smithi121_621349_object29535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.36018994099999, 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84544850299999996, 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29536 2023-09-23T21:16:59.986 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:16:59.987 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510216+0000", 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999866 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510277+0000", 2023-09-23T21:16:59.988 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0891999999999999e-05 2023-09-23T21:16:59.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.989 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:16:59.989 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510295+0000", 2023-09-23T21:16:59.989 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7482999999999998e-05 2023-09-23T21:16:59.989 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.989 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510301+0000", 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4749999999999998e-06 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510359+0000", 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7927000000000003e-05 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.990 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510368+0000", 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9919999999999996e-06 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346282+0000", 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83591370700000001 2023-09-23T21:16:59.991 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346367+0000", 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4648999999999995e-05 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:16:59.992 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346467+0000", 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100895 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351395+0000", 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0049271159999999996 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.993 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355574+0000", 2023-09-23T21:16:59.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041792699999999997 2023-09-23T21:16:59.994 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.994 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.994 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:16:59.994 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355598+0000", 2023-09-23T21:16:59.994 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4198000000000001e-05 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355645+0000", 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6449999999999997e-05 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355678+0000", 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3657e-05 2023-09-23T21:16:59.995 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29540 13.4 13:221f9937:::benchmark_data_smithi121_621349_object29539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.358934643, 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84431284100000004, 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:16:59.996 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29540 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.997 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511484+0000", 2023-09-23T21:16:59.998 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:16:59.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:16:59.999 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:16:59.999 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511579+0000", 2023-09-23T21:16:59.999 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5329999999999997e-05 2023-09-23T21:16:59.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:16:59.999 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511591+0000", 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.145e-05 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511606+0000", 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5325e-05 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.000 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511668+0000", 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1458999999999996e-05 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511677+0000", 2023-09-23T21:17:00.001 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0380000000000005e-06 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347329+0000", 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.835651904 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.002 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347442+0000", 2023-09-23T21:17:00.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011388899999999999 2023-09-23T21:17:00.003 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.003 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.003 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.003 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347532+0000", 2023-09-23T21:17:00.003 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9306000000000002e-05 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351403+0000", 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038715400000000001 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.004 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355708+0000", 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004304446 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355732+0000", 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4731000000000001e-05 2023-09-23T21:17:00.005 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355770+0000", 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7104000000000003e-05 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355798+0000", 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8242000000000002e-05 2023-09-23T21:17:00.006 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29542 13.4 13:25d6cd22:::benchmark_data_smithi121_621349_object29541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.35682465400001, 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.842315747, 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:00.007 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29542 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.008 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513594+0000", 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:00.009 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513646+0000", 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1647999999999997e-05 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513656+0000", 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0375e-05 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.010 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513663+0000", 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6549999999999998e-06 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513727+0000", 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3992999999999995e-05 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.011 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513736+0000", 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2660000000000007e-06 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348465+0000", 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83472863200000003 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.012 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348553+0000", 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8218000000000001e-05 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348636+0000", 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3065000000000005e-05 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.013 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352933+0000", 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042976840000000004 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355826+0000", 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028929429999999998 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.014 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355851+0000", 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4511e-05 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355883+0000", 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2462000000000002e-05 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.015 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355911+0000", 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7293000000000001e-05 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29561 13.4 13:2d5ffecf:::benchmark_data_smithi121_621349_object29560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:17:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.34234538600001, 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82794979099999999, 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29561 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:00.017 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.018 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528073+0000", 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528107+0000", 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4207e-05 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.019 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528127+0000", 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9539999999999999e-05 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528133+0000", 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4559999999999997e-06 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.020 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528211+0000", 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8057000000000002e-05 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528224+0000", 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2646000000000001e-05 2023-09-23T21:17:00.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349494+0000", 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82127059199999997 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349617+0000", 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012300200000000001 2023-09-23T21:17:00.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349779+0000", 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016171800000000001 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352944+0000", 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031647020000000001 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.023 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.024 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.025 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355939+0000", 2023-09-23T21:17:00.025 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029952239999999999 2023-09-23T21:17:00.025 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.025 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.025 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355964+0000", 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4788999999999999e-05 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355997+0000", 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3025999999999999e-05 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.026 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356024+0000", 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7231999999999999e-05 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29582 13.4 13:29c0d8e7:::benchmark_data_smithi121_621349_object29581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:17:00.027 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.322238787, 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80795742199999998, 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29582 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.028 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.029 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548179+0000", 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548222+0000", 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2599999999999999e-05 2023-09-23T21:17:00.030 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548233+0000", 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1537e-05 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548238+0000", 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7700000000000001e-06 2023-09-23T21:17:00.031 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548305+0000", 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6441999999999998e-05 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548314+0000", 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0410000000000003e-06 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.032 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350381+0000", 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80206683400000001 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350462+0000", 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1507000000000004e-05 2023-09-23T21:17:00.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350589+0000", 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012726299999999999 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353762+0000", 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031722870000000002 2023-09-23T21:17:00.034 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356053+0000", 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022911889999999999 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356078+0000", 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4754999999999999e-05 2023-09-23T21:17:00.035 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356110+0000", 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2641999999999997e-05 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356138+0000", 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8017e-05 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.036 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29585 13.4 13:234913ee:::benchmark_data_smithi121_621349_object29584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.321443958, 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80729236000000004, 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:00.037 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29585 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.038 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548975+0000", 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:00.039 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549041+0000", 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6148000000000004e-05 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549057+0000", 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5491000000000001e-05 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.040 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549064+0000", 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8620000000000003e-06 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549105+0000", 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.104e-05 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.041 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549112+0000", 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2879999999999998e-06 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351443+0000", 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80233120000000002 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.042 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351547+0000", 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103696 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351687+0000", 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140529 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.043 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353772+0000", 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020847539999999999 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356167+0000", 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023954800000000002 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.044 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356196+0000", 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8331999999999999e-05 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356229+0000", 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3577000000000001e-05 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.045 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356268+0000", 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8692000000000002e-05 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29594 13.4 13:299d3c93:::benchmark_data_smithi121_621349_object29593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:17:00.046 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.31014626300001, 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.79612242, 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29594 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.047 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.048 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560272+0000", 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560317+0000", 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4302999999999999e-05 2023-09-23T21:17:00.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560329+0000", 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2690999999999999e-05 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560334+0000", 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0429999999999998e-06 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560389+0000", 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4363000000000001e-05 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560395+0000", 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6510000000000004e-06 2023-09-23T21:17:00.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352241+0000", 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.79184583500000005 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352323+0000", 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1592000000000001e-05 2023-09-23T21:17:00.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352403+0000", 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0591000000000001e-05 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353852+0000", 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014487339999999999 2023-09-23T21:17:00.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356303+0000", 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024512969999999999 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356331+0000", 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7708e-05 2023-09-23T21:17:00.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356366+0000", 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5049999999999998e-05 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356396+0000", 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9581999999999999e-05 2023-09-23T21:17:00.055 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29614 13.4 13:26518e14:::benchmark_data_smithi121_621349_object29613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.602767+0000", 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 142.26765220999999, 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75380685999999997, 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:00.056 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29614 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602767+0000", 2023-09-23T21:17:00.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602767+0000", 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602766+0000", 2023-09-23T21:17:00.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602802+0000", 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6000000000000001e-05 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602816+0000", 2023-09-23T21:17:00.059 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3468999999999999e-05 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602821+0000", 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2410000000000003e-06 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602872+0000", 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1474000000000002e-05 2023-09-23T21:17:00.060 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.602881+0000", 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6740000000000003e-06 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352963+0000", 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75008157099999995 2023-09-23T21:17:00.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353042+0000", 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9394000000000006e-05 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353138+0000", 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5440999999999996e-05 2023-09-23T21:17:00.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354455+0000", 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0013174689999999999 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356425+0000", 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0019701699999999998 2023-09-23T21:17:00.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356498+0000", 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3073999999999999e-05 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356541+0000", 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2964999999999999e-05 2023-09-23T21:17:00.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356574+0000", 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3055000000000001e-05 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.065 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:134841 13.11 13:8a9b168c:::benchmark_data_smithi121_621349_object42347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e546)", 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:16:54.048295+0000", 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 5.8221248860000001, 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.247665419, 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 134841 2023-09-23T21:17:00.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048295+0000", 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048295+0000", 2023-09-23T21:17:00.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048294+0000", 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048302+0000", 2023-09-23T21:17:00.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7100000000000007e-06 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048313+0000", 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1603999999999999e-05 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048319+0000", 2023-09-23T21:17:00.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.9329999999999998e-06 2023-09-23T21:17:00.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:00.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048835+0000", 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00051538799999999998 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.048947+0000", 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011226999999999999 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.049356+0000", 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00040947599999999998 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.051927+0000", 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00257085 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.052571+0000", 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00064355200000000003 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.295872+0000", 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.24330111099999999 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.295923+0000", 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1722999999999999e-05 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:54.295960+0000", 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6638999999999999e-05 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.074 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.075 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:00.075 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:00.075 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:17:00.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:00.076 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:00.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:00.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:17:00.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:17:00.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:17:00.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:17:00.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:17:00.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:17:00.092 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:17:00.103 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:16:59 smithi123 ceph-mon[193903]: pgmap v1711: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 728 op/s; 25071/142026 objects degraded (17.652%); 14 MiB/s, 320 objects/s recovering 2023-09-23T21:17:00.124 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:17:00.128 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.128 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.128 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.128 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.128 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.129 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.129 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.129 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:17:00.151 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:16:59 smithi121 ceph-mon[219709]: pgmap v1711: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 728 op/s; 25071/142026 objects degraded (17.652%); 14 MiB/s, 320 objects/s recovering 2023-09-23T21:17:00.152 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:16:59 smithi121 ceph-mon[211526]: pgmap v1711: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 728 op/s; 25071/142026 objects degraded (17.652%); 14 MiB/s, 320 objects/s recovering 2023-09-23T21:17:00.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:00.218 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:00.373 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:00.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:00.479 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:00.508 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:00.558 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:00.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:00.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:00.662 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:00.755 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:00.837 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:17:00.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1994253842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:00.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/629340173' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:17:00.861 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:17:00.869 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:17:00.921 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:00.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:00.947 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:17:01.040 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:01.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1994253842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:01.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/629340173' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:17:01.074 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:17:01.126 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:01.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1994253842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:01.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/629340173' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:17:01.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:17:01.258 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:01.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:01.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:01.373 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:01.466 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:01.589 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:01.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:01.700 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:01.786 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:01.810 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:01.916 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:01.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:01.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:02.029 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:02.089 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:17:02.089 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:17:02.098 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:01 smithi123 ceph-mon[193903]: failed to encode map e549 with expected crc 2023-09-23T21:17:02.098 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:01 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:17:02.098 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:01 smithi123 ceph-mon[193903]: osdmap e549: 8 total, 8 up, 5 in 2023-09-23T21:17:02.098 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:01 smithi123 ceph-mon[193903]: failed to encode map e549 with expected crc 2023-09-23T21:17:02.099 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:01 smithi123 ceph-mon[193903]: pgmap v1713: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 608 op/s; 25071/140952 objects degraded (17.787%); 6.7 MiB/s, 150 objects/s recovering 2023-09-23T21:17:02.099 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 25071/142026 objects degraded (17.652%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:02.129 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:02.134 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[211526]: failed to encode map e549 with expected crc 2023-09-23T21:17:02.135 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:17:02.135 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[211526]: osdmap e549: 8 total, 8 up, 5 in 2023-09-23T21:17:02.135 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[211526]: failed to encode map e549 with expected crc 2023-09-23T21:17:02.135 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[211526]: pgmap v1713: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 608 op/s; 25071/140952 objects degraded (17.787%); 6.7 MiB/s, 150 objects/s recovering 2023-09-23T21:17:02.135 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 25071/142026 objects degraded (17.652%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:02.136 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[219709]: failed to encode map e549 with expected crc 2023-09-23T21:17:02.136 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:17:02.136 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[219709]: osdmap e549: 8 total, 8 up, 5 in 2023-09-23T21:17:02.136 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[219709]: failed to encode map e549 with expected crc 2023-09-23T21:17:02.137 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[219709]: pgmap v1713: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 608 op/s; 25071/140952 objects degraded (17.787%); 6.7 MiB/s, 150 objects/s recovering 2023-09-23T21:17:02.137 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 25071/142026 objects degraded (17.652%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:02.138 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:02.304 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:02.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:02.416 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:02.510 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:02.643 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:02.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:02.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:02.752 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:02.836 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:02.974 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:02.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:03.087 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:03.158 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:03.257 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:03.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:03.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:03.360 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:03.383 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:17:03.391 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:03.468 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:03.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:03.534 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:03.572 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:03.628 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:03.721 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:03.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:03.827 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:03.907 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:04.010 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:04.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:04.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:04.126 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:04.209 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:04.335 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:04.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:04.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:04.440 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:04 smithi123 ceph-mon[193903]: pgmap v1714: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 652 op/s; 24825/137193 objects degraded (18.095%); 7.4 MiB/s, 200 objects/s recovering 2023-09-23T21:17:04.530 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:04.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:04 smithi121 ceph-mon[211526]: pgmap v1714: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 652 op/s; 24825/137193 objects degraded (18.095%); 7.4 MiB/s, 200 objects/s recovering 2023-09-23T21:17:04.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:04 smithi121 ceph-mon[219709]: pgmap v1714: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 652 op/s; 24825/137193 objects degraded (18.095%); 7.4 MiB/s, 200 objects/s recovering 2023-09-23T21:17:04.686 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:04.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:04.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:04.750 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:04.751 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:04.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:04.831 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:05.099 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:05.102 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:17:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:17:04] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.43.0" 2023-09-23T21:17:05.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:05.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:05.168 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:05.203 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:05.285 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:05.357 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:05.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:05.409 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:17:05.462 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:05.488 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:05.572 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:05.579 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/109194263' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:05.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:05.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:05.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/109194263' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:05.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/109194263' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:05.680 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:05.723 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:05.942 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:17:06.457 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:06.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[211526]: pgmap v1715: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 908 op/s; 24376/130098 objects degraded (18.737%); 11 MiB/s, 256 objects/s recovering 2023-09-23T21:17:06.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[211526]: failed to encode map e550 with expected crc 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/109194263' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[211526]: osdmap e550: 8 total, 8 up, 5 in 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[211526]: failed to encode map e550 with expected crc 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[219709]: pgmap v1715: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 908 op/s; 24376/130098 objects degraded (18.737%); 11 MiB/s, 256 objects/s recovering 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[219709]: failed to encode map e550 with expected crc 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/109194263' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[219709]: osdmap e550: 8 total, 8 up, 5 in 2023-09-23T21:17:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:06 smithi121 ceph-mon[219709]: failed to encode map e550 with expected crc 2023-09-23T21:17:06.688 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:06 smithi123 ceph-mon[193903]: pgmap v1715: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 908 op/s; 24376/130098 objects degraded (18.737%); 11 MiB/s, 256 objects/s recovering 2023-09-23T21:17:06.688 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:06 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:06.689 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:06 smithi123 ceph-mon[193903]: failed to encode map e550 with expected crc 2023-09-23T21:17:06.689 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/109194263' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:06.689 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:06 smithi123 ceph-mon[193903]: osdmap e550: 8 total, 8 up, 5 in 2023-09-23T21:17:06.689 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:06 smithi123 ceph-mon[193903]: failed to encode map e550 with expected crc 2023-09-23T21:17:07.133 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:17:07.135 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:17:07.356 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:07.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:07.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:07 smithi123 ceph-mon[193903]: failed to encode map e550 with expected crc 2023-09-23T21:17:07.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:07 smithi123 ceph-mon[193903]: failed to encode map e550 with expected crc 2023-09-23T21:17:07.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:07 smithi123 ceph-mon[193903]: from='client.47267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:07.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:07 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 24376/130098 objects degraded (18.737%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:07.464 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:07.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[219709]: failed to encode map e550 with expected crc 2023-09-23T21:17:07.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[219709]: failed to encode map e550 with expected crc 2023-09-23T21:17:07.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[219709]: from='client.47267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:07.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 24376/130098 objects degraded (18.737%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:07.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[211526]: failed to encode map e550 with expected crc 2023-09-23T21:17:07.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[211526]: failed to encode map e550 with expected crc 2023-09-23T21:17:07.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[211526]: from='client.47267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:07.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:07 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 24376/130098 objects degraded (18.737%), 25 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:07.551 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:07.670 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:07.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:07.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:07.774 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:07.809 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:08.314 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1716,"stamp":"2023-09-23T21:17:05.401055+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2840926136,"num_objects":43366,"num_object_clones":0,"num_object_copies":130098,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":24376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43366,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":126889,"num_write_kb":4954830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24816,"num_bytes_recovered":1475685068,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195640,"ondisk_log_size":195640,"up":327,"acting":311,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":13325696,"kb_used_data":9155928,"kb_used_omap":7,"kb_used_meta":4169272,"kb_avail":455400064,"statfs":{"total":479975178240,"available":466329665536,"internally_reserved":0,"allocated":9375670272,"data_stored":9367111620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4269334609},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-476577792,"num_objects":-7272,"num_object_clones":0,"num_object_copies":-21816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1346,"num_objects_misplaced":-47,"num_objects_unfound":0,"num_objects_dirty":-7272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7272,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2053,"num_bytes_recovered":91291648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.677091"},"pg_stats":[{"pgid":"13.23","version":"549'5545","reported_seq":6998,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.455840+0000","last_change":"2023-09-23T21:16:00.853306+0000","last_active":"2023-09-23T21:17:03.455840+0000","last_peered":"2023-09-23T21:17:03.455840+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:00.848945+0000","last_became_peered":"2023-09-23T21:16:00.848945+0000","last_unstale":"2023-09-23T21:17:03.455840+0000","last_undegraded":"2023-09-23T21:16:00.815130+0000","last_fullsized":"2023-09-23T21:17:03.455840+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45973504,"num_objects":702,"num_object_clones":0,"num_object_copies":2106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":349,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1284,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":651,"num_bytes_recovered":42663936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":225},{"shards":"4,5,7","objects":477}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.22","version":"549'5474","reported_seq":7381,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.346653+0000","last_change":"2023-09-23T21:16:50.849447+0000","last_active":"2023-09-23T21:17:03.346653+0000","last_peered":"2023-09-23T21:17:03.346653+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:00.839299+0000","last_became_peered":"2023-09-23T21:16:00.839299+0000","last_unstale":"2023-09-23T21:17:03.346653+0000","last_undegraded":"2023-09-23T21:16:50.849328+0000","last_fullsized":"2023-09-23T21:17:03.346653+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5474,"log_dups_size":0,"ondisk_log_size":5474,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43581440,"num_objects":665,"num_object_clones":0,"num_object_copies":1995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":68,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2235,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":894,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5,6","objects":665}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.21","version":"549'5352","reported_seq":6442,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:02.790519+0000","last_change":"2023-09-23T21:16:51.863113+0000","last_active":"2023-09-23T21:17:02.790519+0000","last_peered":"2023-09-23T21:17:02.790519+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858774+0000","last_became_peered":"2023-09-23T21:16:51.858774+0000","last_unstale":"2023-09-23T21:17:02.790519+0000","last_undegraded":"2023-09-23T21:16:51.798038+0000","last_fullsized":"2023-09-23T21:16:51.792078+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5352,"log_dups_size":0,"ondisk_log_size":5352,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44941312,"num_objects":686,"num_object_clones":0,"num_object_copies":2058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":750,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2171,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":541},{"shards":"2,4,7","objects":145}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.20","version":"549'5459","reported_seq":8589,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.330549+0000","last_change":"2023-09-23T21:16:51.842022+0000","last_active":"2023-09-23T21:17:03.330549+0000","last_peered":"2023-09-23T21:17:03.330549+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834659+0000","last_became_peered":"2023-09-23T21:16:51.834659+0000","last_unstale":"2023-09-23T21:17:03.330549+0000","last_undegraded":"2023-09-23T21:16:51.797563+0000","last_fullsized":"2023-09-23T21:16:51.792808+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46137344,"num_objects":704,"num_object_clones":0,"num_object_copies":2112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1336,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":32964608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":144},{"shards":"2,7","objects":560}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.14","version":"549'6189","reported_seq":8391,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.025840+0000","last_change":"2023-09-23T21:16:51.065826+0000","last_active":"2023-09-23T21:17:03.025840+0000","last_peered":"2023-09-23T21:17:03.025840+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:16:51.052190+0000","last_became_peered":"2023-09-23T21:16:51.052190+0000","last_unstale":"2023-09-23T21:17:03.025840+0000","last_undegraded":"2023-09-23T21:16:50.808861+0000","last_fullsized":"2023-09-23T21:16:50.789676+0000","mapping_epoch":544,"log_start":"489'328","ondisk_log_start":"489'328","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4478,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":281},{"shards":"2,5","objects":1107}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":151,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:58.076023+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:16:58.076023+0000","last_peered":"2023-09-23T21:16:58.076023+0000","last_clean":"2023-09-23T21:16:58.076023+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:16:58.076023+0000","last_undegraded":"2023-09-23T21:16:58.076023+0000","last_fullsized":"2023-09-23T21:16:58.076023+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.749629+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:16:57.749629+0000","last_peered":"2023-09-23T21:16:57.749629+0000","last_clean":"2023-09-23T21:16:57.749629+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:16:57.749629+0000","last_undegraded":"2023-09-23T21:16:57.749629+0000","last_fullsized":"2023-09-23T21:16:57.749629+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"549'6067","reported_seq":6173,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.405228+0000","last_change":"2023-09-23T21:16:51.036947+0000","last_active":"2023-09-23T21:17:03.405228+0000","last_peered":"2023-09-23T21:17:03.405228+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:16:51.020513+0000","last_became_peered":"2023-09-23T21:16:51.020513+0000","last_unstale":"2023-09-23T21:17:03.405228+0000","last_undegraded":"2023-09-23T21:16:50.811498+0000","last_fullsized":"2023-09-23T21:16:50.789456+0000","mapping_epoch":544,"log_start":"489'844","ondisk_log_start":"489'844","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5223,"log_dups_size":0,"ondisk_log_size":5223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85065728,"num_objects":1298,"num_object_clones":0,"num_object_copies":3894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1586,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4424,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":304},{"shards":"4,7","objects":994}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":336,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.757566+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:16:57.757566+0000","last_peered":"2023-09-23T21:16:57.757566+0000","last_clean":"2023-09-23T21:16:57.757566+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:16:57.757566+0000","last_undegraded":"2023-09-23T21:16:57.757566+0000","last_fullsized":"2023-09-23T21:16:57.757566+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":196,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.744372+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:16:57.744372+0000","last_peered":"2023-09-23T21:16:57.744372+0000","last_clean":"2023-09-23T21:16:57.744372+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:16:57.744372+0000","last_undegraded":"2023-09-23T21:16:57.744372+0000","last_fullsized":"2023-09-23T21:16:57.744372+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"549'6064","reported_seq":10081,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.473446+0000","last_change":"2023-09-23T21:16:36.510340+0000","last_active":"2023-09-23T21:17:03.473446+0000","last_peered":"2023-09-23T21:17:03.473446+0000","last_clean":"2023-09-23T21:17:03.473446+0000","last_became_active":"2023-09-23T21:16:36.498646+0000","last_became_peered":"2023-09-23T21:16:36.498646+0000","last_unstale":"2023-09-23T21:17:03.473446+0000","last_undegraded":"2023-09-23T21:17:03.473446+0000","last_fullsized":"2023-09-23T21:17:03.473446+0000","mapping_epoch":537,"log_start":"493'1465","ondisk_log_start":"493'1465","created":507,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4599,"log_dups_size":0,"ondisk_log_size":4599,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4403,"num_write_kb":184192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1956,"num_bytes_recovered":93519872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":208,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748783+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:16:57.748783+0000","last_peered":"2023-09-23T21:16:57.748783+0000","last_clean":"2023-09-23T21:16:57.748783+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:16:57.748783+0000","last_undegraded":"2023-09-23T21:16:57.748783+0000","last_fullsized":"2023-09-23T21:16:57.748783+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.754651+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:16:57.754651+0000","last_peered":"2023-09-23T21:16:57.754651+0000","last_clean":"2023-09-23T21:16:57.754651+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:16:57.754651+0000","last_undegraded":"2023-09-23T21:16:57.754651+0000","last_fullsized":"2023-09-23T21:16:57.754651+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"549'5974","reported_seq":6087,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.501777+0000","last_change":"2023-09-23T21:16:51.074430+0000","last_active":"2023-09-23T21:17:03.501777+0000","last_peered":"2023-09-23T21:17:03.501777+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:16:51.054035+0000","last_became_peered":"2023-09-23T21:16:51.054035+0000","last_unstale":"2023-09-23T21:17:03.501777+0000","last_undegraded":"2023-09-23T21:16:50.838593+0000","last_fullsized":"2023-09-23T21:16:50.797592+0000","mapping_epoch":544,"log_start":"493'1334","ondisk_log_start":"493'1334","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4640,"log_dups_size":0,"ondisk_log_size":4640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84574208,"num_objects":1291,"num_object_clones":0,"num_object_copies":3873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4339,"num_write_kb":180128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":258},{"shards":"2,5","objects":1033}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":143,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.741365+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:16:57.741365+0000","last_peered":"2023-09-23T21:16:57.741365+0000","last_clean":"2023-09-23T21:16:57.741365+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:16:57.741365+0000","last_undegraded":"2023-09-23T21:16:57.741365+0000","last_fullsized":"2023-09-23T21:16:57.741365+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.749211+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:16:57.749211+0000","last_peered":"2023-09-23T21:16:57.749211+0000","last_clean":"2023-09-23T21:16:57.749211+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:16:57.749211+0000","last_undegraded":"2023-09-23T21:16:57.749211+0000","last_fullsized":"2023-09-23T21:16:57.749211+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.18","version":"549'6069","reported_seq":8120,"reported_epoch":549,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:17:03.072847+0000","last_change":"2023-09-23T21:16:51.096371+0000","last_active":"2023-09-23T21:17:03.072847+0000","last_peered":"2023-09-23T21:17:03.072847+0000","last_clean":"2023-09-23T21:15:36.291044+0000","last_became_active":"2023-09-23T21:16:51.051104+0000","last_became_peered":"2023-09-23T21:16:51.051104+0000","last_unstale":"2023-09-23T21:17:03.072847+0000","last_undegraded":"2023-09-23T21:17:03.072847+0000","last_fullsized":"2023-09-23T21:16:50.790365+0000","mapping_epoch":544,"log_start":"489'228","ondisk_log_start":"489'228","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":5841,"log_dups_size":0,"ondisk_log_size":5841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.211162+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86507520,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2556,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":950,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1320}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":202,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.747373+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:16:57.747373+0000","last_peered":"2023-09-23T21:16:57.747373+0000","last_clean":"2023-09-23T21:16:57.747373+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:16:57.747373+0000","last_undegraded":"2023-09-23T21:16:57.747373+0000","last_fullsized":"2023-09-23T21:16:57.747373+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737586+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:16:57.737586+0000","last_peered":"2023-09-23T21:16:57.737586+0000","last_clean":"2023-09-23T21:16:57.737586+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:16:57.737586+0000","last_undegraded":"2023-09-23T21:16:57.737586+0000","last_fullsized":"2023-09-23T21:16:57.737586+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1b","version":"549'7804","reported_seq":12045,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.025846+0000","last_change":"2023-09-23T21:16:00.814154+0000","last_active":"2023-09-23T21:17:03.025846+0000","last_peered":"2023-09-23T21:17:03.025846+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:00.808957+0000","last_became_peered":"2023-09-23T21:16:00.808957+0000","last_unstale":"2023-09-23T21:17:03.025846+0000","last_undegraded":"2023-09-23T21:16:00.770362+0000","last_fullsized":"2023-09-23T21:17:03.025846+0000","mapping_epoch":525,"log_start":"493'1783","ondisk_log_start":"493'1783","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91193344,"num_objects":1392,"num_object_clones":0,"num_object_copies":4176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":457,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4569,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1579,"num_bytes_recovered":103481344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":457},{"shards":"2,4,5","objects":935}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":144,"reported_epoch":548,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:57.737661+0000","last_change":"2023-09-23T21:16:50.974446+0000","last_active":"2023-09-23T21:16:57.737661+0000","last_peered":"2023-09-23T21:16:57.737661+0000","last_clean":"2023-09-23T21:16:47.791993+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:16:57.737661+0000","last_undegraded":"2023-09-23T21:16:50.810242+0000","last_fullsized":"2023-09-23T21:16:57.737661+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748420+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:16:57.748420+0000","last_peered":"2023-09-23T21:16:57.748420+0000","last_clean":"2023-09-23T21:16:57.748420+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:16:57.748420+0000","last_undegraded":"2023-09-23T21:16:57.748420+0000","last_fullsized":"2023-09-23T21:16:57.748420+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1a","version":"549'7494","reported_seq":9617,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.077021+0000","last_change":"2023-09-23T21:16:00.812483+0000","last_active":"2023-09-23T21:17:03.077021+0000","last_peered":"2023-09-23T21:17:03.077021+0000","last_clean":"2023-09-23T21:17:03.077021+0000","last_became_active":"2023-09-23T21:16:00.806979+0000","last_became_peered":"2023-09-23T21:16:00.806979+0000","last_unstale":"2023-09-23T21:17:03.077021+0000","last_undegraded":"2023-09-23T21:17:03.077021+0000","last_fullsized":"2023-09-23T21:17:03.077021+0000","mapping_epoch":525,"log_start":"493'1441","ondisk_log_start":"493'1441","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6053,"log_dups_size":0,"ondisk_log_size":6053,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87228416,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4382,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":176,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.739110+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:16:57.739110+0000","last_peered":"2023-09-23T21:16:57.739110+0000","last_clean":"2023-09-23T21:16:57.739110+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:16:57.739110+0000","last_undegraded":"2023-09-23T21:16:57.739110+0000","last_fullsized":"2023-09-23T21:16:57.739110+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":477,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.766490+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:16:57.766490+0000","last_peered":"2023-09-23T21:16:57.766490+0000","last_clean":"2023-09-23T21:16:57.766490+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:16:57.766490+0000","last_undegraded":"2023-09-23T21:16:57.766490+0000","last_fullsized":"2023-09-23T21:16:57.766490+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1d","version":"549'7697","reported_seq":12528,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.483207+0000","last_change":"2023-09-23T21:16:03.792593+0000","last_active":"2023-09-23T21:17:03.483207+0000","last_peered":"2023-09-23T21:17:03.483207+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:16:03.773235+0000","last_became_peered":"2023-09-23T21:16:03.773235+0000","last_unstale":"2023-09-23T21:17:03.483207+0000","last_undegraded":"2023-09-23T21:16:03.727693+0000","last_fullsized":"2023-09-23T21:17:03.483207+0000","mapping_epoch":528,"log_start":"497'2164","ondisk_log_start":"497'2164","created":525,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5533,"log_dups_size":0,"ondisk_log_size":5533,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87457804,"num_objects":1335,"num_object_clones":0,"num_object_copies":4005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":625,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1335,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4520,"num_write_kb":187329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1847,"num_bytes_recovered":121044992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":385},{"shards":"4,5,7","objects":950}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":132,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.738056+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:16:57.738056+0000","last_peered":"2023-09-23T21:16:57.738056+0000","last_clean":"2023-09-23T21:16:57.738056+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:16:57.738056+0000","last_undegraded":"2023-09-23T21:16:57.738056+0000","last_fullsized":"2023-09-23T21:16:57.738056+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":200,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.738952+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:16:57.738952+0000","last_peered":"2023-09-23T21:16:57.738952+0000","last_clean":"2023-09-23T21:16:57.738952+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:16:57.738952+0000","last_undegraded":"2023-09-23T21:16:57.738952+0000","last_fullsized":"2023-09-23T21:16:57.738952+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1c","version":"549'7676","reported_seq":9792,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.484094+0000","last_change":"2023-09-23T21:16:00.824659+0000","last_active":"2023-09-23T21:17:03.484094+0000","last_peered":"2023-09-23T21:17:03.484094+0000","last_clean":"2023-09-23T21:17:03.484094+0000","last_became_active":"2023-09-23T21:16:00.820619+0000","last_became_peered":"2023-09-23T21:16:00.820619+0000","last_unstale":"2023-09-23T21:17:03.484094+0000","last_undegraded":"2023-09-23T21:17:03.484094+0000","last_fullsized":"2023-09-23T21:17:03.484094+0000","mapping_epoch":525,"log_start":"500'2344","ondisk_log_start":"500'2344","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5332,"log_dups_size":0,"ondisk_log_size":5332,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90865664,"num_objects":1387,"num_object_clones":0,"num_object_copies":4161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4492,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":128,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.738611+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:16:57.738611+0000","last_peered":"2023-09-23T21:16:57.738611+0000","last_clean":"2023-09-23T21:16:57.738611+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:16:57.738611+0000","last_undegraded":"2023-09-23T21:16:57.738611+0000","last_fullsized":"2023-09-23T21:16:57.738611+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":136,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737974+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:16:57.737974+0000","last_peered":"2023-09-23T21:16:57.737974+0000","last_clean":"2023-09-23T21:16:57.737974+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:16:57.737974+0000","last_undegraded":"2023-09-23T21:16:57.737974+0000","last_fullsized":"2023-09-23T21:16:57.737974+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1f","version":"549'7514","reported_seq":7621,"reported_epoch":549,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:01.192193+0000","last_change":"2023-09-23T21:16:51.848275+0000","last_active":"2023-09-23T21:17:01.192193+0000","last_peered":"2023-09-23T21:17:01.192193+0000","last_clean":"2023-09-23T21:15:36.326799+0000","last_became_active":"2023-09-23T21:16:51.820010+0000","last_became_peered":"2023-09-23T21:16:51.820010+0000","last_unstale":"2023-09-23T21:17:01.192193+0000","last_undegraded":"2023-09-23T21:17:01.192193+0000","last_fullsized":"2023-09-23T21:17:01.192193+0000","mapping_epoch":545,"log_start":"493'1700","ondisk_log_start":"493'1700","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92110848,"num_objects":1406,"num_object_clones":0,"num_object_copies":4218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4384,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","5","7"],"object_location_counts":[{"shards":"2,6,7","objects":1406}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":126,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737804+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:16:57.737804+0000","last_peered":"2023-09-23T21:16:57.737804+0000","last_clean":"2023-09-23T21:16:57.737804+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:16:57.737804+0000","last_undegraded":"2023-09-23T21:16:57.737804+0000","last_fullsized":"2023-09-23T21:16:57.737804+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":150,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.744352+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:16:57.744352+0000","last_peered":"2023-09-23T21:16:57.744352+0000","last_clean":"2023-09-23T21:16:57.744352+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:16:57.744352+0000","last_undegraded":"2023-09-23T21:16:57.744352+0000","last_fullsized":"2023-09-23T21:16:57.744352+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1e","version":"549'7479","reported_seq":7563,"reported_epoch":549,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:03.445436+0000","last_change":"2023-09-23T21:16:00.911470+0000","last_active":"2023-09-23T21:17:03.445436+0000","last_peered":"2023-09-23T21:17:03.445436+0000","last_clean":"2023-09-23T21:15:59.749850+0000","last_became_active":"2023-09-23T21:16:00.844392+0000","last_became_peered":"2023-09-23T21:16:00.844392+0000","last_unstale":"2023-09-23T21:17:03.445436+0000","last_undegraded":"2023-09-23T21:17:03.445436+0000","last_fullsized":"2023-09-23T21:17:03.445436+0000","mapping_epoch":525,"log_start":"493'1633","ondisk_log_start":"493'1633","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87457792,"num_objects":1335,"num_object_clones":0,"num_object_copies":4005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4388,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":1335}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":585,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.749772+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:16:57.749772+0000","last_peered":"2023-09-23T21:16:57.749772+0000","last_clean":"2023-09-23T21:16:57.749772+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:16:57.749772+0000","last_undegraded":"2023-09-23T21:16:57.749772+0000","last_fullsized":"2023-09-23T21:16:57.749772+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":200,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.746130+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:16:57.746130+0000","last_peered":"2023-09-23T21:16:57.746130+0000","last_clean":"2023-09-23T21:16:57.746130+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:16:57.746130+0000","last_undegraded":"2023-09-23T21:16:57.746130+0000","last_fullsized":"2023-09-23T21:16:57.746130+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"549'5383","reported_seq":6516,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:02.929832+0000","last_change":"2023-09-23T21:16:51.863144+0000","last_active":"2023-09-23T21:17:02.929832+0000","last_peered":"2023-09-23T21:17:02.929832+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858843+0000","last_became_peered":"2023-09-23T21:16:51.858843+0000","last_unstale":"2023-09-23T21:17:02.929832+0000","last_undegraded":"2023-09-23T21:16:51.799178+0000","last_fullsized":"2023-09-23T21:16:51.792321+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5383,"log_dups_size":0,"ondisk_log_size":5383,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42909696,"num_objects":654,"num_object_clones":0,"num_object_copies":1962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2201,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":511,"num_bytes_recovered":33488896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":485},{"shards":"2,4,7","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":130,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.746218+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:16:57.746218+0000","last_peered":"2023-09-23T21:16:57.746218+0000","last_clean":"2023-09-23T21:16:57.746218+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:16:57.746218+0000","last_undegraded":"2023-09-23T21:16:57.746218+0000","last_fullsized":"2023-09-23T21:16:57.746218+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":202,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.738992+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:16:57.738992+0000","last_peered":"2023-09-23T21:16:57.738992+0000","last_clean":"2023-09-23T21:16:57.738992+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:16:57.738992+0000","last_undegraded":"2023-09-23T21:16:57.738992+0000","last_fullsized":"2023-09-23T21:16:57.738992+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"549'5478","reported_seq":8584,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.170513+0000","last_change":"2023-09-23T21:16:51.842306+0000","last_active":"2023-09-23T21:17:03.170513+0000","last_peered":"2023-09-23T21:17:03.170513+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834848+0000","last_became_peered":"2023-09-23T21:16:51.834848+0000","last_unstale":"2023-09-23T21:17:03.170513+0000","last_undegraded":"2023-09-23T21:16:51.803926+0000","last_fullsized":"2023-09-23T21:16:51.799106+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5478,"log_dups_size":0,"ondisk_log_size":5478,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44892160,"num_objects":685,"num_object_clones":0,"num_object_copies":2055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1355,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32178176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":161},{"shards":"2,7","objects":524}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":144,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.744452+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:16:57.744452+0000","last_peered":"2023-09-23T21:16:57.744452+0000","last_clean":"2023-09-23T21:16:57.744452+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:16:57.744452+0000","last_undegraded":"2023-09-23T21:16:57.744452+0000","last_fullsized":"2023-09-23T21:16:57.744452+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.750426+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:16:57.750426+0000","last_peered":"2023-09-23T21:16:57.750426+0000","last_clean":"2023-09-23T21:16:57.750426+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:16:57.750426+0000","last_undegraded":"2023-09-23T21:16:57.750426+0000","last_fullsized":"2023-09-23T21:16:57.750426+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3","version":"549'5552","reported_seq":7022,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.473705+0000","last_change":"2023-09-23T21:16:01.735614+0000","last_active":"2023-09-23T21:17:03.473705+0000","last_peered":"2023-09-23T21:17:03.473705+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:01.731847+0000","last_became_peered":"2023-09-23T21:16:01.731847+0000","last_unstale":"2023-09-23T21:17:03.473705+0000","last_undegraded":"2023-09-23T21:16:01.699708+0000","last_fullsized":"2023-09-23T21:17:03.473705+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45514752,"num_objects":694,"num_object_clones":0,"num_object_copies":2082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":364,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1291,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":658,"num_bytes_recovered":43122688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":221},{"shards":"4,5,7","objects":473}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":130,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737984+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:16:57.737984+0000","last_peered":"2023-09-23T21:16:57.737984+0000","last_clean":"2023-09-23T21:16:57.737984+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:16:57.737984+0000","last_undegraded":"2023-09-23T21:16:57.737984+0000","last_fullsized":"2023-09-23T21:16:57.737984+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748845+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:16:57.748845+0000","last_peered":"2023-09-23T21:16:57.748845+0000","last_clean":"2023-09-23T21:16:57.748845+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:16:57.748845+0000","last_undegraded":"2023-09-23T21:16:57.748845+0000","last_fullsized":"2023-09-23T21:16:57.748845+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"549'5428","reported_seq":6781,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.471048+0000","last_change":"2023-09-23T21:16:01.750035+0000","last_active":"2023-09-23T21:17:03.471048+0000","last_peered":"2023-09-23T21:17:03.471048+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:01.742399+0000","last_became_peered":"2023-09-23T21:16:01.742399+0000","last_unstale":"2023-09-23T21:17:03.471048+0000","last_undegraded":"2023-09-23T21:16:01.706463+0000","last_fullsized":"2023-09-23T21:17:03.471048+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5428,"log_dups_size":0,"ondisk_log_size":5428,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46596096,"num_objects":711,"num_object_clones":0,"num_object_copies":2133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":340,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2189,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":617,"num_bytes_recovered":40435712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":207},{"shards":"4,5,6","objects":504}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748314+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:16:57.748314+0000","last_peered":"2023-09-23T21:16:57.748314+0000","last_clean":"2023-09-23T21:16:57.748314+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:16:57.748314+0000","last_undegraded":"2023-09-23T21:16:57.748314+0000","last_fullsized":"2023-09-23T21:16:57.748314+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748263+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:16:57.748263+0000","last_peered":"2023-09-23T21:16:57.748263+0000","last_clean":"2023-09-23T21:16:57.748263+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:16:57.748263+0000","last_undegraded":"2023-09-23T21:16:57.748263+0000","last_fullsized":"2023-09-23T21:16:57.748263+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"549'6075","reported_seq":6284,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.507295+0000","last_change":"2023-09-23T21:16:51.047553+0000","last_active":"2023-09-23T21:17:03.507295+0000","last_peered":"2023-09-23T21:17:03.507295+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:16:51.032827+0000","last_became_peered":"2023-09-23T21:16:51.032827+0000","last_unstale":"2023-09-23T21:17:03.507295+0000","last_undegraded":"2023-09-23T21:16:50.886600+0000","last_fullsized":"2023-09-23T21:16:50.791765+0000","mapping_epoch":544,"log_start":"493'1385","ondisk_log_start":"493'1385","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":4690,"log_dups_size":0,"ondisk_log_size":4690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:01.438609+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88080384,"num_objects":1344,"num_object_clones":0,"num_object_copies":4032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2256,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2519,"num_write_kb":64384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":314},{"shards":"3,4,5","objects":430},{"shards":"4,5","objects":315}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":216,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737931+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:16:57.737931+0000","last_peered":"2023-09-23T21:16:57.737931+0000","last_clean":"2023-09-23T21:16:57.737931+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:16:57.737931+0000","last_undegraded":"2023-09-23T21:16:57.737931+0000","last_fullsized":"2023-09-23T21:16:57.737931+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748456+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:16:57.748456+0000","last_peered":"2023-09-23T21:16:57.748456+0000","last_clean":"2023-09-23T21:16:57.748456+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:16:57.748456+0000","last_undegraded":"2023-09-23T21:16:57.748456+0000","last_fullsized":"2023-09-23T21:16:57.748456+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.4","version":"549'6155","reported_seq":8449,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.070805+0000","last_change":"2023-09-23T21:16:51.065805+0000","last_active":"2023-09-23T21:17:03.070805+0000","last_peered":"2023-09-23T21:17:03.070805+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:16:51.051146+0000","last_became_peered":"2023-09-23T21:16:51.051146+0000","last_unstale":"2023-09-23T21:17:03.070805+0000","last_undegraded":"2023-09-23T21:16:50.809981+0000","last_fullsized":"2023-09-23T21:16:50.790363+0000","mapping_epoch":544,"log_start":"489'365","ondisk_log_start":"489'365","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":5790,"log_dups_size":0,"ondisk_log_size":5790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:23.280602+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689472,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2455,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":242},{"shards":"2,5","objects":1035}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748501+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:16:57.748501+0000","last_peered":"2023-09-23T21:16:57.748501+0000","last_clean":"2023-09-23T21:16:57.748501+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:16:57.748501+0000","last_undegraded":"2023-09-23T21:16:57.748501+0000","last_fullsized":"2023-09-23T21:16:57.748501+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":590,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748543+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:16:57.748543+0000","last_peered":"2023-09-23T21:16:57.748543+0000","last_clean":"2023-09-23T21:16:57.748543+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:16:57.748543+0000","last_undegraded":"2023-09-23T21:16:57.748543+0000","last_fullsized":"2023-09-23T21:16:57.748543+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"549'5999","reported_seq":6110,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.405688+0000","last_change":"2023-09-23T21:16:51.082314+0000","last_active":"2023-09-23T21:17:03.405688+0000","last_peered":"2023-09-23T21:17:03.405688+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:16:51.064800+0000","last_became_peered":"2023-09-23T21:16:51.064800+0000","last_unstale":"2023-09-23T21:17:03.405688+0000","last_undegraded":"2023-09-23T21:16:50.820535+0000","last_fullsized":"2023-09-23T21:16:50.800952+0000","mapping_epoch":544,"log_start":"489'680","ondisk_log_start":"489'680","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5319,"log_dups_size":0,"ondisk_log_size":5319,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88997888,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4356,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":295},{"shards":"4,7","objects":1063}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":186,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.757815+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:16:57.757815+0000","last_peered":"2023-09-23T21:16:57.757815+0000","last_clean":"2023-09-23T21:16:57.757815+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:16:57.757815+0000","last_undegraded":"2023-09-23T21:16:57.757815+0000","last_fullsized":"2023-09-23T21:16:57.757815+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748754+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:16:57.748754+0000","last_peered":"2023-09-23T21:16:57.748754+0000","last_clean":"2023-09-23T21:16:57.748754+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:16:57.748754+0000","last_undegraded":"2023-09-23T21:16:57.748754+0000","last_fullsized":"2023-09-23T21:16:57.748754+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"549'6007","reported_seq":6210,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.490630+0000","last_change":"2023-09-23T21:15:36.787083+0000","last_active":"2023-09-23T21:17:03.490630+0000","last_peered":"2023-09-23T21:17:03.490630+0000","last_clean":"2023-09-23T21:15:09.799249+0000","last_became_active":"2023-09-23T21:15:12.799540+0000","last_became_peered":"2023-09-23T21:15:12.799540+0000","last_unstale":"2023-09-23T21:17:03.490630+0000","last_undegraded":"2023-09-23T21:15:36.785930+0000","last_fullsized":"2023-09-23T21:15:12.679693+0000","mapping_epoch":514,"log_start":"493'1404","ondisk_log_start":"493'1404","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":4603,"log_dups_size":0,"ondisk_log_size":4603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:59.312702+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87490560,"num_objects":1335,"num_object_clones":0,"num_object_copies":4005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2394,"num_write_kb":57088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":443},{"shards":"4,5","objects":-603}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737897+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:16:57.737897+0000","last_peered":"2023-09-23T21:16:57.737897+0000","last_clean":"2023-09-23T21:16:57.737897+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:16:57.737897+0000","last_undegraded":"2023-09-23T21:16:57.737897+0000","last_fullsized":"2023-09-23T21:16:57.737897+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":288,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.745133+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:16:57.745133+0000","last_peered":"2023-09-23T21:16:57.745133+0000","last_clean":"2023-09-23T21:16:57.745133+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:16:57.745133+0000","last_undegraded":"2023-09-23T21:16:57.745133+0000","last_fullsized":"2023-09-23T21:16:57.745133+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"549'7761","reported_seq":11955,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.077512+0000","last_change":"2023-09-23T21:16:01.728998+0000","last_active":"2023-09-23T21:17:03.077512+0000","last_peered":"2023-09-23T21:17:03.077512+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:01.723118+0000","last_became_peered":"2023-09-23T21:16:01.723118+0000","last_unstale":"2023-09-23T21:17:03.077512+0000","last_undegraded":"2023-09-23T21:16:01.697539+0000","last_fullsized":"2023-09-23T21:17:03.077512+0000","mapping_epoch":526,"log_start":"493'1783","ondisk_log_start":"493'1783","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5978,"log_dups_size":0,"ondisk_log_size":5978,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94011392,"num_objects":1434,"num_object_clones":0,"num_object_copies":4302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4525,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1556,"num_bytes_recovered":101974016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":427},{"shards":"2,4,5","objects":1007}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":473,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.739029+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:16:57.739029+0000","last_peered":"2023-09-23T21:16:57.739029+0000","last_clean":"2023-09-23T21:16:57.739029+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:16:57.739029+0000","last_undegraded":"2023-09-23T21:16:57.739029+0000","last_fullsized":"2023-09-23T21:16:57.739029+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":224,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.741659+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:16:57.741659+0000","last_peered":"2023-09-23T21:16:57.741659+0000","last_clean":"2023-09-23T21:16:57.741659+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:16:57.741659+0000","last_undegraded":"2023-09-23T21:16:57.741659+0000","last_fullsized":"2023-09-23T21:16:57.741659+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":98,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.742464+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:16:57.742464+0000","last_peered":"2023-09-23T21:16:57.742464+0000","last_clean":"2023-09-23T21:16:57.742464+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:16:57.742464+0000","last_undegraded":"2023-09-23T21:16:57.742464+0000","last_fullsized":"2023-09-23T21:16:57.742464+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"549'7663","reported_seq":12416,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:03.512532+0000","last_change":"2023-09-23T21:16:01.774573+0000","last_active":"2023-09-23T21:17:03.512532+0000","last_peered":"2023-09-23T21:17:03.512532+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:16:01.764537+0000","last_became_peered":"2023-09-23T21:16:01.764537+0000","last_unstale":"2023-09-23T21:17:03.512532+0000","last_undegraded":"2023-09-23T21:16:01.707917+0000","last_fullsized":"2023-09-23T21:17:03.512532+0000","mapping_epoch":526,"log_start":"497'2164","ondisk_log_start":"497'2164","created":482,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89686028,"num_objects":1369,"num_object_clones":0,"num_object_copies":4107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":674,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4486,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1813,"num_bytes_recovered":118816768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":415},{"shards":"4,5,7","objects":954}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748345+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:16:57.748345+0000","last_peered":"2023-09-23T21:16:57.748345+0000","last_clean":"2023-09-23T21:16:57.748345+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:16:57.748345+0000","last_undegraded":"2023-09-23T21:16:57.748345+0000","last_fullsized":"2023-09-23T21:16:57.748345+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748086+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:16:57.748086+0000","last_peered":"2023-09-23T21:16:57.748086+0000","last_clean":"2023-09-23T21:16:57.748086+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:16:57.748086+0000","last_undegraded":"2023-09-23T21:16:57.748086+0000","last_fullsized":"2023-09-23T21:16:57.748086+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":263,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.750869+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:16:57.750869+0000","last_peered":"2023-09-23T21:16:57.750869+0000","last_clean":"2023-09-23T21:16:57.750869+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:16:57.750869+0000","last_undegraded":"2023-09-23T21:16:57.750869+0000","last_fullsized":"2023-09-23T21:16:57.750869+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"549'7457","reported_seq":7542,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.435056+0000","last_change":"2023-09-23T21:16:00.841315+0000","last_active":"2023-09-23T21:17:03.435056+0000","last_peered":"2023-09-23T21:17:03.435056+0000","last_clean":"2023-09-23T21:17:03.435056+0000","last_became_active":"2023-09-23T21:16:00.829607+0000","last_became_peered":"2023-09-23T21:16:00.829607+0000","last_unstale":"2023-09-23T21:17:03.435056+0000","last_undegraded":"2023-09-23T21:17:03.435056+0000","last_fullsized":"2023-09-23T21:17:03.435056+0000","mapping_epoch":482,"log_start":"493'1633","ondisk_log_start":"493'1633","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88899584,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4365,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":557,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748535+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:16:57.748535+0000","last_peered":"2023-09-23T21:16:57.748535+0000","last_clean":"2023-09-23T21:16:57.748535+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:16:57.748535+0000","last_undegraded":"2023-09-23T21:16:57.748535+0000","last_fullsized":"2023-09-23T21:16:57.748535+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":295,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748119+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:16:57.748119+0000","last_peered":"2023-09-23T21:16:57.748119+0000","last_clean":"2023-09-23T21:16:57.748119+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:16:57.748119+0000","last_undegraded":"2023-09-23T21:16:57.748119+0000","last_fullsized":"2023-09-23T21:16:57.748119+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":202,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.745212+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:16:57.745212+0000","last_peered":"2023-09-23T21:16:57.745212+0000","last_clean":"2023-09-23T21:16:57.745212+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:16:57.745212+0000","last_undegraded":"2023-09-23T21:16:57.745212+0000","last_fullsized":"2023-09-23T21:16:57.745212+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"549'7681","reported_seq":9798,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.442551+0000","last_change":"2023-09-23T21:16:00.748968+0000","last_active":"2023-09-23T21:17:03.442551+0000","last_peered":"2023-09-23T21:17:03.442551+0000","last_clean":"2023-09-23T21:17:03.442551+0000","last_became_active":"2023-09-23T21:16:00.745123+0000","last_became_peered":"2023-09-23T21:16:00.745123+0000","last_unstale":"2023-09-23T21:17:03.442551+0000","last_undegraded":"2023-09-23T21:17:03.442551+0000","last_fullsized":"2023-09-23T21:17:03.442551+0000","mapping_epoch":501,"log_start":"500'2344","ondisk_log_start":"500'2344","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90537984,"num_objects":1381,"num_object_clones":0,"num_object_copies":4143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4496,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":502,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":377,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.749148+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:16:57.749148+0000","last_peered":"2023-09-23T21:16:57.749148+0000","last_clean":"2023-09-23T21:16:57.749148+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:16:57.749148+0000","last_undegraded":"2023-09-23T21:16:57.749148+0000","last_fullsized":"2023-09-23T21:16:57.749148+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":895,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737683+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:16:57.737683+0000","last_peered":"2023-09-23T21:16:57.737683+0000","last_clean":"2023-09-23T21:16:57.737683+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:16:57.737683+0000","last_undegraded":"2023-09-23T21:16:57.737683+0000","last_fullsized":"2023-09-23T21:16:57.737683+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":121,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.740338+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:16:57.740338+0000","last_peered":"2023-09-23T21:16:57.740338+0000","last_clean":"2023-09-23T21:16:57.740338+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:16:57.740338+0000","last_undegraded":"2023-09-23T21:16:57.740338+0000","last_fullsized":"2023-09-23T21:16:57.740338+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.749285+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:16:57.749285+0000","last_peered":"2023-09-23T21:16:57.749285+0000","last_clean":"2023-09-23T21:16:57.749285+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:16:57.749285+0000","last_undegraded":"2023-09-23T21:16:57.749285+0000","last_fullsized":"2023-09-23T21:16:57.749285+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"549'7535","reported_seq":7636,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:01.107072+0000","last_change":"2023-09-23T21:16:51.832676+0000","last_active":"2023-09-23T21:17:01.107072+0000","last_peered":"2023-09-23T21:17:01.107072+0000","last_clean":"2023-09-23T21:17:01.107072+0000","last_became_active":"2023-09-23T21:16:51.803067+0000","last_became_peered":"2023-09-23T21:16:51.803067+0000","last_unstale":"2023-09-23T21:17:01.107072+0000","last_undegraded":"2023-09-23T21:17:01.107072+0000","last_fullsized":"2023-09-23T21:17:01.107072+0000","mapping_epoch":545,"log_start":"493'1700","ondisk_log_start":"493'1700","created":482,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5835,"log_dups_size":0,"ondisk_log_size":5835,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90734592,"num_objects":1384,"num_object_clones":0,"num_object_copies":4152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4404,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":176,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737764+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:16:57.737764+0000","last_peered":"2023-09-23T21:16:57.737764+0000","last_clean":"2023-09-23T21:16:57.737764+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:16:57.737764+0000","last_undegraded":"2023-09-23T21:16:57.737764+0000","last_fullsized":"2023-09-23T21:16:57.737764+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737507+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:16:57.737507+0000","last_peered":"2023-09-23T21:16:57.737507+0000","last_clean":"2023-09-23T21:16:57.737507+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:16:57.737507+0000","last_undegraded":"2023-09-23T21:16:57.737507+0000","last_fullsized":"2023-09-23T21:16:57.737507+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":458,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.749392+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:16:57.749392+0000","last_peered":"2023-09-23T21:16:57.749392+0000","last_clean":"2023-09-23T21:16:57.749392+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:16:57.749392+0000","last_undegraded":"2023-09-23T21:16:57.749392+0000","last_fullsized":"2023-09-23T21:16:57.749392+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"549'7534","reported_seq":9658,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.038245+0000","last_change":"2023-09-23T21:16:00.763134+0000","last_active":"2023-09-23T21:17:03.038245+0000","last_peered":"2023-09-23T21:17:03.038245+0000","last_clean":"2023-09-23T21:17:03.038245+0000","last_became_active":"2023-09-23T21:16:00.758875+0000","last_became_peered":"2023-09-23T21:16:00.758875+0000","last_unstale":"2023-09-23T21:17:03.038245+0000","last_undegraded":"2023-09-23T21:17:03.038245+0000","last_fullsized":"2023-09-23T21:17:03.038245+0000","mapping_epoch":498,"log_start":"493'1441","ondisk_log_start":"493'1441","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6093,"log_dups_size":0,"ondisk_log_size":6093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84606976,"num_objects":1291,"num_object_clones":0,"num_object_copies":3873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4422,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":146,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737875+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:16:57.737875+0000","last_peered":"2023-09-23T21:16:57.737875+0000","last_clean":"2023-09-23T21:16:57.737875+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:16:57.737875+0000","last_undegraded":"2023-09-23T21:16:57.737875+0000","last_fullsized":"2023-09-23T21:16:57.737875+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":362,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.750918+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:16:57.750918+0000","last_peered":"2023-09-23T21:16:57.750918+0000","last_clean":"2023-09-23T21:16:57.750918+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:16:57.750918+0000","last_undegraded":"2023-09-23T21:16:57.750918+0000","last_fullsized":"2023-09-23T21:16:57.750918+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":254,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.743515+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:16:57.743515+0000","last_peered":"2023-09-23T21:16:57.743515+0000","last_clean":"2023-09-23T21:16:57.743515+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:16:57.743515+0000","last_undegraded":"2023-09-23T21:16:57.743515+0000","last_fullsized":"2023-09-23T21:16:57.743515+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"549'6047","reported_seq":6248,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.487477+0000","last_change":"2023-09-23T21:16:51.076997+0000","last_active":"2023-09-23T21:17:03.487477+0000","last_peered":"2023-09-23T21:17:03.487477+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:16:51.057102+0000","last_became_peered":"2023-09-23T21:16:51.057102+0000","last_unstale":"2023-09-23T21:17:03.487477+0000","last_undegraded":"2023-09-23T21:16:50.840109+0000","last_fullsized":"2023-09-23T21:16:50.799660+0000","mapping_epoch":544,"log_start":"493'1517","ondisk_log_start":"493'1517","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":4530,"log_dups_size":0,"ondisk_log_size":4530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.748037+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88866816,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1565,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2485,"num_write_kb":60864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":315},{"shards":"2,5","objects":1041}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":204,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.746312+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:16:57.746312+0000","last_peered":"2023-09-23T21:16:57.746312+0000","last_clean":"2023-09-23T21:16:57.746312+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:16:57.746312+0000","last_undegraded":"2023-09-23T21:16:57.746312+0000","last_fullsized":"2023-09-23T21:16:57.746312+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748687+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:16:57.748687+0000","last_peered":"2023-09-23T21:16:57.748687+0000","last_clean":"2023-09-23T21:16:57.748687+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:16:57.748687+0000","last_undegraded":"2023-09-23T21:16:57.748687+0000","last_fullsized":"2023-09-23T21:16:57.748687+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":229,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.737596+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:16:57.737596+0000","last_peered":"2023-09-23T21:16:57.737596+0000","last_clean":"2023-09-23T21:16:57.737596+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:16:57.737596+0000","last_undegraded":"2023-09-23T21:16:57.737596+0000","last_fullsized":"2023-09-23T21:16:57.737596+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"549'6140","reported_seq":6289,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.052620+0000","last_change":"2023-09-23T21:16:51.063450+0000","last_active":"2023-09-23T21:17:03.052620+0000","last_peered":"2023-09-23T21:17:03.052620+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:16:51.051345+0000","last_became_peered":"2023-09-23T21:16:51.051345+0000","last_unstale":"2023-09-23T21:17:03.052620+0000","last_undegraded":"2023-09-23T21:16:50.807164+0000","last_fullsized":"2023-09-23T21:16:50.788756+0000","mapping_epoch":544,"log_start":"489'357","ondisk_log_start":"489'357","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:15.933191+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91947008,"num_objects":1403,"num_object_clones":0,"num_object_copies":4209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1557,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2652,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1102},{"shards":"2,5,7","objects":301}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":455,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.754780+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:16:57.754780+0000","last_peered":"2023-09-23T21:16:57.754780+0000","last_clean":"2023-09-23T21:16:57.754780+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:16:57.754780+0000","last_undegraded":"2023-09-23T21:16:57.754780+0000","last_fullsized":"2023-09-23T21:16:57.754780+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":86,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.766482+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:16:57.766482+0000","last_peered":"2023-09-23T21:16:57.766482+0000","last_clean":"2023-09-23T21:16:57.766482+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:16:57.766482+0000","last_undegraded":"2023-09-23T21:16:57.766482+0000","last_fullsized":"2023-09-23T21:16:57.766482+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":234,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.739978+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:16:57.739978+0000","last_peered":"2023-09-23T21:16:57.739978+0000","last_clean":"2023-09-23T21:16:57.739978+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:16:57.739978+0000","last_undegraded":"2023-09-23T21:16:57.739978+0000","last_fullsized":"2023-09-23T21:16:57.739978+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"549'6200","reported_seq":6322,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.512121+0000","last_change":"2023-09-23T21:16:51.067304+0000","last_active":"2023-09-23T21:17:03.512121+0000","last_peered":"2023-09-23T21:17:03.512121+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:16:51.048602+0000","last_became_peered":"2023-09-23T21:16:51.048602+0000","last_unstale":"2023-09-23T21:17:03.512121+0000","last_undegraded":"2023-09-23T21:16:50.818351+0000","last_fullsized":"2023-09-23T21:16:50.791414+0000","mapping_epoch":544,"log_start":"493'1660","ondisk_log_start":"493'1660","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4540,"log_dups_size":0,"ondisk_log_size":4540,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86966272,"num_objects":1327,"num_object_clones":0,"num_object_copies":3981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4527,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":293},{"shards":"3,4,5","objects":407},{"shards":"4,5","objects":341}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":176,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.738145+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:16:57.738145+0000","last_peered":"2023-09-23T21:16:57.738145+0000","last_clean":"2023-09-23T21:16:57.738145+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:16:57.738145+0000","last_undegraded":"2023-09-23T21:16:57.738145+0000","last_fullsized":"2023-09-23T21:16:57.738145+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":612,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.750964+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:16:57.750964+0000","last_peered":"2023-09-23T21:16:57.750964+0000","last_clean":"2023-09-23T21:16:57.750964+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:16:57.750964+0000","last_undegraded":"2023-09-23T21:16:57.750964+0000","last_fullsized":"2023-09-23T21:16:57.750964+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"549'6093","reported_seq":10137,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.457583+0000","last_change":"2023-09-23T21:16:37.510873+0000","last_active":"2023-09-23T21:17:03.457583+0000","last_peered":"2023-09-23T21:17:03.457583+0000","last_clean":"2023-09-23T21:17:03.457583+0000","last_became_active":"2023-09-23T21:16:37.500341+0000","last_became_peered":"2023-09-23T21:16:37.500341+0000","last_unstale":"2023-09-23T21:17:03.457583+0000","last_undegraded":"2023-09-23T21:17:03.457583+0000","last_fullsized":"2023-09-23T21:17:03.457583+0000","mapping_epoch":538,"log_start":"493'1453","ondisk_log_start":"493'1453","created":507,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4640,"log_dups_size":0,"ondisk_log_size":4640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85458944,"num_objects":1304,"num_object_clones":0,"num_object_copies":3912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4432,"num_write_kb":183552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1970,"num_bytes_recovered":94175232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":253,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.765536+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:16:57.765536+0000","last_peered":"2023-09-23T21:16:57.765536+0000","last_clean":"2023-09-23T21:16:57.765536+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:16:57.765536+0000","last_undegraded":"2023-09-23T21:16:57.765536+0000","last_fullsized":"2023-09-23T21:16:57.765536+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":206,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748827+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:16:57.748827+0000","last_peered":"2023-09-23T21:16:57.748827+0000","last_clean":"2023-09-23T21:16:57.748827+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:16:57.748827+0000","last_undegraded":"2023-09-23T21:16:57.748827+0000","last_fullsized":"2023-09-23T21:16:57.748827+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"549'6113","reported_seq":8843,"reported_epoch":549,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:03.490612+0000","last_change":"2023-09-23T21:16:50.886093+0000","last_active":"2023-09-23T21:17:03.490612+0000","last_peered":"2023-09-23T21:17:03.490612+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:17:03.490612+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"493'1452","ondisk_log_start":"493'1452","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4661,"log_dups_size":0,"ondisk_log_size":4661,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88834048,"num_objects":1356,"num_object_clones":0,"num_object_copies":4068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4430,"num_write_kb":185120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":61800448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5","objects":733}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":12,"reported_epoch":548,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:16:57.749129+0000","last_change":"2023-09-23T21:16:51.045869+0000","last_active":"2023-09-23T21:16:57.749129+0000","last_peered":"2023-09-23T21:16:57.749129+0000","last_clean":"2023-09-23T20:30:24.124074+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:16:57.749129+0000","last_undegraded":"2023-09-23T21:16:50.824831+0000","last_fullsized":"2023-09-23T21:16:57.749129+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"4,5,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":193,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748345+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:16:57.748345+0000","last_peered":"2023-09-23T21:16:57.748345+0000","last_clean":"2023-09-23T21:16:57.748345+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:16:57.748345+0000","last_undegraded":"2023-09-23T21:16:57.748345+0000","last_fullsized":"2023-09-23T21:16:57.748345+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"549'5977","reported_seq":11838,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:03.071696+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:17:03.071696+0000","last_peered":"2023-09-23T21:17:03.071696+0000","last_clean":"2023-09-23T21:17:03.071696+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:17:03.071696+0000","last_undegraded":"2023-09-23T21:17:03.071696+0000","last_fullsized":"2023-09-23T21:17:03.071696+0000","mapping_epoch":518,"log_start":"489'30","ondisk_log_start":"489'30","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85721088,"num_objects":1308,"num_object_clones":0,"num_object_copies":3924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4314,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":14,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748614+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:16:57.748614+0000","last_peered":"2023-09-23T21:16:57.748614+0000","last_clean":"2023-09-23T21:16:57.748614+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:16:57.748614+0000","last_undegraded":"2023-09-23T21:16:57.748614+0000","last_fullsized":"2023-09-23T21:16:57.748614+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":408,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748837+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:16:57.748837+0000","last_peered":"2023-09-23T21:16:57.748837+0000","last_clean":"2023-09-23T21:16:57.748837+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:16:57.748837+0000","last_undegraded":"2023-09-23T21:16:57.748837+0000","last_fullsized":"2023-09-23T21:16:57.748837+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"549'5966","reported_seq":9923,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:01.194386+0000","last_change":"2023-09-23T21:16:49.855987+0000","last_active":"2023-09-23T21:17:01.194386+0000","last_peered":"2023-09-23T21:17:01.194386+0000","last_clean":"2023-09-23T21:17:01.194386+0000","last_became_active":"2023-09-23T21:16:15.742634+0000","last_became_peered":"2023-09-23T21:16:15.742634+0000","last_unstale":"2023-09-23T21:17:01.194386+0000","last_undegraded":"2023-09-23T21:17:01.194386+0000","last_fullsized":"2023-09-23T21:17:01.194386+0000","mapping_epoch":531,"log_start":"489'45","ondisk_log_start":"489'45","created":507,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5921,"log_dups_size":0,"ondisk_log_size":5921,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93159424,"num_objects":1422,"num_object_clones":0,"num_object_copies":4266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4330,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":505,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.748747+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:16:57.748747+0000","last_peered":"2023-09-23T21:16:57.748747+0000","last_clean":"2023-09-23T21:16:57.748747+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:16:57.748747+0000","last_undegraded":"2023-09-23T21:16:57.748747+0000","last_fullsized":"2023-09-23T21:16:57.748747+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-09-23T21:16:57.755361+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:16:57.755361+0000","last_peered":"2023-09-23T21:16:57.755361+0000","last_clean":"2023-09-23T21:16:57.755361+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:16:57.755361+0000","last_undegraded":"2023-09-23T21:16:57.755361+0000","last_fullsized":"2023-09-23T21:16:57.755361+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":36,"stat_sum":{"num_bytes":2840330264,"num_objects":43342,"num_object_clones":0,"num_object_copies":130026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24373,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43342,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126589,"num_write_kb":4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24668,"num_bytes_recovered":1470332928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10712920064,"data_stored":10712907848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195424,"ondisk_log_size":195424,"up":108,"acting":92,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":130,"num_bytes_recovered":38828,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201860,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2200720,"kb_used_data":1423880,"kb_used_omap":0,"kb_used_meta":776704,"kb_avail":91544432,"statfs":{"total":95995035648,"available":93741498368,"internally_reserved":0,"allocated":1458053120,"data_stored":1456563435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":795344896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":3.5129999999999999,"15min":2.1920000000000002},"min":{"1min":0.218,"5min":0.214,"15min":0.20699999999999999},"max":{"1min":5.7949999999999999,"5min":141.33199999999999,"15min":141.33199999999999},"last":4.0140000000000002},{"interface":"front","average":{"1min":1.9079999999999999,"5min":3.6230000000000002,"15min":2.2280000000000002},"min":{"1min":0.27800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.8330000000000002,"5min":141.31700000000001,"15min":141.31700000000001},"last":4.1219999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":3.0030000000000001,"15min":3.3889999999999998},"min":{"1min":0.317,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.8700000000000001,"5min":76.007999999999996,"15min":114.16500000000001},"last":4.1459999999999999},{"interface":"front","average":{"1min":1.294,"5min":4.077,"15min":3.8610000000000002},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.681,"5min":117.637,"15min":117.637},"last":4.0540000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":5.0549999999999997,"15min":4.4119999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.7999999999999998,"5min":135.529,"15min":183.44200000000001},"last":4.1669999999999998},{"interface":"front","average":{"1min":1.6679999999999999,"5min":6.016,"15min":4.6349999999999998},"min":{"1min":0.318,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":5.7859999999999996,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.34899999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":7.9820000000000002,"15min":4.2640000000000002},"min":{"1min":0.30499999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":5.7279999999999998,"5min":135.66200000000001,"15min":135.66200000000001},"last":4.0819999999999999},{"interface":"front","average":{"1min":1.897,"5min":8.4979999999999993,"15min":4.375},"min":{"1min":0.318,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":5.8600000000000003,"5min":135.47,"15min":135.47},"last":0.375}]},{"osd":4,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1499999999999999,"5min":1.611,"15min":1.1399999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":9.9529999999999994,"5min":10.989000000000001,"15min":15.942},"last":6.1829999999999998},{"interface":"front","average":{"1min":2.984,"5min":1.6020000000000001,"15min":1.0549999999999999},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":11.773,"5min":11.773,"15min":16.725999999999999},"last":0.28899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":1.7689999999999999,"15min":1.3149999999999999},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.7829999999999995,"5min":11.231,"15min":11.231},"last":6.7869999999999999},{"interface":"front","average":{"1min":3.9420000000000002,"5min":1.9510000000000001,"15min":1.274},"min":{"1min":0.435,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":9.468,"5min":10.005000000000001,"15min":13.867000000000001},"last":6.2910000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.2160000000000002,"15min":2.0289999999999999},"min":{"1min":0.51800000000000002,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":13.369,"5min":143.18700000000001,"15min":143.18700000000001},"last":4.7300000000000004},{"interface":"front","average":{"1min":2.266,"5min":2.7549999999999999,"15min":1.8420000000000001},"min":{"1min":0.53900000000000003,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":7.7450000000000001,"5min":142.70400000000001,"15min":142.70400000000001},"last":3.5409999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463480,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2798132,"kb_used_data":2080748,"kb_used_omap":0,"kb_used_meta":717184,"kb_avail":90947020,"statfs":{"total":95995035648,"available":93129748480,"internally_reserved":0,"allocated":2130685952,"data_stored":2129216637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":734396416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":4.5339999999999998,"15min":2.7490000000000001},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":12.659000000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.0369999999999999,"5min":4.899,"15min":2.835},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":16.021999999999998,"5min":149.303,"15min":149.303},"last":1.337}]},{"osd":1,"last update":"Sat Sep 23 21:16:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":3.9359999999999999,"15min":3.5009999999999999},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":8.4860000000000007,"5min":129.37899999999999,"15min":129.37899999999999},"last":1.367},{"interface":"front","average":{"1min":1.4199999999999999,"5min":3.8450000000000002,"15min":3.3919999999999999},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":6.2539999999999996,"5min":129.40100000000001,"15min":129.40100000000001},"last":3.5459999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.577999999999999,"5min":7.6189999999999998,"15min":5.9340000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":120.28700000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":12.118,"5min":7.8120000000000003,"15min":5.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":120.414,"5min":133.33000000000001,"15min":199.08000000000001},"last":1.292}]},{"osd":3,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":5.6669999999999998,"15min":3.3199999999999998},"min":{"1min":0.38,"5min":0.218,"15min":0.218},"max":{"1min":3.5089999999999999,"5min":137.875,"15min":137.875},"last":3.5089999999999999},{"interface":"front","average":{"1min":1.7250000000000001,"5min":5.391,"15min":3.3650000000000002},"min":{"1min":0.30099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.676,"5min":137.60900000000001,"15min":137.60900000000001},"last":0.30099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":8.9060000000000006,"5min":2.9329999999999998,"15min":1.673},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.22600000000000001},"max":{"1min":55.494,"5min":55.494,"15min":55.494},"last":4.4820000000000002},{"interface":"front","average":{"1min":8.7550000000000008,"5min":2.8100000000000001,"15min":1.4970000000000001},"min":{"1min":0.39900000000000002,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":55.305,"5min":55.305,"15min":55.305},"last":3.472}]},{"osd":5,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":15.223000000000001,"5min":4.3630000000000004,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":222.90899999999999,"5min":222.90899999999999,"15min":222.90899999999999},"last":7.8680000000000003},{"interface":"front","average":{"1min":15.375,"5min":4.4930000000000003,"15min":2.1739999999999999},"min":{"1min":0.37,"5min":0.32200000000000001,"15min":0.28499999999999998},"max":{"1min":224.655,"5min":224.655,"15min":224.655},"last":13.007}]},{"osd":7,"last update":"Sat Sep 23 21:16:25 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":0.94299999999999995,"15min":1.002},"min":{"1min":0.45100000000000001,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":6.9729999999999999,"5min":6.9729999999999999,"15min":7.8079999999999998},"last":0.432},{"interface":"front","average":{"1min":2.0470000000000002,"5min":1.2729999999999999,"15min":1.129},"min":{"1min":0.42799999999999999,"5min":0.251,"15min":0.251},"max":{"1min":6.2060000000000004,"5min":7.9119999999999999,"15min":7.9119999999999999},"last":5.2300000000000004}]}]},{"osd":1,"up_from":0,"seq":1185410973927,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991667,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781558,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2823148,"kb_used_data":1758180,"kb_used_omap":7,"kb_used_meta":1064952,"kb_avail":90922004,"statfs":{"total":95995035648,"available":93104132096,"internally_reserved":0,"allocated":1800376320,"data_stored":1797743916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1090510929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.8420000000000001,"15min":2.157},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":8.0079999999999991,"5min":103.021,"15min":103.021},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.29599999999999999,"5min":3.133,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":0.378,"5min":103.154,"15min":103.154},"last":0.26900000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:16:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":3.4449999999999998,"15min":4.3470000000000004},"min":{"1min":0.224,"5min":0.224,"15min":0.19},"max":{"1min":2.9740000000000002,"5min":106.268,"15min":212.12},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":3.8879999999999999,"15min":4.3550000000000004},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":3.246,"5min":106.32299999999999,"15min":213.02799999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:16:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.8300000000000001,"15min":4.3220000000000001},"min":{"1min":0.27100000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.8260000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.312},{"interface":"front","average":{"1min":0.76400000000000001,"5min":3.4820000000000002,"15min":4.4189999999999996},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.2850000000000001,"5min":82.241,"15min":143.27799999999999},"last":0.43099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":13.896000000000001,"5min":4.3010000000000002,"15min":3.1800000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":220.82900000000001,"5min":220.82900000000001,"15min":225.80799999999999},"last":1.546},{"interface":"front","average":{"1min":13.894,"5min":4.9130000000000003,"15min":3.823},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":220.84299999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":1.5269999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.541,"5min":5.1059999999999999,"15min":3.48},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":220.63900000000001,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":16.257000000000001,"5min":4.7380000000000004,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":220.31999999999999,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.597,"5min":4.6210000000000004,"15min":3.762},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":210.74700000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":14.371,"5min":4.4640000000000004,"15min":3.1259999999999999},"min":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":210.72900000000001,"5min":210.72900000000001,"15min":210.72900000000001},"last":3.6379999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.4940000000000002,"15min":1.948},"min":{"1min":0.371,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":14.019,"5min":25.260000000000002,"15min":25.260000000000002},"last":4.298},{"interface":"front","average":{"1min":1.921,"5min":2.4990000000000001,"15min":2.2850000000000001},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":5.0229999999999997,"5min":12.613,"15min":16.875},"last":1.069}]}]},{"osd":3,"up_from":0,"seq":635655160151,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986770,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2584528,"kb_used_data":1806920,"kb_used_omap":0,"kb_used_meta":777472,"kb_avail":91160624,"statfs":{"total":95995035648,"available":93348478976,"internally_reserved":0,"allocated":1850286080,"data_stored":1848796658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":796131328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":5.1909999999999998,"15min":2.7109999999999999},"min":{"1min":0.247,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":10.436,"5min":226.751,"15min":226.751},"last":0.378},{"interface":"front","average":{"1min":2.0750000000000002,"5min":4.8049999999999997,"15min":2.556},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.223},"max":{"1min":6.218,"5min":226.43299999999999,"15min":226.43299999999999},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":4.4980000000000002,"15min":5.282},"min":{"1min":0.29799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.9980000000000002,"5min":119.55800000000001,"15min":127.411},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.665,"5min":4.1390000000000002,"15min":4.5179999999999998},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":8.2550000000000008,"5min":116.78100000000001,"15min":118.075},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":8.9459999999999997,"5min":7.5439999999999996,"15min":5.8710000000000004},"min":{"1min":0.26000000000000001,"5min":0.253,"15min":0.253},"max":{"1min":90.950000000000003,"5min":210.50299999999999,"15min":210.50299999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":9.9280000000000008,"5min":8.5630000000000006,"15min":6.0149999999999997},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":90.846000000000004,"5min":210.59,"15min":210.59},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 21:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":7.5650000000000004,"15min":4.069},"min":{"1min":0.317,"5min":0.317,"15min":0.29299999999999998},"max":{"1min":4.3319999999999999,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.992,"5min":8.6229999999999993,"15min":4.4470000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.25600000000000001},"max":{"1min":8.2799999999999994,"5min":214.54900000000001,"15min":214.54900000000001},"last":0.39900000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.383,"15min":0.98999999999999999},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.757999999999999,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":3.4119999999999999,"5min":2.157,"15min":1.266},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":18.314,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.26700000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.524,"15min":1.333},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.2569999999999997,"5min":10.446999999999999,"15min":36.557000000000002},"last":41.938000000000002},{"interface":"front","average":{"1min":2.2690000000000001,"5min":1.579,"15min":1.3169999999999999},"min":{"1min":0.35799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.8049999999999997,"5min":6.8049999999999997,"15min":38.468000000000004},"last":40.444000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8740000000000001,"5min":1.746,"15min":1.869},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":18.352,"5min":18.352,"15min":145.721},"last":0.317},{"interface":"front","average":{"1min":2.512,"5min":1.421,"15min":1.917},"min":{"1min":0.42199999999999999,"5min":0.33500000000000002,"15min":0.31900000000000001},"max":{"1min":7.5830000000000002,"5min":10.677,"15min":148.45699999999999},"last":0.32800000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725134,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2919168,"kb_used_data":2086200,"kb_used_omap":0,"kb_used_meta":832960,"kb_avail":90825984,"statfs":{"total":95995035648,"available":93005807616,"internally_reserved":0,"allocated":2136268800,"data_stored":2134790974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":852951040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":1.732,"15min":1.6699999999999999},"min":{"1min":0.33500000000000002,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":7.7450000000000001,"5min":20.023,"15min":82.280000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.077,"5min":2.0640000000000001,"15min":1.8680000000000001},"min":{"1min":0.29299999999999998,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.6710000000000003,"5min":19.853000000000002,"15min":85.236000000000004},"last":0.98099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5619999999999998,"5min":2.5659999999999998,"15min":4.6150000000000002},"min":{"1min":0.42799999999999999,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":8.048,"5min":12.032999999999999,"15min":233.77099999999999},"last":0.85099999999999998},{"interface":"front","average":{"1min":1.966,"5min":2.028,"15min":4.3479999999999999},"min":{"1min":0.32300000000000001,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":7.8520000000000003,"5min":12.016,"15min":233.75200000000001},"last":0.64300000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":2.1110000000000002,"15min":3.7080000000000002},"min":{"1min":0.37,"5min":0.251,"15min":0.251},"max":{"1min":7.8220000000000001,"5min":20.411999999999999,"15min":231.34700000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.1859999999999999,"15min":3.9009999999999998},"min":{"1min":0.38600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.0439999999999996,"5min":11.509,"15min":231.381},"last":1.079}]},{"osd":3,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":2.5710000000000002,"15min":4.0469999999999997},"min":{"1min":0.40500000000000003,"5min":0.317,"15min":0.27700000000000002},"max":{"1min":8.077,"5min":11.597,"15min":263.887},"last":0.88700000000000001},{"interface":"front","average":{"1min":2.004,"5min":1.9910000000000001,"15min":3.831},"min":{"1min":0.46100000000000002,"5min":0.32200000000000001,"15min":0.30099999999999999},"max":{"1min":7.8810000000000002,"5min":10.638,"15min":263.70100000000002},"last":0.66900000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":1.595,"15min":0.97599999999999998},"min":{"1min":0.439,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":11.593999999999999,"5min":11.593999999999999,"15min":12.739000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":3.7040000000000002,"5min":1.3839999999999999,"15min":1.026},"min":{"1min":0.505,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":10.259,"5min":10.259,"15min":10.792},"last":3.4329999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9009999999999998,"5min":1.6739999999999999,"15min":1.7130000000000001},"min":{"1min":0.38500000000000001,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":7.8520000000000003,"5min":8.9130000000000003,"15min":83.451999999999998},"last":3.4609999999999999},{"interface":"front","average":{"1min":3.0379999999999998,"5min":2.2669999999999999,"15min":1.9339999999999999},"min":{"1min":0.58499999999999996,"5min":0.30399999999999999,"15min":0.27400000000000002},"max":{"1min":8.0050000000000008,"5min":12.725,"15min":83.484999999999999},"last":6.601}]},{"osd":7,"last update":"Sat Sep 23 21:16:28 2023","interfaces":[{"interface":"back","average":{"1min":3.8599999999999999,"5min":1.7529999999999999,"15min":1.2529999999999999},"min":{"1min":0.56399999999999995,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":3.3929999999999998},{"interface":"front","average":{"1min":2.5470000000000002,"5min":1.3,"15min":0.874},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":11.640000000000001,"5min":11.640000000000001,"15min":11.640000000000001},"last":0.69499999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1782906880,"data_stored":1782906880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1455226880,"data_stored":1455226880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1838878720,"data_stored":1838874648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2105282560,"data_stored":2105278488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2091712512,"data_stored":2091712512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1438912512,"data_stored":1438908440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:17:08.322 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:17:08.416 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:08 smithi123 ceph-mon[193903]: pgmap v1717: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 543 op/s; 24376/128976 objects degraded (18.900%); 5.4 MiB/s, 134 objects/s recovering 2023-09-23T21:17:08.416 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:08 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:17:08.423 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:08.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:08.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:08.513 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:17:08.529 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:08.538 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:17:08.562 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:17:08.592 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:17:08.631 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:17:08.644 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:08.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:08 smithi121 ceph-mon[219709]: pgmap v1717: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 543 op/s; 24376/128976 objects degraded (18.900%); 5.4 MiB/s, 134 objects/s recovering 2023-09-23T21:17:08.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:08 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:17:08.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:08 smithi121 ceph-mon[211526]: pgmap v1717: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 543 op/s; 24376/128976 objects degraded (18.900%); 5.4 MiB/s, 134 objects/s recovering 2023-09-23T21:17:08.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:08 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:17:08.671 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:17:08.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:17:08.734 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:17:08.738 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.739 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.739 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.739 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.739 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.739 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.739 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.739 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:17:08.774 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:08.798 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:08.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:08.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:08.900 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:08.962 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:09.021 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:17:09.022 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:17:09.022 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:17:09.022 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:17:09.022 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:17:09.254 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:09.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:09.357 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:09.437 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:09.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1225913280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1225913280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1225913280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:09.870 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:17:10.099 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:10.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:10.174 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:17:10.211 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:10.301 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:10.367 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:10.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:10.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:10.470 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:10.536 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:10.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:10 smithi121 ceph-mon[211526]: pgmap v1718: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 925 op/s; 23689/118479 objects degraded (19.994%); 10 MiB/s, 255 objects/s recovering 2023-09-23T21:17:10.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:10 smithi121 ceph-mon[219709]: pgmap v1718: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 925 op/s; 23689/118479 objects degraded (19.994%); 10 MiB/s, 255 objects/s recovering 2023-09-23T21:17:10.714 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:10.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:10.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:10.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:10 smithi123 ceph-mon[193903]: pgmap v1718: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 925 op/s; 23689/118479 objects degraded (19.994%); 10 MiB/s, 255 objects/s recovering 2023-09-23T21:17:10.825 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:10.859 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:10.927 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:10.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:10.998 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:17:11.033 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:11.129 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:11.404 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:11.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:11.450 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:11.524 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:11.558 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:11.693 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:17:11.713 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:11.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:11.787 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:17:11.789 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:17:11.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:11.852 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: pgmap v1719: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 777 op/s; 23688/117630 objects degraded (20.138%); 8.6 MiB/s, 216 objects/s recovering 2023-09-23T21:17:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: osdmap e551: 8 total, 8 up, 5 in 2023-09-23T21:17:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 23688/117630 objects degraded (20.138%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:12 smithi123 ceph-mon[193903]: from='client.37305 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:12.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: pgmap v1719: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 777 op/s; 23688/117630 objects degraded (20.138%); 8.6 MiB/s, 216 objects/s recovering 2023-09-23T21:17:12.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: osdmap e551: 8 total, 8 up, 5 in 2023-09-23T21:17:12.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 23688/117630 objects degraded (20.138%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[219709]: from='client.37305 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: pgmap v1719: 109 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 777 op/s; 23688/117630 objects degraded (20.138%); 8.6 MiB/s, 216 objects/s recovering 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: osdmap e551: 8 total, 8 up, 5 in 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: failed to encode map e551 with expected crc 2023-09-23T21:17:12.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 23688/117630 objects degraded (20.138%), 24 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:12.782 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:12 smithi121 ceph-mon[211526]: from='client.37305 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:13.016 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:13.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:13.070 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1720,"stamp":"2023-09-23T21:17:11.429553+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2568558520,"num_objects":39210,"num_object_clones":0,"num_object_copies":117630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23688,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39210,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":131045,"num_write_kb":4954830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25911,"num_bytes_recovered":1520708300,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195881,"ondisk_log_size":195881,"up":327,"acting":311,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":12487560,"kb_used_data":8308576,"kb_used_omap":7,"kb_used_meta":4178616,"kb_avail":456238200,"statfs":{"total":479975178240,"available":467187916800,"internally_reserved":0,"allocated":8507981824,"data_stored":8499400318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4278902865},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-427360256,"num_objects":-6521,"num_object_clones":0,"num_object_copies":-19563,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-1137,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6521,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1544,"num_bytes_recovered":74448896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.092557"},"pg_stats":[{"pgid":"13.23","version":"550'5607","reported_seq":7185,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.505606+0000","last_change":"2023-09-23T21:16:00.853306+0000","last_active":"2023-09-23T21:17:08.505606+0000","last_peered":"2023-09-23T21:17:08.505606+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:00.848945+0000","last_became_peered":"2023-09-23T21:16:00.848945+0000","last_unstale":"2023-09-23T21:17:08.505606+0000","last_undegraded":"2023-09-23T21:16:00.815130+0000","last_fullsized":"2023-09-23T21:17:08.505606+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5607,"log_dups_size":0,"ondisk_log_size":5607,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41910272,"num_objects":640,"num_object_clones":0,"num_object_copies":1920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":287,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1346,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":713,"num_bytes_recovered":46727168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":163},{"shards":"4,5,7","objects":477}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.22","version":"550'5545","reported_seq":7453,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.138660+0000","last_change":"2023-09-23T21:16:50.849447+0000","last_active":"2023-09-23T21:17:08.138660+0000","last_peered":"2023-09-23T21:17:08.138660+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:00.839299+0000","last_became_peered":"2023-09-23T21:16:00.839299+0000","last_unstale":"2023-09-23T21:17:08.138660+0000","last_undegraded":"2023-09-23T21:16:50.849328+0000","last_fullsized":"2023-09-23T21:17:08.138660+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38928384,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":68,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2306,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":894,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5,6","objects":594}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.21","version":"550'5406","reported_seq":6497,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:07.984270+0000","last_change":"2023-09-23T21:16:51.863113+0000","last_active":"2023-09-23T21:17:07.984270+0000","last_peered":"2023-09-23T21:17:07.984270+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858774+0000","last_became_peered":"2023-09-23T21:16:51.858774+0000","last_unstale":"2023-09-23T21:17:07.984270+0000","last_undegraded":"2023-09-23T21:16:51.798038+0000","last_fullsized":"2023-09-23T21:16:51.792078+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5406,"log_dups_size":0,"ondisk_log_size":5406,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41402368,"num_objects":632,"num_object_clones":0,"num_object_copies":1896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":750,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2225,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":487},{"shards":"2,4,7","objects":145}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.20","version":"550'5529","reported_seq":8660,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.317089+0000","last_change":"2023-09-23T21:16:51.842022+0000","last_active":"2023-09-23T21:17:08.317089+0000","last_peered":"2023-09-23T21:17:08.317089+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834659+0000","last_became_peered":"2023-09-23T21:16:51.834659+0000","last_unstale":"2023-09-23T21:17:08.317089+0000","last_undegraded":"2023-09-23T21:16:51.797563+0000","last_fullsized":"2023-09-23T21:16:51.792808+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41549824,"num_objects":634,"num_object_clones":0,"num_object_copies":1902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1406,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":32964608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":144},{"shards":"2,7","objects":490}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.14","version":"550'6306","reported_seq":8509,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.036355+0000","last_change":"2023-09-23T21:16:51.065826+0000","last_active":"2023-09-23T21:17:08.036355+0000","last_peered":"2023-09-23T21:17:08.036355+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:16:51.052190+0000","last_became_peered":"2023-09-23T21:16:51.052190+0000","last_unstale":"2023-09-23T21:17:08.036355+0000","last_undegraded":"2023-09-23T21:16:50.808861+0000","last_fullsized":"2023-09-23T21:16:50.789676+0000","mapping_epoch":544,"log_start":"489'582","ondisk_log_start":"489'582","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4595,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":281},{"shards":"2,5","objects":990}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":152,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863025+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:17:00.863025+0000","last_peered":"2023-09-23T21:17:00.863025+0000","last_clean":"2023-09-23T21:17:00.863025+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:17:00.863025+0000","last_undegraded":"2023-09-23T21:17:00.863025+0000","last_fullsized":"2023-09-23T21:17:00.863025+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863384+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:17:00.863384+0000","last_peered":"2023-09-23T21:17:00.863384+0000","last_clean":"2023-09-23T21:17:00.863384+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:17:00.863384+0000","last_undegraded":"2023-09-23T21:17:00.863384+0000","last_fullsized":"2023-09-23T21:17:00.863384+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"550'6172","reported_seq":6279,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.422192+0000","last_change":"2023-09-23T21:16:51.036947+0000","last_active":"2023-09-23T21:17:08.422192+0000","last_peered":"2023-09-23T21:17:08.422192+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:16:51.020513+0000","last_became_peered":"2023-09-23T21:16:51.020513+0000","last_unstale":"2023-09-23T21:17:08.422192+0000","last_undegraded":"2023-09-23T21:16:50.811498+0000","last_fullsized":"2023-09-23T21:16:50.789456+0000","mapping_epoch":544,"log_start":"492'1046","ondisk_log_start":"492'1046","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5126,"log_dups_size":0,"ondisk_log_size":5126,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78184448,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1586,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4529,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":304},{"shards":"4,7","objects":889}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":337,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.866940+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:17:00.866940+0000","last_peered":"2023-09-23T21:17:00.866940+0000","last_clean":"2023-09-23T21:17:00.866940+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:17:00.866940+0000","last_undegraded":"2023-09-23T21:17:00.866940+0000","last_fullsized":"2023-09-23T21:17:00.866940+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":197,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861794+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:17:00.861794+0000","last_peered":"2023-09-23T21:17:00.861794+0000","last_clean":"2023-09-23T21:17:00.861794+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:17:00.861794+0000","last_undegraded":"2023-09-23T21:17:00.861794+0000","last_fullsized":"2023-09-23T21:17:00.861794+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"550'6204","reported_seq":10222,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.399869+0000","last_change":"2023-09-23T21:16:36.510340+0000","last_active":"2023-09-23T21:17:08.399869+0000","last_peered":"2023-09-23T21:17:08.399869+0000","last_clean":"2023-09-23T21:17:08.399869+0000","last_became_active":"2023-09-23T21:16:36.498646+0000","last_became_peered":"2023-09-23T21:16:36.498646+0000","last_unstale":"2023-09-23T21:17:08.399869+0000","last_undegraded":"2023-09-23T21:17:08.399869+0000","last_fullsized":"2023-09-23T21:17:08.399869+0000","mapping_epoch":537,"log_start":"494'1721","ondisk_log_start":"494'1721","created":507,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4483,"log_dups_size":0,"ondisk_log_size":4483,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79495168,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4543,"num_write_kb":184192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1956,"num_bytes_recovered":93519872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":209,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863782+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:17:00.863782+0000","last_peered":"2023-09-23T21:17:00.863782+0000","last_clean":"2023-09-23T21:17:00.863782+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:17:00.863782+0000","last_undegraded":"2023-09-23T21:17:00.863782+0000","last_fullsized":"2023-09-23T21:17:00.863782+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.864663+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:17:00.864663+0000","last_peered":"2023-09-23T21:17:00.864663+0000","last_clean":"2023-09-23T21:17:00.864663+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:17:00.864663+0000","last_undegraded":"2023-09-23T21:17:00.864663+0000","last_fullsized":"2023-09-23T21:17:00.864663+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"550'6069","reported_seq":6183,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.484350+0000","last_change":"2023-09-23T21:16:51.074430+0000","last_active":"2023-09-23T21:17:08.484350+0000","last_peered":"2023-09-23T21:17:08.484350+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:16:51.054035+0000","last_became_peered":"2023-09-23T21:16:51.054035+0000","last_unstale":"2023-09-23T21:17:08.484350+0000","last_undegraded":"2023-09-23T21:16:50.838593+0000","last_fullsized":"2023-09-23T21:16:50.797592+0000","mapping_epoch":544,"log_start":"493'1534","ondisk_log_start":"493'1534","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4535,"log_dups_size":0,"ondisk_log_size":4535,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78348288,"num_objects":1196,"num_object_clones":0,"num_object_copies":3588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4434,"num_write_kb":180128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":258},{"shards":"2,5","objects":938}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":144,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.875626+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:17:00.875626+0000","last_peered":"2023-09-23T21:17:00.875626+0000","last_clean":"2023-09-23T21:17:00.875626+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:17:00.875626+0000","last_undegraded":"2023-09-23T21:17:00.875626+0000","last_fullsized":"2023-09-23T21:17:00.875626+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863049+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:17:00.863049+0000","last_peered":"2023-09-23T21:17:00.863049+0000","last_clean":"2023-09-23T21:17:00.863049+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:17:00.863049+0000","last_undegraded":"2023-09-23T21:17:00.863049+0000","last_fullsized":"2023-09-23T21:17:00.863049+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.18","version":"550'6197","reported_seq":9063,"reported_epoch":550,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:17:08.067862+0000","last_change":"2023-09-23T21:16:51.096371+0000","last_active":"2023-09-23T21:17:08.067862+0000","last_peered":"2023-09-23T21:17:08.067862+0000","last_clean":"2023-09-23T21:15:36.291044+0000","last_became_active":"2023-09-23T21:16:51.051104+0000","last_became_peered":"2023-09-23T21:16:51.051104+0000","last_unstale":"2023-09-23T21:17:08.067862+0000","last_undegraded":"2023-09-23T21:17:08.067862+0000","last_fullsized":"2023-09-23T21:16:50.790365+0000","mapping_epoch":544,"log_start":"489'503","ondisk_log_start":"489'503","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.211162+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78118912,"num_objects":1192,"num_object_clones":0,"num_object_copies":3576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2684,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1357,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1192}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":203,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.876080+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:17:00.876080+0000","last_peered":"2023-09-23T21:17:00.876080+0000","last_clean":"2023-09-23T21:17:00.876080+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:17:00.876080+0000","last_undegraded":"2023-09-23T21:17:00.876080+0000","last_fullsized":"2023-09-23T21:17:00.876080+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":172,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859878+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:17:00.859878+0000","last_peered":"2023-09-23T21:17:00.859878+0000","last_clean":"2023-09-23T21:17:00.859878+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:17:00.859878+0000","last_undegraded":"2023-09-23T21:17:00.859878+0000","last_fullsized":"2023-09-23T21:17:00.859878+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1b","version":"550'7928","reported_seq":12418,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.059483+0000","last_change":"2023-09-23T21:16:00.814154+0000","last_active":"2023-09-23T21:17:08.059483+0000","last_peered":"2023-09-23T21:17:08.059483+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:00.808957+0000","last_became_peered":"2023-09-23T21:16:00.808957+0000","last_unstale":"2023-09-23T21:17:08.059483+0000","last_undegraded":"2023-09-23T21:16:00.770362+0000","last_fullsized":"2023-09-23T21:17:08.059483+0000","mapping_epoch":525,"log_start":"493'1783","ondisk_log_start":"493'1783","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6145,"log_dups_size":0,"ondisk_log_size":6145,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83066880,"num_objects":1268,"num_object_clones":0,"num_object_copies":3804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":333,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4693,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1703,"num_bytes_recovered":111607808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":333},{"shards":"2,4,5","objects":935}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":145,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:00.860309+0000","last_change":"2023-09-23T21:16:50.974446+0000","last_active":"2023-09-23T21:17:00.860309+0000","last_peered":"2023-09-23T21:17:00.860309+0000","last_clean":"2023-09-23T21:16:47.791993+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:17:00.860309+0000","last_undegraded":"2023-09-23T21:16:50.810242+0000","last_fullsized":"2023-09-23T21:17:00.860309+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861795+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:17:00.861795+0000","last_peered":"2023-09-23T21:17:00.861795+0000","last_clean":"2023-09-23T21:17:00.861795+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:17:00.861795+0000","last_undegraded":"2023-09-23T21:17:00.861795+0000","last_fullsized":"2023-09-23T21:17:00.861795+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1a","version":"550'7605","reported_seq":9729,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.053672+0000","last_change":"2023-09-23T21:16:00.812483+0000","last_active":"2023-09-23T21:17:08.053672+0000","last_peered":"2023-09-23T21:17:08.053672+0000","last_clean":"2023-09-23T21:17:08.053672+0000","last_became_active":"2023-09-23T21:16:00.806979+0000","last_became_peered":"2023-09-23T21:16:00.806979+0000","last_unstale":"2023-09-23T21:17:08.053672+0000","last_undegraded":"2023-09-23T21:17:08.053672+0000","last_fullsized":"2023-09-23T21:17:08.053672+0000","mapping_epoch":525,"log_start":"493'1441","ondisk_log_start":"493'1441","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6164,"log_dups_size":0,"ondisk_log_size":6164,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4493,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":177,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.858750+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:17:00.858750+0000","last_peered":"2023-09-23T21:17:00.858750+0000","last_clean":"2023-09-23T21:17:00.858750+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:17:00.858750+0000","last_undegraded":"2023-09-23T21:17:00.858750+0000","last_fullsized":"2023-09-23T21:17:00.858750+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":478,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.869281+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:17:00.869281+0000","last_peered":"2023-09-23T21:17:00.869281+0000","last_clean":"2023-09-23T21:17:00.869281+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:17:00.869281+0000","last_undegraded":"2023-09-23T21:17:00.869281+0000","last_fullsized":"2023-09-23T21:17:00.869281+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1d","version":"550'7822","reported_seq":12904,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.515917+0000","last_change":"2023-09-23T21:16:03.792593+0000","last_active":"2023-09-23T21:17:08.515917+0000","last_peered":"2023-09-23T21:17:08.515917+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:16:03.773235+0000","last_became_peered":"2023-09-23T21:16:03.773235+0000","last_unstale":"2023-09-23T21:17:08.515917+0000","last_undegraded":"2023-09-23T21:16:03.727693+0000","last_fullsized":"2023-09-23T21:17:08.515917+0000","mapping_epoch":528,"log_start":"497'2164","ondisk_log_start":"497'2164","created":525,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5658,"log_dups_size":0,"ondisk_log_size":5658,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79265804,"num_objects":1210,"num_object_clones":0,"num_object_copies":3630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":500,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1210,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4645,"num_write_kb":187329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1972,"num_bytes_recovered":129236992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":260},{"shards":"4,5,7","objects":950}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":133,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859470+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:17:00.859470+0000","last_peered":"2023-09-23T21:17:00.859470+0000","last_clean":"2023-09-23T21:17:00.859470+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:17:00.859470+0000","last_undegraded":"2023-09-23T21:17:00.859470+0000","last_fullsized":"2023-09-23T21:17:00.859470+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":201,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.858597+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:17:00.858597+0000","last_peered":"2023-09-23T21:17:00.858597+0000","last_clean":"2023-09-23T21:17:00.858597+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:17:00.858597+0000","last_undegraded":"2023-09-23T21:17:00.858597+0000","last_fullsized":"2023-09-23T21:17:00.858597+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1c","version":"550'7801","reported_seq":9918,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.477755+0000","last_change":"2023-09-23T21:16:00.824659+0000","last_active":"2023-09-23T21:17:08.477755+0000","last_peered":"2023-09-23T21:17:08.477755+0000","last_clean":"2023-09-23T21:17:08.477755+0000","last_became_active":"2023-09-23T21:16:00.820619+0000","last_became_peered":"2023-09-23T21:16:00.820619+0000","last_unstale":"2023-09-23T21:17:08.477755+0000","last_undegraded":"2023-09-23T21:17:08.477755+0000","last_fullsized":"2023-09-23T21:17:08.477755+0000","mapping_epoch":525,"log_start":"500'2344","ondisk_log_start":"500'2344","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82673664,"num_objects":1262,"num_object_clones":0,"num_object_copies":3786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4617,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":129,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859124+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:17:00.859124+0000","last_peered":"2023-09-23T21:17:00.859124+0000","last_clean":"2023-09-23T21:17:00.859124+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:17:00.859124+0000","last_undegraded":"2023-09-23T21:17:00.859124+0000","last_fullsized":"2023-09-23T21:17:00.859124+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":137,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.860608+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:17:00.860608+0000","last_peered":"2023-09-23T21:17:00.860608+0000","last_clean":"2023-09-23T21:17:00.860608+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:17:00.860608+0000","last_undegraded":"2023-09-23T21:17:00.860608+0000","last_fullsized":"2023-09-23T21:17:00.860608+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1f","version":"550'7764","reported_seq":7872,"reported_epoch":550,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:11.226515+0000","last_change":"2023-09-23T21:16:51.848275+0000","last_active":"2023-09-23T21:17:11.226515+0000","last_peered":"2023-09-23T21:17:11.226515+0000","last_clean":"2023-09-23T21:15:36.326799+0000","last_became_active":"2023-09-23T21:16:51.820010+0000","last_became_peered":"2023-09-23T21:16:51.820010+0000","last_unstale":"2023-09-23T21:17:11.226515+0000","last_undegraded":"2023-09-23T21:17:11.226515+0000","last_fullsized":"2023-09-23T21:17:11.226515+0000","mapping_epoch":545,"log_start":"493'1700","ondisk_log_start":"493'1700","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75726848,"num_objects":1156,"num_object_clones":0,"num_object_copies":3468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4634,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","5","7"],"object_location_counts":[{"shards":"2,6,7","objects":1156}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":127,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.860053+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:17:00.860053+0000","last_peered":"2023-09-23T21:17:00.860053+0000","last_clean":"2023-09-23T21:17:00.860053+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:17:00.860053+0000","last_undegraded":"2023-09-23T21:17:00.860053+0000","last_fullsized":"2023-09-23T21:17:00.860053+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":151,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861715+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:17:00.861715+0000","last_peered":"2023-09-23T21:17:00.861715+0000","last_clean":"2023-09-23T21:17:00.861715+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:17:00.861715+0000","last_undegraded":"2023-09-23T21:17:00.861715+0000","last_fullsized":"2023-09-23T21:17:00.861715+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1e","version":"550'7601","reported_seq":7686,"reported_epoch":550,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:08.355672+0000","last_change":"2023-09-23T21:16:00.911470+0000","last_active":"2023-09-23T21:17:08.355672+0000","last_peered":"2023-09-23T21:17:08.355672+0000","last_clean":"2023-09-23T21:15:59.749850+0000","last_became_active":"2023-09-23T21:16:00.844392+0000","last_became_peered":"2023-09-23T21:16:00.844392+0000","last_unstale":"2023-09-23T21:17:08.355672+0000","last_undegraded":"2023-09-23T21:17:08.355672+0000","last_fullsized":"2023-09-23T21:17:08.355672+0000","mapping_epoch":525,"log_start":"493'1633","ondisk_log_start":"493'1633","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5968,"log_dups_size":0,"ondisk_log_size":5968,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79462400,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4510,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":1213}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":586,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863693+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:17:00.863693+0000","last_peered":"2023-09-23T21:17:00.863693+0000","last_clean":"2023-09-23T21:17:00.863693+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:17:00.863693+0000","last_undegraded":"2023-09-23T21:17:00.863693+0000","last_fullsized":"2023-09-23T21:17:00.863693+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":201,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.865967+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:17:00.865967+0000","last_peered":"2023-09-23T21:17:00.865967+0000","last_clean":"2023-09-23T21:17:00.865967+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:17:00.865967+0000","last_undegraded":"2023-09-23T21:17:00.865967+0000","last_fullsized":"2023-09-23T21:17:00.865967+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"550'5453","reported_seq":6587,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:07.949063+0000","last_change":"2023-09-23T21:16:51.863144+0000","last_active":"2023-09-23T21:17:07.949063+0000","last_peered":"2023-09-23T21:17:07.949063+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858843+0000","last_became_peered":"2023-09-23T21:16:51.858843+0000","last_unstale":"2023-09-23T21:17:07.949063+0000","last_undegraded":"2023-09-23T21:16:51.799178+0000","last_fullsized":"2023-09-23T21:16:51.792321+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5453,"log_dups_size":0,"ondisk_log_size":5453,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38322176,"num_objects":584,"num_object_clones":0,"num_object_copies":1752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2271,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":511,"num_bytes_recovered":33488896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":415},{"shards":"2,4,7","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":131,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.866053+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:17:00.866053+0000","last_peered":"2023-09-23T21:17:00.866053+0000","last_clean":"2023-09-23T21:17:00.866053+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:17:00.866053+0000","last_undegraded":"2023-09-23T21:17:00.866053+0000","last_fullsized":"2023-09-23T21:17:00.866053+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":203,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.872287+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:17:00.872287+0000","last_peered":"2023-09-23T21:17:00.872287+0000","last_clean":"2023-09-23T21:17:00.872287+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:17:00.872287+0000","last_undegraded":"2023-09-23T21:17:00.872287+0000","last_fullsized":"2023-09-23T21:17:00.872287+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"550'5537","reported_seq":8644,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.422238+0000","last_change":"2023-09-23T21:16:51.842306+0000","last_active":"2023-09-23T21:17:08.422238+0000","last_peered":"2023-09-23T21:17:08.422238+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834848+0000","last_became_peered":"2023-09-23T21:16:51.834848+0000","last_unstale":"2023-09-23T21:17:08.422238+0000","last_undegraded":"2023-09-23T21:16:51.803926+0000","last_fullsized":"2023-09-23T21:16:51.799106+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5537,"log_dups_size":0,"ondisk_log_size":5537,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41025536,"num_objects":626,"num_object_clones":0,"num_object_copies":1878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1414,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32178176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":161},{"shards":"2,7","objects":465}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":145,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861882+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:17:00.861882+0000","last_peered":"2023-09-23T21:17:00.861882+0000","last_clean":"2023-09-23T21:17:00.861882+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:17:00.861882+0000","last_undegraded":"2023-09-23T21:17:00.861882+0000","last_fullsized":"2023-09-23T21:17:00.861882+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861906+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:17:00.861906+0000","last_peered":"2023-09-23T21:17:00.861906+0000","last_clean":"2023-09-23T21:17:00.861906+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:17:00.861906+0000","last_undegraded":"2023-09-23T21:17:00.861906+0000","last_fullsized":"2023-09-23T21:17:00.861906+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3","version":"550'5624","reported_seq":7239,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.501139+0000","last_change":"2023-09-23T21:16:01.735614+0000","last_active":"2023-09-23T21:17:08.501139+0000","last_peered":"2023-09-23T21:17:08.501139+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:01.731847+0000","last_became_peered":"2023-09-23T21:16:01.731847+0000","last_unstale":"2023-09-23T21:17:08.501139+0000","last_undegraded":"2023-09-23T21:16:01.699708+0000","last_fullsized":"2023-09-23T21:17:08.501139+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5624,"log_dups_size":0,"ondisk_log_size":5624,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40796160,"num_objects":622,"num_object_clones":0,"num_object_copies":1866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1363,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":730,"num_bytes_recovered":47841280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":149},{"shards":"4,5,7","objects":473}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":131,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.860683+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:17:00.860683+0000","last_peered":"2023-09-23T21:17:00.860683+0000","last_clean":"2023-09-23T21:17:00.860683+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:17:00.860683+0000","last_undegraded":"2023-09-23T21:17:00.860683+0000","last_fullsized":"2023-09-23T21:17:00.860683+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.862004+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:17:00.862004+0000","last_peered":"2023-09-23T21:17:00.862004+0000","last_clean":"2023-09-23T21:17:00.862004+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:17:00.862004+0000","last_undegraded":"2023-09-23T21:17:00.862004+0000","last_fullsized":"2023-09-23T21:17:00.862004+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"550'5480","reported_seq":6940,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.521361+0000","last_change":"2023-09-23T21:16:01.750035+0000","last_active":"2023-09-23T21:17:08.521361+0000","last_peered":"2023-09-23T21:17:08.521361+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:01.742399+0000","last_became_peered":"2023-09-23T21:16:01.742399+0000","last_unstale":"2023-09-23T21:17:08.521361+0000","last_undegraded":"2023-09-23T21:16:01.706463+0000","last_fullsized":"2023-09-23T21:17:08.521361+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43188224,"num_objects":659,"num_object_clones":0,"num_object_copies":1977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":287,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2241,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":670,"num_bytes_recovered":43909120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":154},{"shards":"4,5,6","objects":505}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863047+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:17:00.863047+0000","last_peered":"2023-09-23T21:17:00.863047+0000","last_clean":"2023-09-23T21:17:00.863047+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:17:00.863047+0000","last_undegraded":"2023-09-23T21:17:00.863047+0000","last_fullsized":"2023-09-23T21:17:00.863047+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861413+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:17:00.861413+0000","last_peered":"2023-09-23T21:17:00.861413+0000","last_clean":"2023-09-23T21:17:00.861413+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:17:00.861413+0000","last_undegraded":"2023-09-23T21:17:00.861413+0000","last_fullsized":"2023-09-23T21:17:00.861413+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"550'6199","reported_seq":6409,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.360638+0000","last_change":"2023-09-23T21:16:51.047553+0000","last_active":"2023-09-23T21:17:08.360638+0000","last_peered":"2023-09-23T21:17:08.360638+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:16:51.032827+0000","last_became_peered":"2023-09-23T21:16:51.032827+0000","last_unstale":"2023-09-23T21:17:08.360638+0000","last_undegraded":"2023-09-23T21:16:50.886600+0000","last_fullsized":"2023-09-23T21:16:50.791765+0000","mapping_epoch":544,"log_start":"493'1626","ondisk_log_start":"493'1626","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":4573,"log_dups_size":0,"ondisk_log_size":4573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:01.438609+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2256,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2643,"num_write_kb":64384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":314},{"shards":"3,4,5","objects":306},{"shards":"4,5","objects":191}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":217,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859333+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:17:00.859333+0000","last_peered":"2023-09-23T21:17:00.859333+0000","last_clean":"2023-09-23T21:17:00.859333+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:17:00.859333+0000","last_undegraded":"2023-09-23T21:17:00.859333+0000","last_fullsized":"2023-09-23T21:17:00.859333+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863200+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:17:00.863200+0000","last_peered":"2023-09-23T21:17:00.863200+0000","last_clean":"2023-09-23T21:17:00.863200+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:17:00.863200+0000","last_undegraded":"2023-09-23T21:17:00.863200+0000","last_fullsized":"2023-09-23T21:17:00.863200+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.4","version":"550'6284","reported_seq":8579,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:07.917714+0000","last_change":"2023-09-23T21:16:51.065805+0000","last_active":"2023-09-23T21:17:07.917714+0000","last_peered":"2023-09-23T21:17:07.917714+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:16:51.051146+0000","last_became_peered":"2023-09-23T21:16:51.051146+0000","last_unstale":"2023-09-23T21:17:07.917714+0000","last_undegraded":"2023-09-23T21:16:50.809981+0000","last_fullsized":"2023-09-23T21:16:50.790363+0000","mapping_epoch":544,"log_start":"489'610","ondisk_log_start":"489'610","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:23.280602+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75235328,"num_objects":1148,"num_object_clones":0,"num_object_copies":3444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2584,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":242},{"shards":"2,5","objects":906}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861828+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:17:00.861828+0000","last_peered":"2023-09-23T21:17:00.861828+0000","last_clean":"2023-09-23T21:17:00.861828+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:17:00.861828+0000","last_undegraded":"2023-09-23T21:17:00.861828+0000","last_fullsized":"2023-09-23T21:17:00.861828+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":592,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:05.502872+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:17:05.502872+0000","last_peered":"2023-09-23T21:17:05.502872+0000","last_clean":"2023-09-23T21:17:05.502872+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:17:05.502872+0000","last_undegraded":"2023-09-23T21:17:05.502872+0000","last_fullsized":"2023-09-23T21:17:05.502872+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"550'6106","reported_seq":6218,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.422737+0000","last_change":"2023-09-23T21:16:51.082314+0000","last_active":"2023-09-23T21:17:08.422737+0000","last_peered":"2023-09-23T21:17:08.422737+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:16:51.064800+0000","last_became_peered":"2023-09-23T21:16:51.064800+0000","last_unstale":"2023-09-23T21:17:08.422737+0000","last_undegraded":"2023-09-23T21:16:50.820535+0000","last_fullsized":"2023-09-23T21:16:50.800952+0000","mapping_epoch":544,"log_start":"489'910","ondisk_log_start":"489'910","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5196,"log_dups_size":0,"ondisk_log_size":5196,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81985536,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4463,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":295},{"shards":"4,7","objects":956}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.867205+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:17:00.867205+0000","last_peered":"2023-09-23T21:17:00.867205+0000","last_clean":"2023-09-23T21:17:00.867205+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:17:00.867205+0000","last_undegraded":"2023-09-23T21:17:00.867205+0000","last_fullsized":"2023-09-23T21:17:00.867205+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.862825+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:17:00.862825+0000","last_peered":"2023-09-23T21:17:00.862825+0000","last_clean":"2023-09-23T21:17:00.862825+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:17:00.862825+0000","last_undegraded":"2023-09-23T21:17:00.862825+0000","last_fullsized":"2023-09-23T21:17:00.862825+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"550'6154","reported_seq":6358,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.521723+0000","last_change":"2023-09-23T21:15:36.787083+0000","last_active":"2023-09-23T21:17:08.521723+0000","last_peered":"2023-09-23T21:17:08.521723+0000","last_clean":"2023-09-23T21:15:09.799249+0000","last_became_active":"2023-09-23T21:15:12.799540+0000","last_became_peered":"2023-09-23T21:15:12.799540+0000","last_unstale":"2023-09-23T21:17:08.521723+0000","last_undegraded":"2023-09-23T21:15:36.785930+0000","last_fullsized":"2023-09-23T21:15:12.679693+0000","mapping_epoch":514,"log_start":"494'1711","ondisk_log_start":"494'1711","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":4443,"log_dups_size":0,"ondisk_log_size":4443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:59.312702+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77856768,"num_objects":1188,"num_object_clones":0,"num_object_copies":3564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":57088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":296},{"shards":"4,5","objects":-750}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":201,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859719+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:17:00.859719+0000","last_peered":"2023-09-23T21:17:00.859719+0000","last_clean":"2023-09-23T21:17:00.859719+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:17:00.859719+0000","last_undegraded":"2023-09-23T21:17:00.859719+0000","last_fullsized":"2023-09-23T21:17:00.859719+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":289,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.864988+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:17:00.864988+0000","last_peered":"2023-09-23T21:17:00.864988+0000","last_clean":"2023-09-23T21:17:00.864988+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:17:00.864988+0000","last_undegraded":"2023-09-23T21:17:00.864988+0000","last_fullsized":"2023-09-23T21:17:00.864988+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"550'7889","reported_seq":12340,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.063541+0000","last_change":"2023-09-23T21:16:01.728998+0000","last_active":"2023-09-23T21:17:08.063541+0000","last_peered":"2023-09-23T21:17:08.063541+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:01.723118+0000","last_became_peered":"2023-09-23T21:16:01.723118+0000","last_unstale":"2023-09-23T21:17:08.063541+0000","last_undegraded":"2023-09-23T21:16:01.697539+0000","last_fullsized":"2023-09-23T21:17:08.063541+0000","mapping_epoch":526,"log_start":"493'1783","ondisk_log_start":"493'1783","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6106,"log_dups_size":0,"ondisk_log_size":6106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85622784,"num_objects":1306,"num_object_clones":0,"num_object_copies":3918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4653,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1683,"num_bytes_recovered":110297088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":300},{"shards":"2,4,5","objects":1006}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":474,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.858669+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:17:00.858669+0000","last_peered":"2023-09-23T21:17:00.858669+0000","last_clean":"2023-09-23T21:17:00.858669+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:17:00.858669+0000","last_undegraded":"2023-09-23T21:17:00.858669+0000","last_fullsized":"2023-09-23T21:17:00.858669+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":225,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.879913+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:17:00.879913+0000","last_peered":"2023-09-23T21:17:00.879913+0000","last_clean":"2023-09-23T21:17:00.879913+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:17:00.879913+0000","last_undegraded":"2023-09-23T21:17:00.879913+0000","last_fullsized":"2023-09-23T21:17:00.879913+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":99,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.873584+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:17:00.873584+0000","last_peered":"2023-09-23T21:17:00.873584+0000","last_clean":"2023-09-23T21:17:00.873584+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:17:00.873584+0000","last_undegraded":"2023-09-23T21:17:00.873584+0000","last_fullsized":"2023-09-23T21:17:00.873584+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"550'7787","reported_seq":12791,"reported_epoch":550,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:08.478506+0000","last_change":"2023-09-23T21:16:01.774573+0000","last_active":"2023-09-23T21:17:08.478506+0000","last_peered":"2023-09-23T21:17:08.478506+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:16:01.764537+0000","last_became_peered":"2023-09-23T21:16:01.764537+0000","last_unstale":"2023-09-23T21:17:08.478506+0000","last_undegraded":"2023-09-23T21:16:01.707917+0000","last_fullsized":"2023-09-23T21:17:08.478506+0000","mapping_epoch":526,"log_start":"497'2164","ondisk_log_start":"497'2164","created":482,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5623,"log_dups_size":0,"ondisk_log_size":5623,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81559564,"num_objects":1245,"num_object_clones":0,"num_object_copies":3735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":550,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4610,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1937,"num_bytes_recovered":126943232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":291},{"shards":"4,5,7","objects":954}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861745+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:17:00.861745+0000","last_peered":"2023-09-23T21:17:00.861745+0000","last_clean":"2023-09-23T21:17:00.861745+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:17:00.861745+0000","last_undegraded":"2023-09-23T21:17:00.861745+0000","last_fullsized":"2023-09-23T21:17:00.861745+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.861502+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:17:00.861502+0000","last_peered":"2023-09-23T21:17:00.861502+0000","last_clean":"2023-09-23T21:17:00.861502+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:17:00.861502+0000","last_undegraded":"2023-09-23T21:17:00.861502+0000","last_fullsized":"2023-09-23T21:17:00.861502+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":264,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.864221+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:17:00.864221+0000","last_peered":"2023-09-23T21:17:00.864221+0000","last_clean":"2023-09-23T21:17:00.864221+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:17:00.864221+0000","last_undegraded":"2023-09-23T21:17:00.864221+0000","last_fullsized":"2023-09-23T21:17:00.864221+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"550'7575","reported_seq":7661,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.382849+0000","last_change":"2023-09-23T21:16:00.841315+0000","last_active":"2023-09-23T21:17:08.382849+0000","last_peered":"2023-09-23T21:17:08.382849+0000","last_clean":"2023-09-23T21:17:08.382849+0000","last_became_active":"2023-09-23T21:16:00.829607+0000","last_became_peered":"2023-09-23T21:16:00.829607+0000","last_unstale":"2023-09-23T21:17:08.382849+0000","last_undegraded":"2023-09-23T21:17:08.382849+0000","last_fullsized":"2023-09-23T21:17:08.382849+0000","mapping_epoch":482,"log_start":"493'1633","ondisk_log_start":"493'1633","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81166336,"num_objects":1238,"num_object_clones":0,"num_object_copies":3714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4483,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":558,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863162+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:17:00.863162+0000","last_peered":"2023-09-23T21:17:00.863162+0000","last_clean":"2023-09-23T21:17:00.863162+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:17:00.863162+0000","last_undegraded":"2023-09-23T21:17:00.863162+0000","last_fullsized":"2023-09-23T21:17:00.863162+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":296,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.862936+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:17:00.862936+0000","last_peered":"2023-09-23T21:17:00.862936+0000","last_clean":"2023-09-23T21:17:00.862936+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:17:00.862936+0000","last_undegraded":"2023-09-23T21:17:00.862936+0000","last_fullsized":"2023-09-23T21:17:00.862936+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":203,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.865066+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:17:00.865066+0000","last_peered":"2023-09-23T21:17:00.865066+0000","last_clean":"2023-09-23T21:17:00.865066+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:17:00.865066+0000","last_undegraded":"2023-09-23T21:17:00.865066+0000","last_fullsized":"2023-09-23T21:17:00.865066+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"550'7814","reported_seq":9932,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.478031+0000","last_change":"2023-09-23T21:16:00.748968+0000","last_active":"2023-09-23T21:17:08.478031+0000","last_peered":"2023-09-23T21:17:08.478031+0000","last_clean":"2023-09-23T21:17:08.478031+0000","last_became_active":"2023-09-23T21:16:00.745123+0000","last_became_peered":"2023-09-23T21:16:00.745123+0000","last_unstale":"2023-09-23T21:17:08.478031+0000","last_undegraded":"2023-09-23T21:17:08.478031+0000","last_fullsized":"2023-09-23T21:17:08.478031+0000","mapping_epoch":501,"log_start":"500'2344","ondisk_log_start":"500'2344","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81821696,"num_objects":1248,"num_object_clones":0,"num_object_copies":3744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4629,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":502,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":378,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.866372+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:17:00.866372+0000","last_peered":"2023-09-23T21:17:00.866372+0000","last_clean":"2023-09-23T21:17:00.866372+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:17:00.866372+0000","last_undegraded":"2023-09-23T21:17:00.866372+0000","last_fullsized":"2023-09-23T21:17:00.866372+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":896,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859476+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:17:00.859476+0000","last_peered":"2023-09-23T21:17:00.859476+0000","last_clean":"2023-09-23T21:17:00.859476+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:17:00.859476+0000","last_undegraded":"2023-09-23T21:17:00.859476+0000","last_fullsized":"2023-09-23T21:17:00.859476+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":122,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.873132+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:17:00.873132+0000","last_peered":"2023-09-23T21:17:00.873132+0000","last_clean":"2023-09-23T21:17:00.873132+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:17:00.873132+0000","last_undegraded":"2023-09-23T21:17:00.873132+0000","last_fullsized":"2023-09-23T21:17:00.873132+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863084+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:17:00.863084+0000","last_peered":"2023-09-23T21:17:00.863084+0000","last_clean":"2023-09-23T21:17:00.863084+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:17:00.863084+0000","last_undegraded":"2023-09-23T21:17:00.863084+0000","last_fullsized":"2023-09-23T21:17:00.863084+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"550'7731","reported_seq":7833,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:11.201089+0000","last_change":"2023-09-23T21:16:51.832676+0000","last_active":"2023-09-23T21:17:11.201089+0000","last_peered":"2023-09-23T21:17:11.201089+0000","last_clean":"2023-09-23T21:17:11.201089+0000","last_became_active":"2023-09-23T21:16:51.803067+0000","last_became_peered":"2023-09-23T21:16:51.803067+0000","last_unstale":"2023-09-23T21:17:11.201089+0000","last_undegraded":"2023-09-23T21:17:11.201089+0000","last_fullsized":"2023-09-23T21:17:11.201089+0000","mapping_epoch":545,"log_start":"493'1700","ondisk_log_start":"493'1700","created":482,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6031,"log_dups_size":0,"ondisk_log_size":6031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77889536,"num_objects":1188,"num_object_clones":0,"num_object_copies":3564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4600,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":177,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859558+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:17:00.859558+0000","last_peered":"2023-09-23T21:17:00.859558+0000","last_clean":"2023-09-23T21:17:00.859558+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:17:00.859558+0000","last_undegraded":"2023-09-23T21:17:00.859558+0000","last_fullsized":"2023-09-23T21:17:00.859558+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":46,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859811+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:17:00.859811+0000","last_peered":"2023-09-23T21:17:00.859811+0000","last_clean":"2023-09-23T21:17:00.859811+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:17:00.859811+0000","last_undegraded":"2023-09-23T21:17:00.859811+0000","last_fullsized":"2023-09-23T21:17:00.859811+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":459,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.865154+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:17:00.865154+0000","last_peered":"2023-09-23T21:17:00.865154+0000","last_clean":"2023-09-23T21:17:00.865154+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:17:00.865154+0000","last_undegraded":"2023-09-23T21:17:00.865154+0000","last_fullsized":"2023-09-23T21:17:00.865154+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"550'7644","reported_seq":9768,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.051779+0000","last_change":"2023-09-23T21:16:00.763134+0000","last_active":"2023-09-23T21:17:08.051779+0000","last_peered":"2023-09-23T21:17:08.051779+0000","last_clean":"2023-09-23T21:17:08.051779+0000","last_became_active":"2023-09-23T21:16:00.758875+0000","last_became_peered":"2023-09-23T21:16:00.758875+0000","last_unstale":"2023-09-23T21:17:08.051779+0000","last_undegraded":"2023-09-23T21:17:08.051779+0000","last_fullsized":"2023-09-23T21:17:08.051779+0000","mapping_epoch":498,"log_start":"493'1441","ondisk_log_start":"493'1441","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6203,"log_dups_size":0,"ondisk_log_size":6203,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4532,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":147,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.860485+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:17:00.860485+0000","last_peered":"2023-09-23T21:17:00.860485+0000","last_clean":"2023-09-23T21:17:00.860485+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:17:00.860485+0000","last_undegraded":"2023-09-23T21:17:00.860485+0000","last_fullsized":"2023-09-23T21:17:00.860485+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":363,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.864324+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:17:00.864324+0000","last_peered":"2023-09-23T21:17:00.864324+0000","last_clean":"2023-09-23T21:17:00.864324+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:17:00.864324+0000","last_undegraded":"2023-09-23T21:17:00.864324+0000","last_fullsized":"2023-09-23T21:17:00.864324+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":255,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.875487+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:17:00.875487+0000","last_peered":"2023-09-23T21:17:00.875487+0000","last_clean":"2023-09-23T21:17:00.875487+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:17:00.875487+0000","last_undegraded":"2023-09-23T21:17:00.875487+0000","last_fullsized":"2023-09-23T21:17:00.875487+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"550'6159","reported_seq":6361,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.438030+0000","last_change":"2023-09-23T21:16:51.076997+0000","last_active":"2023-09-23T21:17:08.438030+0000","last_peered":"2023-09-23T21:17:08.438030+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:16:51.057102+0000","last_became_peered":"2023-09-23T21:16:51.057102+0000","last_unstale":"2023-09-23T21:17:08.438030+0000","last_undegraded":"2023-09-23T21:16:50.840109+0000","last_fullsized":"2023-09-23T21:16:50.799660+0000","mapping_epoch":544,"log_start":"493'1740","ondisk_log_start":"493'1740","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":4419,"log_dups_size":0,"ondisk_log_size":4419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.748037+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81526784,"num_objects":1244,"num_object_clones":0,"num_object_copies":3732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1565,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2597,"num_write_kb":60864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":315},{"shards":"2,5","objects":929}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":205,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.866142+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:17:00.866142+0000","last_peered":"2023-09-23T21:17:00.866142+0000","last_clean":"2023-09-23T21:17:00.866142+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:17:00.866142+0000","last_undegraded":"2023-09-23T21:17:00.866142+0000","last_fullsized":"2023-09-23T21:17:00.866142+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.862757+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:17:00.862757+0000","last_peered":"2023-09-23T21:17:00.862757+0000","last_clean":"2023-09-23T21:17:00.862757+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:17:00.862757+0000","last_undegraded":"2023-09-23T21:17:00.862757+0000","last_fullsized":"2023-09-23T21:17:00.862757+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":230,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859387+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:17:00.859387+0000","last_peered":"2023-09-23T21:17:00.859387+0000","last_clean":"2023-09-23T21:17:00.859387+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:17:00.859387+0000","last_undegraded":"2023-09-23T21:17:00.859387+0000","last_fullsized":"2023-09-23T21:17:00.859387+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"550'6257","reported_seq":6407,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.039371+0000","last_change":"2023-09-23T21:16:51.063450+0000","last_active":"2023-09-23T21:17:08.039371+0000","last_peered":"2023-09-23T21:17:08.039371+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:16:51.051345+0000","last_became_peered":"2023-09-23T21:16:51.051345+0000","last_unstale":"2023-09-23T21:17:08.039371+0000","last_undegraded":"2023-09-23T21:16:50.807164+0000","last_fullsized":"2023-09-23T21:16:50.788756+0000","mapping_epoch":544,"log_start":"489'560","ondisk_log_start":"489'560","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:15.933191+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84279296,"num_objects":1286,"num_object_clones":0,"num_object_copies":3858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1557,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2769,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":985},{"shards":"2,5,7","objects":301}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":456,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.864727+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:17:00.864727+0000","last_peered":"2023-09-23T21:17:00.864727+0000","last_clean":"2023-09-23T21:17:00.864727+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:17:00.864727+0000","last_undegraded":"2023-09-23T21:17:00.864727+0000","last_fullsized":"2023-09-23T21:17:00.864727+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":87,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.871208+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:17:00.871208+0000","last_peered":"2023-09-23T21:17:00.871208+0000","last_clean":"2023-09-23T21:17:00.871208+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:17:00.871208+0000","last_undegraded":"2023-09-23T21:17:00.871208+0000","last_fullsized":"2023-09-23T21:17:00.871208+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":235,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.872840+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:17:00.872840+0000","last_peered":"2023-09-23T21:17:00.872840+0000","last_clean":"2023-09-23T21:17:00.872840+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:17:00.872840+0000","last_undegraded":"2023-09-23T21:17:00.872840+0000","last_fullsized":"2023-09-23T21:17:00.872840+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"550'6304","reported_seq":6427,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.480846+0000","last_change":"2023-09-23T21:16:51.067304+0000","last_active":"2023-09-23T21:17:08.480846+0000","last_peered":"2023-09-23T21:17:08.480846+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:16:51.048602+0000","last_became_peered":"2023-09-23T21:16:51.048602+0000","last_unstale":"2023-09-23T21:17:08.480846+0000","last_undegraded":"2023-09-23T21:16:50.818351+0000","last_fullsized":"2023-09-23T21:16:50.791414+0000","mapping_epoch":544,"log_start":"497'1860","ondisk_log_start":"497'1860","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4444,"log_dups_size":0,"ondisk_log_size":4444,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80150528,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4631,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":293},{"shards":"3,4,5","objects":303},{"shards":"4,5","objects":237}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":177,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.859562+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:17:00.859562+0000","last_peered":"2023-09-23T21:17:00.859562+0000","last_clean":"2023-09-23T21:17:00.859562+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:17:00.859562+0000","last_undegraded":"2023-09-23T21:17:00.859562+0000","last_fullsized":"2023-09-23T21:17:00.859562+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":613,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.864406+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:17:00.864406+0000","last_peered":"2023-09-23T21:17:00.864406+0000","last_clean":"2023-09-23T21:17:00.864406+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:17:00.864406+0000","last_undegraded":"2023-09-23T21:17:00.864406+0000","last_fullsized":"2023-09-23T21:17:00.864406+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"550'6200","reported_seq":10245,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.382468+0000","last_change":"2023-09-23T21:16:37.510873+0000","last_active":"2023-09-23T21:17:08.382468+0000","last_peered":"2023-09-23T21:17:08.382468+0000","last_clean":"2023-09-23T21:17:08.382468+0000","last_became_active":"2023-09-23T21:16:37.500341+0000","last_became_peered":"2023-09-23T21:16:37.500341+0000","last_unstale":"2023-09-23T21:17:08.382468+0000","last_undegraded":"2023-09-23T21:17:08.382468+0000","last_fullsized":"2023-09-23T21:17:08.382468+0000","mapping_epoch":538,"log_start":"493'1688","ondisk_log_start":"493'1688","created":507,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4512,"log_dups_size":0,"ondisk_log_size":4512,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78446592,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4539,"num_write_kb":183552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1970,"num_bytes_recovered":94175232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":254,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.868219+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:17:00.868219+0000","last_peered":"2023-09-23T21:17:00.868219+0000","last_clean":"2023-09-23T21:17:00.868219+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:17:00.868219+0000","last_undegraded":"2023-09-23T21:17:00.868219+0000","last_fullsized":"2023-09-23T21:17:00.868219+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":207,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.865411+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:17:00.865411+0000","last_peered":"2023-09-23T21:17:00.865411+0000","last_clean":"2023-09-23T21:17:00.865411+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:17:00.865411+0000","last_undegraded":"2023-09-23T21:17:00.865411+0000","last_fullsized":"2023-09-23T21:17:00.865411+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"550'6228","reported_seq":8959,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:08.514812+0000","last_change":"2023-09-23T21:16:50.886093+0000","last_active":"2023-09-23T21:17:08.514812+0000","last_peered":"2023-09-23T21:17:08.514812+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:17:08.514812+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"493'1684","ondisk_log_start":"493'1684","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4544,"log_dups_size":0,"ondisk_log_size":4544,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81297408,"num_objects":1241,"num_object_clones":0,"num_object_copies":3723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4545,"num_write_kb":185120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":61800448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5","objects":618}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":19,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:11.215174+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:17:11.215174+0000","last_peered":"2023-09-23T21:17:11.215174+0000","last_clean":"2023-09-23T21:17:11.215174+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:17:11.215174+0000","last_undegraded":"2023-09-23T21:17:11.215174+0000","last_fullsized":"2023-09-23T21:17:11.215174+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":194,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.862831+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:17:00.862831+0000","last_peered":"2023-09-23T21:17:00.862831+0000","last_clean":"2023-09-23T21:17:00.862831+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:17:00.862831+0000","last_undegraded":"2023-09-23T21:17:00.862831+0000","last_fullsized":"2023-09-23T21:17:00.862831+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"550'6099","reported_seq":11961,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:08.020126+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:17:08.020126+0000","last_peered":"2023-09-23T21:17:08.020126+0000","last_clean":"2023-09-23T21:17:08.020126+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:17:08.020126+0000","last_undegraded":"2023-09-23T21:17:08.020126+0000","last_fullsized":"2023-09-23T21:17:08.020126+0000","mapping_epoch":518,"log_start":"489'259","ondisk_log_start":"489'259","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77725696,"num_objects":1186,"num_object_clones":0,"num_object_copies":3558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4436,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.862680+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:17:00.862680+0000","last_peered":"2023-09-23T21:17:00.862680+0000","last_clean":"2023-09-23T21:17:00.862680+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:17:00.862680+0000","last_undegraded":"2023-09-23T21:17:00.862680+0000","last_fullsized":"2023-09-23T21:17:00.862680+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":409,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.863762+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:17:00.863762+0000","last_peered":"2023-09-23T21:17:00.863762+0000","last_clean":"2023-09-23T21:17:00.863762+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:17:00.863762+0000","last_undegraded":"2023-09-23T21:17:00.863762+0000","last_fullsized":"2023-09-23T21:17:00.863762+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"550'6177","reported_seq":10135,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-09-23T21:17:11.086586+0000","last_change":"2023-09-23T21:16:49.855987+0000","last_active":"2023-09-23T21:17:11.086586+0000","last_peered":"2023-09-23T21:17:11.086586+0000","last_clean":"2023-09-23T21:17:11.086586+0000","last_became_active":"2023-09-23T21:16:15.742634+0000","last_became_peered":"2023-09-23T21:16:15.742634+0000","last_unstale":"2023-09-23T21:17:11.086586+0000","last_undegraded":"2023-09-23T21:17:11.086586+0000","last_fullsized":"2023-09-23T21:17:11.086586+0000","mapping_epoch":531,"log_start":"489'428","ondisk_log_start":"489'428","created":507,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79331328,"num_objects":1211,"num_object_clones":0,"num_object_copies":3633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4541,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":506,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.864867+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:17:00.864867+0000","last_peered":"2023-09-23T21:17:00.864867+0000","last_clean":"2023-09-23T21:17:00.864867+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:17:00.864867+0000","last_undegraded":"2023-09-23T21:17:00.864867+0000","last_fullsized":"2023-09-23T21:17:00.864867+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":217,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-09-23T21:17:00.867707+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:17:00.867707+0000","last_peered":"2023-09-23T21:17:00.867707+0000","last_clean":"2023-09-23T21:17:00.867707+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:17:00.867707+0000","last_undegraded":"2023-09-23T21:17:00.867707+0000","last_fullsized":"2023-09-23T21:17:00.867707+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":36,"stat_sum":{"num_bytes":2567962648,"num_objects":39186,"num_object_clones":0,"num_object_copies":117558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39186,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130745,"num_write_kb":4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25762,"num_bytes_recovered":1515356160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9815535616,"data_stored":9815523400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195665,"ondisk_log_size":195665,"up":108,"acting":92,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":38828,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201861,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2083740,"kb_used_data":1312724,"kb_used_omap":0,"kb_used_meta":770880,"kb_avail":91661412,"statfs":{"total":95995035648,"available":93861285888,"internally_reserved":0,"allocated":1344229376,"data_stored":1342735882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":789381120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":3.5129999999999999,"15min":2.1920000000000002},"min":{"1min":0.218,"5min":0.214,"15min":0.20699999999999999},"max":{"1min":5.7949999999999999,"5min":141.33199999999999,"15min":141.33199999999999},"last":1.2150000000000001},{"interface":"front","average":{"1min":1.9079999999999999,"5min":3.6230000000000002,"15min":2.2280000000000002},"min":{"1min":0.27800000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.8330000000000002,"5min":141.31700000000001,"15min":141.31700000000001},"last":1.4970000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":3.0030000000000001,"15min":3.3889999999999998},"min":{"1min":0.317,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.8700000000000001,"5min":76.007999999999996,"15min":114.16500000000001},"last":1.5389999999999999},{"interface":"front","average":{"1min":1.294,"5min":4.077,"15min":3.8610000000000002},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.681,"5min":117.637,"15min":117.637},"last":1.254}]},{"osd":2,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8939999999999999,"5min":5.0549999999999997,"15min":4.4119999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.7999999999999998,"5min":135.529,"15min":183.44200000000001},"last":1.583},{"interface":"front","average":{"1min":1.6679999999999999,"5min":6.016,"15min":4.6349999999999998},"min":{"1min":0.318,"5min":0.27200000000000002,"15min":0.26100000000000001},"max":{"1min":5.7859999999999996,"5min":135.56700000000001,"15min":183.46199999999999},"last":2.5139999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:16:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":7.9820000000000002,"15min":4.2640000000000002},"min":{"1min":0.30499999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":5.7279999999999998,"5min":135.66200000000001,"15min":135.66200000000001},"last":1.282},{"interface":"front","average":{"1min":1.897,"5min":8.4979999999999993,"15min":4.375},"min":{"1min":0.318,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":5.8600000000000003,"5min":135.47,"15min":135.47},"last":2.552}]},{"osd":4,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1499999999999999,"5min":1.611,"15min":1.1399999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":9.9529999999999994,"5min":10.989000000000001,"15min":15.942},"last":1.649},{"interface":"front","average":{"1min":2.984,"5min":1.6020000000000001,"15min":1.0549999999999999},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":11.773,"5min":11.773,"15min":16.725999999999999},"last":1.72}]},{"osd":5,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":1.7689999999999999,"15min":1.3149999999999999},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.7829999999999995,"5min":11.231,"15min":11.231},"last":1.3100000000000001},{"interface":"front","average":{"1min":3.9420000000000002,"5min":1.9510000000000001,"15min":1.274},"min":{"1min":0.435,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":9.468,"5min":10.005000000000001,"15min":13.867000000000001},"last":10.9}]},{"osd":6,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.2160000000000002,"15min":2.0289999999999999},"min":{"1min":0.51800000000000002,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":13.369,"5min":143.18700000000001,"15min":143.18700000000001},"last":1.6160000000000001},{"interface":"front","average":{"1min":2.266,"5min":2.7549999999999999,"15min":1.8420000000000001},"min":{"1min":0.53900000000000003,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":7.7450000000000001,"5min":142.70400000000001,"15min":142.70400000000001},"last":1.4339999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463482,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2516940,"kb_used_data":1792772,"kb_used_omap":0,"kb_used_meta":724160,"kb_avail":91228212,"statfs":{"total":95995035648,"available":93417689088,"internally_reserved":0,"allocated":1835798528,"data_stored":1834321595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":741539840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":4.5339999999999998,"15min":2.7490000000000001},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":12.659000000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.0369999999999999,"5min":4.899,"15min":2.835},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":16.021999999999998,"5min":149.303,"15min":149.303},"last":0.78000000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:16:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":3.9359999999999999,"15min":3.5009999999999999},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":8.4860000000000007,"5min":129.37899999999999,"15min":129.37899999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.4199999999999999,"5min":3.8450000000000002,"15min":3.3919999999999999},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":6.2539999999999996,"5min":129.40100000000001,"15min":129.40100000000001},"last":8.0909999999999993}]},{"osd":2,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.577999999999999,"5min":7.6189999999999998,"15min":5.9340000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":120.28700000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":12.118,"5min":7.8120000000000003,"15min":5.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":120.414,"5min":133.33000000000001,"15min":199.08000000000001},"last":0.81000000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":5.6669999999999998,"15min":3.3199999999999998},"min":{"1min":0.38,"5min":0.218,"15min":0.218},"max":{"1min":3.5089999999999999,"5min":137.875,"15min":137.875},"last":8.1199999999999992},{"interface":"front","average":{"1min":1.7250000000000001,"5min":5.391,"15min":3.3650000000000002},"min":{"1min":0.30099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.676,"5min":137.60900000000001,"15min":137.60900000000001},"last":0.58499999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":8.9060000000000006,"5min":2.9329999999999998,"15min":1.673},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.22600000000000001},"max":{"1min":55.494,"5min":55.494,"15min":55.494},"last":0.75},{"interface":"front","average":{"1min":8.7550000000000008,"5min":2.8100000000000001,"15min":1.4970000000000001},"min":{"1min":0.39900000000000002,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":55.305,"5min":55.305,"15min":55.305},"last":9.282}]},{"osd":5,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":15.223000000000001,"5min":4.3630000000000004,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":222.90899999999999,"5min":222.90899999999999,"15min":222.90899999999999},"last":1.615},{"interface":"front","average":{"1min":15.375,"5min":4.4930000000000003,"15min":2.1739999999999999},"min":{"1min":0.37,"5min":0.32200000000000001,"15min":0.28499999999999998},"max":{"1min":224.655,"5min":224.655,"15min":224.655},"last":8.0340000000000007}]},{"osd":7,"last update":"Sat Sep 23 21:16:25 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":0.94299999999999995,"15min":1.002},"min":{"1min":0.45100000000000001,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":6.9729999999999999,"5min":6.9729999999999999,"15min":7.8079999999999998},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.0470000000000002,"5min":1.2729999999999999,"15min":1.129},"min":{"1min":0.42799999999999999,"5min":0.251,"15min":0.251},"max":{"1min":6.2060000000000004,"5min":7.9119999999999999,"15min":7.9119999999999999},"last":15.878}]}]},{"osd":1,"up_from":0,"seq":1185410973928,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991668,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781559,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2696696,"kb_used_data":1615152,"kb_used_omap":7,"kb_used_meta":1081528,"kb_avail":91048456,"statfs":{"total":95995035648,"available":93233618944,"internally_reserved":0,"allocated":1653915648,"data_stored":1651279435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1107484753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.8420000000000001,"15min":2.157},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":8.0079999999999991,"5min":103.021,"15min":103.021},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":3.133,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":0.378,"5min":103.154,"15min":103.154},"last":0.26400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:16:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":3.4449999999999998,"15min":4.3470000000000004},"min":{"1min":0.224,"5min":0.224,"15min":0.19},"max":{"1min":2.9740000000000002,"5min":106.268,"15min":212.12},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.94699999999999995,"5min":3.8879999999999999,"15min":4.3550000000000004},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":3.246,"5min":106.32299999999999,"15min":213.02799999999999},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:16:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.8300000000000001,"15min":4.3220000000000001},"min":{"1min":0.27100000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.8260000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":3.4820000000000002,"15min":4.4189999999999996},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.2850000000000001,"5min":82.241,"15min":143.27799999999999},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":13.896000000000001,"5min":4.3010000000000002,"15min":3.1800000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":220.82900000000001,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":13.894,"5min":4.9130000000000003,"15min":3.823},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":220.84299999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.541,"5min":5.1059999999999999,"15min":3.48},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":220.63900000000001,"5min":220.63900000000001,"15min":220.63900000000001},"last":4.9820000000000002},{"interface":"front","average":{"1min":16.257000000000001,"5min":4.7380000000000004,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":220.31999999999999,"5min":220.31999999999999,"15min":220.31999999999999},"last":1.8009999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.597,"5min":4.6210000000000004,"15min":3.762},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":210.74700000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":1.944},{"interface":"front","average":{"1min":14.371,"5min":4.4640000000000004,"15min":3.1259999999999999},"min":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":210.72900000000001,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.4940000000000002,"15min":1.948},"min":{"1min":0.371,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":14.019,"5min":25.260000000000002,"15min":25.260000000000002},"last":1.4410000000000001},{"interface":"front","average":{"1min":1.921,"5min":2.4990000000000001,"15min":2.2850000000000001},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":5.0229999999999997,"5min":12.613,"15min":16.875},"last":7.5289999999999999}]}]},{"osd":3,"up_from":0,"seq":635655160153,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986771,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2445980,"kb_used_data":1673108,"kb_used_omap":0,"kb_used_meta":772864,"kb_avail":91299172,"statfs":{"total":95995035648,"available":93490352128,"internally_reserved":0,"allocated":1713262592,"data_stored":1711769361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":791412736},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":5.1909999999999998,"15min":2.7109999999999999},"min":{"1min":0.247,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":10.436,"5min":226.751,"15min":226.751},"last":0.378},{"interface":"front","average":{"1min":2.0750000000000002,"5min":4.8049999999999997,"15min":2.556},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.223},"max":{"1min":6.218,"5min":226.43299999999999,"15min":226.43299999999999},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":4.4980000000000002,"15min":5.282},"min":{"1min":0.29799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.9980000000000002,"5min":119.55800000000001,"15min":127.411},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.665,"5min":4.1390000000000002,"15min":4.5179999999999998},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":8.2550000000000008,"5min":116.78100000000001,"15min":118.075},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":8.9459999999999997,"5min":7.5439999999999996,"15min":5.8710000000000004},"min":{"1min":0.26000000000000001,"5min":0.253,"15min":0.253},"max":{"1min":90.950000000000003,"5min":210.50299999999999,"15min":210.50299999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":9.9280000000000008,"5min":8.5630000000000006,"15min":6.0149999999999997},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":90.846000000000004,"5min":210.59,"15min":210.59},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 21:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":7.5650000000000004,"15min":4.069},"min":{"1min":0.317,"5min":0.317,"15min":0.29299999999999998},"max":{"1min":4.3319999999999999,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.992,"5min":8.6229999999999993,"15min":4.4470000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.25600000000000001},"max":{"1min":8.2799999999999994,"5min":214.54900000000001,"15min":214.54900000000001},"last":0.39900000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.383,"15min":0.98999999999999999},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.757999999999999,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":3.4119999999999999,"5min":2.157,"15min":1.266},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":18.314,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.26700000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.524,"15min":1.333},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.2569999999999997,"5min":10.446999999999999,"15min":36.557000000000002},"last":41.938000000000002},{"interface":"front","average":{"1min":2.2690000000000001,"5min":1.579,"15min":1.3169999999999999},"min":{"1min":0.35799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.8049999999999997,"5min":6.8049999999999997,"15min":38.468000000000004},"last":40.444000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8740000000000001,"5min":1.746,"15min":1.869},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":18.352,"5min":18.352,"15min":145.721},"last":0.317},{"interface":"front","average":{"1min":2.512,"5min":1.421,"15min":1.917},"min":{"1min":0.42199999999999999,"5min":0.33500000000000002,"15min":0.31900000000000001},"max":{"1min":7.5830000000000002,"5min":10.677,"15min":148.45699999999999},"last":0.32800000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725135,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2744204,"kb_used_data":1914820,"kb_used_omap":0,"kb_used_meta":829184,"kb_avail":91000948,"statfs":{"total":95995035648,"available":93184970752,"internally_reserved":0,"allocated":1960775680,"data_stored":1959294045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":849084416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":1.732,"15min":1.6699999999999999},"min":{"1min":0.33500000000000002,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":7.7450000000000001,"5min":20.023,"15min":82.280000000000001},"last":2.6120000000000001},{"interface":"front","average":{"1min":2.077,"5min":2.0640000000000001,"15min":1.8680000000000001},"min":{"1min":0.29299999999999998,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.6710000000000003,"5min":19.853000000000002,"15min":85.236000000000004},"last":0.92900000000000005}]},{"osd":1,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5619999999999998,"5min":2.5659999999999998,"15min":4.6150000000000002},"min":{"1min":0.42799999999999999,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":8.048,"5min":12.032999999999999,"15min":233.77099999999999},"last":1.4350000000000001},{"interface":"front","average":{"1min":1.966,"5min":2.028,"15min":4.3479999999999999},"min":{"1min":0.32300000000000001,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":7.8520000000000003,"5min":12.016,"15min":233.75200000000001},"last":2.6360000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":2.1110000000000002,"15min":3.7080000000000002},"min":{"1min":0.37,"5min":0.251,"15min":0.251},"max":{"1min":7.8220000000000001,"5min":20.411999999999999,"15min":231.34700000000001},"last":2.5779999999999998},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.1859999999999999,"15min":3.9009999999999998},"min":{"1min":0.38600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.0439999999999996,"5min":11.509,"15min":231.381},"last":0.95699999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":2.5710000000000002,"15min":4.0469999999999997},"min":{"1min":0.40500000000000003,"5min":0.317,"15min":0.27700000000000002},"max":{"1min":8.077,"5min":11.597,"15min":263.887},"last":1.468},{"interface":"front","average":{"1min":2.004,"5min":1.9910000000000001,"15min":3.831},"min":{"1min":0.46100000000000002,"5min":0.32200000000000001,"15min":0.30099999999999999},"max":{"1min":7.8810000000000002,"5min":10.638,"15min":263.70100000000002},"last":2.6579999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":1.595,"15min":0.97599999999999998},"min":{"1min":0.439,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":11.593999999999999,"5min":11.593999999999999,"15min":12.739000000000001},"last":2.6800000000000002},{"interface":"front","average":{"1min":3.7040000000000002,"5min":1.3839999999999999,"15min":1.026},"min":{"1min":0.505,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":10.259,"5min":10.259,"15min":10.792},"last":1.179}]},{"osd":6,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9009999999999998,"5min":1.6739999999999999,"15min":1.7130000000000001},"min":{"1min":0.38500000000000001,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":7.8520000000000003,"5min":8.9130000000000003,"15min":83.451999999999998},"last":0.97999999999999998},{"interface":"front","average":{"1min":3.0379999999999998,"5min":2.2669999999999999,"15min":1.9339999999999999},"min":{"1min":0.58499999999999996,"5min":0.30399999999999999,"15min":0.27400000000000002},"max":{"1min":8.0050000000000008,"5min":12.725,"15min":83.484999999999999},"last":1.8580000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:16:28 2023","interfaces":[{"interface":"back","average":{"1min":3.8599999999999999,"5min":1.7529999999999999,"15min":1.2529999999999999},"min":{"1min":0.56399999999999995,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":7.0010000000000003},{"interface":"front","average":{"1min":2.5470000000000002,"5min":1.3,"15min":0.874},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":11.640000000000001,"5min":11.640000000000001,"15min":11.640000000000001},"last":3.7200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1634598912,"data_stored":1634598912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1415905280,"data_stored":1415905280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1704398848,"data_stored":1704394776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1929383936,"data_stored":1929379864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1808531456,"data_stored":1808531456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1322717184,"data_stored":1322713112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:17:13.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:17:13.120 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:13.152 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:13.303 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:13.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:13.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:13.411 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:13.472 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:13.540 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:13.553 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:13.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[211526]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[211526]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[211526]: osdmap e552: 8 total, 8 up, 5 in 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[211526]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[211526]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[211526]: pgmap v1722: 109 pgs: 9 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 646 op/s; 23486/114708 objects degraded (20.475%); 7.0 MiB/s, 190 objects/s recovering 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[219709]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[219709]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[219709]: osdmap e552: 8 total, 8 up, 5 in 2023-09-23T21:17:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[219709]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[219709]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:13 smithi121 ceph-mon[219709]: pgmap v1722: 109 pgs: 9 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 646 op/s; 23486/114708 objects degraded (20.475%); 7.0 MiB/s, 190 objects/s recovering 2023-09-23T21:17:13.727 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:13 smithi123 ceph-mon[193903]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:13 smithi123 ceph-mon[193903]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:13 smithi123 ceph-mon[193903]: osdmap e552: 8 total, 8 up, 5 in 2023-09-23T21:17:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:13 smithi123 ceph-mon[193903]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:13 smithi123 ceph-mon[193903]: failed to encode map e552 with expected crc 2023-09-23T21:17:13.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:13 smithi123 ceph-mon[193903]: pgmap v1722: 109 pgs: 9 active+recovery_wait+degraded, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 646 op/s; 23486/114708 objects degraded (20.475%); 7.0 MiB/s, 190 objects/s recovering 2023-09-23T21:17:13.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:13.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:13.834 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:13.928 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:14.110 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:14.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:14.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:14.223 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:14.265 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:14.337 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:17:14.337 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:17:14.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:17:14.337 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:17:14.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.337 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159372 12.d 12:b6c49147:::benchmark_data_smithi121_563784_object68969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 232.902130726, 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.04656902, 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159372 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.338 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423455+0000", 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423460+0000", 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5589999999999996e-06 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423473+0000", 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2984e-05 2023-09-23T21:17:14.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423479+0000", 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0290000000000002e-06 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423555+0000", 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5654000000000004e-05 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423563+0000", 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4989999999999994e-06 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432320+0000", 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.008756972 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432467+0000", 2023-09-23T21:17:14.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014657299999999999 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432861+0000", 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039391600000000001 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442657+0000", 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.009796674 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469909+0000", 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027251573000000001 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469942+0000", 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3423000000000002e-05 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469983+0000", 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1127000000000001e-05 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470025+0000", 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1706999999999999e-05 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159374 12.d 12:bcbc6391:::benchmark_data_smithi121_563784_object68971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 232.89512380900001, 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0397036749999999, 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159374 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430462+0000", 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430471+0000", 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8060000000000001e-06 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430485+0000", 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4771e-05 2023-09-23T21:17:14.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430492+0000", 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2400000000000004e-06 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430543+0000", 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1551999999999997e-05 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430554+0000", 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0668e-05 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436375+0000", 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0058211749999999 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436460+0000", 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4902999999999997e-05 2023-09-23T21:17:14.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436540+0000", 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9652000000000002e-05 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447278+0000", 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010738077 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470065+0000", 2023-09-23T21:17:14.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022787136999999999 2023-09-23T21:17:14.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470102+0000", 2023-09-23T21:17:14.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6761000000000003e-05 2023-09-23T21:17:14.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470134+0000", 2023-09-23T21:17:14.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2085999999999997e-05 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470167+0000", 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3135e-05 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.354 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159390 12.d 12:b1ecff3e:::benchmark_data_smithi121_563784_object68987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 232.871336004, 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.016060076, 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159390 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454249+0000", 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454265+0000", 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5506999999999999e-05 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454282+0000", 2023-09-23T21:17:14.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6966e-05 2023-09-23T21:17:14.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454288+0000", 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4189999999999999e-06 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454345+0000", 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6897000000000001e-05 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454353+0000", 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2530000000000004e-06 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438507+0000", 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98415336099999995 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438592+0000", 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5316999999999996e-05 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438669+0000", 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6503999999999998e-05 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447287+0000", 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0086180560000000007 2023-09-23T21:17:14.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470085+0000", 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.02279871 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470250+0000", 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016454100000000001 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470281+0000", 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0571000000000003e-05 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470311+0000", 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0451999999999999e-05 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159394 12.d 12:b55656ba:::benchmark_data_smithi121_563784_object68991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 232.86663569699999, 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.011503713, 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159394 2023-09-23T21:17:14.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458950+0000", 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458958+0000", 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4789999999999996e-06 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458972+0000", 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.31e-05 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458978+0000", 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6930000000000001e-06 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.459041+0000", 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3157999999999994e-05 2023-09-23T21:17:14.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.459049+0000", 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0639999999999994e-06 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440532+0000", 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98148217000000004 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440613+0000", 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0908000000000005e-05 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440689+0000", 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6810000000000005e-05 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457467+0000", 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016777255000000001 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470372+0000", 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012905020999999999 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470399+0000", 2023-09-23T21:17:14.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7229999999999998e-05 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470424+0000", 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5514e-05 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470455+0000", 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0576e-05 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159428 12.d 12:b4378ef8:::benchmark_data_smithi121_563784_object69025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 232.81913653000001, 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96414785400000003, 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.376 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159428 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506449+0000", 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506456+0000", 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9489999999999999e-06 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506471+0000", 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5101e-05 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506478+0000", 2023-09-23T21:17:14.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8709999999999999e-06 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506530+0000", 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2335e-05 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506539+0000", 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6319999999999997e-06 2023-09-23T21:17:14.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442702+0000", 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93616331100000005 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442785+0000", 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2917000000000006e-05 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442859+0000", 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4169000000000001e-05 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457477+0000", 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014618133 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470489+0000", 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013011224 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470543+0000", 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4104999999999998e-05 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470575+0000", 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1878000000000002e-05 2023-09-23T21:17:14.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470598+0000", 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3657999999999999e-05 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159431 12.d 12:b9860c03:::benchmark_data_smithi121_563784_object69028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:14.384 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 232.81909444799999, 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96419124899999997, 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159431 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506495+0000", 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6230000000000001e-06 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506501+0000", 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.879e-06 2023-09-23T21:17:14.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506503+0000", 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7319999999999998e-06 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506557+0000", 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3786000000000003e-05 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506561+0000", 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7440000000000001e-06 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445001+0000", 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93843964400000002 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445133+0000", 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013199800000000001 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445286+0000", 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015309000000000001 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462371+0000", 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.017085625 2023-09-23T21:17:14.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470516+0000", 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0081449550000000006 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470630+0000", 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113372 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470654+0000", 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4085000000000001e-05 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470684+0000", 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0071000000000001e-05 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.392 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159432 12.d 12:b2bd39fd:::benchmark_data_smithi121_563784_object69029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 232.80313096899999, 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94835160100000004, 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:14.393 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159432 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:17:14.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522454+0000", 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522466+0000", 2023-09-23T21:17:14.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1997000000000001e-05 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522497+0000", 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1341999999999999e-05 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522511+0000", 2023-09-23T21:17:14.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3417e-05 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522631+0000", 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120647 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522648+0000", 2023-09-23T21:17:14.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7096999999999999e-05 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447334+0000", 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92468556300000004 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447475+0000", 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014071400000000001 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447625+0000", 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015043700000000001 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462380+0000", 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014754949 2023-09-23T21:17:14.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470712+0000", 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083322190000000001 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470763+0000", 2023-09-23T21:17:14.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0980999999999999e-05 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470789+0000", 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6191999999999999e-05 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470808+0000", 2023-09-23T21:17:14.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8159999999999999e-05 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50819 13.0 13:07dbfe5f:::benchmark_data_smithi121_621349_object50818:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.686076+0000", 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.63951103100001, 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0142723499999999, 2023-09-23T21:17:14.402 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50819 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686076+0000", 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686076+0000", 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686074+0000", 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686117+0000", 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3220999999999998e-05 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686133+0000", 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5795999999999999e-05 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686140+0000", 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0090000000000001e-06 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686202+0000", 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1588999999999996e-05 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op must wait for map", 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686209+0000", 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6210000000000004e-06 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.691240+0000", 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050300650000000002 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.691245+0000", 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3369999999999999e-06 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.694345+0000", 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.003100034 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.694506+0000", 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016077499999999999 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.694726+0000", 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000220781 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697158+0000", 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024318690000000001 2023-09-23T21:17:14.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700181+0000", 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003023038 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700239+0000", 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7735999999999997e-05 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700304+0000", 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5214000000000003e-05 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700348+0000", 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3934999999999999e-05 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50835 13.0 13:070b5ba2:::benchmark_data_smithi121_621349_object50834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.697856+0000", 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.62773093199999, 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0027707880000001, 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50835 2023-09-23T21:17:14.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697856+0000", 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697856+0000", 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697855+0000", 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697929+0000", 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.326e-05 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697943+0000", 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4089e-05 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697948+0000", 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9479999999999999e-06 2023-09-23T21:17:14.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.698013+0000", 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5320000000000005e-05 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.698024+0000", 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1396999999999999e-05 2023-09-23T21:17:14.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697199+0000", 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99917456299999996 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697346+0000", 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014690999999999999 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697477+0000", 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013073499999999999 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700411+0000", 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002934716 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700485+0000", 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3169000000000004e-05 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700572+0000", 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7449999999999998e-05 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700604+0000", 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1837e-05 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700627+0000", 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2895e-05 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.421 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50857 13.0 13:0be955d9:::benchmark_data_smithi121_621349_object50856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.611192604, 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98640245000000004, 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50857 2023-09-23T21:17:14.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714426+0000", 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2026999999999997e-05 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714439+0000", 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3606e-05 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714445+0000", 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3550000000000004e-06 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715059+0000", 2023-09-23T21:17:14.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00061476900000000004 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715067+0000", 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0080000000000002e-06 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.698491+0000", 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98342367900000005 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.698626+0000", 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013504700000000001 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.698783+0000", 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015678400000000001 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700546+0000", 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0017630740000000001 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700637+0000", 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1219999999999997e-05 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700688+0000", 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0742999999999999e-05 2023-09-23T21:17:14.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700760+0000", 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2452000000000004e-05 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700797+0000", 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6395e-05 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.430 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50859 13.0 13:0bb990fe:::benchmark_data_smithi121_621349_object50858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.61112683600001, 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99097865500000004, 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.431 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50859 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714524+0000", 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4227999999999996e-05 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714532+0000", 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9489999999999996e-06 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714536+0000", 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1350000000000002e-06 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715080+0000", 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00054414899999999996 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715084+0000", 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.647e-06 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700850+0000", 2023-09-23T21:17:14.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98576657599999995 2023-09-23T21:17:14.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.701010+0000", 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015928399999999999 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.701101+0000", 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1631000000000004e-05 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705290+0000", 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041886479999999997 2023-09-23T21:17:14.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705297+0000", 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5680000000000002e-06 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705346+0000", 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8714000000000001e-05 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705395+0000", 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8738e-05 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705439+0000", 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3816999999999997e-05 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.440 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50889 13.0 13:00c0c20f:::benchmark_data_smithi121_621349_object50888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.736071+0000", 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.58951580199999, 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97526533100000001, 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50889 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736071+0000", 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736071+0000", 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736070+0000", 2023-09-23T21:17:14.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736117+0000", 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6950000000000003e-05 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736129+0000", 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2414e-05 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736138+0000", 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6279999999999994e-06 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736189+0000", 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1014000000000003e-05 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736198+0000", 2023-09-23T21:17:14.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.02e-06 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.702264+0000", 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96606645300000005 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.702350+0000", 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5798999999999996e-05 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.702460+0000", 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110215 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705301+0000", 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002840857 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705322+0000", 2023-09-23T21:17:14.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0720000000000002e-05 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711265+0000", 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005942854 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711305+0000", 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0679999999999997e-05 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711336+0000", 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0973000000000003e-05 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50894 13.0 13:058cc793:::benchmark_data_smithi121_621349_object50893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.738626+0000", 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.58696121899999, 2023-09-23T21:17:14.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97285262100000003, 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50894 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738626+0000", 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738626+0000", 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738625+0000", 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738676+0000", 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0964999999999997e-05 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738687+0000", 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1051e-05 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738691+0000", 2023-09-23T21:17:14.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6890000000000003e-06 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738738+0000", 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6381999999999997e-05 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738744+0000", 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6100000000000002e-06 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.703672+0000", 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96492752400000004 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.703794+0000", 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012215300000000001 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.703910+0000", 2023-09-23T21:17:14.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011578000000000001 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705496+0000", 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001585899 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711165+0000", 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005669716 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711382+0000", 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021679199999999999 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711437+0000", 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4718e-05 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711478+0000", 2023-09-23T21:17:14.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1434999999999999e-05 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50903 13.0 13:0f9f6d1d:::benchmark_data_smithi121_621349_object50902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.747607+0000", 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.577979898, 2023-09-23T21:17:14.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97278066100000005, 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50903 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747607+0000", 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747607+0000", 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747606+0000", 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747684+0000", 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8508000000000001e-05 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747697+0000", 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.296e-05 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747704+0000", 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2200000000000003e-06 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747761+0000", 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6390000000000001e-05 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747771+0000", 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0530000000000001e-05 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705554+0000", 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95778294900000005 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.461 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705662+0000", 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000108287 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705834+0000", 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017180200000000001 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.462 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711216+0000", 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053816890000000003 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715936+0000", 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0047201630000000003 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720289+0000", 2023-09-23T21:17:14.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043527870000000003 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720343+0000", 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3619000000000003e-05 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720388+0000", 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5161999999999999e-05 2023-09-23T21:17:14.464 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50916 13.0 13:049e41a7:::benchmark_data_smithi121_621349_object50915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.757423+0000", 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.56816432100001, 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.963065434, 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.465 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50916 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757423+0000", 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757423+0000", 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757421+0000", 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757474+0000", 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2800000000000003e-05 2023-09-23T21:17:14.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757492+0000", 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7567e-05 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757498+0000", 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5259999999999999e-06 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757560+0000", 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1404999999999999e-05 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757570+0000", 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.007e-05 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.469 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.707628+0000", 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95005845799999999 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.707746+0000", 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117658 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.707840+0000", 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3846999999999998e-05 2023-09-23T21:17:14.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713639+0000", 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0057993460000000004 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715960+0000", 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002320871 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720401+0000", 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044407250000000004 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720452+0000", 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1483000000000002e-05 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720488+0000", 2023-09-23T21:17:14.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5890999999999999e-05 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50919 13.0 13:0850c3dd:::benchmark_data_smithi121_621349_object50918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.759629+0000", 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.56595819200001, 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97667352299999999, 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.473 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50919 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759629+0000", 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.474 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759629+0000", 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759627+0000", 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759690+0000", 2023-09-23T21:17:14.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2829000000000004e-05 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759707+0000", 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7243000000000001e-05 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759714+0000", 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8120000000000002e-06 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.761268+0000", 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001554115 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.761279+0000", 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0654e-05 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.477 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.478 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.478 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.709282+0000", 2023-09-23T21:17:14.478 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94800265699999997 2023-09-23T21:17:14.478 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.478 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.709401+0000", 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119968 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.709570+0000", 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016860200000000001 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.479 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713667+0000", 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040964399999999998 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715988+0000", 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023218010000000001 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736189+0000", 2023-09-23T21:17:14.480 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020200722000000001 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736248+0000", 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8765999999999999e-05 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736302+0000", 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4419000000000002e-05 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.481 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50958 13.0 13:081dfd76:::benchmark_data_smithi121_621349_object50957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.791211+0000", 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.53437566100001, 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94518114399999997, 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.482 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50958 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791211+0000", 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791211+0000", 2023-09-23T21:17:14.483 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791210+0000", 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791278+0000", 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8035999999999997e-05 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.484 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791290+0000", 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1032e-05 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791296+0000", 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1240000000000002e-06 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.485 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791357+0000", 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1005000000000003e-05 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791366+0000", 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3710000000000006e-06 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711515+0000", 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92014889099999997 2023-09-23T21:17:14.486 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711647+0000", 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013166 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711798+0000", 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015160799999999999 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.487 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713692+0000", 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00189364 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.717934+0000", 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042422359999999999 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736312+0000", 2023-09-23T21:17:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018377935000000001 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736356+0000", 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4456999999999998e-05 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736392+0000", 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5951000000000002e-05 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.489 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50971 13.0 13:0a7f1691:::benchmark_data_smithi121_621349_object50970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.523838802, 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93478755099999999, 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.490 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50971 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801776+0000", 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8784000000000001e-05 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.492 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801784+0000", 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3610000000000003e-06 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801788+0000", 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.301e-06 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801839+0000", 2023-09-23T21:17:14.493 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1397999999999998e-05 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801848+0000", 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1140000000000006e-06 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713711+0000", 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.91186365400000002 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.494 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713824+0000", 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000112525 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713961+0000", 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013691699999999999 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.495 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715908+0000", 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001946925 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720538+0000", 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0046308019999999998 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736447+0000", 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015908164999999998 2023-09-23T21:17:14.496 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736497+0000", 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0773000000000003e-05 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736536+0000", 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8309000000000002e-05 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.497 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50977 13.0 13:0e0af701:::benchmark_data_smithi121_621349_object50976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.808462+0000", 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.517124986, 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92816345899999997, 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.498 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50977 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808462+0000", 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808462+0000", 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.499 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808460+0000", 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808504+0000", 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3204000000000002e-05 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808518+0000", 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4809e-05 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808525+0000", 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2759999999999997e-06 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808575+0000", 2023-09-23T21:17:14.501 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0324999999999997e-05 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808588+0000", 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2578999999999999e-05 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.716009+0000", 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90742122400000003 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.502 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.716109+0000", 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010050799999999999 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.716204+0000", 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4594999999999998e-05 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.503 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.717906+0000", 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001702425 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.725391+0000", 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0074848570000000001 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736548+0000", 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01115709 2023-09-23T21:17:14.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736590+0000", 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1406000000000003e-05 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736625+0000", 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5738999999999998e-05 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.505 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50981 13.0 13:0439b986:::benchmark_data_smithi121_621349_object50980:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.811090+0000", 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 137.51449698499999, 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92615377799999998, 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:17:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50981 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811090+0000", 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811090+0000", 2023-09-23T21:17:14.507 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811089+0000", 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811139+0000", 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0235999999999998e-05 2023-09-23T21:17:14.508 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811152+0000", 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2831e-05 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811158+0000", 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3779999999999998e-06 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.509 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811196+0000", 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7651999999999998e-05 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.811202+0000", 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3269999999999997e-06 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:17:14.510 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.717954+0000", 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90675143000000002 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.718057+0000", 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010307899999999999 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.718191+0000", 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013390400000000001 2023-09-23T21:17:14.511 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720585+0000", 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023946950000000001 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.725421+0000", 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048360019999999998 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:17:14.512 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.737162+0000", 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011740532 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.737209+0000", 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6969999999999999e-05 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.737244+0000", 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4900000000000001e-05 2023-09-23T21:17:14.513 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.514 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.514 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.514 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:17:14.514 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:17:14.514 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:17:14.514 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:17:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/780069935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:14.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:14 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:14.733 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:14.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:14.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:14.737 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/780069935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:14.737 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:14 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:14.737 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/780069935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:14.737 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:14 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:14.837 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:14.869 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:15.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:17:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:17:14] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:17:15.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[219709]: failed to encode map e553 with expected crc 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[219709]: failed to encode map e553 with expected crc 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[219709]: osdmap e553: 8 total, 8 up, 5 in 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[219709]: pgmap v1724: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 523 op/s; 22705/109059 objects degraded (20.819%); 7.5 MiB/s, 167 objects/s recovering 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[211526]: failed to encode map e553 with expected crc 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[211526]: failed to encode map e553 with expected crc 2023-09-23T21:17:15.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[211526]: osdmap e553: 8 total, 8 up, 5 in 2023-09-23T21:17:15.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:15 smithi121 ceph-mon[211526]: pgmap v1724: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 523 op/s; 22705/109059 objects degraded (20.819%); 7.5 MiB/s, 167 objects/s recovering 2023-09-23T21:17:15.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:15 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:15.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:15 smithi123 ceph-mon[193903]: failed to encode map e553 with expected crc 2023-09-23T21:17:15.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:15 smithi123 ceph-mon[193903]: failed to encode map e553 with expected crc 2023-09-23T21:17:15.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:15.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:15 smithi123 ceph-mon[193903]: osdmap e553: 8 total, 8 up, 5 in 2023-09-23T21:17:15.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:15 smithi123 ceph-mon[193903]: pgmap v1724: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 523 op/s; 22705/109059 objects degraded (20.819%); 7.5 MiB/s, 167 objects/s recovering 2023-09-23T21:17:15.764 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:17:16.088 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:16.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:16.193 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:16.226 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:16.285 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:16.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:16.389 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:16.420 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:16.549 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:16.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:16.668 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:16.790 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:17.006 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:17.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:17.123 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:17.153 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:16 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 22705/109059 objects degraded (20.819%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:17.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:16 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 22705/109059 objects degraded (20.819%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:17.157 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:17.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:16 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 22705/109059 objects degraded (20.819%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:17.469 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:17.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:17.548 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:17:17.573 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:17.581 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:17:17.606 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:17:17.610 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:17.632 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:17:17.667 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:17.669 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:17:17.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:17:17.720 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:17.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:17.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:17.731 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:17:17.764 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:17:17.765 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.766 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.766 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.766 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.766 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.766 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.766 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.766 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:17:17.825 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:17.860 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:17.975 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:17 smithi121 ceph-mon[211526]: pgmap v1725: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 22705/108024 objects degraded (21.018%); 7.5 MiB/s, 166 objects/s recovering 2023-09-23T21:17:17.976 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1628160491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:17.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:17 smithi121 ceph-mon[219709]: pgmap v1725: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 22705/108024 objects degraded (21.018%); 7.5 MiB/s, 166 objects/s recovering 2023-09-23T21:17:17.976 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1628160491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:18.119 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:17:18.120 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:17 smithi123 ceph-mon[193903]: pgmap v1725: 109 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 22705/108024 objects degraded (21.018%); 7.5 MiB/s, 166 objects/s recovering 2023-09-23T21:17:18.120 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1628160491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:18.184 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:17:18.184 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22705/108024 objects degraded (21.018%), 22 pgs degraded, 2 pgs undersized","count":24},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2294,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":553,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":5,"osd_in_since":1695503710,"num_remapped_pgs":15},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":84},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":15},{"state_name":"active+recovery_wait+degraded","count":7},{"state_name":"active+recovery_wait","count":2},{"state_name":"active+recovering","count":1}],"num_pgs":109,"num_pools":5,"num_objects":36008,"data_bytes":2358712248,"bytes_used":12234547200,"bytes_avail":467740631040,"bytes_total":479975178240,"degraded_objects":22705,"degraded_total":108024,"degraded_ratio":0.21018477375398059,"recovering_objects_per_sec":166,"recovering_bytes_per_sec":7884133,"recovering_keys_per_sec":0,"num_objects_recovered":1002,"num_bytes_recovered":47317082,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":533},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T21:17:03.337132+0000","services":{}},"progress_events":{"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"d33d2ace-2031-48c2-a8f8-693bca5c747e":{"message":"Global Recovery Event (110s)\n [=====================.......] (remaining: 36s)","progress":0.75229358673095703,"add_to_ceph_s":true}}} 2023-09-23T21:17:18.340 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:18.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:18.445 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:18.516 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:18.591 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:18.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:18.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:18.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:18.724 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:19.131 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:19.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3583472159' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:17:19.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:19.136 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:19.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:19.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:19.137 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3583472159' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:17:19.138 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:19.138 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:19.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3583472159' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:17:19.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:19.233 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:19.238 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:19.271 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:19.749 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:19.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:19.852 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:19.882 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:19.971 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:19.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:19.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:20.087 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:20.165 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:20.305 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:20.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:20.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:20.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[211526]: pgmap v1726: 109 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 877 op/s; 21710/96807 objects degraded (22.426%); 15/96807 objects misplaced (0.015%); 12 MiB/s, 245 objects/s recovering 2023-09-23T21:17:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:17:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:17:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[219709]: pgmap v1726: 109 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 877 op/s; 21710/96807 objects degraded (22.426%); 15/96807 objects misplaced (0.015%); 12 MiB/s, 245 objects/s recovering 2023-09-23T21:17:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:17:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:17:20.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:20 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.415 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:20.446 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:20.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:20 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:20 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:20 smithi123 ceph-mon[193903]: pgmap v1726: 109 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 877 op/s; 21710/96807 objects degraded (22.426%); 15/96807 objects misplaced (0.015%); 12 MiB/s, 245 objects/s recovering 2023-09-23T21:17:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:20 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:17:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:20 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:17:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:20 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:20.776 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:17:20.777 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:17:20.959 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:20.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:20.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:21.063 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:21.143 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:21.154 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:21.285 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:21.290 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:21.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:21.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:21.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:21.434 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:21.711 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:21.711 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:21.834 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:21.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:21.939 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:21.974 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:22.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:17:22.233 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:22.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:22.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:22.337 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:22.369 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:22.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:22 smithi121 ceph-mon[211526]: pgmap v1727: 109 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 824 op/s; 21710/95607 objects degraded (22.708%); 15/95607 objects misplaced (0.016%); 10 MiB/s, 217 objects/s recovering 2023-09-23T21:17:22.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 21710/95607 objects degraded (22.708%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:22.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:22 smithi121 ceph-mon[219709]: pgmap v1727: 109 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 824 op/s; 21710/95607 objects degraded (22.708%); 15/95607 objects misplaced (0.016%); 10 MiB/s, 217 objects/s recovering 2023-09-23T21:17:22.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 21710/95607 objects degraded (22.708%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:22.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:22 smithi123 ceph-mon[193903]: pgmap v1727: 109 pgs: 5 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 824 op/s; 21710/95607 objects degraded (22.708%); 15/95607 objects misplaced (0.016%); 10 MiB/s, 217 objects/s recovering 2023-09-23T21:17:22.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 21710/95607 objects degraded (22.708%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:23.522 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:23.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:23.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:23.628 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:23.658 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:24.012 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:17:24.380 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:24.467 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:24.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:24.515 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:24 smithi121 ceph-mon[211526]: pgmap v1728: 109 pgs: 4 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 769 op/s; 21646/91611 objects degraded (23.628%); 15/91611 objects misplaced (0.016%); 8.3 MiB/s, 158 objects/s recovering 2023-09-23T21:17:24.515 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:24 smithi121 ceph-mon[219709]: pgmap v1728: 109 pgs: 4 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 769 op/s; 21646/91611 objects degraded (23.628%); 15/91611 objects misplaced (0.016%); 8.3 MiB/s, 158 objects/s recovering 2023-09-23T21:17:24.575 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:24.609 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:24.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:24 smithi123 ceph-mon[193903]: pgmap v1728: 109 pgs: 4 active+recovery_wait+degraded, 1 active+recovering, 3 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 769 op/s; 21646/91611 objects degraded (23.628%); 15/91611 objects misplaced (0.016%); 8.3 MiB/s, 158 objects/s recovering 2023-09-23T21:17:24.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:17:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:17:24] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:17:25.058 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:25.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:25.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:25.164 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:25.248 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:17:25.250 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:17:25.256 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:25.259 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:25.417 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:25.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:25.433 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:17:25.526 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:25.561 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3277710329' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:25.561 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:25 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:25.561 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3277710329' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:25.561 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:25 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:25.565 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3277710329' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:25.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:25 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:17:25.826 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:25.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:25.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:25.932 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:26.005 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:26.118 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:26.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:26.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:26.227 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:26.297 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:26.401 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:26.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:26.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:26.517 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:26.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:26 smithi123 ceph-mon[193903]: from='client.47306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:26.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:26 smithi123 ceph-mon[193903]: pgmap v1729: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 773 op/s; 19680/83892 objects degraded (23.459%); 150/83892 objects misplaced (0.179%); 6.4 MiB/s, 170 objects/s recovering 2023-09-23T21:17:26.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:26 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:26.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:26.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:26 smithi123 ceph-mon[193903]: osdmap e554: 8 total, 8 up, 5 in 2023-09-23T21:17:26.592 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:26 smithi123 ceph-mon[193903]: failed to encode map e554 with expected crc 2023-09-23T21:17:26.602 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[211526]: from='client.47306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[211526]: pgmap v1729: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 773 op/s; 19680/83892 objects degraded (23.459%); 150/83892 objects misplaced (0.179%); 6.4 MiB/s, 170 objects/s recovering 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[211526]: osdmap e554: 8 total, 8 up, 5 in 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[211526]: failed to encode map e554 with expected crc 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[219709]: from='client.47306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[219709]: pgmap v1729: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 773 op/s; 19680/83892 objects degraded (23.459%); 150/83892 objects misplaced (0.179%); 6.4 MiB/s, 170 objects/s recovering 2023-09-23T21:17:26.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:26.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:17:26.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[219709]: osdmap e554: 8 total, 8 up, 5 in 2023-09-23T21:17:26.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:26 smithi121 ceph-mon[219709]: failed to encode map e554 with expected crc 2023-09-23T21:17:26.731 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:26.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:26.846 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:26.880 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:27.083 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:27.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:27.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:27.188 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:27.265 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:27.369 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:27.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:27.483 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:27.516 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:27 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 19680/83892 objects degraded (23.459%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:27.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:27 smithi121 ceph-mon[219709]: pgmap v1731: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 844 op/s; 19680/82695 objects degraded (23.798%); 150/82695 objects misplaced (0.181%); 7.0 MiB/s, 184 objects/s recovering 2023-09-23T21:17:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:27 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 19680/83892 objects degraded (23.459%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:27 smithi121 ceph-mon[211526]: pgmap v1731: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 844 op/s; 19680/82695 objects degraded (23.798%); 150/82695 objects misplaced (0.181%); 7.0 MiB/s, 184 objects/s recovering 2023-09-23T21:17:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:27 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 19680/83892 objects degraded (23.459%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:27.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:27 smithi123 ceph-mon[193903]: pgmap v1731: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 844 op/s; 19680/82695 objects degraded (23.798%); 150/82695 objects misplaced (0.181%); 7.0 MiB/s, 184 objects/s recovering 2023-09-23T21:17:28.606 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1728,"stamp":"2023-09-23T21:17:23.342874+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2000164792,"num_objects":30537,"num_object_clones":0,"num_object_copies":91611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21646,"num_objects_misplaced":15,"num_objects_unfound":0,"num_objects_dirty":30537,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":139718,"num_write_kb":4954830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27916,"num_bytes_recovered":1620716326,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195576,"ondisk_log_size":195576,"up":327,"acting":312,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":11088648,"kb_used_data":6871264,"kb_used_omap":7,"kb_used_meta":4216504,"kb_avail":457637112,"statfs":{"total":479975178240,"available":468620402688,"internally_reserved":0,"allocated":7036174336,"data_stored":7027523587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4317700177},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504561664,"num_objects":-7699,"num_object_clones":0,"num_object_copies":-23097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1840,"num_objects_misplaced":15,"num_objects_unfound":0,"num_objects_dirty":-7699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7699,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1585,"num_bytes_recovered":86769754,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002994"},"pg_stats":[{"pgid":"13.23","version":"553'5734","reported_seq":7569,"reported_epoch":553,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:18.458940+0000","last_change":"2023-09-23T21:16:00.853306+0000","last_active":"2023-09-23T21:17:18.458940+0000","last_peered":"2023-09-23T21:17:18.458940+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:00.848945+0000","last_became_peered":"2023-09-23T21:16:00.848945+0000","last_unstale":"2023-09-23T21:17:18.458940+0000","last_undegraded":"2023-09-23T21:16:00.815130+0000","last_fullsized":"2023-09-23T21:17:18.458940+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33587200,"num_objects":513,"num_object_clones":0,"num_object_copies":1539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":160,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1473,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":55050240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":36},{"shards":"4,5,7","objects":477}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.22","version":"553'5639","reported_seq":7550,"reported_epoch":553,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:18.409555+0000","last_change":"2023-09-23T21:16:50.849447+0000","last_active":"2023-09-23T21:17:18.409555+0000","last_peered":"2023-09-23T21:17:18.409555+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:00.839299+0000","last_became_peered":"2023-09-23T21:16:00.839299+0000","last_unstale":"2023-09-23T21:17:18.409555+0000","last_undegraded":"2023-09-23T21:16:50.849328+0000","last_fullsized":"2023-09-23T21:17:18.409555+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5639,"log_dups_size":0,"ondisk_log_size":5639,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768000,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":68,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2400,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":894,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5,6","objects":500}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.21","version":"553'5582","reported_seq":6676,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:22.955470+0000","last_change":"2023-09-23T21:16:51.863113+0000","last_active":"2023-09-23T21:17:22.955470+0000","last_peered":"2023-09-23T21:17:22.955470+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858774+0000","last_became_peered":"2023-09-23T21:16:51.858774+0000","last_unstale":"2023-09-23T21:17:22.955470+0000","last_undegraded":"2023-09-23T21:16:51.798038+0000","last_fullsized":"2023-09-23T21:16:51.792078+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5582,"log_dups_size":0,"ondisk_log_size":5582,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29868032,"num_objects":456,"num_object_clones":0,"num_object_copies":1368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":750,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2401,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":311},{"shards":"2,4,7","objects":145}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.20","version":"553'5656","reported_seq":8790,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.395121+0000","last_change":"2023-09-23T21:16:51.842022+0000","last_active":"2023-09-23T21:17:18.395121+0000","last_peered":"2023-09-23T21:17:18.395121+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834659+0000","last_became_peered":"2023-09-23T21:16:51.834659+0000","last_unstale":"2023-09-23T21:17:18.395121+0000","last_undegraded":"2023-09-23T21:16:51.797563+0000","last_fullsized":"2023-09-23T21:16:51.792808+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1533,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":32964608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":144},{"shards":"2,7","objects":363}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.14","version":"553'6673","reported_seq":8879,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:23.104074+0000","last_change":"2023-09-23T21:16:51.065826+0000","last_active":"2023-09-23T21:17:23.104074+0000","last_peered":"2023-09-23T21:17:23.104074+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:16:51.052190+0000","last_became_peered":"2023-09-23T21:16:51.052190+0000","last_unstale":"2023-09-23T21:17:23.104074+0000","last_undegraded":"2023-09-23T21:16:50.808861+0000","last_fullsized":"2023-09-23T21:16:50.789676+0000","mapping_epoch":544,"log_start":"493'1341","ondisk_log_start":"493'1341","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5332,"log_dups_size":0,"ondisk_log_size":5332,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59244544,"num_objects":904,"num_object_clones":0,"num_object_copies":2712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4962,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":281},{"shards":"2,5","objects":623}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":155,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.452707+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:17:12.452707+0000","last_peered":"2023-09-23T21:17:12.452707+0000","last_clean":"2023-09-23T21:17:12.452707+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:17:12.452707+0000","last_undegraded":"2023-09-23T21:17:12.452707+0000","last_fullsized":"2023-09-23T21:17:12.452707+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.444053+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:17:12.444053+0000","last_peered":"2023-09-23T21:17:12.444053+0000","last_clean":"2023-09-23T21:17:12.444053+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:17:12.444053+0000","last_undegraded":"2023-09-23T21:17:12.444053+0000","last_fullsized":"2023-09-23T21:17:12.444053+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"553'6393","reported_seq":6503,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.414723+0000","last_change":"2023-09-23T21:16:51.036947+0000","last_active":"2023-09-23T21:17:18.414723+0000","last_peered":"2023-09-23T21:17:18.414723+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:16:51.020513+0000","last_became_peered":"2023-09-23T21:16:51.020513+0000","last_unstale":"2023-09-23T21:17:18.414723+0000","last_undegraded":"2023-09-23T21:16:50.811498+0000","last_fullsized":"2023-09-23T21:16:50.789456+0000","mapping_epoch":544,"log_start":"493'1496","ondisk_log_start":"493'1496","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4897,"log_dups_size":0,"ondisk_log_size":4897,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63700992,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1586,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4750,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":304},{"shards":"4,7","objects":668}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":340,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438699+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:17:12.438699+0000","last_peered":"2023-09-23T21:17:12.438699+0000","last_clean":"2023-09-23T21:17:12.438699+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:17:12.438699+0000","last_undegraded":"2023-09-23T21:17:12.438699+0000","last_fullsized":"2023-09-23T21:17:12.438699+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":200,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.474428+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:17:12.474428+0000","last_peered":"2023-09-23T21:17:12.474428+0000","last_clean":"2023-09-23T21:17:12.474428+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:17:12.474428+0000","last_undegraded":"2023-09-23T21:17:12.474428+0000","last_fullsized":"2023-09-23T21:17:12.474428+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"553'6427","reported_seq":10448,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:18.296273+0000","last_change":"2023-09-23T21:16:36.510340+0000","last_active":"2023-09-23T21:17:18.296273+0000","last_peered":"2023-09-23T21:17:18.296273+0000","last_clean":"2023-09-23T21:17:18.296273+0000","last_became_active":"2023-09-23T21:16:36.498646+0000","last_became_peered":"2023-09-23T21:16:36.498646+0000","last_unstale":"2023-09-23T21:17:18.296273+0000","last_undegraded":"2023-09-23T21:17:18.296273+0000","last_fullsized":"2023-09-23T21:17:18.296273+0000","mapping_epoch":537,"log_start":"499'2203","ondisk_log_start":"499'2203","created":507,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4224,"log_dups_size":0,"ondisk_log_size":4224,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880640,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4766,"num_write_kb":184192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1956,"num_bytes_recovered":93519872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.452791+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:17:12.452791+0000","last_peered":"2023-09-23T21:17:12.452791+0000","last_clean":"2023-09-23T21:17:12.452791+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:17:12.452791+0000","last_undegraded":"2023-09-23T21:17:12.452791+0000","last_fullsized":"2023-09-23T21:17:12.452791+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.444390+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:17:12.444390+0000","last_peered":"2023-09-23T21:17:12.444390+0000","last_clean":"2023-09-23T21:17:12.444390+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:17:12.444390+0000","last_undegraded":"2023-09-23T21:17:12.444390+0000","last_fullsized":"2023-09-23T21:17:12.444390+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"553'6308","reported_seq":6425,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.450731+0000","last_change":"2023-09-23T21:16:51.074430+0000","last_active":"2023-09-23T21:17:18.450731+0000","last_peered":"2023-09-23T21:17:18.450731+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:16:51.054035+0000","last_became_peered":"2023-09-23T21:16:51.054035+0000","last_unstale":"2023-09-23T21:17:18.450731+0000","last_undegraded":"2023-09-23T21:16:50.838593+0000","last_fullsized":"2023-09-23T21:16:50.797592+0000","mapping_epoch":544,"log_start":"497'2020","ondisk_log_start":"497'2020","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4288,"log_dups_size":0,"ondisk_log_size":4288,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62685184,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4673,"num_write_kb":180128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":258},{"shards":"2,5","objects":699}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":147,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.439305+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:17:12.439305+0000","last_peered":"2023-09-23T21:17:12.439305+0000","last_clean":"2023-09-23T21:17:12.439305+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:17:12.439305+0000","last_undegraded":"2023-09-23T21:17:12.439305+0000","last_fullsized":"2023-09-23T21:17:12.439305+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443146+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:17:12.443146+0000","last_peered":"2023-09-23T21:17:12.443146+0000","last_clean":"2023-09-23T21:17:12.443146+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:17:12.443146+0000","last_undegraded":"2023-09-23T21:17:12.443146+0000","last_fullsized":"2023-09-23T21:17:12.443146+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.18","version":"553'6563","reported_seq":9878,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:22.975971+0000","last_change":"2023-09-23T21:17:12.466376+0000","last_active":"2023-09-23T21:17:22.975971+0000","last_peered":"2023-09-23T21:17:22.975971+0000","last_clean":"2023-09-23T21:17:22.975971+0000","last_became_active":"2023-09-23T21:17:12.454368+0000","last_became_peered":"2023-09-23T21:17:12.454368+0000","last_unstale":"2023-09-23T21:17:22.975971+0000","last_undegraded":"2023-09-23T21:17:22.975971+0000","last_fullsized":"2023-09-23T21:17:22.975971+0000","mapping_epoch":551,"log_start":"493'1217","ondisk_log_start":"493'1217","created":507,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:01.792696+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54132736,"num_objects":826,"num_object_clones":0,"num_object_copies":2478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3050,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1575,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":206,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.439168+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:17:12.439168+0000","last_peered":"2023-09-23T21:17:12.439168+0000","last_clean":"2023-09-23T21:17:12.439168+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:17:12.439168+0000","last_undegraded":"2023-09-23T21:17:12.439168+0000","last_fullsized":"2023-09-23T21:17:12.439168+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":175,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472608+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:17:12.472608+0000","last_peered":"2023-09-23T21:17:12.472608+0000","last_clean":"2023-09-23T21:17:12.472608+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:17:12.472608+0000","last_undegraded":"2023-09-23T21:17:12.472608+0000","last_fullsized":"2023-09-23T21:17:12.472608+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1b","version":"553'8308","reported_seq":13471,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:22.976443+0000","last_change":"2023-09-23T21:17:17.318827+0000","last_active":"2023-09-23T21:17:22.976443+0000","last_peered":"2023-09-23T21:17:22.976443+0000","last_clean":"2023-09-23T21:17:22.976443+0000","last_became_active":"2023-09-23T21:16:00.808957+0000","last_became_peered":"2023-09-23T21:16:00.808957+0000","last_unstale":"2023-09-23T21:17:22.976443+0000","last_undegraded":"2023-09-23T21:17:22.976443+0000","last_fullsized":"2023-09-23T21:17:22.976443+0000","mapping_epoch":525,"log_start":"493'1783","ondisk_log_start":"493'1783","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58163200,"num_objects":888,"num_object_clones":0,"num_object_copies":2664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5073,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133431296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":156,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472819+0000","last_change":"2023-09-23T21:17:11.237055+0000","last_active":"2023-09-23T21:17:12.472819+0000","last_peered":"2023-09-23T21:17:12.472819+0000","last_clean":"2023-09-23T21:17:12.472819+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:17:12.472819+0000","last_undegraded":"2023-09-23T21:17:12.472819+0000","last_fullsized":"2023-09-23T21:17:12.472819+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443336+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:17:12.443336+0000","last_peered":"2023-09-23T21:17:12.443336+0000","last_clean":"2023-09-23T21:17:12.443336+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:17:12.443336+0000","last_undegraded":"2023-09-23T21:17:12.443336+0000","last_fullsized":"2023-09-23T21:17:12.443336+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1a","version":"553'7950","reported_seq":10077,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:22.910972+0000","last_change":"2023-09-23T21:16:00.812483+0000","last_active":"2023-09-23T21:17:22.910972+0000","last_peered":"2023-09-23T21:17:22.910972+0000","last_clean":"2023-09-23T21:17:22.910972+0000","last_became_active":"2023-09-23T21:16:00.806979+0000","last_became_peered":"2023-09-23T21:16:00.806979+0000","last_unstale":"2023-09-23T21:17:22.910972+0000","last_undegraded":"2023-09-23T21:17:22.910972+0000","last_fullsized":"2023-09-23T21:17:22.910972+0000","mapping_epoch":525,"log_start":"493'1441","ondisk_log_start":"493'1441","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6509,"log_dups_size":0,"ondisk_log_size":6509,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57344000,"num_objects":875,"num_object_clones":0,"num_object_copies":2625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4838,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":180,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473141+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:17:12.473141+0000","last_peered":"2023-09-23T21:17:12.473141+0000","last_clean":"2023-09-23T21:17:12.473141+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:17:12.473141+0000","last_undegraded":"2023-09-23T21:17:12.473141+0000","last_fullsized":"2023-09-23T21:17:12.473141+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":481,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438519+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:17:12.438519+0000","last_peered":"2023-09-23T21:17:12.438519+0000","last_clean":"2023-09-23T21:17:12.438519+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:17:12.438519+0000","last_undegraded":"2023-09-23T21:17:12.438519+0000","last_fullsized":"2023-09-23T21:17:12.438519+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1d","version":"553'8026","reported_seq":13602,"reported_epoch":553,"state":"active+recovering","last_fresh":"2023-09-23T21:17:18.539412+0000","last_change":"2023-09-23T21:17:17.355243+0000","last_active":"2023-09-23T21:17:18.539412+0000","last_peered":"2023-09-23T21:17:18.539412+0000","last_clean":"2023-09-23T21:15:09.796802+0000","last_became_active":"2023-09-23T21:16:03.773235+0000","last_became_peered":"2023-09-23T21:16:03.773235+0000","last_unstale":"2023-09-23T21:17:18.539412+0000","last_undegraded":"2023-09-23T21:17:18.539412+0000","last_fullsized":"2023-09-23T21:17:18.539412+0000","mapping_epoch":528,"log_start":"497'2206","ondisk_log_start":"497'2206","created":525,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65896460,"num_objects":1006,"num_object_clones":0,"num_object_copies":3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":15,"num_objects_unfound":0,"num_objects_dirty":1006,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4849,"num_write_kb":187329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2217,"num_bytes_recovered":145293312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":["4","1","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":15},{"shards":"4,5,7","objects":991}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":136,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473753+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:17:12.473753+0000","last_peered":"2023-09-23T21:17:12.473753+0000","last_clean":"2023-09-23T21:17:12.473753+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:17:12.473753+0000","last_undegraded":"2023-09-23T21:17:12.473753+0000","last_fullsized":"2023-09-23T21:17:12.473753+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":204,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472981+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:17:12.472981+0000","last_peered":"2023-09-23T21:17:12.472981+0000","last_clean":"2023-09-23T21:17:12.472981+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:17:12.472981+0000","last_undegraded":"2023-09-23T21:17:12.472981+0000","last_fullsized":"2023-09-23T21:17:12.472981+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1c","version":"553'8031","reported_seq":10151,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:18.495977+0000","last_change":"2023-09-23T21:16:00.824659+0000","last_active":"2023-09-23T21:17:18.495977+0000","last_peered":"2023-09-23T21:17:18.495977+0000","last_clean":"2023-09-23T21:17:18.495977+0000","last_became_active":"2023-09-23T21:16:00.820619+0000","last_became_peered":"2023-09-23T21:16:00.820619+0000","last_unstale":"2023-09-23T21:17:18.495977+0000","last_undegraded":"2023-09-23T21:17:18.495977+0000","last_fullsized":"2023-09-23T21:17:18.495977+0000","mapping_epoch":525,"log_start":"500'2397","ondisk_log_start":"500'2397","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5634,"log_dups_size":0,"ondisk_log_size":5634,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67600384,"num_objects":1032,"num_object_clones":0,"num_object_copies":3096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4847,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":132,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473168+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:17:12.473168+0000","last_peered":"2023-09-23T21:17:12.473168+0000","last_clean":"2023-09-23T21:17:12.473168+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:17:12.473168+0000","last_undegraded":"2023-09-23T21:17:12.473168+0000","last_fullsized":"2023-09-23T21:17:12.473168+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":140,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472991+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:17:12.472991+0000","last_peered":"2023-09-23T21:17:12.472991+0000","last_clean":"2023-09-23T21:17:12.472991+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:17:12.472991+0000","last_undegraded":"2023-09-23T21:17:12.472991+0000","last_fullsized":"2023-09-23T21:17:12.472991+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1f","version":"553'8007","reported_seq":8118,"reported_epoch":553,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:21.195945+0000","last_change":"2023-09-23T21:16:51.848275+0000","last_active":"2023-09-23T21:17:21.195945+0000","last_peered":"2023-09-23T21:17:21.195945+0000","last_clean":"2023-09-23T21:15:36.326799+0000","last_became_active":"2023-09-23T21:16:51.820010+0000","last_became_peered":"2023-09-23T21:16:51.820010+0000","last_unstale":"2023-09-23T21:17:21.195945+0000","last_undegraded":"2023-09-23T21:17:21.195945+0000","last_fullsized":"2023-09-23T21:17:21.195945+0000","mapping_epoch":545,"log_start":"493'1700","ondisk_log_start":"493'1700","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59801600,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4877,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","5","7"],"object_location_counts":[{"shards":"2,6,7","objects":913}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":130,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473285+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:17:12.473285+0000","last_peered":"2023-09-23T21:17:12.473285+0000","last_clean":"2023-09-23T21:17:12.473285+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:17:12.473285+0000","last_undegraded":"2023-09-23T21:17:12.473285+0000","last_fullsized":"2023-09-23T21:17:12.473285+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":154,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.474352+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:17:12.474352+0000","last_peered":"2023-09-23T21:17:12.474352+0000","last_clean":"2023-09-23T21:17:12.474352+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:17:12.474352+0000","last_undegraded":"2023-09-23T21:17:12.474352+0000","last_fullsized":"2023-09-23T21:17:12.474352+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1e","version":"553'7812","reported_seq":7899,"reported_epoch":553,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:18.445459+0000","last_change":"2023-09-23T21:16:00.911470+0000","last_active":"2023-09-23T21:17:18.445459+0000","last_peered":"2023-09-23T21:17:18.445459+0000","last_clean":"2023-09-23T21:15:59.749850+0000","last_became_active":"2023-09-23T21:16:00.844392+0000","last_became_peered":"2023-09-23T21:16:00.844392+0000","last_unstale":"2023-09-23T21:17:18.445459+0000","last_undegraded":"2023-09-23T21:17:18.445459+0000","last_fullsized":"2023-09-23T21:17:18.445459+0000","mapping_epoch":525,"log_start":"493'1633","ondisk_log_start":"493'1633","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6179,"log_dups_size":0,"ondisk_log_size":6179,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65634304,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4721,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"4,5,7","objects":1002}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":589,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443615+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:17:12.443615+0000","last_peered":"2023-09-23T21:17:12.443615+0000","last_clean":"2023-09-23T21:17:12.443615+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:17:12.443615+0000","last_undegraded":"2023-09-23T21:17:12.443615+0000","last_fullsized":"2023-09-23T21:17:12.443615+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":204,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.474045+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:17:12.474045+0000","last_peered":"2023-09-23T21:17:12.474045+0000","last_clean":"2023-09-23T21:17:12.474045+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:17:12.474045+0000","last_undegraded":"2023-09-23T21:17:12.474045+0000","last_fullsized":"2023-09-23T21:17:12.474045+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"553'5630","reported_seq":6767,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:22.967767+0000","last_change":"2023-09-23T21:16:51.863144+0000","last_active":"2023-09-23T21:17:22.967767+0000","last_peered":"2023-09-23T21:17:22.967767+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858843+0000","last_became_peered":"2023-09-23T21:16:51.858843+0000","last_unstale":"2023-09-23T21:17:22.967767+0000","last_undegraded":"2023-09-23T21:16:51.799178+0000","last_fullsized":"2023-09-23T21:16:51.792321+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26722304,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2448,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":511,"num_bytes_recovered":33488896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":238},{"shards":"2,4,7","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":134,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.474124+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:17:12.474124+0000","last_peered":"2023-09-23T21:17:12.474124+0000","last_clean":"2023-09-23T21:17:12.474124+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:17:12.474124+0000","last_undegraded":"2023-09-23T21:17:12.474124+0000","last_fullsized":"2023-09-23T21:17:12.474124+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":206,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.437218+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:17:12.437218+0000","last_peered":"2023-09-23T21:17:12.437218+0000","last_clean":"2023-09-23T21:17:12.437218+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:17:12.437218+0000","last_undegraded":"2023-09-23T21:17:12.437218+0000","last_fullsized":"2023-09-23T21:17:12.437218+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"553'5638","reported_seq":8748,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.322878+0000","last_change":"2023-09-23T21:16:51.842306+0000","last_active":"2023-09-23T21:17:18.322878+0000","last_peered":"2023-09-23T21:17:18.322878+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834848+0000","last_became_peered":"2023-09-23T21:16:51.834848+0000","last_unstale":"2023-09-23T21:17:18.322878+0000","last_undegraded":"2023-09-23T21:16:51.803926+0000","last_fullsized":"2023-09-23T21:16:51.799106+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34406400,"num_objects":525,"num_object_clones":0,"num_object_copies":1575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1515,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32178176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":161},{"shards":"2,7","objects":364}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":148,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.474506+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:17:12.474506+0000","last_peered":"2023-09-23T21:17:12.474506+0000","last_clean":"2023-09-23T21:17:12.474506+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:17:12.474506+0000","last_undegraded":"2023-09-23T21:17:12.474506+0000","last_fullsized":"2023-09-23T21:17:12.474506+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443188+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:17:12.443188+0000","last_peered":"2023-09-23T21:17:12.443188+0000","last_clean":"2023-09-23T21:17:12.443188+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:17:12.443188+0000","last_undegraded":"2023-09-23T21:17:12.443188+0000","last_fullsized":"2023-09-23T21:17:12.443188+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3","version":"553'5733","reported_seq":7569,"reported_epoch":553,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:18.496026+0000","last_change":"2023-09-23T21:16:01.735614+0000","last_active":"2023-09-23T21:17:18.496026+0000","last_peered":"2023-09-23T21:17:18.496026+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:01.731847+0000","last_became_peered":"2023-09-23T21:16:01.731847+0000","last_unstale":"2023-09-23T21:17:18.496026+0000","last_undegraded":"2023-09-23T21:16:01.699708+0000","last_fullsized":"2023-09-23T21:17:18.496026+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33652736,"num_objects":513,"num_object_clones":0,"num_object_copies":1539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":183,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1472,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":839,"num_bytes_recovered":54984704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":40},{"shards":"4,5,7","objects":473}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":134,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473076+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:17:12.473076+0000","last_peered":"2023-09-23T21:17:12.473076+0000","last_clean":"2023-09-23T21:17:12.473076+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:17:12.473076+0000","last_undegraded":"2023-09-23T21:17:12.473076+0000","last_fullsized":"2023-09-23T21:17:12.473076+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443720+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:17:12.443720+0000","last_peered":"2023-09-23T21:17:12.443720+0000","last_clean":"2023-09-23T21:17:12.443720+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:17:12.443720+0000","last_undegraded":"2023-09-23T21:17:12.443720+0000","last_fullsized":"2023-09-23T21:17:12.443720+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"553'5602","reported_seq":7310,"reported_epoch":553,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:18.463459+0000","last_change":"2023-09-23T21:17:11.228334+0000","last_active":"2023-09-23T21:17:18.463459+0000","last_peered":"2023-09-23T21:17:18.463459+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:01.742399+0000","last_became_peered":"2023-09-23T21:16:01.742399+0000","last_unstale":"2023-09-23T21:17:18.463459+0000","last_undegraded":"2023-09-23T21:17:11.225179+0000","last_fullsized":"2023-09-23T21:17:18.463459+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":166,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2363,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":791,"num_bytes_recovered":51838976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"3,4,5","objects":33},{"shards":"4,5,6","objects":504}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.444347+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:17:12.444347+0000","last_peered":"2023-09-23T21:17:12.444347+0000","last_clean":"2023-09-23T21:17:12.444347+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:17:12.444347+0000","last_undegraded":"2023-09-23T21:17:12.444347+0000","last_fullsized":"2023-09-23T21:17:12.444347+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443579+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:17:12.443579+0000","last_peered":"2023-09-23T21:17:12.443579+0000","last_clean":"2023-09-23T21:17:12.443579+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:17:12.443579+0000","last_undegraded":"2023-09-23T21:17:12.443579+0000","last_fullsized":"2023-09-23T21:17:12.443579+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"553'6437","reported_seq":6650,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.505622+0000","last_change":"2023-09-23T21:16:51.047553+0000","last_active":"2023-09-23T21:17:18.505622+0000","last_peered":"2023-09-23T21:17:18.505622+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:16:51.032827+0000","last_became_peered":"2023-09-23T21:16:51.032827+0000","last_unstale":"2023-09-23T21:17:18.505622+0000","last_undegraded":"2023-09-23T21:16:50.886600+0000","last_fullsized":"2023-09-23T21:16:50.791765+0000","mapping_epoch":544,"log_start":"497'2123","ondisk_log_start":"497'2123","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":4314,"log_dups_size":0,"ondisk_log_size":4314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:01.438609+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64356352,"num_objects":982,"num_object_clones":0,"num_object_copies":2946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2256,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2881,"num_write_kb":64384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":314},{"shards":"3,4,5","objects":68},{"shards":"4,5","objects":-47}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":220,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473963+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:17:12.473963+0000","last_peered":"2023-09-23T21:17:12.473963+0000","last_clean":"2023-09-23T21:17:12.473963+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:17:12.473963+0000","last_undegraded":"2023-09-23T21:17:12.473963+0000","last_fullsized":"2023-09-23T21:17:12.473963+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.444405+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:17:12.444405+0000","last_peered":"2023-09-23T21:17:12.444405+0000","last_clean":"2023-09-23T21:17:12.444405+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:17:12.444405+0000","last_undegraded":"2023-09-23T21:17:12.444405+0000","last_fullsized":"2023-09-23T21:17:12.444405+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.4","version":"553'6620","reported_seq":8918,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:22.988944+0000","last_change":"2023-09-23T21:16:51.065805+0000","last_active":"2023-09-23T21:17:22.988944+0000","last_peered":"2023-09-23T21:17:22.988944+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:16:51.051146+0000","last_became_peered":"2023-09-23T21:16:51.051146+0000","last_unstale":"2023-09-23T21:17:22.988944+0000","last_undegraded":"2023-09-23T21:16:50.809981+0000","last_fullsized":"2023-09-23T21:16:50.790363+0000","mapping_epoch":544,"log_start":"493'1263","ondisk_log_start":"493'1263","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:23.280602+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53215232,"num_objects":812,"num_object_clones":0,"num_object_copies":2436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2920,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":242},{"shards":"2,5","objects":570}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443412+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:17:12.443412+0000","last_peered":"2023-09-23T21:17:12.443412+0000","last_clean":"2023-09-23T21:17:12.443412+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:17:12.443412+0000","last_undegraded":"2023-09-23T21:17:12.443412+0000","last_fullsized":"2023-09-23T21:17:12.443412+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":595,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.453359+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:17:12.453359+0000","last_peered":"2023-09-23T21:17:12.453359+0000","last_clean":"2023-09-23T21:17:12.453359+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:17:12.453359+0000","last_undegraded":"2023-09-23T21:17:12.453359+0000","last_fullsized":"2023-09-23T21:17:12.453359+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"553'6348","reported_seq":6463,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.298505+0000","last_change":"2023-09-23T21:16:51.082314+0000","last_active":"2023-09-23T21:17:18.298505+0000","last_peered":"2023-09-23T21:17:18.298505+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:16:51.064800+0000","last_became_peered":"2023-09-23T21:16:51.064800+0000","last_unstale":"2023-09-23T21:17:18.298505+0000","last_undegraded":"2023-09-23T21:16:50.820535+0000","last_fullsized":"2023-09-23T21:16:50.800952+0000","mapping_epoch":544,"log_start":"493'1414","ondisk_log_start":"493'1414","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66125824,"num_objects":1009,"num_object_clones":0,"num_object_copies":3027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4705,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":295},{"shards":"4,7","objects":714}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":190,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438949+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:17:12.438949+0000","last_peered":"2023-09-23T21:17:12.438949+0000","last_clean":"2023-09-23T21:17:12.438949+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:17:12.438949+0000","last_undegraded":"2023-09-23T21:17:12.438949+0000","last_fullsized":"2023-09-23T21:17:12.438949+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443932+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:17:12.443932+0000","last_peered":"2023-09-23T21:17:12.443932+0000","last_clean":"2023-09-23T21:17:12.443932+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:17:12.443932+0000","last_undegraded":"2023-09-23T21:17:12.443932+0000","last_fullsized":"2023-09-23T21:17:12.443932+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"553'6388","reported_seq":6595,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.247611+0000","last_change":"2023-09-23T21:15:36.787083+0000","last_active":"2023-09-23T21:17:18.247611+0000","last_peered":"2023-09-23T21:17:18.247611+0000","last_clean":"2023-09-23T21:15:09.799249+0000","last_became_active":"2023-09-23T21:15:12.799540+0000","last_became_peered":"2023-09-23T21:15:12.799540+0000","last_unstale":"2023-09-23T21:17:18.247611+0000","last_undegraded":"2023-09-23T21:15:36.785930+0000","last_fullsized":"2023-09-23T21:15:12.679693+0000","mapping_epoch":514,"log_start":"499'2148","ondisk_log_start":"499'2148","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":4240,"log_dups_size":0,"ondisk_log_size":4240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:59.312702+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62521344,"num_objects":954,"num_object_clones":0,"num_object_copies":2862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2775,"num_write_kb":57088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":62},{"shards":"4,5","objects":-984}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":204,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473363+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:17:12.473363+0000","last_peered":"2023-09-23T21:17:12.473363+0000","last_clean":"2023-09-23T21:17:12.473363+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:17:12.473363+0000","last_undegraded":"2023-09-23T21:17:12.473363+0000","last_fullsized":"2023-09-23T21:17:12.473363+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":292,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473552+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:17:12.473552+0000","last_peered":"2023-09-23T21:17:12.473552+0000","last_clean":"2023-09-23T21:17:12.473552+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:17:12.473552+0000","last_undegraded":"2023-09-23T21:17:12.473552+0000","last_fullsized":"2023-09-23T21:17:12.473552+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"553'8247","reported_seq":13301,"reported_epoch":553,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:23.019219+0000","last_change":"2023-09-23T21:17:21.110094+0000","last_active":"2023-09-23T21:17:23.019219+0000","last_peered":"2023-09-23T21:17:23.019219+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:01.723118+0000","last_became_peered":"2023-09-23T21:16:01.723118+0000","last_unstale":"2023-09-23T21:17:23.019219+0000","last_undegraded":"2023-09-23T21:17:23.019219+0000","last_fullsized":"2023-09-23T21:17:23.019219+0000","mapping_epoch":526,"log_start":"493'1783","ondisk_log_start":"493'1783","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62160896,"num_objects":948,"num_object_clones":0,"num_object_copies":2844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5011,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":129957888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":948}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":477,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473055+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:17:12.473055+0000","last_peered":"2023-09-23T21:17:12.473055+0000","last_clean":"2023-09-23T21:17:12.473055+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:17:12.473055+0000","last_undegraded":"2023-09-23T21:17:12.473055+0000","last_fullsized":"2023-09-23T21:17:12.473055+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":228,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438843+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:17:12.438843+0000","last_peered":"2023-09-23T21:17:12.438843+0000","last_clean":"2023-09-23T21:17:12.438843+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:17:12.438843+0000","last_undegraded":"2023-09-23T21:17:12.438843+0000","last_fullsized":"2023-09-23T21:17:12.438843+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.437776+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:17:12.437776+0000","last_peered":"2023-09-23T21:17:12.437776+0000","last_clean":"2023-09-23T21:17:12.437776+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:17:12.437776+0000","last_undegraded":"2023-09-23T21:17:12.437776+0000","last_fullsized":"2023-09-23T21:17:12.437776+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"553'8010","reported_seq":14121,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:18.517908+0000","last_change":"2023-09-23T21:17:15.462587+0000","last_active":"2023-09-23T21:17:18.517908+0000","last_peered":"2023-09-23T21:17:18.517908+0000","last_clean":"2023-09-23T21:17:18.517908+0000","last_became_active":"2023-09-23T21:16:01.764537+0000","last_became_peered":"2023-09-23T21:16:01.764537+0000","last_unstale":"2023-09-23T21:17:18.517908+0000","last_undegraded":"2023-09-23T21:17:18.517908+0000","last_fullsized":"2023-09-23T21:17:18.517908+0000","mapping_epoch":526,"log_start":"500'2330","ondisk_log_start":"500'2330","created":482,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66945036,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4833,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2487,"num_bytes_recovered":146014208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443271+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:17:12.443271+0000","last_peered":"2023-09-23T21:17:12.443271+0000","last_clean":"2023-09-23T21:17:12.443271+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:17:12.443271+0000","last_undegraded":"2023-09-23T21:17:12.443271+0000","last_fullsized":"2023-09-23T21:17:12.443271+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443661+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:17:12.443661+0000","last_peered":"2023-09-23T21:17:12.443661+0000","last_clean":"2023-09-23T21:17:12.443661+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:17:12.443661+0000","last_undegraded":"2023-09-23T21:17:12.443661+0000","last_fullsized":"2023-09-23T21:17:12.443661+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":267,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.454289+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:17:12.454289+0000","last_peered":"2023-09-23T21:17:12.454289+0000","last_clean":"2023-09-23T21:17:12.454289+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:17:12.454289+0000","last_undegraded":"2023-09-23T21:17:12.454289+0000","last_fullsized":"2023-09-23T21:17:12.454289+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"553'7823","reported_seq":7912,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:18.422091+0000","last_change":"2023-09-23T21:16:00.841315+0000","last_active":"2023-09-23T21:17:18.422091+0000","last_peered":"2023-09-23T21:17:18.422091+0000","last_clean":"2023-09-23T21:17:18.422091+0000","last_became_active":"2023-09-23T21:16:00.829607+0000","last_became_peered":"2023-09-23T21:16:00.829607+0000","last_unstale":"2023-09-23T21:17:18.422091+0000","last_undegraded":"2023-09-23T21:17:18.422091+0000","last_fullsized":"2023-09-23T21:17:18.422091+0000","mapping_epoch":482,"log_start":"493'1633","ondisk_log_start":"493'1633","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6190,"log_dups_size":0,"ondisk_log_size":6190,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64913408,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4731,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":561,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438368+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:17:12.438368+0000","last_peered":"2023-09-23T21:17:12.438368+0000","last_clean":"2023-09-23T21:17:12.438368+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:17:12.438368+0000","last_undegraded":"2023-09-23T21:17:12.438368+0000","last_fullsized":"2023-09-23T21:17:12.438368+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":299,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438132+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:17:12.438132+0000","last_peered":"2023-09-23T21:17:12.438132+0000","last_clean":"2023-09-23T21:17:12.438132+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:17:12.438132+0000","last_undegraded":"2023-09-23T21:17:12.438132+0000","last_fullsized":"2023-09-23T21:17:12.438132+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":206,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473631+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:17:12.473631+0000","last_peered":"2023-09-23T21:17:12.473631+0000","last_clean":"2023-09-23T21:17:12.473631+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:17:12.473631+0000","last_undegraded":"2023-09-23T21:17:12.473631+0000","last_fullsized":"2023-09-23T21:17:12.473631+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"553'8020","reported_seq":10141,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:18.398227+0000","last_change":"2023-09-23T21:16:00.748968+0000","last_active":"2023-09-23T21:17:18.398227+0000","last_peered":"2023-09-23T21:17:18.398227+0000","last_clean":"2023-09-23T21:17:18.398227+0000","last_became_active":"2023-09-23T21:16:00.745123+0000","last_became_peered":"2023-09-23T21:16:00.745123+0000","last_unstale":"2023-09-23T21:17:18.398227+0000","last_undegraded":"2023-09-23T21:17:18.398227+0000","last_fullsized":"2023-09-23T21:17:18.398227+0000","mapping_epoch":501,"log_start":"500'2344","ondisk_log_start":"500'2344","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68321280,"num_objects":1042,"num_object_clones":0,"num_object_copies":3126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4835,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":502,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":381,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.453103+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:17:12.453103+0000","last_peered":"2023-09-23T21:17:12.453103+0000","last_clean":"2023-09-23T21:17:12.453103+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:17:12.453103+0000","last_undegraded":"2023-09-23T21:17:12.453103+0000","last_fullsized":"2023-09-23T21:17:12.453103+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":899,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472995+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:17:12.472995+0000","last_peered":"2023-09-23T21:17:12.472995+0000","last_clean":"2023-09-23T21:17:12.472995+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:17:12.472995+0000","last_undegraded":"2023-09-23T21:17:12.472995+0000","last_fullsized":"2023-09-23T21:17:12.472995+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":125,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.437332+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:17:12.437332+0000","last_peered":"2023-09-23T21:17:12.437332+0000","last_clean":"2023-09-23T21:17:12.437332+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:17:12.437332+0000","last_undegraded":"2023-09-23T21:17:12.437332+0000","last_fullsized":"2023-09-23T21:17:12.437332+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443914+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:17:12.443914+0000","last_peered":"2023-09-23T21:17:12.443914+0000","last_clean":"2023-09-23T21:17:12.443914+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:17:12.443914+0000","last_undegraded":"2023-09-23T21:17:12.443914+0000","last_fullsized":"2023-09-23T21:17:12.443914+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"553'7970","reported_seq":8075,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:21.105584+0000","last_change":"2023-09-23T21:16:51.832676+0000","last_active":"2023-09-23T21:17:21.105584+0000","last_peered":"2023-09-23T21:17:21.105584+0000","last_clean":"2023-09-23T21:17:21.105584+0000","last_became_active":"2023-09-23T21:16:51.803067+0000","last_became_peered":"2023-09-23T21:16:51.803067+0000","last_unstale":"2023-09-23T21:17:21.105584+0000","last_undegraded":"2023-09-23T21:17:21.105584+0000","last_fullsized":"2023-09-23T21:17:21.105584+0000","mapping_epoch":545,"log_start":"493'1700","ondisk_log_start":"493'1700","created":482,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6270,"log_dups_size":0,"ondisk_log_size":6270,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62226432,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4839,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":180,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473072+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:17:12.473072+0000","last_peered":"2023-09-23T21:17:12.473072+0000","last_clean":"2023-09-23T21:17:12.473072+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:17:12.473072+0000","last_undegraded":"2023-09-23T21:17:12.473072+0000","last_fullsized":"2023-09-23T21:17:12.473072+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":49,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472579+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:17:12.472579+0000","last_peered":"2023-09-23T21:17:12.472579+0000","last_clean":"2023-09-23T21:17:12.472579+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:17:12.472579+0000","last_undegraded":"2023-09-23T21:17:12.472579+0000","last_fullsized":"2023-09-23T21:17:12.472579+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":462,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.453516+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:17:12.453516+0000","last_peered":"2023-09-23T21:17:12.453516+0000","last_clean":"2023-09-23T21:17:12.453516+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:17:12.453516+0000","last_undegraded":"2023-09-23T21:17:12.453516+0000","last_fullsized":"2023-09-23T21:17:12.453516+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"553'7994","reported_seq":10122,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:23.003685+0000","last_change":"2023-09-23T21:16:00.763134+0000","last_active":"2023-09-23T21:17:23.003685+0000","last_peered":"2023-09-23T21:17:23.003685+0000","last_clean":"2023-09-23T21:17:23.003685+0000","last_became_active":"2023-09-23T21:16:00.758875+0000","last_became_peered":"2023-09-23T21:16:00.758875+0000","last_unstale":"2023-09-23T21:17:23.003685+0000","last_undegraded":"2023-09-23T21:17:23.003685+0000","last_fullsized":"2023-09-23T21:17:23.003685+0000","mapping_epoch":498,"log_start":"493'1441","ondisk_log_start":"493'1441","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6553,"log_dups_size":0,"ondisk_log_size":6553,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4882,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":150,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472884+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:17:12.472884+0000","last_peered":"2023-09-23T21:17:12.472884+0000","last_clean":"2023-09-23T21:17:12.472884+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:17:12.472884+0000","last_undegraded":"2023-09-23T21:17:12.472884+0000","last_fullsized":"2023-09-23T21:17:12.472884+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":366,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.454313+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:17:12.454313+0000","last_peered":"2023-09-23T21:17:12.454313+0000","last_clean":"2023-09-23T21:17:12.454313+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:17:12.454313+0000","last_undegraded":"2023-09-23T21:17:12.454313+0000","last_fullsized":"2023-09-23T21:17:12.454313+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":258,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438204+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:17:12.438204+0000","last_peered":"2023-09-23T21:17:12.438204+0000","last_clean":"2023-09-23T21:17:12.438204+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:17:12.438204+0000","last_undegraded":"2023-09-23T21:17:12.438204+0000","last_fullsized":"2023-09-23T21:17:12.438204+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"553'6386","reported_seq":6591,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.362863+0000","last_change":"2023-09-23T21:16:51.076997+0000","last_active":"2023-09-23T21:17:18.362863+0000","last_peered":"2023-09-23T21:17:18.362863+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:16:51.057102+0000","last_became_peered":"2023-09-23T21:16:51.057102+0000","last_unstale":"2023-09-23T21:17:18.362863+0000","last_undegraded":"2023-09-23T21:16:50.840109+0000","last_fullsized":"2023-09-23T21:16:50.799660+0000","mapping_epoch":544,"log_start":"499'2191","ondisk_log_start":"499'2191","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":4195,"log_dups_size":0,"ondisk_log_size":4195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.748037+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66650112,"num_objects":1017,"num_object_clones":0,"num_object_copies":3051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1565,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2824,"num_write_kb":60864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":315},{"shards":"2,5","objects":702}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":208,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.474204+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:17:12.474204+0000","last_peered":"2023-09-23T21:17:12.474204+0000","last_clean":"2023-09-23T21:17:12.474204+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:17:12.474204+0000","last_undegraded":"2023-09-23T21:17:12.474204+0000","last_fullsized":"2023-09-23T21:17:12.474204+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443825+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:17:12.443825+0000","last_peered":"2023-09-23T21:17:12.443825+0000","last_clean":"2023-09-23T21:17:12.443825+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:17:12.443825+0000","last_undegraded":"2023-09-23T21:17:12.443825+0000","last_fullsized":"2023-09-23T21:17:12.443825+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":233,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.472916+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:17:12.472916+0000","last_peered":"2023-09-23T21:17:12.472916+0000","last_clean":"2023-09-23T21:17:12.472916+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:17:12.472916+0000","last_undegraded":"2023-09-23T21:17:12.472916+0000","last_fullsized":"2023-09-23T21:17:12.472916+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"553'6648","reported_seq":6801,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:22.957799+0000","last_change":"2023-09-23T21:16:51.063450+0000","last_active":"2023-09-23T21:17:22.957799+0000","last_peered":"2023-09-23T21:17:22.957799+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:16:51.051345+0000","last_became_peered":"2023-09-23T21:16:51.051345+0000","last_unstale":"2023-09-23T21:17:22.957799+0000","last_undegraded":"2023-09-23T21:16:50.807164+0000","last_fullsized":"2023-09-23T21:16:50.788756+0000","mapping_epoch":544,"log_start":"493'1358","ondisk_log_start":"493'1358","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":5290,"log_dups_size":0,"ondisk_log_size":5290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:15.933191+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58654720,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1557,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3160,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":594},{"shards":"2,5,7","objects":301}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":459,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.444489+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:17:12.444489+0000","last_peered":"2023-09-23T21:17:12.444489+0000","last_clean":"2023-09-23T21:17:12.444489+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:17:12.444489+0000","last_undegraded":"2023-09-23T21:17:12.444489+0000","last_fullsized":"2023-09-23T21:17:12.444489+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":90,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.440174+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:17:12.440174+0000","last_peered":"2023-09-23T21:17:12.440174+0000","last_clean":"2023-09-23T21:17:12.440174+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:17:12.440174+0000","last_undegraded":"2023-09-23T21:17:12.440174+0000","last_fullsized":"2023-09-23T21:17:12.440174+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":238,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.437249+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:17:12.437249+0000","last_peered":"2023-09-23T21:17:12.437249+0000","last_clean":"2023-09-23T21:17:12.437249+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:17:12.437249+0000","last_undegraded":"2023-09-23T21:17:12.437249+0000","last_fullsized":"2023-09-23T21:17:12.437249+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"553'6543","reported_seq":6669,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.502952+0000","last_change":"2023-09-23T21:16:51.067304+0000","last_active":"2023-09-23T21:17:18.502952+0000","last_peered":"2023-09-23T21:17:18.502952+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:16:51.048602+0000","last_became_peered":"2023-09-23T21:16:51.048602+0000","last_unstale":"2023-09-23T21:17:18.502952+0000","last_undegraded":"2023-09-23T21:16:50.818351+0000","last_fullsized":"2023-09-23T21:16:50.791414+0000","mapping_epoch":544,"log_start":"500'2323","ondisk_log_start":"500'2323","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4220,"log_dups_size":0,"ondisk_log_size":4220,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64487424,"num_objects":984,"num_object_clones":0,"num_object_copies":2952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4870,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":293},{"shards":"3,4,5","objects":64},{"shards":"4,5","objects":-2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":180,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.473835+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:17:12.473835+0000","last_peered":"2023-09-23T21:17:12.473835+0000","last_clean":"2023-09-23T21:17:12.473835+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:17:12.473835+0000","last_undegraded":"2023-09-23T21:17:12.473835+0000","last_fullsized":"2023-09-23T21:17:12.473835+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":616,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.454342+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:17:12.454342+0000","last_peered":"2023-09-23T21:17:12.454342+0000","last_clean":"2023-09-23T21:17:12.454342+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:17:12.454342+0000","last_undegraded":"2023-09-23T21:17:12.454342+0000","last_fullsized":"2023-09-23T21:17:12.454342+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"553'6427","reported_seq":10475,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:18.481219+0000","last_change":"2023-09-23T21:16:37.510873+0000","last_active":"2023-09-23T21:17:18.481219+0000","last_peered":"2023-09-23T21:17:18.481219+0000","last_clean":"2023-09-23T21:17:18.481219+0000","last_became_active":"2023-09-23T21:16:37.500341+0000","last_became_peered":"2023-09-23T21:16:37.500341+0000","last_unstale":"2023-09-23T21:17:18.481219+0000","last_undegraded":"2023-09-23T21:17:18.481219+0000","last_fullsized":"2023-09-23T21:17:18.481219+0000","mapping_epoch":538,"log_start":"497'2119","ondisk_log_start":"497'2119","created":507,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4308,"log_dups_size":0,"ondisk_log_size":4308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63569920,"num_objects":970,"num_object_clones":0,"num_object_copies":2910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4766,"num_write_kb":183552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1970,"num_bytes_recovered":94175232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":257,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.439302+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:17:12.439302+0000","last_peered":"2023-09-23T21:17:12.439302+0000","last_clean":"2023-09-23T21:17:12.439302+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:17:12.439302+0000","last_undegraded":"2023-09-23T21:17:12.439302+0000","last_fullsized":"2023-09-23T21:17:12.439302+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":210,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.452700+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:17:12.452700+0000","last_peered":"2023-09-23T21:17:12.452700+0000","last_clean":"2023-09-23T21:17:12.452700+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:17:12.452700+0000","last_undegraded":"2023-09-23T21:17:12.452700+0000","last_fullsized":"2023-09-23T21:17:12.452700+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"553'6464","reported_seq":9198,"reported_epoch":553,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:18.519836+0000","last_change":"2023-09-23T21:16:50.886093+0000","last_active":"2023-09-23T21:17:18.519836+0000","last_peered":"2023-09-23T21:17:18.519836+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:17:18.519836+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"497'2165","ondisk_log_start":"497'2165","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4299,"log_dups_size":0,"ondisk_log_size":4299,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65830912,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4781,"num_write_kb":185120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":61800448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5","objects":382}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":22,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443819+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:17:12.443819+0000","last_peered":"2023-09-23T21:17:12.443819+0000","last_clean":"2023-09-23T21:17:12.443819+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:17:12.443819+0000","last_undegraded":"2023-09-23T21:17:12.443819+0000","last_fullsized":"2023-09-23T21:17:12.443819+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":197,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438426+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:17:12.438426+0000","last_peered":"2023-09-23T21:17:12.438426+0000","last_clean":"2023-09-23T21:17:12.438426+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:17:12.438426+0000","last_undegraded":"2023-09-23T21:17:12.438426+0000","last_fullsized":"2023-09-23T21:17:12.438426+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"553'6453","reported_seq":12318,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:22.912136+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:17:22.912136+0000","last_peered":"2023-09-23T21:17:22.912136+0000","last_clean":"2023-09-23T21:17:22.912136+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:17:22.912136+0000","last_undegraded":"2023-09-23T21:17:22.912136+0000","last_fullsized":"2023-09-23T21:17:22.912136+0000","mapping_epoch":518,"log_start":"492'963","ondisk_log_start":"492'963","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525952,"num_objects":832,"num_object_clones":0,"num_object_copies":2496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4790,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":18,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.443741+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:17:12.443741+0000","last_peered":"2023-09-23T21:17:12.443741+0000","last_clean":"2023-09-23T21:17:12.443741+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:17:12.443741+0000","last_undegraded":"2023-09-23T21:17:12.443741+0000","last_fullsized":"2023-09-23T21:17:12.443741+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":412,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.452834+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:17:12.452834+0000","last_peered":"2023-09-23T21:17:12.452834+0000","last_clean":"2023-09-23T21:17:12.452834+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:17:12.452834+0000","last_undegraded":"2023-09-23T21:17:12.452834+0000","last_fullsized":"2023-09-23T21:17:12.452834+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"553'6440","reported_seq":10401,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:21.177891+0000","last_change":"2023-09-23T21:16:49.855987+0000","last_active":"2023-09-23T21:17:21.177891+0000","last_peered":"2023-09-23T21:17:21.177891+0000","last_clean":"2023-09-23T21:17:21.177891+0000","last_became_active":"2023-09-23T21:16:15.742634+0000","last_became_peered":"2023-09-23T21:16:15.742634+0000","last_unstale":"2023-09-23T21:17:21.177891+0000","last_undegraded":"2023-09-23T21:17:21.177891+0000","last_fullsized":"2023-09-23T21:17:21.177891+0000","mapping_epoch":531,"log_start":"489'835","ondisk_log_start":"489'835","created":507,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62095360,"num_objects":948,"num_object_clones":0,"num_object_copies":2844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4804,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":509,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.454240+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:17:12.454240+0000","last_peered":"2023-09-23T21:17:12.454240+0000","last_clean":"2023-09-23T21:17:12.454240+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:17:12.454240+0000","last_undegraded":"2023-09-23T21:17:12.454240+0000","last_fullsized":"2023-09-23T21:17:12.454240+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":220,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T21:17:12.438131+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:17:12.438131+0000","last_peered":"2023-09-23T21:17:12.438131+0000","last_clean":"2023-09-23T21:17:12.438131+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:17:12.438131+0000","last_undegraded":"2023-09-23T21:17:12.438131+0000","last_fullsized":"2023-09-23T21:17:12.438131+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":36,"stat_sum":{"num_bytes":1999568920,"num_objects":30513,"num_object_clones":0,"num_object_copies":91539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21646,"num_objects_misplaced":15,"num_objects_unfound":0,"num_objects_dirty":30513,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139418,"num_write_kb":4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27765,"num_bytes_recovered":1615364096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8259579904,"data_stored":8259567688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195360,"ondisk_log_size":195360,"up":108,"acting":93,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":38918,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201863,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884992,"kb_used_data":1101304,"kb_used_omap":0,"kb_used_meta":783424,"kb_avail":91860160,"statfs":{"total":95995035648,"available":94064803840,"internally_reserved":0,"allocated":1127735296,"data_stored":1126230288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":802226176},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":3.0539999999999998,"15min":2.2519999999999998},"min":{"1min":0.26300000000000001,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":7.726,"5min":141.33199999999999,"15min":141.33199999999999},"last":1.885},{"interface":"front","average":{"1min":2.2109999999999999,"5min":3.3530000000000002,"15min":2.3199999999999998},"min":{"1min":0.307,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.3940000000000001,"5min":141.31700000000001,"15min":141.31700000000001},"last":2.391}]},{"osd":1,"last update":"Sat Sep 23 21:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":3.0030000000000001,"15min":3.3889999999999998},"min":{"1min":0.317,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.8700000000000001,"5min":76.007999999999996,"15min":114.16500000000001},"last":4.694},{"interface":"front","average":{"1min":1.294,"5min":4.077,"15min":3.8610000000000002},"min":{"1min":0.27000000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.681,"5min":117.637,"15min":117.637},"last":1.827}]},{"osd":2,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":4.7679999999999998,"15min":4.4850000000000003},"min":{"1min":0.246,"5min":0.224,"15min":0.224},"max":{"1min":5.4160000000000004,"5min":135.529,"15min":183.44200000000001},"last":4.7460000000000004},{"interface":"front","average":{"1min":1.5740000000000001,"5min":5.6550000000000002,"15min":4.6909999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2370000000000001,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.69499999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":7.1909999999999998,"15min":4.1210000000000004},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":7.7960000000000003,"5min":135.66200000000001,"15min":135.66200000000001},"last":1.798},{"interface":"front","average":{"1min":1.613,"5min":7.6120000000000001,"15min":4.2199999999999998},"min":{"1min":0.375,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":5.2779999999999996,"5min":135.47,"15min":135.47},"last":0.71899999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1499999999999999,"5min":1.611,"15min":1.1399999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":9.9529999999999994,"5min":10.989000000000001,"15min":15.942},"last":2.4289999999999998},{"interface":"front","average":{"1min":2.984,"5min":1.6020000000000001,"15min":1.0549999999999999},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":11.773,"5min":11.773,"15min":16.725999999999999},"last":0.63700000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":1.7689999999999999,"15min":1.3149999999999999},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.7829999999999995,"5min":11.231,"15min":11.231},"last":2.3279999999999998},{"interface":"front","average":{"1min":3.9420000000000002,"5min":1.9510000000000001,"15min":1.274},"min":{"1min":0.435,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":9.468,"5min":10.005000000000001,"15min":13.867000000000001},"last":2.5910000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.2160000000000002,"15min":2.0289999999999999},"min":{"1min":0.51800000000000002,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":13.369,"5min":143.18700000000001,"15min":143.18700000000001},"last":4.7770000000000001},{"interface":"front","average":{"1min":2.266,"5min":2.7549999999999999,"15min":1.8420000000000001},"min":{"1min":0.53900000000000003,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":7.7450000000000001,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.67000000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463484,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2270644,"kb_used_data":1544172,"kb_used_omap":0,"kb_used_meta":726464,"kb_avail":91474508,"statfs":{"total":95995035648,"available":93669896192,"internally_reserved":0,"allocated":1581232128,"data_stored":1579739675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":743899136},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":4.5339999999999998,"15min":2.7490000000000001},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":12.659000000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":1.3180000000000001},{"interface":"front","average":{"1min":2.0369999999999999,"5min":4.899,"15min":2.835},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":16.021999999999998,"5min":149.303,"15min":149.303},"last":2.637}]},{"osd":1,"last update":"Sat Sep 23 21:16:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":3.9359999999999999,"15min":3.5009999999999999},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":8.4860000000000007,"5min":129.37899999999999,"15min":129.37899999999999},"last":2.6760000000000002},{"interface":"front","average":{"1min":1.4199999999999999,"5min":3.8450000000000002,"15min":3.3919999999999999},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.23100000000000001},"max":{"1min":6.2539999999999996,"5min":129.40100000000001,"15min":129.40100000000001},"last":0.70399999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.577999999999999,"5min":7.6189999999999998,"15min":5.9340000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":120.28700000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":1.25},{"interface":"front","average":{"1min":12.118,"5min":7.8120000000000003,"15min":5.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":120.414,"5min":133.33000000000001,"15min":199.08000000000001},"last":2.7050000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":5.6669999999999998,"15min":3.3199999999999998},"min":{"1min":0.38,"5min":0.218,"15min":0.218},"max":{"1min":3.5089999999999999,"5min":137.875,"15min":137.875},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.7250000000000001,"5min":5.391,"15min":3.3650000000000002},"min":{"1min":0.30099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.676,"5min":137.60900000000001,"15min":137.60900000000001},"last":1.282}]},{"osd":4,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":8.9060000000000006,"5min":2.9329999999999998,"15min":1.673},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.22600000000000001},"max":{"1min":55.494,"5min":55.494,"15min":55.494},"last":2.7360000000000002},{"interface":"front","average":{"1min":8.7550000000000008,"5min":2.8100000000000001,"15min":1.4970000000000001},"min":{"1min":0.39900000000000002,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":55.305,"5min":55.305,"15min":55.305},"last":0.755}]},{"osd":5,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":15.223000000000001,"5min":4.3630000000000004,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":222.90899999999999,"5min":222.90899999999999,"15min":222.90899999999999},"last":5.1580000000000004},{"interface":"front","average":{"1min":15.375,"5min":4.4930000000000003,"15min":2.1739999999999999},"min":{"1min":0.37,"5min":0.32200000000000001,"15min":0.28499999999999998},"max":{"1min":224.655,"5min":224.655,"15min":224.655},"last":5.774}]},{"osd":7,"last update":"Sat Sep 23 21:16:25 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":0.94299999999999995,"15min":1.002},"min":{"1min":0.45100000000000001,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":6.9729999999999999,"5min":6.9729999999999999,"15min":7.8079999999999998},"last":1.6639999999999999},{"interface":"front","average":{"1min":2.0470000000000002,"5min":1.2729999999999999,"15min":1.129},"min":{"1min":0.42799999999999999,"5min":0.251,"15min":0.251},"max":{"1min":6.2060000000000004,"5min":7.9119999999999999,"15min":7.9119999999999999},"last":0.66700000000000004}]}]},{"osd":1,"up_from":0,"seq":1185410973931,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991670,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781562,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2252324,"kb_used_data":1174876,"kb_used_omap":7,"kb_used_meta":1077432,"kb_avail":91492828,"statfs":{"total":95995035648,"available":93688655872,"internally_reserved":0,"allocated":1203073024,"data_stored":1200417630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1103290449},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.8420000000000001,"15min":2.157},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":8.0079999999999991,"5min":103.021,"15min":103.021},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.29599999999999999,"5min":3.133,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":0.378,"5min":103.154,"15min":103.154},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:16:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":3.4449999999999998,"15min":4.3470000000000004},"min":{"1min":0.224,"5min":0.224,"15min":0.19},"max":{"1min":2.9740000000000002,"5min":106.268,"15min":212.12},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.94699999999999995,"5min":3.8879999999999999,"15min":4.3550000000000004},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":3.246,"5min":106.32299999999999,"15min":213.02799999999999},"last":0.69799999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.5670000000000002,"15min":4.0339999999999998},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5190000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.2250000000000001,"15min":4.2519999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.2189999999999994,"5min":82.241,"15min":143.27799999999999},"last":1.373}]},{"osd":4,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":13.896000000000001,"5min":4.3010000000000002,"15min":3.1800000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":220.82900000000001,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":13.894,"5min":4.9130000000000003,"15min":3.823},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":220.84299999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.541,"5min":5.1059999999999999,"15min":3.48},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":220.63900000000001,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":16.257000000000001,"5min":4.7380000000000004,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":220.31999999999999,"5min":220.31999999999999,"15min":220.31999999999999},"last":1.8879999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.597,"5min":4.6210000000000004,"15min":3.762},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":210.74700000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":3.5299999999999998},{"interface":"front","average":{"1min":14.371,"5min":4.4640000000000004,"15min":3.1259999999999999},"min":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":210.72900000000001,"5min":210.72900000000001,"15min":210.72900000000001},"last":7.1619999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.4940000000000002,"15min":1.948},"min":{"1min":0.371,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":14.019,"5min":25.260000000000002,"15min":25.260000000000002},"last":2.4489999999999998},{"interface":"front","average":{"1min":1.921,"5min":2.4990000000000001,"15min":2.2850000000000001},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":5.0229999999999997,"5min":12.613,"15min":16.875},"last":2.3380000000000001}]}]},{"osd":3,"up_from":0,"seq":635655160155,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986773,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2251712,"kb_used_data":1460536,"kb_used_omap":0,"kb_used_meta":791168,"kb_avail":91493440,"statfs":{"total":95995035648,"available":93689282560,"internally_reserved":0,"allocated":1495588864,"data_stored":1494084119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":810156032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":5.1909999999999998,"15min":2.7109999999999999},"min":{"1min":0.247,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":10.436,"5min":226.751,"15min":226.751},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.0750000000000002,"5min":4.8049999999999997,"15min":2.556},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.223},"max":{"1min":6.218,"5min":226.43299999999999,"15min":226.43299999999999},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":4.4980000000000002,"15min":5.282},"min":{"1min":0.29799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.9980000000000002,"5min":119.55800000000001,"15min":127.411},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.665,"5min":4.1390000000000002,"15min":4.5179999999999998},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":8.2550000000000008,"5min":116.78100000000001,"15min":118.075},"last":0.48299999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":8.9459999999999997,"5min":7.5439999999999996,"15min":5.8710000000000004},"min":{"1min":0.26000000000000001,"5min":0.253,"15min":0.253},"max":{"1min":90.950000000000003,"5min":210.50299999999999,"15min":210.50299999999999},"last":33.427},{"interface":"front","average":{"1min":9.9280000000000008,"5min":8.5630000000000006,"15min":6.0149999999999997},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":90.846000000000004,"5min":210.59,"15min":210.59},"last":33.203000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:16:22 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":7.5650000000000004,"15min":4.069},"min":{"1min":0.317,"5min":0.317,"15min":0.29299999999999998},"max":{"1min":4.3319999999999999,"5min":214.64099999999999,"15min":214.64099999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.992,"5min":8.6229999999999993,"15min":4.4470000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.25600000000000001},"max":{"1min":8.2799999999999994,"5min":214.54900000000001,"15min":214.54900000000001},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.383,"15min":0.98999999999999999},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.757999999999999,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":3.4119999999999999,"5min":2.157,"15min":1.266},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":18.314,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.524,"15min":1.333},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.2569999999999997,"5min":10.446999999999999,"15min":36.557000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.2690000000000001,"5min":1.579,"15min":1.3169999999999999},"min":{"1min":0.35799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.8049999999999997,"5min":6.8049999999999997,"15min":38.468000000000004},"last":0.439}]},{"osd":7,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8740000000000001,"5min":1.746,"15min":1.869},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":18.352,"5min":18.352,"15min":145.721},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.512,"5min":1.421,"15min":1.917},"min":{"1min":0.42199999999999999,"5min":0.33500000000000002,"15min":0.31900000000000001},"max":{"1min":7.5830000000000002,"5min":10.677,"15min":148.45699999999999},"last":0.47399999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725137,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2428976,"kb_used_data":1590376,"kb_used_omap":0,"kb_used_meta":838016,"kb_avail":91316176,"statfs":{"total":95995035648,"available":93507764224,"internally_reserved":0,"allocated":1628545024,"data_stored":1627051875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":858128384},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":1.732,"15min":1.6699999999999999},"min":{"1min":0.33500000000000002,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":7.7450000000000001,"5min":20.023,"15min":82.280000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.077,"5min":2.0640000000000001,"15min":1.8680000000000001},"min":{"1min":0.29299999999999998,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.6710000000000003,"5min":19.853000000000002,"15min":85.236000000000004},"last":0.52600000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5619999999999998,"5min":2.5659999999999998,"15min":4.6150000000000002},"min":{"1min":0.42799999999999999,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":8.048,"5min":12.032999999999999,"15min":233.77099999999999},"last":3.0419999999999998},{"interface":"front","average":{"1min":1.966,"5min":2.028,"15min":4.3479999999999999},"min":{"1min":0.32300000000000001,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":7.8520000000000003,"5min":12.016,"15min":233.75200000000001},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":2.1110000000000002,"15min":3.7080000000000002},"min":{"1min":0.37,"5min":0.251,"15min":0.251},"max":{"1min":7.8220000000000001,"5min":20.411999999999999,"15min":231.34700000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.1859999999999999,"15min":3.9009999999999998},"min":{"1min":0.38600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.0439999999999996,"5min":11.509,"15min":231.381},"last":0.503}]},{"osd":3,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":2.5710000000000002,"15min":4.0469999999999997},"min":{"1min":0.40500000000000003,"5min":0.317,"15min":0.27700000000000002},"max":{"1min":8.077,"5min":11.597,"15min":263.887},"last":3.073},{"interface":"front","average":{"1min":2.004,"5min":1.9910000000000001,"15min":3.831},"min":{"1min":0.46100000000000002,"5min":0.32200000000000001,"15min":0.30099999999999999},"max":{"1min":7.8810000000000002,"5min":10.638,"15min":263.70100000000002},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":1.595,"15min":0.97599999999999998},"min":{"1min":0.439,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":11.593999999999999,"5min":11.593999999999999,"15min":12.739000000000001},"last":1.5349999999999999},{"interface":"front","average":{"1min":3.7040000000000002,"5min":1.3839999999999999,"15min":1.026},"min":{"1min":0.505,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":10.259,"5min":10.259,"15min":10.792},"last":1.587}]},{"osd":6,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9009999999999998,"5min":1.6739999999999999,"15min":1.7130000000000001},"min":{"1min":0.38500000000000001,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":7.8520000000000003,"5min":8.9130000000000003,"15min":83.451999999999998},"last":1.5660000000000001},{"interface":"front","average":{"1min":3.0379999999999998,"5min":2.2669999999999999,"15min":1.9339999999999999},"min":{"1min":0.58499999999999996,"5min":0.30399999999999999,"15min":0.27400000000000002},"max":{"1min":8.0050000000000008,"5min":12.725,"15min":83.484999999999999},"last":5.3570000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:16:28 2023","interfaces":[{"interface":"back","average":{"1min":3.8599999999999999,"5min":1.7529999999999999,"15min":1.2529999999999999},"min":{"1min":0.56399999999999995,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":3.7789999999999999},{"interface":"front","average":{"1min":2.5470000000000002,"5min":1.3,"15min":0.874},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":11.640000000000001,"5min":11.640000000000001,"15min":11.640000000000001},"last":1.506}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1180827648,"data_stored":1180827648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1357053952,"data_stored":1357053952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1456476160,"data_stored":1456472088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1604063232,"data_stored":1604059160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1558052864,"data_stored":1558052864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1103106048,"data_stored":1103101976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:17:28.622 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:28.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:28.715 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:28.715 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:28.727 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:28.785 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:28.912 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:28.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:28.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:28.998 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:17:29.025 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:17:29.032 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:29.049 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:17:29.074 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:17:29.110 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:17:29.137 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:29.141 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:17:29.159 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:17:29.172 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:17:29.208 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:17:29.209 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.209 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.209 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.210 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.210 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.210 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.210 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.210 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:17:29.247 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:17:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:17:29.247 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:17:29.248 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:17:29.248 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:17:29.248 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:29.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:29.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:29.358 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:29.395 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:29.453 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1956356487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:29.453 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:29 smithi123 ceph-mon[193903]: pgmap v1732: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.4 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 857 op/s; 19241/71073 objects degraded (27.072%); 3.3 MiB/s, 172 objects/s recovering 2023-09-23T21:17:29.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1956356487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:29 smithi121 ceph-mon[211526]: pgmap v1732: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.4 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 857 op/s; 19241/71073 objects degraded (27.072%); 3.3 MiB/s, 172 objects/s recovering 2023-09-23T21:17:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1956356487' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:29 smithi121 ceph-mon[219709]: pgmap v1732: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.4 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 857 op/s; 19241/71073 objects degraded (27.072%); 3.3 MiB/s, 172 objects/s recovering 2023-09-23T21:17:30.128 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:30.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:30.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:30.233 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:30.302 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:30.445 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:30.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:30.559 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:30.637 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:30.670 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:17:30.715 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:30.818 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:30.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:30.933 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:30.972 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:31.196 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:31.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:31.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:31.301 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:31.389 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:31.481 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:31.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:31.587 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:31.614 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:17:31.615 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:31.749 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:31.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:31.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:31.871 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:31.901 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:31.949 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:31.968 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:31.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:32.076 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:32.106 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:32.138 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:32.320 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:32.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:32.425 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:32.463 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:32.512 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:32 smithi121 ceph-mon[211526]: pgmap v1733: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 858 op/s; 19241/69858 objects degraded (27.543%); 3.3 MiB/s, 172 objects/s recovering 2023-09-23T21:17:32.513 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 19241/69858 objects degraded (27.543%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:32.513 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:32 smithi121 ceph-mon[219709]: pgmap v1733: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 858 op/s; 19241/69858 objects degraded (27.543%); 3.3 MiB/s, 172 objects/s recovering 2023-09-23T21:17:32.513 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 19241/69858 objects degraded (27.543%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:32.523 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:32.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:32.621 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:17:32.628 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:32.700 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:32 smithi123 ceph-mon[193903]: pgmap v1733: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 858 op/s; 19241/69858 objects degraded (27.543%); 3.3 MiB/s, 172 objects/s recovering 2023-09-23T21:17:32.700 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 19241/69858 objects degraded (27.543%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:32.715 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:32.866 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:32.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:32.977 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:33.007 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:33.411 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1234779000' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:33.425 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:17:33.617 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:33.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:33.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1234779000' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:33.722 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:33.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1234779000' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:33.801 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:33.934 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:33.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:33.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:34.040 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:34.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[211526]: pgmap v1734: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 858 op/s; 19241/65862 objects degraded (29.214%); 2.9 MiB/s, 165 objects/s recovering 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[211526]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1234779000' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[211526]: osdmap e555: 8 total, 8 up, 5 in 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[211526]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[211526]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[219709]: pgmap v1734: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 858 op/s; 19241/65862 objects degraded (29.214%); 2.9 MiB/s, 165 objects/s recovering 2023-09-23T21:17:34.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:34.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[219709]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1234779000' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:34.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[219709]: osdmap e555: 8 total, 8 up, 5 in 2023-09-23T21:17:34.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[219709]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:34 smithi121 ceph-mon[219709]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:34 smithi123 ceph-mon[193903]: pgmap v1734: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 858 op/s; 19241/65862 objects degraded (29.214%); 2.9 MiB/s, 165 objects/s recovering 2023-09-23T21:17:34.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:34 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:17:34.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:34 smithi123 ceph-mon[193903]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:34 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1234779000' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:34.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:34 smithi123 ceph-mon[193903]: osdmap e555: 8 total, 8 up, 5 in 2023-09-23T21:17:34.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:34 smithi123 ceph-mon[193903]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:34 smithi123 ceph-mon[193903]: failed to encode map e555 with expected crc 2023-09-23T21:17:34.785 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:34.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:34.893 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:34.922 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:35.158 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:17:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:17:34] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T21:17:35.370 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:35.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:35.474 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:35.571 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:35.634 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:17:35.634 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:17:35.634 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:17:35.634 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:17:35.635 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.635 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64073 12.7 12:ecfffd2a:::benchmark_data_smithi121_563784_object64072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:17:35.636 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.257742+0000", 2023-09-23T21:17:35.636 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 409.37408266300002, 2023-09-23T21:17:35.636 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32869984099999999, 2023-09-23T21:17:35.636 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.636 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.636 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.636 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64073 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257742+0000", 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.637 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257742+0000", 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257740+0000", 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:35.638 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257764+0000", 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3506000000000002e-05 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257777+0000", 2023-09-23T21:17:35.639 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3035e-05 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257784+0000", 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.374e-06 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257831+0000", 2023-09-23T21:17:35.640 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.638e-05 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.257948+0000", 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000117759 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.258072+0000", 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012374400000000001 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.641 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.264437+0000", 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0063651180000000003 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.264469+0000", 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1847000000000002e-05 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.642 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.586341+0000", 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.32187152099999999 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.586397+0000", 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6854999999999997e-05 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.586442+0000", 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4381000000000001e-05 2023-09-23T21:17:35.643 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99892 12.6 12:6ec6e1ac:::benchmark_data_smithi121_563784_object9489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 358.97803048999998, 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.75238461400000001, 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.644 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99892 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.645 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653794+0000", 2023-09-23T21:17:35.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653798+0000", 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3279999999999999e-06 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653806+0000", 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4330000000000004e-06 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653812+0000", 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.088e-06 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653898+0000", 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5995999999999995e-05 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.653908+0000", 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0057e-05 2023-09-23T21:17:35.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.388228+0000", 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.73431940200000001 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.388361+0000", 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013329600000000001 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.388479+0000", 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011790000000000001 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402136+0000", 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013657463 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402579+0000", 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00044286800000000002 2023-09-23T21:17:35.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406104+0000", 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035243060000000001 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406146+0000", 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2633000000000003e-05 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406179+0000", 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2461e-05 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99922 12.6 12:65825bec:::benchmark_data_smithi121_563784_object9519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.734372+0000", 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 358.89745294599999, 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.671888288, 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.652 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99922 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734372+0000", 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734372+0000", 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734371+0000", 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734377+0000", 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.122e-06 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734390+0000", 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3426e-05 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734396+0000", 2023-09-23T21:17:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1809999999999998e-06 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734451+0000", 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4364999999999999e-05 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.734464+0000", 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3217999999999999e-05 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.391809+0000", 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65734495199999998 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.391934+0000", 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000125165 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.392044+0000", 2023-09-23T21:17:35.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000109831 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402295+0000", 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010250815 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402607+0000", 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00031272899999999998 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406188+0000", 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035809829999999998 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406228+0000", 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9366999999999998e-05 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406260+0000", 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2237000000000003e-05 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99923 12.6 12:6ca689b1:::benchmark_data_smithi121_563784_object9520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.741432+0000", 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 358.89039305400001, 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.664912222, 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.660 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99923 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741432+0000", 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741432+0000", 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741429+0000", 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999976 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741442+0000", 2023-09-23T21:17:35.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2733e-05 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741468+0000", 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6463000000000001e-05 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741481+0000", 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2344000000000001e-05 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741551+0000", 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0297999999999997e-05 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.741567+0000", 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6226e-05 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.395439+0000", 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.65387172699999996 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.395545+0000", 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106679 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.395652+0000", 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010650999999999999 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402425+0000", 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0067725019999999997 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.402637+0000", 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000212621 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406273+0000", 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036357099999999999 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406312+0000", 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9579999999999997e-05 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406344+0000", 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1365000000000002e-05 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.667 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:99931 12.6 12:6cc7051a:::benchmark_data_smithi121_563784_object9528:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:36.785517+0000", 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 358.84630776699998, 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.62893938400000005, 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.668 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 99931 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785517+0000", 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785517+0000", 2023-09-23T21:17:35.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785514+0000", 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999971 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785526+0000", 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2447e-05 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785553+0000", 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6906999999999998e-05 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785567+0000", 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3652000000000001e-05 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785642+0000", 2023-09-23T21:17:35.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4760999999999998e-05 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:36.785657+0000", 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5546e-05 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.398524+0000", 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.61286635599999995 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.398647+0000", 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012347000000000001 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.398768+0000", 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121278 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406353+0000", 2023-09-23T21:17:35.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075846949999999998 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.406412+0000", 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8974000000000002e-05 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.414368+0000", 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079556490000000004 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.414418+0000", 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9959000000000001e-05 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:37.414456+0000", 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8525999999999997e-05 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:105582 12.6 12:6288e697:::benchmark_data_smithi121_563784_object15179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e446)", 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:45.643648+0000", 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 349.98817710100002, 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25724708899999998, 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 105582 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643648+0000", 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643648+0000", 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643646+0000", 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.643654+0000", 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3399999999999998e-06 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817186+0000", 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.173532038 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817220+0000", 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4078999999999997e-05 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817316+0000", 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6385999999999995e-05 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817436+0000", 2023-09-23T21:17:35.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119918 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.817510+0000", 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4039e-05 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.821079+0000", 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035684340000000001 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.821107+0000", 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8014999999999999e-05 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.900806+0000", 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.079699018999999996 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.900856+0000", 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0627999999999997e-05 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:45.900895+0000", 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8126000000000001e-05 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151138 12.6 12:653b9060:::benchmark_data_smithi121_563784_object60735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:35.682 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.188555+0000", 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 270.44326918100001, 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41179959799999999, 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151138 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188555+0000", 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188555+0000", 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188554+0000", 2023-09-23T21:17:35.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188560+0000", 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8250000000000001e-06 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188572+0000", 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1725e-05 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188579+0000", 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7370000000000006e-06 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193786+0000", 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052063190000000001 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193898+0000", 2023-09-23T21:17:35.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011223899999999999 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193995+0000", 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6858e-05 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.599986+0000", 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.40599103199999997 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600200+0000", 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000213787 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600259+0000", 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9061999999999998e-05 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600317+0000", 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7729999999999998e-05 2023-09-23T21:17:35.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600355+0000", 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8516999999999997e-05 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151146 12.7 12:ee3dd0cc:::benchmark_data_smithi121_563784_object60743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:17:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 270.44322561500002, 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.427544013, 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151146 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188602+0000", 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.971e-06 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188609+0000", 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1029999999999996e-06 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188615+0000", 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9649999999999996e-06 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193889+0000", 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00527473 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.194011+0000", 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121808 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.194126+0000", 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011511 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202346+0000", 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0082199960000000002 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202466+0000", 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119311 2023-09-23T21:17:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600412+0000", 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39794612699999998 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.615972+0000", 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015560289 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.616026+0000", 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4135999999999998e-05 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.616143+0000", 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116902 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151147 12.d 12:b1facac4:::benchmark_data_smithi121_563784_object60744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 270.443201617, 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41190085500000001, 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151147 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188625+0000", 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5409999999999999e-06 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188631+0000", 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1040000000000003e-06 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188636+0000", 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0479999999999998e-06 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.189097+0000", 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000460932 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.189209+0000", 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011203499999999999 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.189324+0000", 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011435299999999999 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202379+0000", 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013055628 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202388+0000", 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9260000000000006e-06 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600174+0000", 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.397786272 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600358+0000", 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000183852 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600407+0000", 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8649000000000001e-05 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600524+0000", 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116909 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPushReply(12.7 467/461 [PushReplyOp(12:e9f18e90:::benchmark_data_smithi121_563784_object80001:head)])", 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 270.42936418800002, 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.397904694, 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:17:35.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202464+0000", 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7620000000000001e-06 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202471+0000", 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7240000000000003e-06 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600012+0000", 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.397541533 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600067+0000", 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4707999999999999e-05 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600365+0000", 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00029816199999999998 2023-09-23T21:17:35.707 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:151146 12.7 e467/461 ondisk, result = 0)", 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.204794+0000", 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 270.42703069300001, 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39562049399999999, 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204794+0000", 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204794+0000", 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204793+0000", 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:17:35.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204799+0000", 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1059999999999996e-06 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.599985+0000", 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39518625200000002 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600002+0000", 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6297999999999999e-05 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600395+0000", 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039376 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600404+0000", 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3699999999999995e-06 2023-09-23T21:17:35.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600414+0000", 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0599e-05 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:151138 12.6 e467/444 ondisk, result = 0)", 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.206919+0000", 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 270.42490592199999, 2023-09-23T21:17:35.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39328686699999998, 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206919+0000", 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206919+0000", 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206918+0000", 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206924+0000", 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7509999999999997e-06 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600012+0000", 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39308785499999999 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600021+0000", 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3899999999999999e-06 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600187+0000", 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000165701 2023-09-23T21:17:35.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600195+0000", 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4360000000000002e-06 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600206+0000", 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0424999999999999e-05 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.716 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151879 12.7 12:ec8f3f31:::benchmark_data_smithi121_563784_object61476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.957213+0000", 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 268.674611583, 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27218313300000002, 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151879 2023-09-23T21:17:35.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957213+0000", 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957213+0000", 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957211+0000", 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957220+0000", 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2430000000000005e-06 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957233+0000", 2023-09-23T21:17:35.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3592e-05 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957242+0000", 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1509999999999993e-06 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962402+0000", 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051592410000000002 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962522+0000", 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962634+0000", 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112538 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967748+0000", 2023-09-23T21:17:35.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051140680000000003 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.971176+0000", 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003427491 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.972496+0000", 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001320203 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229210+0000", 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25671398000000001 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229277+0000", 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6853999999999994e-05 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229396+0000", 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119454 2023-09-23T21:17:35.723 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151880 12.6 12:66a3ff02:::benchmark_data_smithi121_563784_object61477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.957616+0000", 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 268.67420860700003, 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27141181800000003, 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151880 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957616+0000", 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957616+0000", 2023-09-23T21:17:35.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957615+0000", 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957619+0000", 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0709999999999996e-06 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957628+0000", 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7819999999999999e-06 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957635+0000", 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7349999999999999e-06 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957683+0000", 2023-09-23T21:17:35.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8510999999999997e-05 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957798+0000", 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114957 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957904+0000", 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010575500000000001 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962235+0000", 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0043305610000000001 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967416+0000", 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051811959999999999 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228948+0000", 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.261531658 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228992+0000", 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4322e-05 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229028+0000", 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5941e-05 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.730 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151882 12.d 12:bc206b0d:::benchmark_data_smithi121_563784_object61479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.962177+0000", 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 268.66964733100002, 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27157668299999999, 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.731 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151882 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962177+0000", 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962177+0000", 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962175+0000", 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962184+0000", 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6009999999999995e-06 2023-09-23T21:17:35.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962197+0000", 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3162e-05 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962207+0000", 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6590000000000002e-06 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962283+0000", 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6058999999999998e-05 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962438+0000", 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000154976 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962576+0000", 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000137601 2023-09-23T21:17:35.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.971496+0000", 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089202540000000007 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.976117+0000", 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046207180000000002 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.980884+0000", 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047674350000000004 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.233602+0000", 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25271828000000002 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.233647+0000", 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4326999999999998e-05 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.233754+0000", 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010741700000000001 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:151887 12.5 e467/426 12:a0b219a3:::benchmark_data_smithi121_563784_object61484:head v 467'9404, mlcod=467'9400)", 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.967400+0000", 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 268.664424967, 2023-09-23T21:17:35.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26153186699999997, 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967400+0000", 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967400+0000", 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967399+0000", 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967402+0000", 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7010000000000001e-06 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967407+0000", 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4880000000000001e-06 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967410+0000", 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.709e-06 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.976250+0000", 2023-09-23T21:17:35.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088392139999999998 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.976277+0000", 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7022e-05 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228893+0000", 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25261600000000001 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228932+0000", 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8902999999999997e-05 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151897 12.d 12:b8ef00ef:::benchmark_data_smithi121_563784_object61494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.982469+0000", 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 268.64935602899999, 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.263483784, 2023-09-23T21:17:35.743 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151897 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982469+0000", 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982469+0000", 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982467+0000", 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:35.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982474+0000", 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0690000000000004e-06 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228881+0000", 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24640680400000001 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228894+0000", 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3123e-05 2023-09-23T21:17:35.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228965+0000", 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1372999999999996e-05 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229134+0000", 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016843000000000001 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229277+0000", 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014363099999999999 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.238499+0000", 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092214940000000002 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.243619+0000", 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051197930000000001 2023-09-23T21:17:35.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.244134+0000", 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00051562099999999996 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.245785+0000", 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016509020000000001 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:35.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.245842+0000", 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6743000000000002e-05 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.245952+0000", 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110326 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:35.750 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:17:35.751 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:35.757 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:35.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:35.870 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:35.956 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:36.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:36.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:36.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:36.198 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:36.227 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:36.635 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:17:36.637 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:17:36.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:36 smithi121 ceph-mon[211526]: pgmap v1736: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.2 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 872 op/s; 18724/57918 objects degraded (32.328%); 968 KiB/s, 170 objects/s recovering 2023-09-23T21:17:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:36 smithi121 ceph-mon[219709]: pgmap v1736: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.2 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 872 op/s; 18724/57918 objects degraded (32.328%); 968 KiB/s, 170 objects/s recovering 2023-09-23T21:17:36.678 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:36.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:36.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:36 smithi123 ceph-mon[193903]: pgmap v1736: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.2 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 872 op/s; 18724/57918 objects degraded (32.328%); 968 KiB/s, 170 objects/s recovering 2023-09-23T21:17:36.783 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:36.816 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:37.287 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:17:37.477 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:37.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:37.578 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18724/57918 objects degraded (32.328%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:37.578 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18724/57918 objects degraded (32.328%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:37.581 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:37.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:17:37.648 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1736,"stamp":"2023-09-23T21:17:35.346618+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1264129976,"num_objects":19306,"num_object_clones":0,"num_object_copies":57918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18724,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19306,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":150949,"num_write_kb":4954830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30450,"num_bytes_recovered":1650797350,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":189964,"ondisk_log_size":189964,"up":327,"acting":312,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8999044,"kb_used_data":4732252,"kb_used_omap":7,"kb_used_meta":4266360,"kb_avail":459726716,"statfs":{"total":479975178240,"available":470760157184,"internally_reserved":0,"allocated":4845826048,"data_stored":4837156109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4368752721},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,2,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-567410688,"num_objects":-8658,"num_object_clones":0,"num_object_copies":-25974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-956,"num_objects_misplaced":-150,"num_objects_unfound":0,"num_objects_dirty":-8658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8658,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1690,"num_bytes_recovered":9830400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.919359"},"pg_stats":[{"pgid":"13.23","version":"555'5965","reported_seq":7874,"reported_epoch":555,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:33.549645+0000","last_change":"2023-09-23T21:16:00.853306+0000","last_active":"2023-09-23T21:17:33.549645+0000","last_peered":"2023-09-23T21:17:33.549645+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:00.848945+0000","last_became_peered":"2023-09-23T21:16:00.848945+0000","last_unstale":"2023-09-23T21:17:33.549645+0000","last_undegraded":"2023-09-23T21:16:00.815130+0000","last_fullsized":"2023-09-23T21:17:33.549645+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5965,"log_dups_size":0,"ondisk_log_size":5965,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18448384,"num_objects":282,"num_object_clones":0,"num_object_copies":846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":124,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1704,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":876,"num_bytes_recovered":57409536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5,7","objects":282}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.22","version":"555'5836","reported_seq":7889,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.468930+0000","last_change":"2023-09-23T21:17:20.794596+0000","last_active":"2023-09-23T21:17:33.468930+0000","last_peered":"2023-09-23T21:17:33.468930+0000","last_clean":"2023-09-23T21:17:33.468930+0000","last_became_active":"2023-09-23T21:16:00.839299+0000","last_became_peered":"2023-09-23T21:16:00.839299+0000","last_unstale":"2023-09-23T21:17:33.468930+0000","last_undegraded":"2023-09-23T21:17:33.468930+0000","last_fullsized":"2023-09-23T21:17:33.468930+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5836,"log_dups_size":0,"ondisk_log_size":5836,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19857408,"num_objects":303,"num_object_clones":0,"num_object_copies":909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2597,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":962,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.21","version":"554'5711","reported_seq":6806,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:32.838257+0000","last_change":"2023-09-23T21:16:51.863113+0000","last_active":"2023-09-23T21:17:32.838257+0000","last_peered":"2023-09-23T21:17:32.838257+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858774+0000","last_became_peered":"2023-09-23T21:16:51.858774+0000","last_unstale":"2023-09-23T21:17:32.838257+0000","last_undegraded":"2023-09-23T21:16:51.798038+0000","last_fullsized":"2023-09-23T21:16:51.792078+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5711,"log_dups_size":0,"ondisk_log_size":5711,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21413888,"num_objects":327,"num_object_clones":0,"num_object_copies":981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":750,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2530,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":182},{"shards":"2,4,7","objects":145}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.20","version":"554'5856","reported_seq":8992,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.448070+0000","last_change":"2023-09-23T21:16:51.842022+0000","last_active":"2023-09-23T21:17:33.448070+0000","last_peered":"2023-09-23T21:17:33.448070+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834659+0000","last_became_peered":"2023-09-23T21:16:51.834659+0000","last_unstale":"2023-09-23T21:17:33.448070+0000","last_undegraded":"2023-09-23T21:16:51.797563+0000","last_fullsized":"2023-09-23T21:16:51.792808+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5856,"log_dups_size":0,"ondisk_log_size":5856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20119552,"num_objects":307,"num_object_clones":0,"num_object_copies":921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1733,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":32964608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":144},{"shards":"2,7","objects":163}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.14","version":"554'6925","reported_seq":9132,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.047716+0000","last_change":"2023-09-23T21:16:51.065826+0000","last_active":"2023-09-23T21:17:33.047716+0000","last_peered":"2023-09-23T21:17:33.047716+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:16:51.052190+0000","last_became_peered":"2023-09-23T21:16:51.052190+0000","last_unstale":"2023-09-23T21:17:33.047716+0000","last_undegraded":"2023-09-23T21:16:50.808861+0000","last_fullsized":"2023-09-23T21:16:50.789676+0000","mapping_epoch":544,"log_start":"494'1834","ondisk_log_start":"494'1834","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42729472,"num_objects":652,"num_object_clones":0,"num_object_copies":1956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5214,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":281},{"shards":"2,5","objects":371}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":156,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.504965+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:17:14.504965+0000","last_peered":"2023-09-23T21:17:14.504965+0000","last_clean":"2023-09-23T21:17:14.504965+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:17:14.504965+0000","last_undegraded":"2023-09-23T21:17:14.504965+0000","last_fullsized":"2023-09-23T21:17:14.504965+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.520433+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:17:14.520433+0000","last_peered":"2023-09-23T21:17:14.520433+0000","last_clean":"2023-09-23T21:17:14.520433+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:17:14.520433+0000","last_undegraded":"2023-09-23T21:17:14.520433+0000","last_fullsized":"2023-09-23T21:17:14.520433+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"555'6798","reported_seq":6910,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.447822+0000","last_change":"2023-09-23T21:16:51.036947+0000","last_active":"2023-09-23T21:17:33.447822+0000","last_peered":"2023-09-23T21:17:33.447822+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:16:51.020513+0000","last_became_peered":"2023-09-23T21:16:51.020513+0000","last_unstale":"2023-09-23T21:17:33.447822+0000","last_undegraded":"2023-09-23T21:16:50.811498+0000","last_fullsized":"2023-09-23T21:16:50.789456+0000","mapping_epoch":544,"log_start":"500'2322","ondisk_log_start":"500'2322","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4476,"log_dups_size":0,"ondisk_log_size":4476,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1586,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5155,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":304},{"shards":"4,7","objects":263}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":342,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.447791+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:17:25.447791+0000","last_peered":"2023-09-23T21:17:25.447791+0000","last_clean":"2023-09-23T21:17:25.447791+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:17:25.447791+0000","last_undegraded":"2023-09-23T21:17:25.447791+0000","last_fullsized":"2023-09-23T21:17:25.447791+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":202,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.443916+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:17:25.443916+0000","last_peered":"2023-09-23T21:17:25.443916+0000","last_clean":"2023-09-23T21:17:25.443916+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:17:25.443916+0000","last_undegraded":"2023-09-23T21:17:25.443916+0000","last_fullsized":"2023-09-23T21:17:25.443916+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"555'6829","reported_seq":10852,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.459588+0000","last_change":"2023-09-23T21:16:36.510340+0000","last_active":"2023-09-23T21:17:33.459588+0000","last_peered":"2023-09-23T21:17:33.459588+0000","last_clean":"2023-09-23T21:17:33.459588+0000","last_became_active":"2023-09-23T21:16:36.498646+0000","last_became_peered":"2023-09-23T21:16:36.498646+0000","last_unstale":"2023-09-23T21:17:33.459588+0000","last_undegraded":"2023-09-23T21:17:33.459588+0000","last_fullsized":"2023-09-23T21:17:33.459588+0000","mapping_epoch":537,"log_start":"503'3009","ondisk_log_start":"503'3009","created":507,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3820,"log_dups_size":0,"ondisk_log_size":3820,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38535168,"num_objects":588,"num_object_clones":0,"num_object_copies":1764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5168,"num_write_kb":184192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1956,"num_bytes_recovered":93519872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":213,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.504706+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:17:14.504706+0000","last_peered":"2023-09-23T21:17:14.504706+0000","last_clean":"2023-09-23T21:17:14.504706+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:17:14.504706+0000","last_undegraded":"2023-09-23T21:17:14.504706+0000","last_fullsized":"2023-09-23T21:17:14.504706+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.512642+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:17:14.512642+0000","last_peered":"2023-09-23T21:17:14.512642+0000","last_clean":"2023-09-23T21:17:14.512642+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:17:14.512642+0000","last_undegraded":"2023-09-23T21:17:14.512642+0000","last_fullsized":"2023-09-23T21:17:14.512642+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"555'6697","reported_seq":6816,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.436165+0000","last_change":"2023-09-23T21:16:51.074430+0000","last_active":"2023-09-23T21:17:33.436165+0000","last_peered":"2023-09-23T21:17:33.436165+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:16:51.054035+0000","last_became_peered":"2023-09-23T21:16:51.054035+0000","last_unstale":"2023-09-23T21:17:33.436165+0000","last_undegraded":"2023-09-23T21:16:50.838593+0000","last_fullsized":"2023-09-23T21:16:50.797592+0000","mapping_epoch":544,"log_start":"502'2792","ondisk_log_start":"502'2792","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3905,"log_dups_size":0,"ondisk_log_size":3905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37191680,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5062,"num_write_kb":180128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":258},{"shards":"2,5","objects":310}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":149,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.451520+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:17:25.451520+0000","last_peered":"2023-09-23T21:17:25.451520+0000","last_clean":"2023-09-23T21:17:25.451520+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:17:25.451520+0000","last_undegraded":"2023-09-23T21:17:25.451520+0000","last_fullsized":"2023-09-23T21:17:25.451520+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.511344+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:17:14.511344+0000","last_peered":"2023-09-23T21:17:14.511344+0000","last_clean":"2023-09-23T21:17:14.511344+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:17:14.511344+0000","last_undegraded":"2023-09-23T21:17:14.511344+0000","last_fullsized":"2023-09-23T21:17:14.511344+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.18","version":"554'6812","reported_seq":10129,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:32.836034+0000","last_change":"2023-09-23T21:17:12.466376+0000","last_active":"2023-09-23T21:17:32.836034+0000","last_peered":"2023-09-23T21:17:32.836034+0000","last_clean":"2023-09-23T21:17:32.836034+0000","last_became_active":"2023-09-23T21:17:12.454368+0000","last_became_peered":"2023-09-23T21:17:12.454368+0000","last_unstale":"2023-09-23T21:17:32.836034+0000","last_undegraded":"2023-09-23T21:17:32.836034+0000","last_fullsized":"2023-09-23T21:17:32.836034+0000","mapping_epoch":551,"log_start":"494'1702","ondisk_log_start":"494'1702","created":507,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":5110,"log_dups_size":0,"ondisk_log_size":5110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:01.792696+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37814272,"num_objects":577,"num_object_clones":0,"num_object_copies":1731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3299,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1575,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":208,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.450708+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:17:25.450708+0000","last_peered":"2023-09-23T21:17:25.450708+0000","last_clean":"2023-09-23T21:17:25.450708+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:17:25.450708+0000","last_undegraded":"2023-09-23T21:17:25.450708+0000","last_fullsized":"2023-09-23T21:17:25.450708+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":177,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.441408+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:17:25.441408+0000","last_peered":"2023-09-23T21:17:25.441408+0000","last_clean":"2023-09-23T21:17:25.441408+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:17:25.441408+0000","last_undegraded":"2023-09-23T21:17:25.441408+0000","last_fullsized":"2023-09-23T21:17:25.441408+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1b","version":"554'8575","reported_seq":13739,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.084545+0000","last_change":"2023-09-23T21:17:17.318827+0000","last_active":"2023-09-23T21:17:33.084545+0000","last_peered":"2023-09-23T21:17:33.084545+0000","last_clean":"2023-09-23T21:17:33.084545+0000","last_became_active":"2023-09-23T21:16:00.808957+0000","last_became_peered":"2023-09-23T21:16:00.808957+0000","last_unstale":"2023-09-23T21:17:33.084545+0000","last_undegraded":"2023-09-23T21:17:33.084545+0000","last_fullsized":"2023-09-23T21:17:33.084545+0000","mapping_epoch":525,"log_start":"497'2174","ondisk_log_start":"497'2174","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6401,"log_dups_size":0,"ondisk_log_size":6401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40665088,"num_objects":621,"num_object_clones":0,"num_object_copies":1863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5340,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133431296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":158,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.445158+0000","last_change":"2023-09-23T21:17:11.237055+0000","last_active":"2023-09-23T21:17:25.445158+0000","last_peered":"2023-09-23T21:17:25.445158+0000","last_clean":"2023-09-23T21:17:25.445158+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:17:25.445158+0000","last_undegraded":"2023-09-23T21:17:25.445158+0000","last_fullsized":"2023-09-23T21:17:25.445158+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.511669+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:17:14.511669+0000","last_peered":"2023-09-23T21:17:14.511669+0000","last_clean":"2023-09-23T21:17:14.511669+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:17:14.511669+0000","last_undegraded":"2023-09-23T21:17:14.511669+0000","last_fullsized":"2023-09-23T21:17:14.511669+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1a","version":"554'8223","reported_seq":10351,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.012677+0000","last_change":"2023-09-23T21:16:00.812483+0000","last_active":"2023-09-23T21:17:33.012677+0000","last_peered":"2023-09-23T21:17:33.012677+0000","last_clean":"2023-09-23T21:17:33.012677+0000","last_became_active":"2023-09-23T21:16:00.806979+0000","last_became_peered":"2023-09-23T21:16:00.806979+0000","last_unstale":"2023-09-23T21:17:33.012677+0000","last_undegraded":"2023-09-23T21:17:33.012677+0000","last_fullsized":"2023-09-23T21:17:33.012677+0000","mapping_epoch":525,"log_start":"493'1608","ondisk_log_start":"493'1608","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39452672,"num_objects":602,"num_object_clones":0,"num_object_copies":1806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5111,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":182,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442312+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:17:25.442312+0000","last_peered":"2023-09-23T21:17:25.442312+0000","last_clean":"2023-09-23T21:17:25.442312+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:17:25.442312+0000","last_undegraded":"2023-09-23T21:17:25.442312+0000","last_fullsized":"2023-09-23T21:17:25.442312+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":483,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.447472+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:17:25.447472+0000","last_peered":"2023-09-23T21:17:25.447472+0000","last_clean":"2023-09-23T21:17:25.447472+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:17:25.447472+0000","last_undegraded":"2023-09-23T21:17:25.447472+0000","last_fullsized":"2023-09-23T21:17:25.447472+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1d","version":"554'8416","reported_seq":14507,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.397921+0000","last_change":"2023-09-23T21:17:20.608316+0000","last_active":"2023-09-23T21:17:33.397921+0000","last_peered":"2023-09-23T21:17:33.397921+0000","last_clean":"2023-09-23T21:17:33.397921+0000","last_became_active":"2023-09-23T21:16:03.773235+0000","last_became_peered":"2023-09-23T21:16:03.773235+0000","last_unstale":"2023-09-23T21:17:33.397921+0000","last_undegraded":"2023-09-23T21:17:33.397921+0000","last_fullsized":"2023-09-23T21:17:33.397921+0000","mapping_epoch":528,"log_start":"502'2984","ondisk_log_start":"502'2984","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40337420,"num_objects":616,"num_object_clones":0,"num_object_copies":1848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":616,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5239,"num_write_kb":187329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":146276352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":138,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.445281+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:17:25.445281+0000","last_peered":"2023-09-23T21:17:25.445281+0000","last_clean":"2023-09-23T21:17:25.445281+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:17:25.445281+0000","last_undegraded":"2023-09-23T21:17:25.445281+0000","last_fullsized":"2023-09-23T21:17:25.445281+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":206,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442185+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:17:25.442185+0000","last_peered":"2023-09-23T21:17:25.442185+0000","last_clean":"2023-09-23T21:17:25.442185+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:17:25.442185+0000","last_undegraded":"2023-09-23T21:17:25.442185+0000","last_fullsized":"2023-09-23T21:17:25.442185+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1c","version":"554'8438","reported_seq":10560,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.436325+0000","last_change":"2023-09-23T21:16:00.824659+0000","last_active":"2023-09-23T21:17:33.436325+0000","last_peered":"2023-09-23T21:17:33.436325+0000","last_clean":"2023-09-23T21:17:33.436325+0000","last_became_active":"2023-09-23T21:16:00.820619+0000","last_became_peered":"2023-09-23T21:16:00.820619+0000","last_unstale":"2023-09-23T21:17:33.436325+0000","last_undegraded":"2023-09-23T21:17:33.436325+0000","last_fullsized":"2023-09-23T21:17:33.436325+0000","mapping_epoch":525,"log_start":"505'3231","ondisk_log_start":"505'3231","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5207,"log_dups_size":0,"ondisk_log_size":5207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40927232,"num_objects":625,"num_object_clones":0,"num_object_copies":1875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5254,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":134,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442343+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:17:25.442343+0000","last_peered":"2023-09-23T21:17:25.442343+0000","last_clean":"2023-09-23T21:17:25.442343+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:17:25.442343+0000","last_undegraded":"2023-09-23T21:17:25.442343+0000","last_fullsized":"2023-09-23T21:17:25.442343+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":142,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.445424+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:17:25.445424+0000","last_peered":"2023-09-23T21:17:25.445424+0000","last_clean":"2023-09-23T21:17:25.445424+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:17:25.445424+0000","last_undegraded":"2023-09-23T21:17:25.445424+0000","last_fullsized":"2023-09-23T21:17:25.445424+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1f","version":"554'8280","reported_seq":8392,"reported_epoch":554,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:31.242705+0000","last_change":"2023-09-23T21:16:51.848275+0000","last_active":"2023-09-23T21:17:31.242705+0000","last_peered":"2023-09-23T21:17:31.242705+0000","last_clean":"2023-09-23T21:15:36.326799+0000","last_became_active":"2023-09-23T21:16:51.820010+0000","last_became_peered":"2023-09-23T21:16:51.820010+0000","last_unstale":"2023-09-23T21:17:31.242705+0000","last_undegraded":"2023-09-23T21:17:31.242705+0000","last_fullsized":"2023-09-23T21:17:31.242705+0000","mapping_epoch":545,"log_start":"493'1725","ondisk_log_start":"493'1725","created":525,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6555,"log_dups_size":0,"ondisk_log_size":6555,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41910272,"num_objects":640,"num_object_clones":0,"num_object_copies":1920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5150,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":["6","2","5","7"],"object_location_counts":[{"shards":"2,6,7","objects":640}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":132,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.441898+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:17:25.441898+0000","last_peered":"2023-09-23T21:17:25.441898+0000","last_clean":"2023-09-23T21:17:25.441898+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:17:25.441898+0000","last_undegraded":"2023-09-23T21:17:25.441898+0000","last_fullsized":"2023-09-23T21:17:25.441898+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":156,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.443837+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:17:25.443837+0000","last_peered":"2023-09-23T21:17:25.443837+0000","last_clean":"2023-09-23T21:17:25.443837+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:17:25.443837+0000","last_undegraded":"2023-09-23T21:17:25.443837+0000","last_fullsized":"2023-09-23T21:17:25.443837+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1e","version":"554'8201","reported_seq":8295,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.372627+0000","last_change":"2023-09-23T21:17:20.608819+0000","last_active":"2023-09-23T21:17:33.372627+0000","last_peered":"2023-09-23T21:17:33.372627+0000","last_clean":"2023-09-23T21:17:33.372627+0000","last_became_active":"2023-09-23T21:16:00.844392+0000","last_became_peered":"2023-09-23T21:16:00.844392+0000","last_unstale":"2023-09-23T21:17:33.372627+0000","last_undegraded":"2023-09-23T21:17:33.372627+0000","last_fullsized":"2023-09-23T21:17:33.372627+0000","mapping_epoch":525,"log_start":"497'2095","ondisk_log_start":"497'2095","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6106,"log_dups_size":0,"ondisk_log_size":6106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40140800,"num_objects":613,"num_object_clones":0,"num_object_copies":1839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5110,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":590,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.512192+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:17:14.512192+0000","last_peered":"2023-09-23T21:17:14.512192+0000","last_clean":"2023-09-23T21:17:14.512192+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:17:14.512192+0000","last_undegraded":"2023-09-23T21:17:14.512192+0000","last_fullsized":"2023-09-23T21:17:14.512192+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":206,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.444802+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:17:25.444802+0000","last_peered":"2023-09-23T21:17:25.444802+0000","last_clean":"2023-09-23T21:17:25.444802+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:17:25.444802+0000","last_undegraded":"2023-09-23T21:17:25.444802+0000","last_fullsized":"2023-09-23T21:17:25.444802+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"554'5772","reported_seq":6910,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.078744+0000","last_change":"2023-09-23T21:16:51.863144+0000","last_active":"2023-09-23T21:17:33.078744+0000","last_peered":"2023-09-23T21:17:33.078744+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858843+0000","last_became_peered":"2023-09-23T21:16:51.858843+0000","last_unstale":"2023-09-23T21:17:33.078744+0000","last_undegraded":"2023-09-23T21:16:51.799178+0000","last_fullsized":"2023-09-23T21:16:51.792321+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17416192,"num_objects":265,"num_object_clones":0,"num_object_copies":795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2590,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":511,"num_bytes_recovered":33488896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":96},{"shards":"2,4,7","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":136,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.444827+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:17:25.444827+0000","last_peered":"2023-09-23T21:17:25.444827+0000","last_clean":"2023-09-23T21:17:25.444827+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:17:25.444827+0000","last_undegraded":"2023-09-23T21:17:25.444827+0000","last_fullsized":"2023-09-23T21:17:25.444827+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":208,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.450272+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:17:25.450272+0000","last_peered":"2023-09-23T21:17:25.450272+0000","last_clean":"2023-09-23T21:17:25.450272+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:17:25.450272+0000","last_undegraded":"2023-09-23T21:17:25.450272+0000","last_fullsized":"2023-09-23T21:17:25.450272+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"554'5858","reported_seq":8970,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.377988+0000","last_change":"2023-09-23T21:16:51.842306+0000","last_active":"2023-09-23T21:17:33.377988+0000","last_peered":"2023-09-23T21:17:33.377988+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834848+0000","last_became_peered":"2023-09-23T21:16:51.834848+0000","last_unstale":"2023-09-23T21:17:33.377988+0000","last_undegraded":"2023-09-23T21:16:51.803926+0000","last_fullsized":"2023-09-23T21:16:51.799106+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19988480,"num_objects":305,"num_object_clones":0,"num_object_copies":915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1735,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32178176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":161},{"shards":"2,7","objects":144}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":150,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.443997+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:17:25.443997+0000","last_peered":"2023-09-23T21:17:25.443997+0000","last_clean":"2023-09-23T21:17:25.443997+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:17:25.443997+0000","last_undegraded":"2023-09-23T21:17:25.443997+0000","last_fullsized":"2023-09-23T21:17:25.443997+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.514154+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:17:14.514154+0000","last_peered":"2023-09-23T21:17:14.514154+0000","last_clean":"2023-09-23T21:17:14.514154+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:17:14.514154+0000","last_undegraded":"2023-09-23T21:17:14.514154+0000","last_fullsized":"2023-09-23T21:17:14.514154+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3","version":"555'5956","reported_seq":7874,"reported_epoch":555,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:33.472650+0000","last_change":"2023-09-23T21:16:01.735614+0000","last_active":"2023-09-23T21:17:33.472650+0000","last_peered":"2023-09-23T21:17:33.472650+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:01.731847+0000","last_became_peered":"2023-09-23T21:16:01.731847+0000","last_unstale":"2023-09-23T21:17:33.472650+0000","last_undegraded":"2023-09-23T21:16:01.699708+0000","last_fullsized":"2023-09-23T21:17:33.472650+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":5956,"log_dups_size":0,"ondisk_log_size":5956,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19038208,"num_objects":290,"num_object_clones":0,"num_object_copies":870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":143,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1695,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":879,"num_bytes_recovered":57606144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5,7","objects":290}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":136,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.445545+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:17:25.445545+0000","last_peered":"2023-09-23T21:17:25.445545+0000","last_clean":"2023-09-23T21:17:25.445545+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:17:25.445545+0000","last_undegraded":"2023-09-23T21:17:25.445545+0000","last_fullsized":"2023-09-23T21:17:25.445545+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.512083+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:17:14.512083+0000","last_peered":"2023-09-23T21:17:14.512083+0000","last_clean":"2023-09-23T21:17:14.512083+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:17:14.512083+0000","last_undegraded":"2023-09-23T21:17:14.512083+0000","last_fullsized":"2023-09-23T21:17:14.512083+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"555'5836","reported_seq":7612,"reported_epoch":555,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:33.443989+0000","last_change":"2023-09-23T21:17:11.228334+0000","last_active":"2023-09-23T21:17:33.443989+0000","last_peered":"2023-09-23T21:17:33.443989+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:01.742399+0000","last_became_peered":"2023-09-23T21:16:01.742399+0000","last_unstale":"2023-09-23T21:17:33.443989+0000","last_undegraded":"2023-09-23T21:17:11.225179+0000","last_fullsized":"2023-09-23T21:17:33.443989+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5836,"log_dups_size":0,"ondisk_log_size":5836,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19857408,"num_objects":303,"num_object_clones":0,"num_object_copies":909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":133,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2597,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":54001664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5,6","objects":303}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.512368+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:17:14.512368+0000","last_peered":"2023-09-23T21:17:14.512368+0000","last_clean":"2023-09-23T21:17:14.512368+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:17:14.512368+0000","last_undegraded":"2023-09-23T21:17:14.512368+0000","last_fullsized":"2023-09-23T21:17:14.512368+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.511866+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:17:14.511866+0000","last_peered":"2023-09-23T21:17:14.511866+0000","last_clean":"2023-09-23T21:17:14.511866+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:17:14.511866+0000","last_undegraded":"2023-09-23T21:17:14.511866+0000","last_fullsized":"2023-09-23T21:17:14.511866+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"555'6833","reported_seq":8865,"reported_epoch":555,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.535692+0000","last_change":"2023-09-23T21:17:20.815638+0000","last_active":"2023-09-23T21:17:33.535692+0000","last_peered":"2023-09-23T21:17:33.535692+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:16:51.032827+0000","last_became_peered":"2023-09-23T21:16:51.032827+0000","last_unstale":"2023-09-23T21:17:33.535692+0000","last_undegraded":"2023-09-23T21:16:50.886600+0000","last_fullsized":"2023-09-23T21:16:50.791765+0000","mapping_epoch":544,"log_start":"502'2930","ondisk_log_start":"502'2930","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":3903,"log_dups_size":0,"ondisk_log_size":3903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:01.438609+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38404096,"num_objects":586,"num_object_clones":0,"num_object_copies":1758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":705,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3277,"num_write_kb":64384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":909,"num_bytes_recovered":21233664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5","objects":-119}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":222,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.445129+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:17:25.445129+0000","last_peered":"2023-09-23T21:17:25.445129+0000","last_clean":"2023-09-23T21:17:25.445129+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:17:25.445129+0000","last_undegraded":"2023-09-23T21:17:25.445129+0000","last_fullsized":"2023-09-23T21:17:25.445129+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.512513+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:17:14.512513+0000","last_peered":"2023-09-23T21:17:14.512513+0000","last_clean":"2023-09-23T21:17:14.512513+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:17:14.512513+0000","last_undegraded":"2023-09-23T21:17:14.512513+0000","last_fullsized":"2023-09-23T21:17:14.512513+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.4","version":"554'6896","reported_seq":9195,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.118654+0000","last_change":"2023-09-23T21:16:51.065805+0000","last_active":"2023-09-23T21:17:33.118654+0000","last_peered":"2023-09-23T21:17:33.118654+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:16:51.051146+0000","last_became_peered":"2023-09-23T21:16:51.051146+0000","last_unstale":"2023-09-23T21:17:33.118654+0000","last_undegraded":"2023-09-23T21:16:50.809981+0000","last_fullsized":"2023-09-23T21:16:50.790363+0000","mapping_epoch":544,"log_start":"493'1813","ondisk_log_start":"493'1813","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:23.280602+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35127296,"num_objects":536,"num_object_clones":0,"num_object_copies":1608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3196,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":242},{"shards":"2,5","objects":294}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.511756+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:17:14.511756+0000","last_peered":"2023-09-23T21:17:14.511756+0000","last_clean":"2023-09-23T21:17:14.511756+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:17:14.511756+0000","last_undegraded":"2023-09-23T21:17:14.511756+0000","last_fullsized":"2023-09-23T21:17:14.511756+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":596,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506401+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:17:14.506401+0000","last_peered":"2023-09-23T21:17:14.506401+0000","last_clean":"2023-09-23T21:17:14.506401+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:17:14.506401+0000","last_undegraded":"2023-09-23T21:17:14.506401+0000","last_fullsized":"2023-09-23T21:17:14.506401+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"554'6741","reported_seq":6858,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.417431+0000","last_change":"2023-09-23T21:16:51.082314+0000","last_active":"2023-09-23T21:17:33.417431+0000","last_peered":"2023-09-23T21:17:33.417431+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:16:51.064800+0000","last_became_peered":"2023-09-23T21:16:51.064800+0000","last_unstale":"2023-09-23T21:17:33.417431+0000","last_undegraded":"2023-09-23T21:16:50.820535+0000","last_fullsized":"2023-09-23T21:16:50.800952+0000","mapping_epoch":544,"log_start":"498'2178","ondisk_log_start":"498'2178","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4563,"log_dups_size":0,"ondisk_log_size":4563,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40370176,"num_objects":616,"num_object_clones":0,"num_object_copies":1848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5098,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":295},{"shards":"4,7","objects":321}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.448024+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:17:25.448024+0000","last_peered":"2023-09-23T21:17:25.448024+0000","last_clean":"2023-09-23T21:17:25.448024+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:17:25.448024+0000","last_undegraded":"2023-09-23T21:17:25.448024+0000","last_fullsized":"2023-09-23T21:17:25.448024+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.515162+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:17:14.515162+0000","last_peered":"2023-09-23T21:17:14.515162+0000","last_clean":"2023-09-23T21:17:14.515162+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:17:14.515162+0000","last_undegraded":"2023-09-23T21:17:14.515162+0000","last_fullsized":"2023-09-23T21:17:14.515162+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"555'6785","reported_seq":9383,"reported_epoch":555,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.553142+0000","last_change":"2023-09-23T21:17:20.609014+0000","last_active":"2023-09-23T21:17:33.553142+0000","last_peered":"2023-09-23T21:17:33.553142+0000","last_clean":"2023-09-23T21:15:09.799249+0000","last_became_active":"2023-09-23T21:15:12.799540+0000","last_became_peered":"2023-09-23T21:15:12.799540+0000","last_unstale":"2023-09-23T21:17:33.553142+0000","last_undegraded":"2023-09-23T21:15:36.785930+0000","last_fullsized":"2023-09-23T21:15:12.679693+0000","mapping_epoch":514,"log_start":"503'2934","ondisk_log_start":"503'2934","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":3851,"log_dups_size":0,"ondisk_log_size":3851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:59.312702+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36503552,"num_objects":557,"num_object_clones":0,"num_object_copies":1671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":744,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3172,"num_write_kb":57088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1201,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5","objects":-187}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":206,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442690+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:17:25.442690+0000","last_peered":"2023-09-23T21:17:25.442690+0000","last_clean":"2023-09-23T21:17:25.442690+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:17:25.442690+0000","last_undegraded":"2023-09-23T21:17:25.442690+0000","last_fullsized":"2023-09-23T21:17:25.442690+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":294,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.443933+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:17:25.443933+0000","last_peered":"2023-09-23T21:17:25.443933+0000","last_clean":"2023-09-23T21:17:25.443933+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:17:25.443933+0000","last_undegraded":"2023-09-23T21:17:25.443933+0000","last_fullsized":"2023-09-23T21:17:25.443933+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"554'8530","reported_seq":13586,"reported_epoch":554,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:33.074423+0000","last_change":"2023-09-23T21:17:21.110094+0000","last_active":"2023-09-23T21:17:33.074423+0000","last_peered":"2023-09-23T21:17:33.074423+0000","last_clean":"2023-09-23T21:15:08.158120+0000","last_became_active":"2023-09-23T21:16:01.723118+0000","last_became_peered":"2023-09-23T21:16:01.723118+0000","last_unstale":"2023-09-23T21:17:33.074423+0000","last_undegraded":"2023-09-23T21:17:33.074423+0000","last_fullsized":"2023-09-23T21:17:33.074423+0000","mapping_epoch":526,"log_start":"497'1937","ondisk_log_start":"497'1937","created":482,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43614208,"num_objects":665,"num_object_clones":0,"num_object_copies":1995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5294,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":129957888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":["2","3","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":665}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":479,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442231+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:17:25.442231+0000","last_peered":"2023-09-23T21:17:25.442231+0000","last_clean":"2023-09-23T21:17:25.442231+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:17:25.442231+0000","last_undegraded":"2023-09-23T21:17:25.442231+0000","last_fullsized":"2023-09-23T21:17:25.442231+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":230,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.452007+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:17:25.452007+0000","last_peered":"2023-09-23T21:17:25.452007+0000","last_clean":"2023-09-23T21:17:25.452007+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:17:25.452007+0000","last_undegraded":"2023-09-23T21:17:25.452007+0000","last_fullsized":"2023-09-23T21:17:25.452007+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":104,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.450862+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:17:25.450862+0000","last_peered":"2023-09-23T21:17:25.450862+0000","last_clean":"2023-09-23T21:17:25.450862+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:17:25.450862+0000","last_undegraded":"2023-09-23T21:17:25.450862+0000","last_fullsized":"2023-09-23T21:17:25.450862+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"555'8468","reported_seq":14581,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.472625+0000","last_change":"2023-09-23T21:17:15.462587+0000","last_active":"2023-09-23T21:17:33.472625+0000","last_peered":"2023-09-23T21:17:33.472625+0000","last_clean":"2023-09-23T21:17:33.472625+0000","last_became_active":"2023-09-23T21:16:01.764537+0000","last_became_peered":"2023-09-23T21:16:01.764537+0000","last_unstale":"2023-09-23T21:17:33.472625+0000","last_undegraded":"2023-09-23T21:17:33.472625+0000","last_fullsized":"2023-09-23T21:17:33.472625+0000","mapping_epoch":526,"log_start":"505'3271","ondisk_log_start":"505'3271","created":482,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5197,"log_dups_size":0,"ondisk_log_size":5197,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36929548,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5291,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2487,"num_bytes_recovered":146014208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.511561+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:17:14.511561+0000","last_peered":"2023-09-23T21:17:14.511561+0000","last_clean":"2023-09-23T21:17:14.511561+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:17:14.511561+0000","last_undegraded":"2023-09-23T21:17:14.511561+0000","last_fullsized":"2023-09-23T21:17:14.511561+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.511738+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:17:14.511738+0000","last_peered":"2023-09-23T21:17:14.511738+0000","last_clean":"2023-09-23T21:17:14.511738+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:17:14.511738+0000","last_undegraded":"2023-09-23T21:17:14.511738+0000","last_fullsized":"2023-09-23T21:17:14.511738+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":268,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506170+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:17:14.506170+0000","last_peered":"2023-09-23T21:17:14.506170+0000","last_clean":"2023-09-23T21:17:14.506170+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:17:14.506170+0000","last_undegraded":"2023-09-23T21:17:14.506170+0000","last_fullsized":"2023-09-23T21:17:14.506170+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"555'8236","reported_seq":8326,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.471914+0000","last_change":"2023-09-23T21:16:00.841315+0000","last_active":"2023-09-23T21:17:33.471914+0000","last_peered":"2023-09-23T21:17:33.471914+0000","last_clean":"2023-09-23T21:17:33.471914+0000","last_became_active":"2023-09-23T21:16:00.829607+0000","last_became_peered":"2023-09-23T21:16:00.829607+0000","last_unstale":"2023-09-23T21:17:33.471914+0000","last_undegraded":"2023-09-23T21:17:33.471914+0000","last_fullsized":"2023-09-23T21:17:33.471914+0000","mapping_epoch":482,"log_start":"500'2428","ondisk_log_start":"500'2428","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5808,"log_dups_size":0,"ondisk_log_size":5808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37847040,"num_objects":577,"num_object_clones":0,"num_object_copies":1731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5144,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":562,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.505144+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:17:14.505144+0000","last_peered":"2023-09-23T21:17:14.505144+0000","last_clean":"2023-09-23T21:17:14.505144+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:17:14.505144+0000","last_undegraded":"2023-09-23T21:17:14.505144+0000","last_fullsized":"2023-09-23T21:17:14.505144+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":300,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.504543+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:17:14.504543+0000","last_peered":"2023-09-23T21:17:14.504543+0000","last_clean":"2023-09-23T21:17:14.504543+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:17:14.504543+0000","last_undegraded":"2023-09-23T21:17:14.504543+0000","last_fullsized":"2023-09-23T21:17:14.504543+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":208,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.444004+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:17:25.444004+0000","last_peered":"2023-09-23T21:17:25.444004+0000","last_clean":"2023-09-23T21:17:25.444004+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:17:25.444004+0000","last_undegraded":"2023-09-23T21:17:25.444004+0000","last_fullsized":"2023-09-23T21:17:25.444004+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"555'8456","reported_seq":10579,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.436835+0000","last_change":"2023-09-23T21:16:00.748968+0000","last_active":"2023-09-23T21:17:33.436835+0000","last_peered":"2023-09-23T21:17:33.436835+0000","last_clean":"2023-09-23T21:17:33.436835+0000","last_became_active":"2023-09-23T21:16:00.745123+0000","last_became_peered":"2023-09-23T21:16:00.745123+0000","last_unstale":"2023-09-23T21:17:33.436835+0000","last_undegraded":"2023-09-23T21:17:33.436835+0000","last_fullsized":"2023-09-23T21:17:33.436835+0000","mapping_epoch":501,"log_start":"503'3188","ondisk_log_start":"503'3188","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39747584,"num_objects":606,"num_object_clones":0,"num_object_copies":1818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5271,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":502,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":382,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506306+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:17:14.506306+0000","last_peered":"2023-09-23T21:17:14.506306+0000","last_clean":"2023-09-23T21:17:14.506306+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:17:14.506306+0000","last_undegraded":"2023-09-23T21:17:14.506306+0000","last_fullsized":"2023-09-23T21:17:14.506306+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":901,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442087+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:17:25.442087+0000","last_peered":"2023-09-23T21:17:25.442087+0000","last_clean":"2023-09-23T21:17:25.442087+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:17:25.442087+0000","last_undegraded":"2023-09-23T21:17:25.442087+0000","last_fullsized":"2023-09-23T21:17:25.442087+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.450462+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:17:25.450462+0000","last_peered":"2023-09-23T21:17:25.450462+0000","last_clean":"2023-09-23T21:17:25.450462+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:17:25.450462+0000","last_undegraded":"2023-09-23T21:17:25.450462+0000","last_fullsized":"2023-09-23T21:17:25.450462+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.520115+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:17:14.520115+0000","last_peered":"2023-09-23T21:17:14.520115+0000","last_clean":"2023-09-23T21:17:14.520115+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:17:14.520115+0000","last_undegraded":"2023-09-23T21:17:14.520115+0000","last_fullsized":"2023-09-23T21:17:14.520115+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"554'8234","reported_seq":8340,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:31.206472+0000","last_change":"2023-09-23T21:16:51.832676+0000","last_active":"2023-09-23T21:17:31.206472+0000","last_peered":"2023-09-23T21:17:31.206472+0000","last_clean":"2023-09-23T21:17:31.206472+0000","last_became_active":"2023-09-23T21:16:51.803067+0000","last_became_peered":"2023-09-23T21:16:51.803067+0000","last_unstale":"2023-09-23T21:17:31.206472+0000","last_undegraded":"2023-09-23T21:17:31.206472+0000","last_fullsized":"2023-09-23T21:17:31.206472+0000","mapping_epoch":545,"log_start":"493'1700","ondisk_log_start":"493'1700","created":482,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44924928,"num_objects":685,"num_object_clones":0,"num_object_copies":2055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5103,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":182,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442167+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:17:25.442167+0000","last_peered":"2023-09-23T21:17:25.442167+0000","last_clean":"2023-09-23T21:17:25.442167+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:17:25.442167+0000","last_undegraded":"2023-09-23T21:17:25.442167+0000","last_fullsized":"2023-09-23T21:17:25.442167+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.441326+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:17:25.441326+0000","last_peered":"2023-09-23T21:17:25.441326+0000","last_clean":"2023-09-23T21:17:25.441326+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:17:25.441326+0000","last_undegraded":"2023-09-23T21:17:25.441326+0000","last_fullsized":"2023-09-23T21:17:25.441326+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":463,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506867+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:17:14.506867+0000","last_peered":"2023-09-23T21:17:14.506867+0000","last_clean":"2023-09-23T21:17:14.506867+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:17:14.506867+0000","last_undegraded":"2023-09-23T21:17:14.506867+0000","last_fullsized":"2023-09-23T21:17:14.506867+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"554'8252","reported_seq":10381,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:32.994650+0000","last_change":"2023-09-23T21:16:00.763134+0000","last_active":"2023-09-23T21:17:32.994650+0000","last_peered":"2023-09-23T21:17:32.994650+0000","last_clean":"2023-09-23T21:17:32.994650+0000","last_became_active":"2023-09-23T21:16:00.758875+0000","last_became_peered":"2023-09-23T21:16:00.758875+0000","last_unstale":"2023-09-23T21:17:32.994650+0000","last_undegraded":"2023-09-23T21:17:32.994650+0000","last_fullsized":"2023-09-23T21:17:32.994650+0000","mapping_epoch":498,"log_start":"493'1741","ondisk_log_start":"493'1741","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6511,"log_dups_size":0,"ondisk_log_size":6511,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37552128,"num_objects":573,"num_object_clones":0,"num_object_copies":1719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5140,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":152,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.445291+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:17:25.445291+0000","last_peered":"2023-09-23T21:17:25.445291+0000","last_clean":"2023-09-23T21:17:25.445291+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:17:25.445291+0000","last_undegraded":"2023-09-23T21:17:25.445291+0000","last_fullsized":"2023-09-23T21:17:25.445291+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":367,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506245+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:17:14.506245+0000","last_peered":"2023-09-23T21:17:14.506245+0000","last_clean":"2023-09-23T21:17:14.506245+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:17:14.506245+0000","last_undegraded":"2023-09-23T21:17:14.506245+0000","last_fullsized":"2023-09-23T21:17:14.506245+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":260,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.452879+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:17:25.452879+0000","last_peered":"2023-09-23T21:17:25.452879+0000","last_clean":"2023-09-23T21:17:25.452879+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:17:25.452879+0000","last_undegraded":"2023-09-23T21:17:25.452879+0000","last_fullsized":"2023-09-23T21:17:25.452879+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"555'6789","reported_seq":6996,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.450103+0000","last_change":"2023-09-23T21:16:51.076997+0000","last_active":"2023-09-23T21:17:33.450103+0000","last_peered":"2023-09-23T21:17:33.450103+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:16:51.057102+0000","last_became_peered":"2023-09-23T21:16:51.057102+0000","last_unstale":"2023-09-23T21:17:33.450103+0000","last_undegraded":"2023-09-23T21:16:50.840109+0000","last_fullsized":"2023-09-23T21:16:50.799660+0000","mapping_epoch":544,"log_start":"503'3003","ondisk_log_start":"503'3003","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":3786,"log_dups_size":0,"ondisk_log_size":3786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.748037+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1565,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3227,"num_write_kb":60864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":315},{"shards":"2,5","objects":299}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":210,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.444892+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:17:25.444892+0000","last_peered":"2023-09-23T21:17:25.444892+0000","last_clean":"2023-09-23T21:17:25.444892+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:17:25.444892+0000","last_undegraded":"2023-09-23T21:17:25.444892+0000","last_fullsized":"2023-09-23T21:17:25.444892+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.515093+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:17:14.515093+0000","last_peered":"2023-09-23T21:17:14.515093+0000","last_clean":"2023-09-23T21:17:14.515093+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:17:14.515093+0000","last_undegraded":"2023-09-23T21:17:14.515093+0000","last_fullsized":"2023-09-23T21:17:14.515093+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":235,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.442006+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:17:25.442006+0000","last_peered":"2023-09-23T21:17:25.442006+0000","last_clean":"2023-09-23T21:17:25.442006+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:17:25.442006+0000","last_undegraded":"2023-09-23T21:17:25.442006+0000","last_fullsized":"2023-09-23T21:17:25.442006+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"554'6920","reported_seq":7074,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.126321+0000","last_change":"2023-09-23T21:16:51.063450+0000","last_active":"2023-09-23T21:17:33.126321+0000","last_peered":"2023-09-23T21:17:33.126321+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:16:51.051345+0000","last_became_peered":"2023-09-23T21:16:51.051345+0000","last_unstale":"2023-09-23T21:17:33.126321+0000","last_undegraded":"2023-09-23T21:16:50.807164+0000","last_fullsized":"2023-09-23T21:16:50.788756+0000","mapping_epoch":544,"log_start":"497'1917","ondisk_log_start":"497'1917","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:15.933191+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40828928,"num_objects":623,"num_object_clones":0,"num_object_copies":1869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1557,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3432,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":322},{"shards":"2,5,7","objects":301}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":460,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.512778+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:17:14.512778+0000","last_peered":"2023-09-23T21:17:14.512778+0000","last_clean":"2023-09-23T21:17:14.512778+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:17:14.512778+0000","last_undegraded":"2023-09-23T21:17:14.512778+0000","last_fullsized":"2023-09-23T21:17:14.512778+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":92,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.448839+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:17:25.448839+0000","last_peered":"2023-09-23T21:17:25.448839+0000","last_clean":"2023-09-23T21:17:25.448839+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:17:25.448839+0000","last_undegraded":"2023-09-23T21:17:25.448839+0000","last_fullsized":"2023-09-23T21:17:25.448839+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":240,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.450638+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:17:25.450638+0000","last_peered":"2023-09-23T21:17:25.450638+0000","last_clean":"2023-09-23T21:17:25.450638+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:17:25.450638+0000","last_undegraded":"2023-09-23T21:17:25.450638+0000","last_fullsized":"2023-09-23T21:17:25.450638+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"555'6960","reported_seq":7088,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.490484+0000","last_change":"2023-09-23T21:16:51.067304+0000","last_active":"2023-09-23T21:17:33.490484+0000","last_peered":"2023-09-23T21:17:33.490484+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:16:51.048602+0000","last_became_peered":"2023-09-23T21:16:51.048602+0000","last_unstale":"2023-09-23T21:17:33.490484+0000","last_undegraded":"2023-09-23T21:16:50.818351+0000","last_fullsized":"2023-09-23T21:16:50.791414+0000","mapping_epoch":544,"log_start":"503'3150","ondisk_log_start":"503'3150","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3810,"log_dups_size":0,"ondisk_log_size":3810,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5287,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":293},{"shards":"4,5","objects":-419}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":182,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.445373+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:17:25.445373+0000","last_peered":"2023-09-23T21:17:25.445373+0000","last_clean":"2023-09-23T21:17:25.445373+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:17:25.445373+0000","last_undegraded":"2023-09-23T21:17:25.445373+0000","last_fullsized":"2023-09-23T21:17:25.445373+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":617,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506337+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:17:14.506337+0000","last_peered":"2023-09-23T21:17:14.506337+0000","last_clean":"2023-09-23T21:17:14.506337+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:17:14.506337+0000","last_undegraded":"2023-09-23T21:17:14.506337+0000","last_fullsized":"2023-09-23T21:17:14.506337+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"555'6820","reported_seq":10870,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.517738+0000","last_change":"2023-09-23T21:16:37.510873+0000","last_active":"2023-09-23T21:17:33.517738+0000","last_peered":"2023-09-23T21:17:33.517738+0000","last_clean":"2023-09-23T21:17:33.517738+0000","last_became_active":"2023-09-23T21:16:37.500341+0000","last_became_peered":"2023-09-23T21:16:37.500341+0000","last_unstale":"2023-09-23T21:17:33.517738+0000","last_undegraded":"2023-09-23T21:17:33.517738+0000","last_fullsized":"2023-09-23T21:17:33.517738+0000","mapping_epoch":538,"log_start":"503'2894","ondisk_log_start":"503'2894","created":507,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3926,"log_dups_size":0,"ondisk_log_size":3926,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37814272,"num_objects":577,"num_object_clones":0,"num_object_copies":1731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5159,"num_write_kb":183552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1970,"num_bytes_recovered":94175232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":259,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.447354+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:17:25.447354+0000","last_peered":"2023-09-23T21:17:25.447354+0000","last_clean":"2023-09-23T21:17:25.447354+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:17:25.447354+0000","last_undegraded":"2023-09-23T21:17:25.447354+0000","last_fullsized":"2023-09-23T21:17:25.447354+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":211,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506043+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:17:14.506043+0000","last_peered":"2023-09-23T21:17:14.506043+0000","last_clean":"2023-09-23T21:17:14.506043+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:17:14.506043+0000","last_undegraded":"2023-09-23T21:17:14.506043+0000","last_fullsized":"2023-09-23T21:17:14.506043+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"555'6839","reported_seq":9575,"reported_epoch":555,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:33.439960+0000","last_change":"2023-09-23T21:16:50.886093+0000","last_active":"2023-09-23T21:17:33.439960+0000","last_peered":"2023-09-23T21:17:33.439960+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:17:33.439960+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"503'2967","ondisk_log_start":"503'2967","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3872,"log_dups_size":0,"ondisk_log_size":3872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41254912,"num_objects":630,"num_object_clones":0,"num_object_copies":1890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5156,"num_write_kb":185120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":61800448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5","objects":7}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":23,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.520044+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:17:14.520044+0000","last_peered":"2023-09-23T21:17:14.520044+0000","last_clean":"2023-09-23T21:17:14.520044+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:17:14.520044+0000","last_undegraded":"2023-09-23T21:17:14.520044+0000","last_fullsized":"2023-09-23T21:17:14.520044+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":198,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.504789+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:17:14.504789+0000","last_peered":"2023-09-23T21:17:14.504789+0000","last_clean":"2023-09-23T21:17:14.504789+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:17:14.504789+0000","last_undegraded":"2023-09-23T21:17:14.504789+0000","last_fullsized":"2023-09-23T21:17:14.504789+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"554'6711","reported_seq":12578,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:33.073047+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:17:33.073047+0000","last_peered":"2023-09-23T21:17:33.073047+0000","last_clean":"2023-09-23T21:17:33.073047+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:17:33.073047+0000","last_undegraded":"2023-09-23T21:17:33.073047+0000","last_fullsized":"2023-09-23T21:17:33.073047+0000","mapping_epoch":518,"log_start":"493'1484","ondisk_log_start":"493'1484","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5227,"log_dups_size":0,"ondisk_log_size":5227,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37617664,"num_objects":574,"num_object_clones":0,"num_object_copies":1722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5048,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.515089+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:17:14.515089+0000","last_peered":"2023-09-23T21:17:14.515089+0000","last_clean":"2023-09-23T21:17:14.515089+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:17:14.515089+0000","last_undegraded":"2023-09-23T21:17:14.515089+0000","last_fullsized":"2023-09-23T21:17:14.515089+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":413,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.505720+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:17:14.505720+0000","last_peered":"2023-09-23T21:17:14.505720+0000","last_clean":"2023-09-23T21:17:14.505720+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:17:14.505720+0000","last_undegraded":"2023-09-23T21:17:14.505720+0000","last_fullsized":"2023-09-23T21:17:14.505720+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"554'6707","reported_seq":10669,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:31.128357+0000","last_change":"2023-09-23T21:16:49.855987+0000","last_active":"2023-09-23T21:17:31.128357+0000","last_peered":"2023-09-23T21:17:31.128357+0000","last_clean":"2023-09-23T21:17:31.128357+0000","last_became_active":"2023-09-23T21:16:15.742634+0000","last_became_peered":"2023-09-23T21:16:15.742634+0000","last_unstale":"2023-09-23T21:17:31.128357+0000","last_undegraded":"2023-09-23T21:17:31.128357+0000","last_fullsized":"2023-09-23T21:17:31.128357+0000","mapping_epoch":531,"log_start":"493'1402","ondisk_log_start":"493'1402","created":507,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44597248,"num_objects":681,"num_object_clones":0,"num_object_copies":2043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5071,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":510,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T21:17:14.506852+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:17:14.506852+0000","last_peered":"2023-09-23T21:17:14.506852+0000","last_clean":"2023-09-23T21:17:14.506852+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:17:14.506852+0000","last_undegraded":"2023-09-23T21:17:14.506852+0000","last_fullsized":"2023-09-23T21:17:14.506852+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":222,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-09-23T21:17:25.446816+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:17:25.446816+0000","last_peered":"2023-09-23T21:17:25.446816+0000","last_clean":"2023-09-23T21:17:25.446816+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:17:25.446816+0000","last_undegraded":"2023-09-23T21:17:25.446816+0000","last_fullsized":"2023-09-23T21:17:25.446816+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":36,"stat_sum":{"num_bytes":1263534104,"num_objects":19282,"num_object_clones":0,"num_object_copies":57846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18724,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19282,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150649,"num_write_kb":4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30299,"num_bytes_recovered":1645445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5950615552,"data_stored":5950603336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":189748,"ondisk_log_size":189748,"up":108,"acting":93,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":38918,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201866,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1470604,"kb_used_data":686788,"kb_used_omap":0,"kb_used_meta":783744,"kb_avail":92274548,"statfs":{"total":95995035648,"available":94489137152,"internally_reserved":0,"allocated":703270912,"data_stored":701762066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":802553856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":3.0539999999999998,"15min":2.2519999999999998},"min":{"1min":0.26300000000000001,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":7.726,"5min":141.33199999999999,"15min":141.33199999999999},"last":3.0670000000000002},{"interface":"front","average":{"1min":2.2109999999999999,"5min":3.3530000000000002,"15min":2.3199999999999998},"min":{"1min":0.307,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.3940000000000001,"5min":141.31700000000001,"15min":141.31700000000001},"last":1.0700000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:19 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.1280000000000001,"15min":3.4239999999999999},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.4329999999999998,"5min":16.684000000000001,"15min":114.16500000000001},"last":1.0449999999999999},{"interface":"front","average":{"1min":1.5920000000000001,"5min":2.8799999999999999,"15min":3.867},"min":{"1min":0.307,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.7699999999999996,"5min":117.637,"15min":117.637},"last":3.109}]},{"osd":2,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":4.7679999999999998,"15min":4.4850000000000003},"min":{"1min":0.246,"5min":0.224,"15min":0.224},"max":{"1min":5.4160000000000004,"5min":135.529,"15min":183.44200000000001},"last":1.0149999999999999},{"interface":"front","average":{"1min":1.5740000000000001,"5min":5.6550000000000002,"15min":4.6909999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2370000000000001,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.27500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":7.1909999999999998,"15min":4.1210000000000004},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":7.7960000000000003,"5min":135.66200000000001,"15min":135.66200000000001},"last":3.1459999999999999},{"interface":"front","average":{"1min":1.613,"5min":7.6120000000000001,"15min":4.2199999999999998},"min":{"1min":0.375,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":5.2779999999999996,"5min":135.47,"15min":135.47},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1499999999999999,"5min":1.611,"15min":1.1399999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":9.9529999999999994,"5min":10.989000000000001,"15min":15.942},"last":0.89900000000000002},{"interface":"front","average":{"1min":2.984,"5min":1.6020000000000001,"15min":1.0549999999999999},"min":{"1min":0.28999999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":11.773,"5min":11.773,"15min":16.725999999999999},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":1.7689999999999999,"15min":1.3149999999999999},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":8.7829999999999995,"5min":11.231,"15min":11.231},"last":3.1749999999999998},{"interface":"front","average":{"1min":3.9420000000000002,"5min":1.9510000000000001,"15min":1.274},"min":{"1min":0.435,"5min":0.32900000000000001,"15min":0.28399999999999997},"max":{"1min":9.468,"5min":10.005000000000001,"15min":13.867000000000001},"last":0.98599999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:16:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5880000000000001,"5min":3.2160000000000002,"15min":2.0289999999999999},"min":{"1min":0.51800000000000002,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":13.369,"5min":143.18700000000001,"15min":143.18700000000001},"last":0.94299999999999995},{"interface":"front","average":{"1min":2.266,"5min":2.7549999999999999,"15min":1.8420000000000001},"min":{"1min":0.53900000000000003,"5min":0.32100000000000001,"15min":0.29099999999999998},"max":{"1min":7.7450000000000001,"5min":142.70400000000001,"15min":142.70400000000001},"last":0.31}]}]},{"osd":6,"up_from":47,"seq":201863463486,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010240,"kb_used_data":1276216,"kb_used_omap":0,"kb_used_meta":733760,"kb_avail":91734912,"statfs":{"total":95995035648,"available":93936549888,"internally_reserved":0,"allocated":1306845184,"data_stored":1305348893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":751370240},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":4.5339999999999998,"15min":2.7490000000000001},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":12.659000000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":1.5580000000000001},{"interface":"front","average":{"1min":2.0369999999999999,"5min":4.899,"15min":2.835},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":16.021999999999998,"5min":149.303,"15min":149.303},"last":2.1549999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":2.379,"15min":3.6080000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.22700000000000001},"max":{"1min":16.126999999999999,"5min":25.122,"15min":129.37899999999999},"last":2.2309999999999999},{"interface":"front","average":{"1min":1.986,"5min":1.8420000000000001,"15min":3.387},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":10.977,"5min":25.193999999999999,"15min":129.40100000000001},"last":6.6749999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.577999999999999,"5min":7.6189999999999998,"15min":5.9340000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":120.28700000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":1.4870000000000001},{"interface":"front","average":{"1min":12.118,"5min":7.8120000000000003,"15min":5.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":120.414,"5min":133.33000000000001,"15min":199.08000000000001},"last":2.198}]},{"osd":3,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":5.6669999999999998,"15min":3.3199999999999998},"min":{"1min":0.38,"5min":0.218,"15min":0.218},"max":{"1min":3.5089999999999999,"5min":137.875,"15min":137.875},"last":6.7400000000000002},{"interface":"front","average":{"1min":1.7250000000000001,"5min":5.391,"15min":3.3650000000000002},"min":{"1min":0.30099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.676,"5min":137.60900000000001,"15min":137.60900000000001},"last":1.5820000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":8.9060000000000006,"5min":2.9329999999999998,"15min":1.673},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.22600000000000001},"max":{"1min":55.494,"5min":55.494,"15min":55.494},"last":2.2890000000000001},{"interface":"front","average":{"1min":8.7550000000000008,"5min":2.8100000000000001,"15min":1.4970000000000001},"min":{"1min":0.39900000000000002,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":55.305,"5min":55.305,"15min":55.305},"last":8.3719999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":15.223000000000001,"5min":4.3630000000000004,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":222.90899999999999,"5min":222.90899999999999,"15min":222.90899999999999},"last":2.2570000000000001},{"interface":"front","average":{"1min":15.375,"5min":4.4930000000000003,"15min":2.1739999999999999},"min":{"1min":0.37,"5min":0.32200000000000001,"15min":0.28499999999999998},"max":{"1min":224.655,"5min":224.655,"15min":224.655},"last":6.6319999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":13.882999999999999,"5min":3.629,"15min":1.883},"min":{"1min":0.39600000000000002,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":165.96799999999999,"5min":165.96799999999999,"15min":165.96799999999999},"last":1.53},{"interface":"front","average":{"1min":13.532,"5min":3.859,"15min":1.98},"min":{"1min":0.442,"5min":0.251,"15min":0.251},"max":{"1min":165.82400000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":6.7149999999999999}]}]},{"osd":1,"up_from":0,"seq":1185410973933,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991673,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781564,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1937456,"kb_used_data":829672,"kb_used_omap":7,"kb_used_meta":1107768,"kb_avail":91807696,"statfs":{"total":95995035648,"available":94011080704,"internally_reserved":0,"allocated":849584128,"data_stored":846924896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1134354513},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.8420000000000001,"15min":2.157},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":8.0079999999999991,"5min":103.021,"15min":103.021},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.29599999999999999,"5min":3.133,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":0.378,"5min":103.154,"15min":103.154},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.2000000000000002,"15min":4.0869999999999997},"min":{"1min":0.24099999999999999,"5min":0.224,"15min":0.19},"max":{"1min":2.4409999999999998,"5min":54.521000000000001,"15min":212.12},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":2.4820000000000002,"15min":4.1399999999999997},"min":{"1min":0.254,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":8.048,"5min":54.598999999999997,"15min":213.02799999999999},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.5670000000000002,"15min":4.0339999999999998},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5190000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.2250000000000001,"15min":4.2519999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.2189999999999994,"5min":82.241,"15min":143.27799999999999},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":13.896000000000001,"5min":4.3010000000000002,"15min":3.1800000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":220.82900000000001,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":13.894,"5min":4.9130000000000003,"15min":3.823},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":220.84299999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.541,"5min":5.1059999999999999,"15min":3.48},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":220.63900000000001,"5min":220.63900000000001,"15min":220.63900000000001},"last":1.9279999999999999},{"interface":"front","average":{"1min":16.257000000000001,"5min":4.7380000000000004,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":220.31999999999999,"5min":220.31999999999999,"15min":220.31999999999999},"last":1.2330000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.597,"5min":4.6210000000000004,"15min":3.762},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":210.74700000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":14.371,"5min":4.4640000000000004,"15min":3.1259999999999999},"min":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":210.72900000000001,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.4940000000000002,"15min":1.948},"min":{"1min":0.371,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":14.019,"5min":25.260000000000002,"15min":25.260000000000002},"last":4.2640000000000002},{"interface":"front","average":{"1min":1.921,"5min":2.4990000000000001,"15min":2.2850000000000001},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":5.0229999999999997,"5min":12.613,"15min":16.875},"last":1.756}]}]},{"osd":3,"up_from":0,"seq":635655160157,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986776,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1758732,"kb_used_data":965380,"kb_used_omap":0,"kb_used_meta":793344,"kb_avail":91986420,"statfs":{"total":95995035648,"available":94194094080,"internally_reserved":0,"allocated":988549120,"data_stored":987040537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":812384256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":5.1909999999999998,"15min":2.7109999999999999},"min":{"1min":0.247,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":10.436,"5min":226.751,"15min":226.751},"last":1.2270000000000001},{"interface":"front","average":{"1min":2.0750000000000002,"5min":4.8049999999999997,"15min":2.556},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.223},"max":{"1min":6.218,"5min":226.43299999999999,"15min":226.43299999999999},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":3.802,"15min":5.2999999999999998},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2510000000000003,"5min":119.55800000000001,"15min":127.411},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.8050000000000002,"5min":3.71,"15min":4.5830000000000002},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":10.506,"5min":116.78100000000001,"15min":118.075},"last":1.264}]},{"osd":2,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":8.9459999999999997,"5min":7.5439999999999996,"15min":5.8710000000000004},"min":{"1min":0.26000000000000001,"5min":0.253,"15min":0.253},"max":{"1min":90.950000000000003,"5min":210.50299999999999,"15min":210.50299999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":9.9280000000000008,"5min":8.5630000000000006,"15min":6.0149999999999997},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":90.846000000000004,"5min":210.59,"15min":210.59},"last":1.1850000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":3.2440000000000002,"15min":4.1200000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0650000000000004,"5min":111.22199999999999,"15min":214.64099999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.0659999999999998,"5min":4.508,"15min":4.5209999999999999},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":10.529,"5min":111.39400000000001,"15min":214.54900000000001},"last":1.2949999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.383,"15min":0.98999999999999999},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.757999999999999,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.316},{"interface":"front","average":{"1min":3.4119999999999999,"5min":2.157,"15min":1.266},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":18.314,"5min":34.883000000000003,"15min":34.883000000000003},"last":14.675000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.524,"15min":1.333},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.2569999999999997,"5min":10.446999999999999,"15min":36.557000000000002},"last":4.8159999999999998},{"interface":"front","average":{"1min":2.2690000000000001,"5min":1.579,"15min":1.3169999999999999},"min":{"1min":0.35799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.8049999999999997,"5min":6.8049999999999997,"15min":38.468000000000004},"last":0.33800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8740000000000001,"5min":1.746,"15min":1.869},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":18.352,"5min":18.352,"15min":145.721},"last":1.3340000000000001},{"interface":"front","average":{"1min":2.512,"5min":1.421,"15min":1.917},"min":{"1min":0.42199999999999999,"5min":0.33500000000000002,"15min":0.31900000000000001},"max":{"1min":7.5830000000000002,"5min":10.677,"15min":148.45699999999999},"last":0.47599999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725140,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1822012,"kb_used_data":974196,"kb_used_omap":0,"kb_used_meta":847744,"kb_avail":91923140,"statfs":{"total":95995035648,"available":94129295360,"internally_reserved":0,"allocated":997576704,"data_stored":996079717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":868089856},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":1.732,"15min":1.6699999999999999},"min":{"1min":0.33500000000000002,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":7.7450000000000001,"5min":20.023,"15min":82.280000000000001},"last":1.6379999999999999},{"interface":"front","average":{"1min":2.077,"5min":2.0640000000000001,"15min":1.8680000000000001},"min":{"1min":0.29299999999999998,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.6710000000000003,"5min":19.853000000000002,"15min":85.236000000000004},"last":1.5489999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5619999999999998,"5min":2.5659999999999998,"15min":4.6150000000000002},"min":{"1min":0.42799999999999999,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":8.048,"5min":12.032999999999999,"15min":233.77099999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.966,"5min":2.028,"15min":4.3479999999999999},"min":{"1min":0.32300000000000001,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":7.8520000000000003,"5min":12.016,"15min":233.75200000000001},"last":1.667}]},{"osd":2,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":2.1110000000000002,"15min":3.7080000000000002},"min":{"1min":0.37,"5min":0.251,"15min":0.251},"max":{"1min":7.8220000000000001,"5min":20.411999999999999,"15min":231.34700000000001},"last":1.694},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.1859999999999999,"15min":3.9009999999999998},"min":{"1min":0.38600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.0439999999999996,"5min":11.509,"15min":231.381},"last":1.5820000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":2.5710000000000002,"15min":4.0469999999999997},"min":{"1min":0.40500000000000003,"5min":0.317,"15min":0.27700000000000002},"max":{"1min":8.077,"5min":11.597,"15min":263.887},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.004,"5min":1.9910000000000001,"15min":3.831},"min":{"1min":0.46100000000000002,"5min":0.32200000000000001,"15min":0.30099999999999999},"max":{"1min":7.8810000000000002,"5min":10.638,"15min":263.70100000000002},"last":1.7150000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":1.595,"15min":0.97599999999999998},"min":{"1min":0.439,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":11.593999999999999,"5min":11.593999999999999,"15min":12.739000000000001},"last":1.998},{"interface":"front","average":{"1min":3.7040000000000002,"5min":1.3839999999999999,"15min":1.026},"min":{"1min":0.505,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":10.259,"5min":10.259,"15min":10.792},"last":8.2140000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9009999999999998,"5min":1.6739999999999999,"15min":1.7130000000000001},"min":{"1min":0.38500000000000001,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":7.8520000000000003,"5min":8.9130000000000003,"15min":83.451999999999998},"last":5.9109999999999996},{"interface":"front","average":{"1min":3.0379999999999998,"5min":2.2669999999999999,"15min":1.9339999999999999},"min":{"1min":0.58499999999999996,"5min":0.30399999999999999,"15min":0.27400000000000002},"max":{"1min":8.0050000000000008,"5min":12.725,"15min":83.484999999999999},"last":3.206}]},{"osd":7,"last update":"Sat Sep 23 21:16:28 2023","interfaces":[{"interface":"back","average":{"1min":3.8599999999999999,"5min":1.7529999999999999,"15min":1.2529999999999999},"min":{"1min":0.56399999999999995,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":1.6060000000000001},{"interface":"front","average":{"1min":2.5470000000000002,"5min":1.3,"15min":0.874},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.28899999999999998},"max":{"1min":11.640000000000001,"5min":11.640000000000001,"15min":11.640000000000001},"last":2.0270000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":829620224,"data_stored":829620224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1243152384,"data_stored":1243152384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":954601472,"data_stored":954597400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":966463488,"data_stored":966459416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1275461632,"data_stored":1275461632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":681316352,"data_stored":681312280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:17:37.657 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:17:37.657 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:37.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18724/57918 objects degraded (32.328%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:37.771 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:37.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:37.881 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:37.913 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:38.332 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:38.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:38.460 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:38.510 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:38.601 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:38.623 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:38.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:38.627 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:38 smithi121 ceph-mon[211526]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:38.627 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:38 smithi121 ceph-mon[211526]: pgmap v1737: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.2 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 864 op/s; 18724/56763 objects degraded (32.986%); 960 KiB/s, 168 objects/s recovering 2023-09-23T21:17:38.627 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:38 smithi121 ceph-mon[219709]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:38.628 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:38 smithi121 ceph-mon[219709]: pgmap v1737: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.2 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 864 op/s; 18724/56763 objects degraded (32.986%); 960 KiB/s, 168 objects/s recovering 2023-09-23T21:17:38.726 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:38.728 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:38.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:38 smithi123 ceph-mon[193903]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:38.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:38 smithi123 ceph-mon[193903]: pgmap v1737: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.2 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 864 op/s; 18724/56763 objects degraded (32.986%); 960 KiB/s, 168 objects/s recovering 2023-09-23T21:17:38.817 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:38.879 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:38.964 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:38.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:38.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:38.988 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:17:39.011 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:17:39.053 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:17:39.059 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:17:39.079 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:39.098 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:17:39.129 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:17:39.156 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:39.159 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:17:39.186 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:17:39.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.187 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.188 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.188 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.188 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.188 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:17:39.287 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:39.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:39.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:39.396 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:39.436 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1419096145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:39.481 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:39.633 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:39.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:39.749 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:39.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1419096145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:39.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1419096145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:39.780 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:40.541 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:40 smithi121 ceph-mon[211526]: pgmap v1738: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 937 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 867 op/s; 18115/45033 objects degraded (40.226%); 0 B/s, 196 objects/s recovering 2023-09-23T21:17:40.542 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:40 smithi121 ceph-mon[219709]: pgmap v1738: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 937 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 867 op/s; 18115/45033 objects degraded (40.226%); 0 B/s, 196 objects/s recovering 2023-09-23T21:17:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:40 smithi123 ceph-mon[193903]: pgmap v1738: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 937 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 867 op/s; 18115/45033 objects degraded (40.226%); 0 B/s, 196 objects/s recovering 2023-09-23T21:17:41.333 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:41.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:41.436 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:41.464 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: osdmap e556: 8 total, 8 up, 5 in 2023-09-23T21:17:41.464 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.464 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.465 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[211526]: pgmap v1740: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 913 MiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 915 op/s; 18115/43893 objects degraded (41.271%); 0 B/s, 245 objects/s recovering 2023-09-23T21:17:41.466 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: osdmap e556: 8 total, 8 up, 5 in 2023-09-23T21:17:41.466 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.466 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.466 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.466 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.467 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.467 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.467 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.467 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:41 smithi121 ceph-mon[219709]: pgmap v1740: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 913 MiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 915 op/s; 18115/43893 objects degraded (41.271%); 0 B/s, 245 objects/s recovering 2023-09-23T21:17:41.467 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:41.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: osdmap e556: 8 total, 8 up, 5 in 2023-09-23T21:17:41.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: failed to encode map e556 with expected crc 2023-09-23T21:17:41.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:41 smithi123 ceph-mon[193903]: pgmap v1740: 109 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 913 MiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 915 op/s; 18115/43893 objects degraded (41.271%); 0 B/s, 245 objects/s recovering 2023-09-23T21:17:42.072 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:42.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:42.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:42.177 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:42.271 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:42.424 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:17:42.452 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:42.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:42.568 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:42.607 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:42.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[219709]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[219709]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[219709]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[219709]: osdmap e557: 8 total, 8 up, 5 in 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18115/43893 objects degraded (41.271%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[211526]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[211526]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[211526]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[211526]: osdmap e557: 8 total, 8 up, 5 in 2023-09-23T21:17:42.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18115/43893 objects degraded (41.271%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:42 smithi123 ceph-mon[193903]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:42 smithi123 ceph-mon[193903]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:42 smithi123 ceph-mon[193903]: failed to encode map e557 with expected crc 2023-09-23T21:17:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:42 smithi123 ceph-mon[193903]: osdmap e557: 8 total, 8 up, 5 in 2023-09-23T21:17:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18115/43893 objects degraded (41.271%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:42.770 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:42.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:42.775 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:42.775 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:42.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:42.971 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:43.118 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:43.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:43.236 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:43.317 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:43.433 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:43.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:43.498 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:17:43.546 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:43.620 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:43.620 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:43.620 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4217592995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:43.620 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:43 smithi123 ceph-mon[193903]: pgmap v1742: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 836 MiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 739 op/s; 18115/40170 objects degraded (45.096%); 0 B/s, 136 objects/s recovering 2023-09-23T21:17:43.631 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4217592995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:43.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[211526]: pgmap v1742: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 836 MiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 739 op/s; 18115/40170 objects degraded (45.096%); 0 B/s, 136 objects/s recovering 2023-09-23T21:17:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:43.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4217592995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:17:43.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:43 smithi121 ceph-mon[219709]: pgmap v1742: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 836 MiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 739 op/s; 18115/40170 objects degraded (45.096%); 0 B/s, 136 objects/s recovering 2023-09-23T21:17:43.794 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:43.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:43.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:43.908 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:43.946 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:17:43.947 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18115/40170 objects degraded (45.096%), 18 pgs degraded, 2 pgs undersized","count":20},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2320,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":558,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":5,"osd_in_since":1695503710,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":90},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":13},{"state_name":"active+recovery_wait+degraded","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+recovery_wait","count":1}],"num_pgs":109,"num_pools":5,"num_objects":13390,"data_bytes":876419000,"bytes_used":8131682304,"bytes_avail":471843495936,"bytes_total":479975178240,"degraded_objects":18115,"degraded_total":40170,"degraded_ratio":0.45095842668658209,"recovering_objects_per_sec":136,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":1089,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":739},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T21:17:03.337132+0000","services":{}},"progress_events":{"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"d33d2ace-2031-48c2-a8f8-693bca5c747e":{"message":"Global Recovery Event (2m)\n [=======================.....] (remaining: 29s)","progress":0.82568806409835815,"add_to_ceph_s":true}}} 2023-09-23T21:17:43.981 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:44.098 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:44.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:44.209 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:44.238 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:44.428 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:17:44.485 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:44.485 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.485 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4217592995' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:44.485 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: osdmap e558: 8 total, 8 up, 5 in 2023-09-23T21:17:44.485 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.485 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.486 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.486 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.486 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4068021913' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:17:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4217592995' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: osdmap e558: 8 total, 8 up, 5 in 2023-09-23T21:17:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4068021913' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:17:44.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:44.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4217592995' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:17:44.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: osdmap e558: 8 total, 8 up, 5 in 2023-09-23T21:17:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: failed to encode map e558 with expected crc 2023-09-23T21:17:44.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4068021913' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:17:45.113 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:45.115 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:17:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:17:44] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.43.0" 2023-09-23T21:17:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:45.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:45.217 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:45.244 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:45.284 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:17:45.285 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:17:45.293 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:45.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:45.377 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:45.397 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:45.427 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:45.623 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:45.632 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:45.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:45.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:45.736 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:45.763 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[219709]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.763 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[219709]: osdmap e559: 8 total, 8 up, 5 in 2023-09-23T21:17:45.763 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[219709]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.763 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[219709]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.764 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[219709]: pgmap v1745: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 686 MiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 818 op/s; 17397/32994 objects degraded (52.728%); 0 B/s, 179 objects/s recovering 2023-09-23T21:17:45.764 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[211526]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.764 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[211526]: osdmap e559: 8 total, 8 up, 5 in 2023-09-23T21:17:45.764 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[211526]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.764 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[211526]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.764 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:45 smithi121 ceph-mon[211526]: pgmap v1745: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 686 MiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 818 op/s; 17397/32994 objects degraded (52.728%); 0 B/s, 179 objects/s recovering 2023-09-23T21:17:45.767 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:45 smithi123 ceph-mon[193903]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:45 smithi123 ceph-mon[193903]: osdmap e559: 8 total, 8 up, 5 in 2023-09-23T21:17:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:45 smithi123 ceph-mon[193903]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:45 smithi123 ceph-mon[193903]: failed to encode map e559 with expected crc 2023-09-23T21:17:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:45 smithi123 ceph-mon[193903]: pgmap v1745: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 686 MiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 818 op/s; 17397/32994 objects degraded (52.728%); 0 B/s, 179 objects/s recovering 2023-09-23T21:17:46.026 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:46.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:46.132 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:46.195 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:46.339 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:46.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:46.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:46.457 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:46.484 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:46.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[211526]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[211526]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[211526]: osdmap e560: 8 total, 8 up, 5 in 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[211526]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[211526]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[211526]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[219709]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[219709]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[219709]: osdmap e560: 8 total, 8 up, 5 in 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[219709]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[219709]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:46 smithi121 ceph-mon[219709]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:46 smithi123 ceph-mon[193903]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:46 smithi123 ceph-mon[193903]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:46 smithi123 ceph-mon[193903]: osdmap e560: 8 total, 8 up, 5 in 2023-09-23T21:17:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:46 smithi123 ceph-mon[193903]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:46 smithi123 ceph-mon[193903]: failed to encode map e560 with expected crc 2023-09-23T21:17:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:46 smithi123 ceph-mon[193903]: failed to encode map e560 with expected crc 2023-09-23T21:17:47.239 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:17:47.383 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:47.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:47.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:47.435 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:47.492 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:47.516 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:47 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 17397/32994 objects degraded (52.728%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:17:47.516 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:47 smithi121 ceph-mon[219709]: pgmap v1747: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 666 MiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 17397/32013 objects degraded (54.344%); 0 B/s, 149 objects/s recovering 2023-09-23T21:17:47.516 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:47 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 17397/32994 objects degraded (52.728%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:17:47.516 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:47 smithi121 ceph-mon[211526]: pgmap v1747: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 666 MiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 17397/32013 objects degraded (54.344%); 0 B/s, 149 objects/s recovering 2023-09-23T21:17:47.540 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:47.595 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:47.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:47.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:47.699 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:47.743 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:47.795 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:47.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:47.900 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:47.972 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:47 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 17397/32994 objects degraded (52.728%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:17:47.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:47 smithi123 ceph-mon[193903]: pgmap v1747: 109 pgs: 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 666 MiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 671 op/s; 17397/32013 objects degraded (54.344%); 0 B/s, 149 objects/s recovering 2023-09-23T21:17:47.979 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:48.097 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:48.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:48.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:48.220 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:48.297 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:48.377 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:48.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:48.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:48.480 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:48.522 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:48.648 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:48.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:48.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:48.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:48.790 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:48.924 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:48.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:48.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:49.028 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:49.098 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:49.202 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:49.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:49.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:49.314 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:17:49.314 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:17:49.318 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:49.357 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:49.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:17:49.611 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:49.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:49.631 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:17:49.658 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:17:49.682 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:17:49.706 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:17:49.716 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:49.742 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:17:49.776 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:17:49.787 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:49.799 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.801 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:17:49.940 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:49.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:49.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:50.050 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:50.119 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:50.163 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:50.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:50.202 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:17:50.204 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:17:50.270 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:50.297 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:50.375 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:50.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:50.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:50.428 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:50.491 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:50.521 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:50 smithi121 ceph-mon[211526]: pgmap v1748: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 459 MiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.00k op/s; 15689/22071 objects degraded (71.084%); 3.2 MiB/s, 245 objects/s recovering 2023-09-23T21:17:50.521 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1977583742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:50.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:50 smithi121 ceph-mon[219709]: pgmap v1748: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 459 MiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.00k op/s; 15689/22071 objects degraded (71.084%); 3.2 MiB/s, 245 objects/s recovering 2023-09-23T21:17:50.521 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1977583742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:50.524 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:50.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:50 smithi123 ceph-mon[193903]: pgmap v1748: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 459 MiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.00k op/s; 15689/22071 objects degraded (71.084%); 3.2 MiB/s, 245 objects/s recovering 2023-09-23T21:17:50.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1977583742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:51.138 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:51.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:51.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:51.242 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:17:51.268 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:51.682 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:51 smithi121 ceph-mon[219709]: from='client.47354 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:51.682 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:51 smithi121 ceph-mon[211526]: from='client.47354 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:51.684 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:17:51.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:51 smithi123 ceph-mon[193903]: from='client.47354 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:17:51.925 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:17:51.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:51.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:17:52.029 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:52.066 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:52.081 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1748,"stamp":"2023-09-23T21:17:49.350767+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":481040312,"num_objects":7357,"num_object_clones":0,"num_object_copies":22071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15689,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7357,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":162898,"num_write_kb":4954830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33013,"num_bytes_recovered":1670720294,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":183176,"ondisk_log_size":183176,"up":327,"acting":314,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":6790836,"kb_used_data":2458764,"kb_used_omap":7,"kb_used_meta":4331768,"kb_avail":461934924,"statfs":{"total":479975178240,"available":473021362176,"internally_reserved":0,"allocated":2517774336,"data_stored":2508980567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4435730513},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-395378688,"num_objects":-6033,"num_object_clones":0,"num_object_copies":-18099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2426,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6033,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1474,"num_bytes_recovered":19922944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001564"},"pg_stats":[{"pgid":"13.23","version":"560'6149","reported_seq":8063,"reported_epoch":560,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:48.296721+0000","last_change":"2023-09-23T21:16:00.853306+0000","last_active":"2023-09-23T21:17:48.296721+0000","last_peered":"2023-09-23T21:17:48.296721+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:00.848945+0000","last_became_peered":"2023-09-23T21:16:00.848945+0000","last_unstale":"2023-09-23T21:17:48.296721+0000","last_undegraded":"2023-09-23T21:16:00.815130+0000","last_fullsized":"2023-09-23T21:17:48.296721+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":6149,"log_dups_size":0,"ondisk_log_size":6149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6389760,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":124,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1888,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":876,"num_bytes_recovered":57409536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5,7","objects":98}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.22","version":"560'6019","reported_seq":8077,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.226047+0000","last_change":"2023-09-23T21:17:20.794596+0000","last_active":"2023-09-23T21:17:48.226047+0000","last_peered":"2023-09-23T21:17:48.226047+0000","last_clean":"2023-09-23T21:17:48.226047+0000","last_became_active":"2023-09-23T21:16:00.839299+0000","last_became_peered":"2023-09-23T21:16:00.839299+0000","last_unstale":"2023-09-23T21:17:48.226047+0000","last_undegraded":"2023-09-23T21:17:48.226047+0000","last_fullsized":"2023-09-23T21:17:48.226047+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7864320,"num_objects":120,"num_object_clones":0,"num_object_copies":360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2780,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":962,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.21","version":"560'5888","reported_seq":6989,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.143066+0000","last_change":"2023-09-23T21:16:51.863113+0000","last_active":"2023-09-23T21:17:48.143066+0000","last_peered":"2023-09-23T21:17:48.143066+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858774+0000","last_became_peered":"2023-09-23T21:16:51.858774+0000","last_unstale":"2023-09-23T21:17:48.143066+0000","last_undegraded":"2023-09-23T21:16:51.798038+0000","last_fullsized":"2023-09-23T21:16:51.792078+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9814016,"num_objects":150,"num_object_clones":0,"num_object_copies":450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":750,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2707,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":5},{"shards":"2,4,7","objects":113}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.20","version":"560'6048","reported_seq":9189,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.287333+0000","last_change":"2023-09-23T21:16:51.842022+0000","last_active":"2023-09-23T21:17:48.287333+0000","last_peered":"2023-09-23T21:17:48.287333+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834659+0000","last_became_peered":"2023-09-23T21:16:51.834659+0000","last_unstale":"2023-09-23T21:17:48.287333+0000","last_undegraded":"2023-09-23T21:16:51.797563+0000","last_fullsized":"2023-09-23T21:16:51.792808+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":6048,"log_dups_size":0,"ondisk_log_size":6048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7536640,"num_objects":115,"num_object_clones":0,"num_object_copies":345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1925,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":32964608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":111},{"shards":"2,7","objects":-29}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.14","version":"560'7318","reported_seq":9531,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.120217+0000","last_change":"2023-09-23T21:16:51.065826+0000","last_active":"2023-09-23T21:17:48.120217+0000","last_peered":"2023-09-23T21:17:48.120217+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:16:51.052190+0000","last_became_peered":"2023-09-23T21:16:51.052190+0000","last_unstale":"2023-09-23T21:17:48.120217+0000","last_undegraded":"2023-09-23T21:16:50.808861+0000","last_fullsized":"2023-09-23T21:16:50.789676+0000","mapping_epoch":544,"log_start":"500'2585","ondisk_log_start":"500'2585","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4733,"log_dups_size":0,"ondisk_log_size":4733,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16973824,"num_objects":259,"num_object_clones":0,"num_object_copies":777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5607,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":220},{"shards":"2,5","objects":-22}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":162,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.511941+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:17:44.511941+0000","last_peered":"2023-09-23T21:17:44.511941+0000","last_clean":"2023-09-23T21:17:44.511941+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:17:44.511941+0000","last_undegraded":"2023-09-23T21:17:44.511941+0000","last_fullsized":"2023-09-23T21:17:44.511941+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514503+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:17:44.514503+0000","last_peered":"2023-09-23T21:17:44.514503+0000","last_clean":"2023-09-23T21:17:44.514503+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:17:44.514503+0000","last_undegraded":"2023-09-23T21:17:44.514503+0000","last_fullsized":"2023-09-23T21:17:44.514503+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"560'7164","reported_seq":8222,"reported_epoch":560,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:17:48.485182+0000","last_change":"2023-09-23T21:17:44.343789+0000","last_active":"2023-09-23T21:17:48.485182+0000","last_peered":"2023-09-23T21:17:48.485182+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:16:51.020513+0000","last_became_peered":"2023-09-23T21:16:51.020513+0000","last_unstale":"2023-09-23T21:17:48.485182+0000","last_undegraded":"2023-09-23T21:17:48.485182+0000","last_fullsized":"2023-09-23T21:16:50.789456+0000","mapping_epoch":544,"log_start":"503'3023","ondisk_log_start":"503'3023","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4141,"log_dups_size":0,"ondisk_log_size":4141,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13172736,"num_objects":201,"num_object_clones":0,"num_object_copies":603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5521,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":470,"num_bytes_recovered":19922944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":201}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":347,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.507878+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:17:44.507878+0000","last_peered":"2023-09-23T21:17:44.507878+0000","last_clean":"2023-09-23T21:17:44.507878+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:17:44.507878+0000","last_undegraded":"2023-09-23T21:17:44.507878+0000","last_fullsized":"2023-09-23T21:17:44.507878+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":207,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526601+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:17:44.526601+0000","last_peered":"2023-09-23T21:17:44.526601+0000","last_clean":"2023-09-23T21:17:44.526601+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:17:44.526601+0000","last_undegraded":"2023-09-23T21:17:44.526601+0000","last_fullsized":"2023-09-23T21:17:44.526601+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"560'7204","reported_seq":11232,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.539321+0000","last_change":"2023-09-23T21:16:36.510340+0000","last_active":"2023-09-23T21:17:48.539321+0000","last_peered":"2023-09-23T21:17:48.539321+0000","last_clean":"2023-09-23T21:17:48.539321+0000","last_became_active":"2023-09-23T21:16:36.498646+0000","last_became_peered":"2023-09-23T21:16:36.498646+0000","last_unstale":"2023-09-23T21:17:48.539321+0000","last_undegraded":"2023-09-23T21:17:48.539321+0000","last_fullsized":"2023-09-23T21:17:48.539321+0000","mapping_epoch":537,"log_start":"509'3489","ondisk_log_start":"509'3489","created":507,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3715,"log_dups_size":0,"ondisk_log_size":3715,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13959168,"num_objects":213,"num_object_clones":0,"num_object_copies":639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5543,"num_write_kb":184192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1956,"num_bytes_recovered":93519872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":220,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.512138+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:17:44.512138+0000","last_peered":"2023-09-23T21:17:44.512138+0000","last_clean":"2023-09-23T21:17:44.512138+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:17:44.512138+0000","last_undegraded":"2023-09-23T21:17:44.512138+0000","last_fullsized":"2023-09-23T21:17:44.512138+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514841+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:17:44.514841+0000","last_peered":"2023-09-23T21:17:44.514841+0000","last_clean":"2023-09-23T21:17:44.514841+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:17:44.514841+0000","last_undegraded":"2023-09-23T21:17:44.514841+0000","last_fullsized":"2023-09-23T21:17:44.514841+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"560'7062","reported_seq":7186,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.542294+0000","last_change":"2023-09-23T21:16:51.074430+0000","last_active":"2023-09-23T21:17:48.542294+0000","last_peered":"2023-09-23T21:17:48.542294+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:16:51.054035+0000","last_became_peered":"2023-09-23T21:16:51.054035+0000","last_unstale":"2023-09-23T21:17:48.542294+0000","last_undegraded":"2023-09-23T21:16:50.838593+0000","last_fullsized":"2023-09-23T21:16:50.797592+0000","mapping_epoch":544,"log_start":"508'3358","ondisk_log_start":"508'3358","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3704,"log_dups_size":0,"ondisk_log_size":3704,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13271040,"num_objects":203,"num_object_clones":0,"num_object_copies":609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5427,"num_write_kb":180128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":190},{"shards":"2,5","objects":-55}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":154,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.516289+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:17:44.516289+0000","last_peered":"2023-09-23T21:17:44.516289+0000","last_clean":"2023-09-23T21:17:44.516289+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:17:44.516289+0000","last_undegraded":"2023-09-23T21:17:44.516289+0000","last_fullsized":"2023-09-23T21:17:44.516289+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.513501+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:17:44.513501+0000","last_peered":"2023-09-23T21:17:44.513501+0000","last_clean":"2023-09-23T21:17:44.513501+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:17:44.513501+0000","last_undegraded":"2023-09-23T21:17:44.513501+0000","last_fullsized":"2023-09-23T21:17:44.513501+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.18","version":"560'7176","reported_seq":10499,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.149309+0000","last_change":"2023-09-23T21:17:12.466376+0000","last_active":"2023-09-23T21:17:48.149309+0000","last_peered":"2023-09-23T21:17:48.149309+0000","last_clean":"2023-09-23T21:17:48.149309+0000","last_became_active":"2023-09-23T21:17:12.454368+0000","last_became_peered":"2023-09-23T21:17:12.454368+0000","last_unstale":"2023-09-23T21:17:48.149309+0000","last_undegraded":"2023-09-23T21:17:48.149309+0000","last_fullsized":"2023-09-23T21:17:48.149309+0000","mapping_epoch":551,"log_start":"500'2452","ondisk_log_start":"500'2452","created":507,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":4724,"log_dups_size":0,"ondisk_log_size":4724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:01.792696+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13959168,"num_objects":213,"num_object_clones":0,"num_object_copies":639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3663,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1575,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.509656+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:17:44.509656+0000","last_peered":"2023-09-23T21:17:44.509656+0000","last_clean":"2023-09-23T21:17:44.509656+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:17:44.509656+0000","last_undegraded":"2023-09-23T21:17:44.509656+0000","last_fullsized":"2023-09-23T21:17:44.509656+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":182,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520066+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:17:44.520066+0000","last_peered":"2023-09-23T21:17:44.520066+0000","last_clean":"2023-09-23T21:17:44.520066+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:17:44.520066+0000","last_undegraded":"2023-09-23T21:17:44.520066+0000","last_fullsized":"2023-09-23T21:17:44.520066+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1b","version":"560'8987","reported_seq":14157,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.144672+0000","last_change":"2023-09-23T21:17:17.318827+0000","last_active":"2023-09-23T21:17:48.144672+0000","last_peered":"2023-09-23T21:17:48.144672+0000","last_clean":"2023-09-23T21:17:48.144672+0000","last_became_active":"2023-09-23T21:16:00.808957+0000","last_became_peered":"2023-09-23T21:16:00.808957+0000","last_unstale":"2023-09-23T21:17:48.144672+0000","last_undegraded":"2023-09-23T21:17:48.144672+0000","last_fullsized":"2023-09-23T21:17:48.144672+0000","mapping_epoch":525,"log_start":"502'2957","ondisk_log_start":"502'2957","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6030,"log_dups_size":0,"ondisk_log_size":6030,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13664256,"num_objects":209,"num_object_clones":0,"num_object_copies":627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5752,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133431296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":163,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520523+0000","last_change":"2023-09-23T21:17:11.237055+0000","last_active":"2023-09-23T21:17:44.520523+0000","last_peered":"2023-09-23T21:17:44.520523+0000","last_clean":"2023-09-23T21:17:44.520523+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:17:44.520523+0000","last_undegraded":"2023-09-23T21:17:44.520523+0000","last_fullsized":"2023-09-23T21:17:44.520523+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514107+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:17:44.514107+0000","last_peered":"2023-09-23T21:17:44.514107+0000","last_clean":"2023-09-23T21:17:44.514107+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:17:44.514107+0000","last_undegraded":"2023-09-23T21:17:44.514107+0000","last_fullsized":"2023-09-23T21:17:44.514107+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1a","version":"560'8584","reported_seq":10717,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.146325+0000","last_change":"2023-09-23T21:16:00.812483+0000","last_active":"2023-09-23T21:17:48.146325+0000","last_peered":"2023-09-23T21:17:48.146325+0000","last_clean":"2023-09-23T21:17:48.146325+0000","last_became_active":"2023-09-23T21:16:00.806979+0000","last_became_peered":"2023-09-23T21:16:00.806979+0000","last_unstale":"2023-09-23T21:17:48.146325+0000","last_undegraded":"2023-09-23T21:17:48.146325+0000","last_fullsized":"2023-09-23T21:17:48.146325+0000","mapping_epoch":525,"log_start":"500'2393","ondisk_log_start":"500'2393","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6191,"log_dups_size":0,"ondisk_log_size":6191,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15794176,"num_objects":241,"num_object_clones":0,"num_object_copies":723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5472,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":187,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520839+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:17:44.520839+0000","last_peered":"2023-09-23T21:17:44.520839+0000","last_clean":"2023-09-23T21:17:44.520839+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:17:44.520839+0000","last_undegraded":"2023-09-23T21:17:44.520839+0000","last_fullsized":"2023-09-23T21:17:44.520839+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":488,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.510918+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:17:44.510918+0000","last_peered":"2023-09-23T21:17:44.510918+0000","last_clean":"2023-09-23T21:17:44.510918+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:17:44.510918+0000","last_undegraded":"2023-09-23T21:17:44.510918+0000","last_fullsized":"2023-09-23T21:17:44.510918+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1d","version":"560'8777","reported_seq":14873,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.315528+0000","last_change":"2023-09-23T21:17:20.608316+0000","last_active":"2023-09-23T21:17:48.315528+0000","last_peered":"2023-09-23T21:17:48.315528+0000","last_clean":"2023-09-23T21:17:48.315528+0000","last_became_active":"2023-09-23T21:16:03.773235+0000","last_became_peered":"2023-09-23T21:16:03.773235+0000","last_unstale":"2023-09-23T21:17:48.315528+0000","last_undegraded":"2023-09-23T21:17:48.315528+0000","last_fullsized":"2023-09-23T21:17:48.315528+0000","mapping_epoch":528,"log_start":"509'3723","ondisk_log_start":"509'3723","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5054,"log_dups_size":0,"ondisk_log_size":5054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16678924,"num_objects":255,"num_object_clones":0,"num_object_copies":765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":255,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5600,"num_write_kb":187329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":146276352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":143,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520216+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:17:44.520216+0000","last_peered":"2023-09-23T21:17:44.520216+0000","last_clean":"2023-09-23T21:17:44.520216+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:17:44.520216+0000","last_undegraded":"2023-09-23T21:17:44.520216+0000","last_fullsized":"2023-09-23T21:17:44.520216+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":211,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520690+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:17:44.520690+0000","last_peered":"2023-09-23T21:17:44.520690+0000","last_clean":"2023-09-23T21:17:44.520690+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:17:44.520690+0000","last_undegraded":"2023-09-23T21:17:44.520690+0000","last_fullsized":"2023-09-23T21:17:44.520690+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1c","version":"560'8831","reported_seq":10958,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.325671+0000","last_change":"2023-09-23T21:16:00.824659+0000","last_active":"2023-09-23T21:17:48.325671+0000","last_peered":"2023-09-23T21:17:48.325671+0000","last_clean":"2023-09-23T21:17:48.325671+0000","last_became_active":"2023-09-23T21:16:00.820619+0000","last_became_peered":"2023-09-23T21:16:00.820619+0000","last_unstale":"2023-09-23T21:17:48.325671+0000","last_undegraded":"2023-09-23T21:17:48.325671+0000","last_fullsized":"2023-09-23T21:17:48.325671+0000","mapping_epoch":525,"log_start":"515'3984","ondisk_log_start":"515'3984","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4847,"log_dups_size":0,"ondisk_log_size":4847,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15171584,"num_objects":232,"num_object_clones":0,"num_object_copies":696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5647,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":139,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.521593+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:17:44.521593+0000","last_peered":"2023-09-23T21:17:44.521593+0000","last_clean":"2023-09-23T21:17:44.521593+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:17:44.521593+0000","last_undegraded":"2023-09-23T21:17:44.521593+0000","last_fullsized":"2023-09-23T21:17:44.521593+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":147,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520758+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:17:44.520758+0000","last_peered":"2023-09-23T21:17:44.520758+0000","last_clean":"2023-09-23T21:17:44.520758+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:17:44.520758+0000","last_undegraded":"2023-09-23T21:17:44.520758+0000","last_fullsized":"2023-09-23T21:17:44.520758+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1f","version":"560'8631","reported_seq":8753,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:46.145530+0000","last_change":"2023-09-23T21:17:39.461735+0000","last_active":"2023-09-23T21:17:46.145530+0000","last_peered":"2023-09-23T21:17:46.145530+0000","last_clean":"2023-09-23T21:17:46.145530+0000","last_became_active":"2023-09-23T21:16:51.820010+0000","last_became_peered":"2023-09-23T21:16:51.820010+0000","last_unstale":"2023-09-23T21:17:46.145530+0000","last_undegraded":"2023-09-23T21:17:46.145530+0000","last_fullsized":"2023-09-23T21:17:46.145530+0000","mapping_epoch":545,"log_start":"500'2442","ondisk_log_start":"500'2442","created":525,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18907136,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5501,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520280+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:17:44.520280+0000","last_peered":"2023-09-23T21:17:44.520280+0000","last_clean":"2023-09-23T21:17:44.520280+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:17:44.520280+0000","last_undegraded":"2023-09-23T21:17:44.520280+0000","last_fullsized":"2023-09-23T21:17:44.520280+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":161,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526516+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:17:44.526516+0000","last_peered":"2023-09-23T21:17:44.526516+0000","last_clean":"2023-09-23T21:17:44.526516+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:17:44.526516+0000","last_undegraded":"2023-09-23T21:17:44.526516+0000","last_fullsized":"2023-09-23T21:17:44.526516+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1e","version":"560'8550","reported_seq":8649,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.353209+0000","last_change":"2023-09-23T21:17:20.608819+0000","last_active":"2023-09-23T21:17:48.353209+0000","last_peered":"2023-09-23T21:17:48.353209+0000","last_clean":"2023-09-23T21:17:48.353209+0000","last_became_active":"2023-09-23T21:16:00.844392+0000","last_became_peered":"2023-09-23T21:16:00.844392+0000","last_unstale":"2023-09-23T21:17:48.353209+0000","last_undegraded":"2023-09-23T21:17:48.353209+0000","last_fullsized":"2023-09-23T21:17:48.353209+0000","mapping_epoch":525,"log_start":"502'2802","ondisk_log_start":"502'2802","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17268736,"num_objects":264,"num_object_clones":0,"num_object_copies":792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5459,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":596,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.513756+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:17:44.513756+0000","last_peered":"2023-09-23T21:17:44.513756+0000","last_clean":"2023-09-23T21:17:44.513756+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:17:44.513756+0000","last_undegraded":"2023-09-23T21:17:44.513756+0000","last_fullsized":"2023-09-23T21:17:44.513756+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":211,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526479+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:17:44.526479+0000","last_peered":"2023-09-23T21:17:44.526479+0000","last_clean":"2023-09-23T21:17:44.526479+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:17:44.526479+0000","last_undegraded":"2023-09-23T21:17:44.526479+0000","last_fullsized":"2023-09-23T21:17:44.526479+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"560'5964","reported_seq":7109,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:47.994654+0000","last_change":"2023-09-23T21:16:51.863144+0000","last_active":"2023-09-23T21:17:47.994654+0000","last_peered":"2023-09-23T21:17:47.994654+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858843+0000","last_became_peered":"2023-09-23T21:16:51.858843+0000","last_unstale":"2023-09-23T21:17:47.994654+0000","last_undegraded":"2023-09-23T21:16:51.799178+0000","last_fullsized":"2023-09-23T21:16:51.792321+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4833280,"num_objects":73,"num_object_clones":0,"num_object_copies":219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2782,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":511,"num_bytes_recovered":33488896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":-96},{"shards":"2,4,7","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":141,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526565+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:17:44.526565+0000","last_peered":"2023-09-23T21:17:44.526565+0000","last_clean":"2023-09-23T21:17:44.526565+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:17:44.526565+0000","last_undegraded":"2023-09-23T21:17:44.526565+0000","last_fullsized":"2023-09-23T21:17:44.526565+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":213,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.507235+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:17:44.507235+0000","last_peered":"2023-09-23T21:17:44.507235+0000","last_clean":"2023-09-23T21:17:44.507235+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:17:44.507235+0000","last_undegraded":"2023-09-23T21:17:44.507235+0000","last_fullsized":"2023-09-23T21:17:44.507235+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"560'6050","reported_seq":9167,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.140910+0000","last_change":"2023-09-23T21:16:51.842306+0000","last_active":"2023-09-23T21:17:48.140910+0000","last_peered":"2023-09-23T21:17:48.140910+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834848+0000","last_became_peered":"2023-09-23T21:16:51.834848+0000","last_unstale":"2023-09-23T21:17:48.140910+0000","last_undegraded":"2023-09-23T21:16:51.803926+0000","last_fullsized":"2023-09-23T21:16:51.799106+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":6050,"log_dups_size":0,"ondisk_log_size":6050,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7405568,"num_objects":113,"num_object_clones":0,"num_object_copies":339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1927,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32178176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,4,7","objects":117},{"shards":"2,7","objects":-48}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":155,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526685+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:17:44.526685+0000","last_peered":"2023-09-23T21:17:44.526685+0000","last_clean":"2023-09-23T21:17:44.526685+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:17:44.526685+0000","last_undegraded":"2023-09-23T21:17:44.526685+0000","last_fullsized":"2023-09-23T21:17:44.526685+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.513589+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:17:44.513589+0000","last_peered":"2023-09-23T21:17:44.513589+0000","last_clean":"2023-09-23T21:17:44.513589+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:17:44.513589+0000","last_undegraded":"2023-09-23T21:17:44.513589+0000","last_fullsized":"2023-09-23T21:17:44.513589+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3","version":"560'6138","reported_seq":8061,"reported_epoch":560,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:48.041977+0000","last_change":"2023-09-23T21:16:01.735614+0000","last_active":"2023-09-23T21:17:48.041977+0000","last_peered":"2023-09-23T21:17:48.041977+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:01.731847+0000","last_became_peered":"2023-09-23T21:16:01.731847+0000","last_unstale":"2023-09-23T21:17:48.041977+0000","last_undegraded":"2023-09-23T21:16:01.699708+0000","last_fullsized":"2023-09-23T21:17:48.041977+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":6138,"log_dups_size":0,"ondisk_log_size":6138,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7110656,"num_objects":108,"num_object_clones":0,"num_object_copies":324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":143,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1877,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":879,"num_bytes_recovered":57606144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5,7","objects":108}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":141,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520833+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:17:44.520833+0000","last_peered":"2023-09-23T21:17:44.520833+0000","last_clean":"2023-09-23T21:17:44.520833+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:17:44.520833+0000","last_undegraded":"2023-09-23T21:17:44.520833+0000","last_fullsized":"2023-09-23T21:17:44.520833+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514725+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:17:44.514725+0000","last_peered":"2023-09-23T21:17:44.514725+0000","last_clean":"2023-09-23T21:17:44.514725+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:17:44.514725+0000","last_undegraded":"2023-09-23T21:17:44.514725+0000","last_fullsized":"2023-09-23T21:17:44.514725+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"560'6026","reported_seq":7807,"reported_epoch":560,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:48.229538+0000","last_change":"2023-09-23T21:17:11.228334+0000","last_active":"2023-09-23T21:17:48.229538+0000","last_peered":"2023-09-23T21:17:48.229538+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:01.742399+0000","last_became_peered":"2023-09-23T21:16:01.742399+0000","last_unstale":"2023-09-23T21:17:48.229538+0000","last_undegraded":"2023-09-23T21:17:11.225179+0000","last_fullsized":"2023-09-23T21:17:48.229538+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7405568,"num_objects":113,"num_object_clones":0,"num_object_copies":339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":133,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2787,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":54001664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5,6","objects":113}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514664+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:17:44.514664+0000","last_peered":"2023-09-23T21:17:44.514664+0000","last_clean":"2023-09-23T21:17:44.514664+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:17:44.514664+0000","last_undegraded":"2023-09-23T21:17:44.514664+0000","last_fullsized":"2023-09-23T21:17:44.514664+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514309+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:17:44.514309+0000","last_peered":"2023-09-23T21:17:44.514309+0000","last_clean":"2023-09-23T21:17:44.514309+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:17:44.514309+0000","last_undegraded":"2023-09-23T21:17:44.514309+0000","last_fullsized":"2023-09-23T21:17:44.514309+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"560'7192","reported_seq":11935,"reported_epoch":560,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:48.337282+0000","last_change":"2023-09-23T21:17:45.677588+0000","last_active":"2023-09-23T21:17:48.337282+0000","last_peered":"2023-09-23T21:17:48.337282+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:17:45.633940+0000","last_became_peered":"2023-09-23T21:17:45.633940+0000","last_unstale":"2023-09-23T21:17:48.337282+0000","last_undegraded":"2023-09-23T21:17:48.337282+0000","last_fullsized":"2023-09-23T21:17:48.337282+0000","mapping_epoch":559,"log_start":"508'3420","ondisk_log_start":"508'3420","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":3772,"log_dups_size":0,"ondisk_log_size":3772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:50.441594+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14876672,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3636,"num_write_kb":64384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2258,"num_bytes_recovered":21233664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5","6"],"object_location_counts":[{"shards":"4,5,6","objects":227}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":227,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520091+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:17:44.520091+0000","last_peered":"2023-09-23T21:17:44.520091+0000","last_clean":"2023-09-23T21:17:44.520091+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:17:44.520091+0000","last_undegraded":"2023-09-23T21:17:44.520091+0000","last_fullsized":"2023-09-23T21:17:44.520091+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514827+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:17:44.514827+0000","last_peered":"2023-09-23T21:17:44.514827+0000","last_clean":"2023-09-23T21:17:44.514827+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:17:44.514827+0000","last_undegraded":"2023-09-23T21:17:44.514827+0000","last_fullsized":"2023-09-23T21:17:44.514827+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.4","version":"560'7246","reported_seq":9551,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.059887+0000","last_change":"2023-09-23T21:16:51.065805+0000","last_active":"2023-09-23T21:17:48.059887+0000","last_peered":"2023-09-23T21:17:48.059887+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:16:51.051146+0000","last_became_peered":"2023-09-23T21:16:51.051146+0000","last_unstale":"2023-09-23T21:17:48.059887+0000","last_undegraded":"2023-09-23T21:16:50.809981+0000","last_fullsized":"2023-09-23T21:16:50.790363+0000","mapping_epoch":544,"log_start":"500'2552","ondisk_log_start":"500'2552","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":4694,"log_dups_size":0,"ondisk_log_size":4694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:23.280602+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12189696,"num_objects":186,"num_object_clones":0,"num_object_copies":558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3546,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":186},{"shards":"2,5","objects":-56}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514371+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:17:44.514371+0000","last_peered":"2023-09-23T21:17:44.514371+0000","last_clean":"2023-09-23T21:17:44.514371+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:17:44.514371+0000","last_undegraded":"2023-09-23T21:17:44.514371+0000","last_fullsized":"2023-09-23T21:17:44.514371+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":602,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.513002+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:17:44.513002+0000","last_peered":"2023-09-23T21:17:44.513002+0000","last_clean":"2023-09-23T21:17:44.513002+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:17:44.513002+0000","last_undegraded":"2023-09-23T21:17:44.513002+0000","last_fullsized":"2023-09-23T21:17:44.513002+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"560'7108","reported_seq":7230,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.269764+0000","last_change":"2023-09-23T21:16:51.082314+0000","last_active":"2023-09-23T21:17:48.269764+0000","last_peered":"2023-09-23T21:17:48.269764+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:16:51.064800+0000","last_became_peered":"2023-09-23T21:16:51.064800+0000","last_unstale":"2023-09-23T21:17:48.269764+0000","last_undegraded":"2023-09-23T21:16:50.820535+0000","last_fullsized":"2023-09-23T21:16:50.800952+0000","mapping_epoch":544,"log_start":"503'2932","ondisk_log_start":"503'2932","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4176,"log_dups_size":0,"ondisk_log_size":4176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16318464,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5465,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,5,7","objects":222},{"shards":"4,7","objects":-46}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.508135+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:17:44.508135+0000","last_peered":"2023-09-23T21:17:44.508135+0000","last_clean":"2023-09-23T21:17:44.508135+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:17:44.508135+0000","last_undegraded":"2023-09-23T21:17:44.508135+0000","last_fullsized":"2023-09-23T21:17:44.508135+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514618+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:17:44.514618+0000","last_peered":"2023-09-23T21:17:44.514618+0000","last_clean":"2023-09-23T21:17:44.514618+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:17:44.514618+0000","last_undegraded":"2023-09-23T21:17:44.514618+0000","last_fullsized":"2023-09-23T21:17:44.514618+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"560'7128","reported_seq":11230,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.539444+0000","last_change":"2023-09-23T21:17:41.511480+0000","last_active":"2023-09-23T21:17:48.539444+0000","last_peered":"2023-09-23T21:17:48.539444+0000","last_clean":"2023-09-23T21:17:48.539444+0000","last_became_active":"2023-09-23T21:17:41.488891+0000","last_became_peered":"2023-09-23T21:17:41.488891+0000","last_unstale":"2023-09-23T21:17:48.539444+0000","last_undegraded":"2023-09-23T21:17:48.539444+0000","last_fullsized":"2023-09-23T21:17:48.539444+0000","mapping_epoch":556,"log_start":"508'3412","ondisk_log_start":"508'3412","created":482,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":3716,"log_dups_size":0,"ondisk_log_size":3716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.993310+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14024704,"num_objects":214,"num_object_clones":0,"num_object_copies":642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3515,"num_write_kb":57088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":211,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.523911+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:17:44.523911+0000","last_peered":"2023-09-23T21:17:44.523911+0000","last_clean":"2023-09-23T21:17:44.523911+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:17:44.523911+0000","last_undegraded":"2023-09-23T21:17:44.523911+0000","last_fullsized":"2023-09-23T21:17:44.523911+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":299,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526091+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:17:44.526091+0000","last_peered":"2023-09-23T21:17:44.526091+0000","last_clean":"2023-09-23T21:17:44.526091+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:17:44.526091+0000","last_undegraded":"2023-09-23T21:17:44.526091+0000","last_fullsized":"2023-09-23T21:17:44.526091+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"560'8944","reported_seq":14010,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.134420+0000","last_change":"2023-09-23T21:17:44.322885+0000","last_active":"2023-09-23T21:17:48.134420+0000","last_peered":"2023-09-23T21:17:48.134420+0000","last_clean":"2023-09-23T21:17:48.134420+0000","last_became_active":"2023-09-23T21:16:01.723118+0000","last_became_peered":"2023-09-23T21:16:01.723118+0000","last_unstale":"2023-09-23T21:17:48.134420+0000","last_undegraded":"2023-09-23T21:17:48.134420+0000","last_fullsized":"2023-09-23T21:17:48.134420+0000","mapping_epoch":526,"log_start":"500'2811","ondisk_log_start":"500'2811","created":482,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6133,"log_dups_size":0,"ondisk_log_size":6133,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16482304,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5708,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":129957888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":484,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520760+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:17:44.520760+0000","last_peered":"2023-09-23T21:17:44.520760+0000","last_clean":"2023-09-23T21:17:44.520760+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:17:44.520760+0000","last_undegraded":"2023-09-23T21:17:44.520760+0000","last_fullsized":"2023-09-23T21:17:44.520760+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":235,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.508624+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:17:44.508624+0000","last_peered":"2023-09-23T21:17:44.508624+0000","last_clean":"2023-09-23T21:17:44.508624+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:17:44.508624+0000","last_undegraded":"2023-09-23T21:17:44.508624+0000","last_fullsized":"2023-09-23T21:17:44.508624+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":109,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.508774+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:17:44.508774+0000","last_peered":"2023-09-23T21:17:44.508774+0000","last_clean":"2023-09-23T21:17:44.508774+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:17:44.508774+0000","last_undegraded":"2023-09-23T21:17:44.508774+0000","last_fullsized":"2023-09-23T21:17:44.508774+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"560'8844","reported_seq":14962,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.229766+0000","last_change":"2023-09-23T21:17:15.462587+0000","last_active":"2023-09-23T21:17:48.229766+0000","last_peered":"2023-09-23T21:17:48.229766+0000","last_clean":"2023-09-23T21:17:48.229766+0000","last_became_active":"2023-09-23T21:16:01.764537+0000","last_became_peered":"2023-09-23T21:16:01.764537+0000","last_unstale":"2023-09-23T21:17:48.229766+0000","last_undegraded":"2023-09-23T21:17:48.229766+0000","last_fullsized":"2023-09-23T21:17:48.229766+0000","mapping_epoch":526,"log_start":"515'3980","ondisk_log_start":"515'3980","created":482,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4864,"log_dups_size":0,"ondisk_log_size":4864,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12288012,"num_objects":188,"num_object_clones":0,"num_object_copies":564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5667,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2487,"num_bytes_recovered":146014208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514017+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:17:44.514017+0000","last_peered":"2023-09-23T21:17:44.514017+0000","last_clean":"2023-09-23T21:17:44.514017+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:17:44.514017+0000","last_undegraded":"2023-09-23T21:17:44.514017+0000","last_fullsized":"2023-09-23T21:17:44.514017+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514399+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:17:44.514399+0000","last_peered":"2023-09-23T21:17:44.514399+0000","last_clean":"2023-09-23T21:17:44.514399+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:17:44.514399+0000","last_undegraded":"2023-09-23T21:17:44.514399+0000","last_fullsized":"2023-09-23T21:17:44.514399+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.512930+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:17:44.512930+0000","last_peered":"2023-09-23T21:17:44.512930+0000","last_clean":"2023-09-23T21:17:44.512930+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:17:44.512930+0000","last_undegraded":"2023-09-23T21:17:44.512930+0000","last_fullsized":"2023-09-23T21:17:44.512930+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"560'8608","reported_seq":8704,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.288560+0000","last_change":"2023-09-23T21:16:00.841315+0000","last_active":"2023-09-23T21:17:48.288560+0000","last_peered":"2023-09-23T21:17:48.288560+0000","last_clean":"2023-09-23T21:17:48.288560+0000","last_became_active":"2023-09-23T21:16:00.829607+0000","last_became_peered":"2023-09-23T21:16:00.829607+0000","last_unstale":"2023-09-23T21:17:48.288560+0000","last_undegraded":"2023-09-23T21:17:48.288560+0000","last_fullsized":"2023-09-23T21:17:48.288560+0000","mapping_epoch":482,"log_start":"505'3176","ondisk_log_start":"505'3176","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13467648,"num_objects":205,"num_object_clones":0,"num_object_copies":615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5516,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":569,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.511807+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:17:44.511807+0000","last_peered":"2023-09-23T21:17:44.511807+0000","last_clean":"2023-09-23T21:17:44.511807+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:17:44.511807+0000","last_undegraded":"2023-09-23T21:17:44.511807+0000","last_fullsized":"2023-09-23T21:17:44.511807+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":306,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.511591+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:17:44.511591+0000","last_peered":"2023-09-23T21:17:44.511591+0000","last_clean":"2023-09-23T21:17:44.511591+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:17:44.511591+0000","last_undegraded":"2023-09-23T21:17:44.511591+0000","last_fullsized":"2023-09-23T21:17:44.511591+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":213,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526140+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:17:44.526140+0000","last_peered":"2023-09-23T21:17:44.526140+0000","last_clean":"2023-09-23T21:17:44.526140+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:17:44.526140+0000","last_undegraded":"2023-09-23T21:17:44.526140+0000","last_fullsized":"2023-09-23T21:17:44.526140+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"560'8836","reported_seq":10962,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.575349+0000","last_change":"2023-09-23T21:16:00.748968+0000","last_active":"2023-09-23T21:17:48.575349+0000","last_peered":"2023-09-23T21:17:48.575349+0000","last_clean":"2023-09-23T21:17:48.575349+0000","last_became_active":"2023-09-23T21:16:00.745123+0000","last_became_peered":"2023-09-23T21:16:00.745123+0000","last_unstale":"2023-09-23T21:17:48.575349+0000","last_undegraded":"2023-09-23T21:17:48.575349+0000","last_fullsized":"2023-09-23T21:17:48.575349+0000","mapping_epoch":501,"log_start":"515'3973","ondisk_log_start":"515'3973","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4863,"log_dups_size":0,"ondisk_log_size":4863,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14843904,"num_objects":226,"num_object_clones":0,"num_object_copies":678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5651,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":502,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":389,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523383+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:17:45.523383+0000","last_peered":"2023-09-23T21:17:45.523383+0000","last_clean":"2023-09-23T21:17:45.523383+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:17:45.523383+0000","last_undegraded":"2023-09-23T21:17:45.523383+0000","last_fullsized":"2023-09-23T21:17:45.523383+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":906,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.523312+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:17:44.523312+0000","last_peered":"2023-09-23T21:17:44.523312+0000","last_clean":"2023-09-23T21:17:44.523312+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:17:44.523312+0000","last_undegraded":"2023-09-23T21:17:44.523312+0000","last_fullsized":"2023-09-23T21:17:44.523312+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":132,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.508365+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:17:44.508365+0000","last_peered":"2023-09-23T21:17:44.508365+0000","last_clean":"2023-09-23T21:17:44.508365+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:17:44.508365+0000","last_undegraded":"2023-09-23T21:17:44.508365+0000","last_fullsized":"2023-09-23T21:17:44.508365+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514306+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:17:44.514306+0000","last_peered":"2023-09-23T21:17:44.514306+0000","last_clean":"2023-09-23T21:17:44.514306+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:17:44.514306+0000","last_undegraded":"2023-09-23T21:17:44.514306+0000","last_fullsized":"2023-09-23T21:17:44.514306+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"560'8614","reported_seq":8725,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:46.254715+0000","last_change":"2023-09-23T21:16:51.832676+0000","last_active":"2023-09-23T21:17:46.254715+0000","last_peered":"2023-09-23T21:17:46.254715+0000","last_clean":"2023-09-23T21:17:46.254715+0000","last_became_active":"2023-09-23T21:16:51.803067+0000","last_became_peered":"2023-09-23T21:16:51.803067+0000","last_unstale":"2023-09-23T21:17:46.254715+0000","last_undegraded":"2023-09-23T21:17:46.254715+0000","last_fullsized":"2023-09-23T21:17:46.254715+0000","mapping_epoch":545,"log_start":"497'2133","ondisk_log_start":"497'2133","created":482,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6481,"log_dups_size":0,"ondisk_log_size":6481,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20021248,"num_objects":305,"num_object_clones":0,"num_object_copies":915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5483,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":187,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.523384+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:17:44.523384+0000","last_peered":"2023-09-23T21:17:44.523384+0000","last_clean":"2023-09-23T21:17:44.523384+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:17:44.523384+0000","last_undegraded":"2023-09-23T21:17:44.523384+0000","last_fullsized":"2023-09-23T21:17:44.523384+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":56,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.519989+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:17:44.519989+0000","last_peered":"2023-09-23T21:17:44.519989+0000","last_clean":"2023-09-23T21:17:44.519989+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:17:44.519989+0000","last_undegraded":"2023-09-23T21:17:44.519989+0000","last_fullsized":"2023-09-23T21:17:44.519989+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":469,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.513498+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:17:44.513498+0000","last_peered":"2023-09-23T21:17:44.513498+0000","last_clean":"2023-09-23T21:17:44.513498+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:17:44.513498+0000","last_undegraded":"2023-09-23T21:17:44.513498+0000","last_fullsized":"2023-09-23T21:17:44.513498+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"560'8620","reported_seq":10755,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.099983+0000","last_change":"2023-09-23T21:16:00.763134+0000","last_active":"2023-09-23T21:17:48.099983+0000","last_peered":"2023-09-23T21:17:48.099983+0000","last_clean":"2023-09-23T21:17:48.099983+0000","last_became_active":"2023-09-23T21:16:00.758875+0000","last_became_peered":"2023-09-23T21:16:00.758875+0000","last_unstale":"2023-09-23T21:17:48.099983+0000","last_undegraded":"2023-09-23T21:17:48.099983+0000","last_fullsized":"2023-09-23T21:17:48.099983+0000","mapping_epoch":498,"log_start":"500'2427","ondisk_log_start":"500'2427","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6193,"log_dups_size":0,"ondisk_log_size":6193,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13434880,"num_objects":205,"num_object_clones":0,"num_object_copies":615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5508,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":157,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520640+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:17:44.520640+0000","last_peered":"2023-09-23T21:17:44.520640+0000","last_clean":"2023-09-23T21:17:44.520640+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:17:44.520640+0000","last_undegraded":"2023-09-23T21:17:44.520640+0000","last_fullsized":"2023-09-23T21:17:44.520640+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":373,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.512976+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:17:44.512976+0000","last_peered":"2023-09-23T21:17:44.512976+0000","last_clean":"2023-09-23T21:17:44.512976+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:17:44.512976+0000","last_undegraded":"2023-09-23T21:17:44.512976+0000","last_fullsized":"2023-09-23T21:17:44.512976+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":265,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.508911+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:17:44.508911+0000","last_peered":"2023-09-23T21:17:44.508911+0000","last_clean":"2023-09-23T21:17:44.508911+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:17:44.508911+0000","last_undegraded":"2023-09-23T21:17:44.508911+0000","last_fullsized":"2023-09-23T21:17:44.508911+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"560'7173","reported_seq":7385,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.344019+0000","last_change":"2023-09-23T21:16:51.076997+0000","last_active":"2023-09-23T21:17:48.344019+0000","last_peered":"2023-09-23T21:17:48.344019+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:16:51.057102+0000","last_became_peered":"2023-09-23T21:16:51.057102+0000","last_unstale":"2023-09-23T21:17:48.344019+0000","last_undegraded":"2023-09-23T21:16:50.840109+0000","last_fullsized":"2023-09-23T21:16:50.799660+0000","mapping_epoch":544,"log_start":"509'3490","ondisk_log_start":"509'3490","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":3683,"log_dups_size":0,"ondisk_log_size":3683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.748037+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15073280,"num_objects":230,"num_object_clones":0,"num_object_copies":690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1565,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3611,"num_write_kb":60864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":230},{"shards":"2,5","objects":-85}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":215,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.526646+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:17:44.526646+0000","last_peered":"2023-09-23T21:17:44.526646+0000","last_clean":"2023-09-23T21:17:44.526646+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:17:44.526646+0000","last_undegraded":"2023-09-23T21:17:44.526646+0000","last_fullsized":"2023-09-23T21:17:44.526646+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514248+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:17:44.514248+0000","last_peered":"2023-09-23T21:17:44.514248+0000","last_clean":"2023-09-23T21:17:44.514248+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:17:44.514248+0000","last_undegraded":"2023-09-23T21:17:44.514248+0000","last_fullsized":"2023-09-23T21:17:44.514248+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":240,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.523227+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:17:44.523227+0000","last_peered":"2023-09-23T21:17:44.523227+0000","last_clean":"2023-09-23T21:17:44.523227+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:17:44.523227+0000","last_undegraded":"2023-09-23T21:17:44.523227+0000","last_fullsized":"2023-09-23T21:17:44.523227+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"560'7301","reported_seq":7461,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.073770+0000","last_change":"2023-09-23T21:16:51.063450+0000","last_active":"2023-09-23T21:17:48.073770+0000","last_peered":"2023-09-23T21:17:48.073770+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:16:51.051345+0000","last_became_peered":"2023-09-23T21:16:51.051345+0000","last_unstale":"2023-09-23T21:17:48.073770+0000","last_undegraded":"2023-09-23T21:16:50.807164+0000","last_fullsized":"2023-09-23T21:16:50.788756+0000","mapping_epoch":544,"log_start":"500'2684","ondisk_log_start":"500'2684","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":4617,"log_dups_size":0,"ondisk_log_size":4617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:15.933191+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15859712,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1557,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3813,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-59},{"shards":"2,5,7","objects":242}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":466,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514823+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:17:44.514823+0000","last_peered":"2023-09-23T21:17:44.514823+0000","last_clean":"2023-09-23T21:17:44.514823+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:17:44.514823+0000","last_undegraded":"2023-09-23T21:17:44.514823+0000","last_fullsized":"2023-09-23T21:17:44.514823+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":97,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.511673+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:17:44.511673+0000","last_peered":"2023-09-23T21:17:44.511673+0000","last_clean":"2023-09-23T21:17:44.511673+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:17:44.511673+0000","last_undegraded":"2023-09-23T21:17:44.511673+0000","last_fullsized":"2023-09-23T21:17:44.511673+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.507397+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:17:44.507397+0000","last_peered":"2023-09-23T21:17:44.507397+0000","last_clean":"2023-09-23T21:17:44.507397+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:17:44.507397+0000","last_undegraded":"2023-09-23T21:17:44.507397+0000","last_fullsized":"2023-09-23T21:17:44.507397+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"560'7343","reported_seq":7476,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.321628+0000","last_change":"2023-09-23T21:16:51.067304+0000","last_active":"2023-09-23T21:17:48.321628+0000","last_peered":"2023-09-23T21:17:48.321628+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:16:51.048602+0000","last_became_peered":"2023-09-23T21:16:51.048602+0000","last_unstale":"2023-09-23T21:17:48.321628+0000","last_undegraded":"2023-09-23T21:16:50.818351+0000","last_fullsized":"2023-09-23T21:16:50.791414+0000","mapping_epoch":544,"log_start":"509'3637","ondisk_log_start":"509'3637","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3706,"log_dups_size":0,"ondisk_log_size":3706,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12058624,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5670,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"2,4,5","objects":216},{"shards":"4,5","objects":-802}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":187,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.520303+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:17:44.520303+0000","last_peered":"2023-09-23T21:17:44.520303+0000","last_clean":"2023-09-23T21:17:44.520303+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:17:44.520303+0000","last_undegraded":"2023-09-23T21:17:44.520303+0000","last_fullsized":"2023-09-23T21:17:44.520303+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":623,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.513033+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:17:44.513033+0000","last_peered":"2023-09-23T21:17:44.513033+0000","last_clean":"2023-09-23T21:17:44.513033+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:17:44.513033+0000","last_undegraded":"2023-09-23T21:17:44.513033+0000","last_fullsized":"2023-09-23T21:17:44.513033+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"560'7213","reported_seq":11268,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.318595+0000","last_change":"2023-09-23T21:16:37.510873+0000","last_active":"2023-09-23T21:17:48.318595+0000","last_peered":"2023-09-23T21:17:48.318595+0000","last_clean":"2023-09-23T21:17:48.318595+0000","last_became_active":"2023-09-23T21:16:37.500341+0000","last_became_peered":"2023-09-23T21:16:37.500341+0000","last_unstale":"2023-09-23T21:17:48.318595+0000","last_undegraded":"2023-09-23T21:17:48.318595+0000","last_fullsized":"2023-09-23T21:17:48.318595+0000","mapping_epoch":538,"log_start":"509'3449","ondisk_log_start":"509'3449","created":507,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3764,"log_dups_size":0,"ondisk_log_size":3764,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12058624,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5552,"num_write_kb":183552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1970,"num_bytes_recovered":94175232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":264,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514412+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:17:44.514412+0000","last_peered":"2023-09-23T21:17:44.514412+0000","last_clean":"2023-09-23T21:17:44.514412+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:17:44.514412+0000","last_undegraded":"2023-09-23T21:17:44.514412+0000","last_fullsized":"2023-09-23T21:17:44.514412+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":218,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.512572+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:17:44.512572+0000","last_peered":"2023-09-23T21:17:44.512572+0000","last_clean":"2023-09-23T21:17:44.512572+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:17:44.512572+0000","last_undegraded":"2023-09-23T21:17:44.512572+0000","last_fullsized":"2023-09-23T21:17:44.512572+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"560'7244","reported_seq":9985,"reported_epoch":560,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:48.322765+0000","last_change":"2023-09-23T21:16:50.886093+0000","last_active":"2023-09-23T21:17:48.322765+0000","last_peered":"2023-09-23T21:17:48.322765+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:17:48.322765+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"509'3534","ondisk_log_start":"509'3534","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3710,"log_dups_size":0,"ondisk_log_size":3710,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14712832,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5561,"num_write_kb":185120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":61800448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5","objects":-398}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":29,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514205+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:17:44.514205+0000","last_peered":"2023-09-23T21:17:44.514205+0000","last_clean":"2023-09-23T21:17:44.514205+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:17:44.514205+0000","last_undegraded":"2023-09-23T21:17:44.514205+0000","last_fullsized":"2023-09-23T21:17:44.514205+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.511776+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:17:44.511776+0000","last_peered":"2023-09-23T21:17:44.511776+0000","last_clean":"2023-09-23T21:17:44.511776+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:17:44.511776+0000","last_undegraded":"2023-09-23T21:17:44.511776+0000","last_fullsized":"2023-09-23T21:17:44.511776+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"560'7062","reported_seq":12935,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.104385+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:17:48.104385+0000","last_peered":"2023-09-23T21:17:48.104385+0000","last_clean":"2023-09-23T21:17:48.104385+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:17:48.104385+0000","last_undegraded":"2023-09-23T21:17:48.104385+0000","last_fullsized":"2023-09-23T21:17:48.104385+0000","mapping_epoch":518,"log_start":"499'2191","ondisk_log_start":"499'2191","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4871,"log_dups_size":0,"ondisk_log_size":4871,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14614528,"num_objects":223,"num_object_clones":0,"num_object_copies":669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5399,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":25,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.514225+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:17:44.514225+0000","last_peered":"2023-09-23T21:17:44.514225+0000","last_clean":"2023-09-23T21:17:44.514225+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:17:44.514225+0000","last_undegraded":"2023-09-23T21:17:44.514225+0000","last_fullsized":"2023-09-23T21:17:44.514225+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":420,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.512083+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:17:44.512083+0000","last_peered":"2023-09-23T21:17:44.512083+0000","last_clean":"2023-09-23T21:17:44.512083+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:17:44.512083+0000","last_undegraded":"2023-09-23T21:17:44.512083+0000","last_fullsized":"2023-09-23T21:17:44.512083+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"560'7068","reported_seq":11036,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:46.172849+0000","last_change":"2023-09-23T21:16:49.855987+0000","last_active":"2023-09-23T21:17:46.172849+0000","last_peered":"2023-09-23T21:17:46.172849+0000","last_clean":"2023-09-23T21:17:46.172849+0000","last_became_active":"2023-09-23T21:16:15.742634+0000","last_became_peered":"2023-09-23T21:16:15.742634+0000","last_unstale":"2023-09-23T21:17:46.172849+0000","last_undegraded":"2023-09-23T21:17:46.172849+0000","last_fullsized":"2023-09-23T21:17:46.172849+0000","mapping_epoch":531,"log_start":"497'2141","ondisk_log_start":"497'2141","created":507,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20938752,"num_objects":320,"num_object_clones":0,"num_object_copies":960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5432,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":516,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.512902+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:17:44.512902+0000","last_peered":"2023-09-23T21:17:44.512902+0000","last_clean":"2023-09-23T21:17:44.512902+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:17:44.512902+0000","last_undegraded":"2023-09-23T21:17:44.512902+0000","last_fullsized":"2023-09-23T21:17:44.512902+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":227,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-09-23T21:17:44.507917+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:17:44.507917+0000","last_peered":"2023-09-23T21:17:44.507917+0000","last_clean":"2023-09-23T21:17:44.507917+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:17:44.507917+0000","last_undegraded":"2023-09-23T21:17:44.507917+0000","last_fullsized":"2023-09-23T21:17:44.507917+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":36,"stat_sum":{"num_bytes":480444440,"num_objects":7333,"num_object_clones":0,"num_object_copies":21999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15689,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7333,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162598,"num_write_kb":4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32862,"num_bytes_recovered":1665368064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3450155008,"data_stored":3450142792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":182960,"ondisk_log_size":182960,"up":108,"acting":95,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":38918,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201869,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101716,"kb_used_data":309580,"kb_used_omap":0,"kb_used_meta":792064,"kb_avail":92643436,"statfs":{"total":95995035648,"available":94866878464,"internally_reserved":0,"allocated":317009920,"data_stored":315477843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":811073536},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":3.0539999999999998,"15min":2.2519999999999998},"min":{"1min":0.26300000000000001,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":7.726,"5min":141.33199999999999,"15min":141.33199999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.2109999999999999,"5min":3.3530000000000002,"15min":2.3199999999999998},"min":{"1min":0.307,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.3940000000000001,"5min":141.31700000000001,"15min":141.31700000000001},"last":0.73599999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:19 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.1280000000000001,"15min":3.4239999999999999},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.4329999999999998,"5min":16.684000000000001,"15min":114.16500000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.5920000000000001,"5min":2.8799999999999999,"15min":3.867},"min":{"1min":0.307,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.7699999999999996,"5min":117.637,"15min":117.637},"last":0.52900000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":4.7679999999999998,"15min":4.4850000000000003},"min":{"1min":0.246,"5min":0.224,"15min":0.224},"max":{"1min":5.4160000000000004,"5min":135.529,"15min":183.44200000000001},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.5740000000000001,"5min":5.6550000000000002,"15min":4.6909999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2370000000000001,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":7.1909999999999998,"15min":4.1210000000000004},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":7.7960000000000003,"5min":135.66200000000001,"15min":135.66200000000001},"last":0.497},{"interface":"front","average":{"1min":1.613,"5min":7.6120000000000001,"15min":4.2199999999999998},"min":{"1min":0.375,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":5.2779999999999996,"5min":135.47,"15min":135.47},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.9430000000000001,"15min":1.2090000000000001},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1829999999999998,"5min":10.989000000000001,"15min":13.739000000000001},"last":8.7940000000000005},{"interface":"front","average":{"1min":3.5840000000000001,"5min":2.0950000000000002,"15min":1.2},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":20.306999999999999,"5min":20.306999999999999,"15min":20.306999999999999},"last":2.6829999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":2.6389999999999998,"15min":1.6080000000000001},"min":{"1min":0.53100000000000003,"5min":0.29099999999999998,"15min":0.26900000000000002},"max":{"1min":14.105,"5min":14.105,"15min":14.105},"last":3.3540000000000001},{"interface":"front","average":{"1min":3.9630000000000001,"5min":2.5339999999999998,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":17.585999999999999,"5min":17.585999999999999,"15min":17.585999999999999},"last":5.7770000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.2930000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":9.9969999999999999,"5min":13.369,"15min":143.18700000000001},"last":2.7280000000000002},{"interface":"front","average":{"1min":2.9380000000000002,"5min":1.819,"15min":1.929},"min":{"1min":0.31,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":15.566000000000001,"5min":15.566000000000001,"15min":142.70400000000001},"last":5.7190000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463489,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1639432,"kb_used_data":882368,"kb_used_omap":0,"kb_used_meta":757056,"kb_avail":92105720,"statfs":{"total":95995035648,"available":94316257280,"internally_reserved":0,"allocated":903544832,"data_stored":902025310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":775225344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":4.5339999999999998,"15min":2.7490000000000001},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":12.659000000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.0369999999999999,"5min":4.899,"15min":2.835},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":16.021999999999998,"5min":149.303,"15min":149.303},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":2.379,"15min":3.6080000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.22700000000000001},"max":{"1min":16.126999999999999,"5min":25.122,"15min":129.37899999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.986,"5min":1.8420000000000001,"15min":3.387},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":10.977,"5min":25.193999999999999,"15min":129.40100000000001},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.577999999999999,"5min":7.6189999999999998,"15min":5.9340000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":120.28700000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":181.846},{"interface":"front","average":{"1min":12.118,"5min":7.8120000000000003,"15min":5.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":120.414,"5min":133.33000000000001,"15min":199.08000000000001},"last":182.14500000000001}]},{"osd":3,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":5.6669999999999998,"15min":3.3199999999999998},"min":{"1min":0.38,"5min":0.218,"15min":0.218},"max":{"1min":3.5089999999999999,"5min":137.875,"15min":137.875},"last":0.435},{"interface":"front","average":{"1min":1.7250000000000001,"5min":5.391,"15min":3.3650000000000002},"min":{"1min":0.30099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.676,"5min":137.60900000000001,"15min":137.60900000000001},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":8.9060000000000006,"5min":2.9329999999999998,"15min":1.673},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.22600000000000001},"max":{"1min":55.494,"5min":55.494,"15min":55.494},"last":0.28799999999999998},{"interface":"front","average":{"1min":8.7550000000000008,"5min":2.8100000000000001,"15min":1.4970000000000001},"min":{"1min":0.39900000000000002,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":55.305,"5min":55.305,"15min":55.305},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":15.223000000000001,"5min":4.3630000000000004,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":222.90899999999999,"5min":222.90899999999999,"15min":222.90899999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":15.375,"5min":4.4930000000000003,"15min":2.1739999999999999},"min":{"1min":0.37,"5min":0.32200000000000001,"15min":0.28499999999999998},"max":{"1min":224.655,"5min":224.655,"15min":224.655},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":13.882999999999999,"5min":3.629,"15min":1.883},"min":{"1min":0.39600000000000002,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":165.96799999999999,"5min":165.96799999999999,"15min":165.96799999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":13.532,"5min":3.859,"15min":1.98},"min":{"1min":0.442,"5min":0.251,"15min":0.251},"max":{"1min":165.82400000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":0.48399999999999999}]}]},{"osd":1,"up_from":0,"seq":1185410973936,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991676,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781567,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1471360,"kb_used_data":363576,"kb_used_omap":7,"kb_used_meta":1107768,"kb_avail":92273792,"statfs":{"total":95995035648,"available":94488363008,"internally_reserved":0,"allocated":372301824,"data_stored":369611686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1134354513},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.8420000000000001,"15min":2.157},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":8.0079999999999991,"5min":103.021,"15min":103.021},"last":0.17899999999999999},{"interface":"front","average":{"1min":0.29599999999999999,"5min":3.133,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":0.378,"5min":103.154,"15min":103.154},"last":0.25900000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.2000000000000002,"15min":4.0869999999999997},"min":{"1min":0.24099999999999999,"5min":0.224,"15min":0.19},"max":{"1min":2.4409999999999998,"5min":54.521000000000001,"15min":212.12},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.98999999999999999,"5min":2.4820000000000002,"15min":4.1399999999999997},"min":{"1min":0.254,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":8.048,"5min":54.598999999999997,"15min":213.02799999999999},"last":0.214}]},{"osd":3,"last update":"Sat Sep 23 21:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.5670000000000002,"15min":4.0339999999999998},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5190000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.2250000000000001,"15min":4.2519999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.2189999999999994,"5min":82.241,"15min":143.27799999999999},"last":0.27900000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":13.896000000000001,"5min":4.3010000000000002,"15min":3.1800000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":220.82900000000001,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":13.894,"5min":4.9130000000000003,"15min":3.823},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":220.84299999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.34300000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.541,"5min":5.1059999999999999,"15min":3.48},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":220.63900000000001,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.31},{"interface":"front","average":{"1min":16.257000000000001,"5min":4.7380000000000004,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":220.31999999999999,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.42599999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.597,"5min":4.6210000000000004,"15min":3.762},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":210.74700000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":1.202},{"interface":"front","average":{"1min":14.371,"5min":4.4640000000000004,"15min":3.1259999999999999},"min":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":210.72900000000001,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.48399999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.4940000000000002,"15min":1.948},"min":{"1min":0.371,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":14.019,"5min":25.260000000000002,"15min":25.260000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.921,"5min":2.4990000000000001,"15min":2.2850000000000001},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":5.0229999999999997,"5min":12.613,"15min":16.875},"last":1.397}]}]},{"osd":3,"up_from":0,"seq":635655160160,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986779,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1312276,"kb_used_data":505292,"kb_used_omap":0,"kb_used_meta":806912,"kb_avail":92432876,"statfs":{"total":95995035648,"available":94651265024,"internally_reserved":0,"allocated":517419008,"data_stored":515887194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":826277888},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":5.1909999999999998,"15min":2.7109999999999999},"min":{"1min":0.247,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":10.436,"5min":226.751,"15min":226.751},"last":0.997},{"interface":"front","average":{"1min":2.0750000000000002,"5min":4.8049999999999997,"15min":2.556},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.223},"max":{"1min":6.218,"5min":226.43299999999999,"15min":226.43299999999999},"last":6.0919999999999996}]},{"osd":1,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":3.802,"15min":5.2999999999999998},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2510000000000003,"5min":119.55800000000001,"15min":127.411},"last":6.1280000000000001},{"interface":"front","average":{"1min":2.8050000000000002,"5min":3.71,"15min":4.5830000000000002},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":10.506,"5min":116.78100000000001,"15min":118.075},"last":1.0880000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":8.9459999999999997,"5min":7.5439999999999996,"15min":5.8710000000000004},"min":{"1min":0.26000000000000001,"5min":0.253,"15min":0.253},"max":{"1min":90.950000000000003,"5min":210.50299999999999,"15min":210.50299999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":9.9280000000000008,"5min":8.5630000000000006,"15min":6.0149999999999997},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":90.846000000000004,"5min":210.59,"15min":210.59},"last":1.0309999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":3.2440000000000002,"15min":4.1200000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0650000000000004,"5min":111.22199999999999,"15min":214.64099999999999},"last":0.311},{"interface":"front","average":{"1min":3.0659999999999998,"5min":4.508,"15min":4.5209999999999999},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":10.529,"5min":111.39400000000001,"15min":214.54900000000001},"last":1.1459999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7829999999999999,"5min":1.7749999999999999,"15min":1.139},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.417,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.8399999999999999,"15min":1.49},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":14.675000000000001,"5min":34.883000000000003,"15min":34.883000000000003},"last":1.117}]},{"osd":6,"last update":"Sat Sep 23 21:16:53 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.524,"15min":1.333},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.2569999999999997,"5min":10.446999999999999,"15min":36.557000000000002},"last":6.1609999999999996},{"interface":"front","average":{"1min":2.2690000000000001,"5min":1.579,"15min":1.3169999999999999},"min":{"1min":0.35799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.8049999999999997,"5min":6.8049999999999997,"15min":38.468000000000004},"last":1.373}]},{"osd":7,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":2.2170000000000001,"15min":1.8740000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":11.859,"5min":18.352,"15min":145.721},"last":1.177},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.8600000000000001,"15min":1.893},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"max":{"1min":8.5570000000000004,"5min":10.677,"15min":148.45699999999999},"last":6.1989999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725143,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266052,"kb_used_data":397948,"kb_used_omap":0,"kb_used_meta":867968,"kb_avail":92479100,"statfs":{"total":95995035648,"available":94698598400,"internally_reserved":0,"allocated":407498752,"data_stored":405978534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":888799232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":1.732,"15min":1.6699999999999999},"min":{"1min":0.33500000000000002,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":7.7450000000000001,"5min":20.023,"15min":82.280000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.077,"5min":2.0640000000000001,"15min":1.8680000000000001},"min":{"1min":0.29299999999999998,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.6710000000000003,"5min":19.853000000000002,"15min":85.236000000000004},"last":0.36299999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3479999999999999,"15min":4.7039999999999997},"min":{"1min":0.255,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":11.374000000000001,"5min":12.032999999999999,"15min":233.77099999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.3979999999999999,"5min":1.6399999999999999,"15min":4.359},"min":{"1min":0.39600000000000002,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":4.0590000000000002,"5min":12.016,"15min":233.75200000000001},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":2.1110000000000002,"15min":3.7080000000000002},"min":{"1min":0.37,"5min":0.251,"15min":0.251},"max":{"1min":7.8220000000000001,"5min":20.411999999999999,"15min":231.34700000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.0840000000000001,"5min":2.1859999999999999,"15min":3.9009999999999998},"min":{"1min":0.38600000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.0439999999999996,"5min":11.509,"15min":231.381},"last":2.2639999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":2.5710000000000002,"15min":4.0469999999999997},"min":{"1min":0.40500000000000003,"5min":0.317,"15min":0.27700000000000002},"max":{"1min":8.077,"5min":11.597,"15min":263.887},"last":0.379},{"interface":"front","average":{"1min":2.004,"5min":1.9910000000000001,"15min":3.831},"min":{"1min":0.46100000000000002,"5min":0.32200000000000001,"15min":0.30099999999999999},"max":{"1min":7.8810000000000002,"5min":10.638,"15min":263.70100000000002},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.7709999999999999,"15min":1.0129999999999999},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":4.0289999999999999,"5min":11.593999999999999,"15min":11.593999999999999},"last":77.804000000000002},{"interface":"front","average":{"1min":3.3860000000000001,"5min":1.95,"15min":1.1899999999999999},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.26400000000000001},"max":{"1min":12.49,"5min":12.49,"15min":12.49},"last":77.825999999999993}]},{"osd":6,"last update":"Sat Sep 23 21:16:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9009999999999998,"5min":1.6739999999999999,"15min":1.7130000000000001},"min":{"1min":0.38500000000000001,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":7.8520000000000003,"5min":8.9130000000000003,"15min":83.451999999999998},"last":2.1669999999999998},{"interface":"front","average":{"1min":3.0379999999999998,"5min":2.2669999999999999,"15min":1.9339999999999999},"min":{"1min":0.58499999999999996,"5min":0.30399999999999999,"15min":0.27400000000000002},"max":{"1min":8.0050000000000008,"5min":12.725,"15min":83.484999999999999},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:17:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1000000000000001,"5min":2.161,"15min":1.419},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":7.4020000000000001,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":2.2200000000000002},{"interface":"front","average":{"1min":2.431,"5min":1.585,"15min":0.995},"min":{"1min":0.45500000000000002,"5min":0.34899999999999998,"15min":0.28899999999999998},"max":{"1min":10.196999999999999,"5min":11.640000000000001,"15min":11.640000000000001},"last":0.60499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":334430208,"data_stored":334430208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1064501248,"data_stored":1064501248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":490344448,"data_stored":490340376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":377819136,"data_stored":377815064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":891551744,"data_stored":891551744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":291508224,"data_stored":291504152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:17:52.227 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:52.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:52.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:52.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:52.364 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:52.415 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:52.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:52.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:52.519 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:52.545 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:52.588 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:52.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:52.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:52.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:52 smithi121 ceph-mon[211526]: pgmap v1749: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 432 MiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 821 op/s; 15689/20787 objects degraded (75.475%); 2.4 MiB/s, 187 objects/s recovering 2023-09-23T21:17:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15689/20787 objects degraded (75.475%), 15 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:52.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:52 smithi121 ceph-mon[219709]: pgmap v1749: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 432 MiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 821 op/s; 15689/20787 objects degraded (75.475%); 2.4 MiB/s, 187 objects/s recovering 2023-09-23T21:17:52.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15689/20787 objects degraded (75.475%), 15 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:52.691 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:52.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:52 smithi123 ceph-mon[193903]: pgmap v1749: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 432 MiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 821 op/s; 15689/20787 objects degraded (75.475%); 2.4 MiB/s, 187 objects/s recovering 2023-09-23T21:17:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15689/20787 objects degraded (75.475%), 15 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:52.738 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:52.844 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:52.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:52.958 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:53.016 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:53.126 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:53.144 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:53.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:53.258 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:53.290 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:53.464 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:53.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:53.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:53.569 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:53.603 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:53.783 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:17:53.962 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:53.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:53.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:54.067 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:54.094 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:54.141 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:54.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:54.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:54.246 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:54.277 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:54.441 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:17:54.445 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:54 smithi123 ceph-mon[193903]: pgmap v1750: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 353 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 602 op/s; 15689/16983 objects degraded (92.381%); 2.1 MiB/s, 66 objects/s recovering 2023-09-23T21:17:54.445 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2427930503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:54.622 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:54.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:54 smithi121 ceph-mon[219709]: pgmap v1750: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 353 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 602 op/s; 15689/16983 objects degraded (92.381%); 2.1 MiB/s, 66 objects/s recovering 2023-09-23T21:17:54.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2427930503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:54.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:54.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:54 smithi121 ceph-mon[211526]: pgmap v1750: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 353 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 602 op/s; 15689/16983 objects degraded (92.381%); 2.1 MiB/s, 66 objects/s recovering 2023-09-23T21:17:54.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2427930503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:17:54.729 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:54.782 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:54.879 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:54.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:54.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:54.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:17:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:17:54] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.43.0" 2023-09-23T21:17:55.000 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:55.081 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:55.089 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:17:55.199 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:55.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:55.316 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:17:55.351 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:55.440 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:17:55.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:55.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:17:55.460 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:55.460 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:55 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:55.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:55 smithi123 ceph-mon[193903]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:55 smithi123 ceph-mon[193903]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2427930503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:55.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:55 smithi123 ceph-mon[193903]: osdmap e561: 8 total, 8 up, 5 in 2023-09-23T21:17:55.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:55 smithi123 ceph-mon[193903]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:55 smithi123 ceph-mon[193903]: pgmap v1752: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 193 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 801 op/s; 15689/9303 objects degraded (168.645%); 1.9 MiB/s, 112 objects/s recovering 2023-09-23T21:17:55.547 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:55.628 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:55.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:55.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[211526]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[211526]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2427930503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[211526]: osdmap e561: 8 total, 8 up, 5 in 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[211526]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[211526]: pgmap v1752: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 193 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 801 op/s; 15689/9303 objects degraded (168.645%); 1.9 MiB/s, 112 objects/s recovering 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[219709]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[219709]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2427930503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:17:55.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[219709]: osdmap e561: 8 total, 8 up, 5 in 2023-09-23T21:17:55.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[219709]: failed to encode map e561 with expected crc 2023-09-23T21:17:55.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:55 smithi121 ceph-mon[219709]: pgmap v1752: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 193 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 801 op/s; 15689/9303 objects degraded (168.645%); 1.9 MiB/s, 112 objects/s recovering 2023-09-23T21:17:55.768 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:55.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:55.822 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:17:55.822 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:17:55.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:17:55.823 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:17:55.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.823 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5656 12.e 12:763500b5:::benchmark_data_smithi121_563784_object5655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:17:55.823 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.187588+0000", 2023-09-23T21:17:55.824 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 480.63193296100002, 2023-09-23T21:17:55.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26195850999999998, 2023-09-23T21:17:55.824 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.824 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.824 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.825 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.825 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.825 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5656 2023-09-23T21:17:55.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.825 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187588+0000", 2023-09-23T21:17:55.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187588+0000", 2023-09-23T21:17:55.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.828 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.828 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.828 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187587+0000", 2023-09-23T21:17:55.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:17:55.828 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.829 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.829 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.829 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187608+0000", 2023-09-23T21:17:55.829 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0772e-05 2023-09-23T21:17:55.829 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.830 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.830 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.830 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187618+0000", 2023-09-23T21:17:55.830 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8719999999999997e-06 2023-09-23T21:17:55.830 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.830 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.830 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.187625+0000", 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8809999999999998e-06 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.188379+0000", 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00075411100000000002 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.188487+0000", 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000108107 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.188629+0000", 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014239799999999999 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.189770+0000", 2023-09-23T21:17:55.832 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001140624 2023-09-23T21:17:55.833 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.833 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.833 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.190075+0000", 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00030567000000000002 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449465+0000", 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25938914899999999 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449508+0000", 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2911999999999998e-05 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.449547+0000", 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9008000000000003e-05 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5668 12.e 12:73426eaa:::benchmark_data_smithi121_563784_object5667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.191572+0000", 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 480.62794946899999, 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26293898500000001, 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5668 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.836 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191572+0000", 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191572+0000", 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191571+0000", 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191597+0000", 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.652e-05 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191608+0000", 2023-09-23T21:17:55.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0976e-05 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191615+0000", 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3259999999999998e-06 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191655+0000", 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9363000000000002e-05 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191781+0000", 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012657599999999999 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,4", 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.448896+0000", 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25711451899999999 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.840 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452403+0000", 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003506852 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.452411+0000", 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1950000000000003e-06 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.454435+0000", 2023-09-23T21:17:55.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00202381 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.454475+0000", 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0309000000000003e-05 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.454511+0000", 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.54e-05 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.842 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39674 12.3 12:c3c1b119:::benchmark_data_smithi121_563784_object39673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.119423+0000", 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 456.70009839900001, 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27609113200000002, 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.843 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39674 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119423+0000", 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119423+0000", 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119421+0000", 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119448+0000", 2023-09-23T21:17:55.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7127e-05 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119462+0000", 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3419e-05 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119471+0000", 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3700000000000001e-06 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.126557+0000", 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0070855839999999998 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.126680+0000", 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000123134 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:17:55.847 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.126873+0000", 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019315900000000001 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.130909+0000", 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004035772 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.134583+0000", 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036738930000000001 2023-09-23T21:17:55.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393864+0000", 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25928128299999997 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393920+0000", 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5995000000000002e-05 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.395514+0000", 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001593726 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39678 12.3 12:c3b9d452:::benchmark_data_smithi121_563784_object39677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 456.70003885199998, 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27732642000000002, 2023-09-23T21:17:55.850 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39678 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119482+0000", 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119502+0000", 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0346000000000001e-05 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119519+0000", 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6858999999999999e-05 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119526+0000", 2023-09-23T21:17:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7719999999999997e-06 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.128199+0000", 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0086730639999999994 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.128325+0000", 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012607699999999999 2023-09-23T21:17:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.128436+0000", 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000111576 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.130914+0000", 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024774319999999999 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.134631+0000", 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003717321 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.395566+0000", 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26093502200000002 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.395613+0000", 2023-09-23T21:17:55.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6745000000000001e-05 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.396809+0000", 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0011957440000000001 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39679 12.3 12:c2c45921:::benchmark_data_smithi121_563784_object39678:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:17:55.857 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.120298+0000", 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 456.69922328500002, 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27795734500000002, 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39679 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120298+0000", 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120298+0000", 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120297+0000", 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120316+0000", 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8694000000000001e-05 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.860 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120327+0000", 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0954999999999999e-05 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120338+0000", 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1939e-05 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.861 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.129660+0000", 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0093211189999999992 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.129825+0000", 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016582500000000001 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.130013+0000", 2023-09-23T21:17:55.862 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000187243 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.131019+0000", 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001006297 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.134666+0000", 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036467190000000001 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.863 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.396869+0000", 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26220349100000001 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.396924+0000", 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5009000000000001e-05 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.864 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.398255+0000", 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0013308720000000001 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:42856 12.c 12:39b21385:::benchmark_data_smithi121_563784_object42855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:22.268596+0000", 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 453.55092510100002, 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28378214000000002, 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.865 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 42856 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268596+0000", 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.866 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268596+0000", 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268594+0000", 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.867 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268618+0000", 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3674999999999999e-05 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268628+0000", 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6770000000000007e-06 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268634+0000", 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3589999999999996e-06 2023-09-23T21:17:55.868 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268771+0000", 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013668799999999999 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.268910+0000", 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139715 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.869 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.269058+0000", 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148126 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.270664+0000", 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016054019999999999 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.870 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.273197+0000", 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025333159999999999 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.552284+0000", 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27908699199999998 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.552337+0000", 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3019999999999997e-05 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.871 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:22.552378+0000", 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0812999999999997e-05 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64067 12.c 12:3da455b5:::benchmark_data_smithi121_563784_object64066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.247796+0000", 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 429.57172529299999, 2023-09-23T21:17:55.872 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.34022702199999999, 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64067 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247796+0000", 2023-09-23T21:17:55.873 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247796+0000", 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247795+0000", 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.874 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247816+0000", 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0786999999999999e-05 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247823+0000", 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0060000000000003e-06 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.875 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247828+0000", 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3990000000000003e-06 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247873+0000", 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4922000000000001e-05 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247991+0000", 2023-09-23T21:17:55.876 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011751700000000001 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.248122+0000", 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013134099999999999 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.250546+0000", 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024234019999999998 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.877 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.252464+0000", 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0019186310000000001 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587936+0000", 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33547226699999999 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.878 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587987+0000", 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0667000000000001e-05 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.588023+0000", 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5578999999999999e-05 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.879 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.880 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89065 12.c 12:3dd530e7:::benchmark_data_smithi121_563784_object89064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:17:55.880 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.116722+0000", 2023-09-23T21:17:55.880 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 394.70279936700001, 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.29870879099999997, 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89065 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116722+0000", 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116722+0000", 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116720+0000", 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116744+0000", 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3669999999999999e-05 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116753+0000", 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6549999999999999e-06 2023-09-23T21:17:55.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116760+0000", 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.224e-06 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116798+0000", 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8661999999999997e-05 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116928+0000", 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000129698 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.117063+0000", 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013555899999999999 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.123190+0000", 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0061264270000000003 2023-09-23T21:17:55.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.127854+0000", 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046641260000000002 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.415336+0000", 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28748226700000001 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.415388+0000", 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1604000000000003e-05 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.415430+0000", 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2508e-05 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89066 12.c 12:356a92a0:::benchmark_data_smithi121_563784_object89065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.116765+0000", 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 394.70275583799997, 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.29923028600000001, 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89066 2023-09-23T21:17:55.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116765+0000", 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116765+0000", 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116765+0000", 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116786+0000", 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1500000000000001e-05 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116792+0000", 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1820000000000003e-06 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116799+0000", 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.2029999999999997e-06 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.120191+0000", 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033921870000000001 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.120316+0000", 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012497300000000001 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.120445+0000", 2023-09-23T21:17:55.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012880199999999999 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130360+0000", 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0099149540000000001 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.133512+0000", 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031525699999999999 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.415895+0000", 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28238268500000002 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.415946+0000", 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0804999999999999e-05 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.415995+0000", 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9867000000000002e-05 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89071 12.c 12:3467bfe0:::benchmark_data_smithi121_563784_object89070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.116803+0000", 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 394.70271797599997, 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.301316154, 2023-09-23T21:17:55.895 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89071 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116803+0000", 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.896 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116803+0000", 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116803+0000", 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116849+0000", 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6088000000000003e-05 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116856+0000", 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8340000000000001e-06 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116861+0000", 2023-09-23T21:17:55.898 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2209999999999996e-06 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124335+0000", 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0074742130000000004 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124489+0000", 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015369 2023-09-23T21:17:55.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124637+0000", 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148715 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.133518+0000", 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088805029999999997 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.133889+0000", 2023-09-23T21:17:55.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00037127599999999997 2023-09-23T21:17:55.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.418032+0000", 2023-09-23T21:17:55.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28414238000000003 2023-09-23T21:17:55.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.418082+0000", 2023-09-23T21:17:55.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9947000000000001e-05 2023-09-23T21:17:55.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.418119+0000", 2023-09-23T21:17:55.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7608999999999999e-05 2023-09-23T21:17:55.905 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.905 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.905 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.906 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89080 12.c 12:3e0edbd7:::benchmark_data_smithi121_563784_object89079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:17:55.906 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.130363+0000", 2023-09-23T21:17:55.906 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 394.689157782, 2023-09-23T21:17:55.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.290647777, 2023-09-23T21:17:55.906 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.907 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.907 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.907 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.907 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.907 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89080 2023-09-23T21:17:55.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.907 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130363+0000", 2023-09-23T21:17:55.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130363+0000", 2023-09-23T21:17:55.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130362+0000", 2023-09-23T21:17:55.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:55.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130387+0000", 2023-09-23T21:17:55.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5143999999999998e-05 2023-09-23T21:17:55.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130404+0000", 2023-09-23T21:17:55.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7345000000000001e-05 2023-09-23T21:17:55.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130413+0000", 2023-09-23T21:17:55.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6959999999999994e-06 2023-09-23T21:17:55.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130521+0000", 2023-09-23T21:17:55.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010772699999999999 2023-09-23T21:17:55.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130725+0000", 2023-09-23T21:17:55.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020399200000000001 2023-09-23T21:17:55.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.130861+0000", 2023-09-23T21:17:55.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000136747 2023-09-23T21:17:55.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.133909+0000", 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003047269 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.137522+0000", 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003613088 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.420936+0000", 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28341368700000003 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.420978+0000", 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2190000000000001e-05 2023-09-23T21:17:55.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.421011+0000", 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3278000000000002e-05 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.917 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116385 12.c 12:363a4924:::benchmark_data_smithi121_563784_object25982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.866444+0000", 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.95307661800001, 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25497925599999999, 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116385 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.918 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866444+0000", 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866444+0000", 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866443+0000", 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866449+0000", 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3130000000000004e-06 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866459+0000", 2023-09-23T21:17:55.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9729999999999992e-06 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866465+0000", 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8359999999999997e-06 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866510+0000", 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5351e-05 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866647+0000", 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000136978 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866743+0000", 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5886999999999999e-05 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.871616+0000", 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048731260000000002 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.871642+0000", 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5123e-05 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121343+0000", 2023-09-23T21:17:55.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24970186799999999 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121386+0000", 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2595000000000001e-05 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121424+0000", 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7610000000000001e-05 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.924 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116386 12.e 12:734fd74f:::benchmark_data_smithi121_563784_object25983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.866475+0000", 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.95304578600002, 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.254453928, 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.925 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116386 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866475+0000", 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866475+0000", 2023-09-23T21:17:55.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866475+0000", 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866477+0000", 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4760000000000001e-06 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866482+0000", 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0699999999999997e-06 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866485+0000", 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.095e-06 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866528+0000", 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2413999999999997e-05 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866647+0000", 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011864 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.866743+0000", 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6613000000000005e-05 2023-09-23T21:17:55.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874722+0000", 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0079788959999999992 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874734+0000", 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2058999999999999e-05 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.120458+0000", 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.245724258 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.120532+0000", 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3728999999999998e-05 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.120929+0000", 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039706000000000002 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116391 12.e 12:7ff9fff6:::benchmark_data_smithi121_563784_object25988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.870695+0000", 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.94882575700001, 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25051252600000001, 2023-09-23T21:17:55.932 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116391 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870695+0000", 2023-09-23T21:17:55.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870695+0000", 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870694+0000", 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870699+0000", 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8509999999999998e-06 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870707+0000", 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5860000000000005e-06 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870714+0000", 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4989999999999999e-06 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870753+0000", 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8844999999999998e-05 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870870+0000", 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011731 2023-09-23T21:17:55.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.870947+0000", 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6619000000000006e-05 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874742+0000", 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003795807 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.876150+0000", 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0014078350000000001 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.120972+0000", 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24482177599999999 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121018+0000", 2023-09-23T21:17:55.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.621e-05 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121208+0000", 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000189598 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116395 12.c 12:3dd3c065:::benchmark_data_smithi121_563784_object25992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:17:55.939 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.873322+0000", 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.94619871999998, 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25213229100000001, 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116395 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873322+0000", 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873322+0000", 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873321+0000", 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:17:55.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873327+0000", 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.866e-06 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873339+0000", 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1719000000000001e-05 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873347+0000", 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5660000000000001e-06 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873390+0000", 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2778999999999998e-05 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873515+0000", 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000125411 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.873638+0000", 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012340800000000001 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121290+0000", 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24765194300000001 2023-09-23T21:17:55.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121453+0000", 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000162497 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.125367+0000", 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003914173 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.125417+0000", 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9564999999999997e-05 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.125455+0000", 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8053999999999999e-05 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116397 12.3 12:c906893f:::benchmark_data_smithi121_563784_object25994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.874197+0000", 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.94532415399999, 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24734350099999999, 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116397 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.947 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874197+0000", 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874197+0000", 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874196+0000", 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874201+0000", 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4689999999999996e-06 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874213+0000", 2023-09-23T21:17:55.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1557e-05 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874221+0000", 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6109999999999996e-06 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874353+0000", 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000132137 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874465+0000", 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112261 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874535+0000", 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9917000000000002e-05 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.120602+0000", 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24606683900000001 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.120609+0000", 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2840000000000001e-06 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121490+0000", 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00088074500000000005 2023-09-23T21:17:55.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121520+0000", 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0552999999999997e-05 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121540+0000", 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0078e-05 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.953 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:116390 12.8 e449/426 12:18d953d1:::benchmark_data_smithi121_563784_object25987:head v 449'7268, mlcod=449'7266)", 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.877423+0000", 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.94209840100001, 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.255300211, 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877423+0000", 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877423+0000", 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877422+0000", 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877429+0000", 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4950000000000002e-06 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877439+0000", 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6500000000000008e-06 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877445+0000", 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3949999999999998e-06 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.119185+0000", 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.241739378 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.119211+0000", 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6016e-05 2023-09-23T21:17:55.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.132692+0000", 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013480637 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.132723+0000", 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1288999999999998e-05 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.958 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:116385 12.c e449/426 ondisk, result = 0)", 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.877923+0000", 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.94159818600002, 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.243505254, 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877923+0000", 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877923+0000", 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877922+0000", 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.877927+0000", 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6310000000000002e-06 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.119680+0000", 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24175317900000001 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.119691+0000", 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1094000000000001e-05 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121330+0000", 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016390689999999999 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121338+0000", 2023-09-23T21:17:55.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3810000000000001e-06 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.121428+0000", 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0535e-05 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.963 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:116396 12.8 e449/426 12:19288291:::benchmark_data_smithi121_563784_object25993:head v 449'7269, mlcod=449'7266)", 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.878478+0000", 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 351.94104288599999, 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.254265721, 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.878478+0000", 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.878478+0000", 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.878478+0000", 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.878484+0000", 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0750000000000003e-06 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.119260+0000", 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.240776517 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.119283+0000", 2023-09-23T21:17:55.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2399999999999999e-05 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.122831+0000", 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035489559999999998 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.122849+0000", 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8034e-05 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.132726+0000", 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.009876157 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.132744+0000", 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8182999999999999e-05 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151904 12.3 12:ce488415:::benchmark_data_smithi121_563784_object61501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.987815+0000", 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 288.83170595600001, 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25388111800000002, 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:17:55.969 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151904 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987815+0000", 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987815+0000", 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:17:55.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987814+0000", 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987820+0000", 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7259999999999999e-06 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987831+0000", 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0838e-05 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987839+0000", 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0339999999999997e-06 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:17:55.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987885+0000", 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6505999999999998e-05 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.988001+0000", 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116183 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.988114+0000", 2023-09-23T21:17:55.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000113099 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.992852+0000", 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047377039999999997 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.995655+0000", 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028032199999999999 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241616+0000", 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.245961033 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241659+0000", 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2435999999999998e-05 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:17:55.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241696+0000", 2023-09-23T21:17:55.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7279e-05 2023-09-23T21:17:55.976 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.976 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.976 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.976 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:17:55.976 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:17:55.976 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:17:55.976 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:17:55.977 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:56.012 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:56.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 92491 objects 2023-09-23T21:17:56.680 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :126.993 2023-09-23T21:17:56.712 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:17:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:56.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:17:56.816 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:56.842 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:56.889 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:56.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:56.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:56.995 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:57.022 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.071 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:57.076 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 15689/9303 objects degraded (168.645%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:57.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 15689/9303 objects degraded (168.645%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 15689/9303 objects degraded (168.645%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T21:17:57.177 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:57.206 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.233 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:17:57.254 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:57.359 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:57.380 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:17:57.386 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.433 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:57.540 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:57.567 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.614 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:57.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:57.719 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:57.746 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.790 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:57.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:57.898 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:17:57.925 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:57.972 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:17:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:57.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:17:58.077 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:58.102 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:57 smithi123 ceph-mon[193903]: pgmap v1753: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 167 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 15689/8055 objects degraded (194.773%); 1.9 MiB/s, 110 objects/s recovering 2023-09-23T21:17:58.102 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:58.102 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:58.104 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.150 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:58.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:58.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:57 smithi121 ceph-mon[219709]: pgmap v1753: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 167 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 15689/8055 objects degraded (194.773%); 1.9 MiB/s, 110 objects/s recovering 2023-09-23T21:17:58.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:58.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:57 smithi121 ceph-mon[211526]: pgmap v1753: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 167 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 15689/8055 objects degraded (194.773%); 1.9 MiB/s, 110 objects/s recovering 2023-09-23T21:17:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:17:58.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:17:58.256 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:17:58.284 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.332 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:17:58.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:58.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:17:58.439 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:58.475 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.534 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:58.650 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:17:58.712 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:58.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:58.717 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:17:58.742 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:17:58.761 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:17:58.781 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:17:58.790 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[219709]: failed to encode map e562 with expected crc 2023-09-23T21:17:58.790 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[219709]: failed to encode map e562 with expected crc 2023-09-23T21:17:58.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[219709]: osdmap e562: 8 total, 8 up, 5 in 2023-09-23T21:17:58.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[219709]: failed to encode map e562 with expected crc 2023-09-23T21:17:58.791 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/145822606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:58.802 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:17:58.816 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:58.824 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:17:58.845 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:58.845 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:17:58.864 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.866 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:17:58.895 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:58.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:59.000 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:59.027 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:59.074 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:59.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:58 smithi123 ceph-mon[193903]: failed to encode map e562 with expected crc 2023-09-23T21:17:59.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:58 smithi123 ceph-mon[193903]: failed to encode map e562 with expected crc 2023-09-23T21:17:59.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:58 smithi123 ceph-mon[193903]: osdmap e562: 8 total, 8 up, 5 in 2023-09-23T21:17:59.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:58 smithi123 ceph-mon[193903]: failed to encode map e562 with expected crc 2023-09-23T21:17:59.079 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/145822606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:59.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:59.094 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-09-23T21:17:59.094 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-09-23T21:17:59.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[211526]: failed to encode map e562 with expected crc 2023-09-23T21:17:59.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[211526]: failed to encode map e562 with expected crc 2023-09-23T21:17:59.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[211526]: osdmap e562: 8 total, 8 up, 5 in 2023-09-23T21:17:59.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[211526]: failed to encode map e562 with expected crc 2023-09-23T21:17:59.131 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/145822606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:17:59.181 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:17:59.215 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:59.509 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:17:59.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:59.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:17:59.612 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:17:59.642 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:59.690 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:17:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:59.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:17:59.795 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:17:59.826 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:17:59.874 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:17:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:17:59.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:17:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:59 smithi123 ceph-mon[193903]: failed to encode map e562 with expected crc 2023-09-23T21:17:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:59 smithi123 ceph-mon[193903]: failed to encode map e563 with expected crc 2023-09-23T21:17:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:59 smithi123 ceph-mon[193903]: failed to encode map e563 with expected crc 2023-09-23T21:17:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:59 smithi123 ceph-mon[193903]: osdmap e563: 8 total, 8 up, 5 in 2023-09-23T21:17:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:59 smithi123 ceph-mon[193903]: failed to encode map e563 with expected crc 2023-09-23T21:17:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:17:59 smithi123 ceph-mon[193903]: pgmap v1756: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 23 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 922 op/s; 14251/1167 objects degraded (1221.165%); 0 B/s, 281 objects/s recovering 2023-09-23T21:17:59.983 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:00.012 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:00.043 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[211526]: failed to encode map e562 with expected crc 2023-09-23T21:18:00.043 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[211526]: failed to encode map e563 with expected crc 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[211526]: failed to encode map e563 with expected crc 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[211526]: osdmap e563: 8 total, 8 up, 5 in 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[211526]: failed to encode map e563 with expected crc 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[211526]: pgmap v1756: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 23 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 922 op/s; 14251/1167 objects degraded (1221.165%); 0 B/s, 281 objects/s recovering 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[219709]: failed to encode map e562 with expected crc 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[219709]: failed to encode map e563 with expected crc 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[219709]: failed to encode map e563 with expected crc 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[219709]: osdmap e563: 8 total, 8 up, 5 in 2023-09-23T21:18:00.044 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[219709]: failed to encode map e563 with expected crc 2023-09-23T21:18:00.045 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:17:59 smithi121 ceph-mon[219709]: pgmap v1756: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 23 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 922 op/s; 14251/1167 objects degraded (1221.165%); 0 B/s, 281 objects/s recovering 2023-09-23T21:18:00.111 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:00.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:00.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:00.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:00.241 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:00.671 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:18:00.675 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:18:00.690 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:00.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:00.795 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:00.822 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:00.873 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:00.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:00.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:00.978 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:01.004 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:00 smithi121 ceph-mon[219709]: failed to encode map e563 with expected crc 2023-09-23T21:18:01.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:00 smithi121 ceph-mon[211526]: failed to encode map e563 with expected crc 2023-09-23T21:18:01.007 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:01.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:00 smithi123 ceph-mon[193903]: failed to encode map e563 with expected crc 2023-09-23T21:18:01.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[211526]: from='client.47369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[211526]: pgmap v1757: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 436 op/s; 14251/1104 objects degraded (1290.851%); 0 B/s, 170 objects/s recovering 2023-09-23T21:18:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14251/1167 objects degraded (1221.165%), 14 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[211526]: osdmap e564: 8 total, 8 up, 5 in 2023-09-23T21:18:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[219709]: from='client.47369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[219709]: pgmap v1757: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 436 op/s; 14251/1104 objects degraded (1290.851%); 0 B/s, 170 objects/s recovering 2023-09-23T21:18:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14251/1167 objects degraded (1221.165%), 14 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:01 smithi121 ceph-mon[219709]: osdmap e564: 8 total, 8 up, 5 in 2023-09-23T21:18:01.958 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:01.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:02.049 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:18:02.062 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:02.123 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:02.151 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1756,"stamp":"2023-09-23T21:17:59.359475+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":24532884,"num_objects":389,"num_object_clones":0,"num_object_copies":1167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14251,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":389,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":170138,"num_write_kb":4954830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34707,"num_bytes_recovered":1670720294,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":179828,"ondisk_log_size":179828,"up":327,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":5403836,"kb_used_data":1064980,"kb_used_omap":7,"kb_used_meta":4338744,"kb_avail":463321924,"statfs":{"total":479975178240,"available":474441650176,"internally_reserved":0,"allocated":1090539520,"data_stored":1081730637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4442873937},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,2,4,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-345358372,"num_objects":-5272,"num_object_clones":0,"num_object_copies":-15816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1438,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1694,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.007387"},"pg_stats":[{"pgid":"13.23","version":"561'6258","reported_seq":8426,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:57.253033+0000","last_change":"2023-09-23T21:17:57.252727+0000","last_active":"2023-09-23T21:17:57.253033+0000","last_peered":"2023-09-23T21:17:57.253033+0000","last_clean":"2023-09-23T21:17:57.253033+0000","last_became_active":"2023-09-23T21:16:00.848945+0000","last_became_peered":"2023-09-23T21:16:00.848945+0000","last_unstale":"2023-09-23T21:17:57.253033+0000","last_undegraded":"2023-09-23T21:17:57.253033+0000","last_fullsized":"2023-09-23T21:17:57.253033+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1997,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1000,"num_bytes_recovered":57409536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.22","version":"561'6143","reported_seq":8203,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.555556+0000","last_change":"2023-09-23T21:17:20.794596+0000","last_active":"2023-09-23T21:17:56.555556+0000","last_peered":"2023-09-23T21:17:56.555556+0000","last_clean":"2023-09-23T21:17:56.555556+0000","last_became_active":"2023-09-23T21:16:00.839299+0000","last_became_peered":"2023-09-23T21:16:00.839299+0000","last_unstale":"2023-09-23T21:17:56.555556+0000","last_undegraded":"2023-09-23T21:17:56.555556+0000","last_fullsized":"2023-09-23T21:17:56.555556+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6143,"log_dups_size":0,"ondisk_log_size":6143,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2904,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":962,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.21","version":"561'6001","reported_seq":7103,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.496279+0000","last_change":"2023-09-23T21:16:51.863113+0000","last_active":"2023-09-23T21:17:56.496279+0000","last_peered":"2023-09-23T21:17:56.496279+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858774+0000","last_became_peered":"2023-09-23T21:16:51.858774+0000","last_unstale":"2023-09-23T21:17:56.496279+0000","last_undegraded":"2023-09-23T21:16:51.798038+0000","last_fullsized":"2023-09-23T21:16:51.792078+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2408448,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":750,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2820,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":-108}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.20","version":"561'6159","reported_seq":9302,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.641846+0000","last_change":"2023-09-23T21:16:51.842022+0000","last_active":"2023-09-23T21:17:56.641846+0000","last_peered":"2023-09-23T21:17:56.641846+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834659+0000","last_became_peered":"2023-09-23T21:16:51.834659+0000","last_unstale":"2023-09-23T21:17:56.641846+0000","last_undegraded":"2023-09-23T21:16:51.797563+0000","last_fullsized":"2023-09-23T21:16:51.792808+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":6,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":6159,"log_dups_size":0,"ondisk_log_size":6159,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":770,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2036,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":32964608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-140}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.14","version":"561'7538","reported_seq":9752,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.652129+0000","last_change":"2023-09-23T21:16:51.065826+0000","last_active":"2023-09-23T21:17:56.652129+0000","last_peered":"2023-09-23T21:17:56.652129+0000","last_clean":"2023-09-23T21:15:36.398296+0000","last_became_active":"2023-09-23T21:16:51.052190+0000","last_became_peered":"2023-09-23T21:16:51.052190+0000","last_unstale":"2023-09-23T21:17:56.652129+0000","last_undegraded":"2023-09-23T21:16:50.808861+0000","last_fullsized":"2023-09-23T21:16:50.789676+0000","mapping_epoch":544,"log_start":"503'3049","ondisk_log_start":"503'3049","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4489,"log_dups_size":0,"ondisk_log_size":4489,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1575,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5827,"num_write_kb":187712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34111488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-242}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":163,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522821+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:17:45.522821+0000","last_peered":"2023-09-23T21:17:45.522821+0000","last_clean":"2023-09-23T21:17:45.522821+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:17:45.522821+0000","last_undegraded":"2023-09-23T21:17:45.522821+0000","last_fullsized":"2023-09-23T21:17:45.522821+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523128+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:17:45.523128+0000","last_peered":"2023-09-23T21:17:45.523128+0000","last_clean":"2023-09-23T21:17:45.523128+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:17:45.523128+0000","last_undegraded":"2023-09-23T21:17:45.523128+0000","last_fullsized":"2023-09-23T21:17:45.523128+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.17","version":"561'7392","reported_seq":10687,"reported_epoch":562,"state":"peering","last_fresh":"2023-09-23T21:17:57.804230+0000","last_change":"2023-09-23T21:17:57.805072+0000","last_active":"2023-09-23T21:17:57.804196+0000","last_peered":"2023-09-23T21:17:57.055980+0000","last_clean":"2023-09-23T21:15:36.267177+0000","last_became_active":"2023-09-23T21:16:51.020513+0000","last_became_peered":"2023-09-23T21:16:51.020513+0000","last_unstale":"2023-09-23T21:17:57.804230+0000","last_undegraded":"2023-09-23T21:17:57.804230+0000","last_fullsized":"2023-09-23T21:17:57.804230+0000","mapping_epoch":562,"log_start":"508'3298","ondisk_log_start":"508'3298","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5749,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1586,"num_bytes_recovered":19922944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,5,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":349,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.450952+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:17:54.450952+0000","last_peered":"2023-09-23T21:17:54.450952+0000","last_clean":"2023-09-23T21:17:54.450952+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:17:54.450952+0000","last_undegraded":"2023-09-23T21:17:54.450952+0000","last_fullsized":"2023-09-23T21:17:54.450952+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":209,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449165+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:17:54.449165+0000","last_peered":"2023-09-23T21:17:54.449165+0000","last_clean":"2023-09-23T21:17:54.449165+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:17:54.449165+0000","last_undegraded":"2023-09-23T21:17:54.449165+0000","last_fullsized":"2023-09-23T21:17:54.449165+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"561'7424","reported_seq":11454,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.679312+0000","last_change":"2023-09-23T21:16:36.510340+0000","last_active":"2023-09-23T21:17:56.679312+0000","last_peered":"2023-09-23T21:17:56.679312+0000","last_clean":"2023-09-23T21:17:56.679312+0000","last_became_active":"2023-09-23T21:16:36.498646+0000","last_became_peered":"2023-09-23T21:16:36.498646+0000","last_unstale":"2023-09-23T21:17:56.679312+0000","last_undegraded":"2023-09-23T21:17:56.679312+0000","last_fullsized":"2023-09-23T21:17:56.679312+0000","mapping_epoch":537,"log_start":"515'3689","ondisk_log_start":"515'3689","created":507,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3735,"log_dups_size":0,"ondisk_log_size":3735,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5763,"num_write_kb":184192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1956,"num_bytes_recovered":93519872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":221,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522126+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:17:45.522126+0000","last_peered":"2023-09-23T21:17:45.522126+0000","last_clean":"2023-09-23T21:17:45.522126+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:17:45.522126+0000","last_undegraded":"2023-09-23T21:17:45.522126+0000","last_fullsized":"2023-09-23T21:17:45.522126+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523198+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:17:45.523198+0000","last_peered":"2023-09-23T21:17:45.523198+0000","last_clean":"2023-09-23T21:17:45.523198+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:17:45.523198+0000","last_undegraded":"2023-09-23T21:17:45.523198+0000","last_fullsized":"2023-09-23T21:17:45.523198+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"561'7252","reported_seq":7378,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.644738+0000","last_change":"2023-09-23T21:16:51.074430+0000","last_active":"2023-09-23T21:17:56.644738+0000","last_peered":"2023-09-23T21:17:56.644738+0000","last_clean":"2023-09-23T21:15:36.374828+0000","last_became_active":"2023-09-23T21:16:51.054035+0000","last_became_peered":"2023-09-23T21:16:51.054035+0000","last_unstale":"2023-09-23T21:17:56.644738+0000","last_undegraded":"2023-09-23T21:16:50.838593+0000","last_fullsized":"2023-09-23T21:16:50.797592+0000","mapping_epoch":544,"log_start":"509'3558","ondisk_log_start":"509'3558","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3694,"log_dups_size":0,"ondisk_log_size":3694,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":819200,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5617,"num_write_kb":180128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":-245}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":156,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451795+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:17:54.451795+0000","last_peered":"2023-09-23T21:17:54.451795+0000","last_clean":"2023-09-23T21:17:54.451795+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:17:54.451795+0000","last_undegraded":"2023-09-23T21:17:54.451795+0000","last_fullsized":"2023-09-23T21:17:54.451795+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522234+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:17:45.522234+0000","last_peered":"2023-09-23T21:17:45.522234+0000","last_clean":"2023-09-23T21:17:45.522234+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:17:45.522234+0000","last_undegraded":"2023-09-23T21:17:45.522234+0000","last_fullsized":"2023-09-23T21:17:45.522234+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.18","version":"561'7389","reported_seq":10713,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.551387+0000","last_change":"2023-09-23T21:17:12.466376+0000","last_active":"2023-09-23T21:17:56.551387+0000","last_peered":"2023-09-23T21:17:56.551387+0000","last_clean":"2023-09-23T21:17:56.551387+0000","last_became_active":"2023-09-23T21:17:12.454368+0000","last_became_peered":"2023-09-23T21:17:12.454368+0000","last_unstale":"2023-09-23T21:17:56.551387+0000","last_undegraded":"2023-09-23T21:17:56.551387+0000","last_fullsized":"2023-09-23T21:17:56.551387+0000","mapping_epoch":551,"log_start":"502'2860","ondisk_log_start":"502'2860","created":507,"last_epoch_clean":552,"parent":"0.0","parent_split_bits":5,"last_scrub":"509'3518","last_scrub_stamp":"2023-09-23T21:15:05.073551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:05.073551+0000","objects_scrubbed":1854,"log_size":4529,"log_dups_size":0,"ondisk_log_size":4529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:01.792696+0000","scrub_duration":0.38156216700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3876,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1575,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":215,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.453033+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:17:54.453033+0000","last_peered":"2023-09-23T21:17:54.453033+0000","last_clean":"2023-09-23T21:17:54.453033+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:17:54.453033+0000","last_undegraded":"2023-09-23T21:17:54.453033+0000","last_fullsized":"2023-09-23T21:17:54.453033+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.446396+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:17:54.446396+0000","last_peered":"2023-09-23T21:17:54.446396+0000","last_clean":"2023-09-23T21:17:54.446396+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:17:54.446396+0000","last_undegraded":"2023-09-23T21:17:54.446396+0000","last_fullsized":"2023-09-23T21:17:54.446396+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1b","version":"561'9238","reported_seq":14409,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.614545+0000","last_change":"2023-09-23T21:17:17.318827+0000","last_active":"2023-09-23T21:17:56.614545+0000","last_peered":"2023-09-23T21:17:56.614545+0000","last_clean":"2023-09-23T21:17:56.614545+0000","last_became_active":"2023-09-23T21:16:00.808957+0000","last_became_peered":"2023-09-23T21:16:00.808957+0000","last_unstale":"2023-09-23T21:17:56.614545+0000","last_undegraded":"2023-09-23T21:17:56.614545+0000","last_fullsized":"2023-09-23T21:17:56.614545+0000","mapping_epoch":525,"log_start":"508'3473","ondisk_log_start":"508'3473","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5765,"log_dups_size":0,"ondisk_log_size":5765,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6003,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133431296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":165,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449514+0000","last_change":"2023-09-23T21:17:11.237055+0000","last_active":"2023-09-23T21:17:54.449514+0000","last_peered":"2023-09-23T21:17:54.449514+0000","last_clean":"2023-09-23T21:17:54.449514+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:17:54.449514+0000","last_undegraded":"2023-09-23T21:17:54.449514+0000","last_fullsized":"2023-09-23T21:17:54.449514+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522429+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:17:45.522429+0000","last_peered":"2023-09-23T21:17:45.522429+0000","last_clean":"2023-09-23T21:17:45.522429+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:17:45.522429+0000","last_undegraded":"2023-09-23T21:17:45.522429+0000","last_fullsized":"2023-09-23T21:17:45.522429+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1a","version":"561'8825","reported_seq":10960,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.664694+0000","last_change":"2023-09-23T21:16:00.812483+0000","last_active":"2023-09-23T21:17:56.664694+0000","last_peered":"2023-09-23T21:17:56.664694+0000","last_clean":"2023-09-23T21:17:56.664694+0000","last_became_active":"2023-09-23T21:16:00.806979+0000","last_became_peered":"2023-09-23T21:16:00.806979+0000","last_unstale":"2023-09-23T21:17:56.664694+0000","last_undegraded":"2023-09-23T21:17:56.664694+0000","last_fullsized":"2023-09-23T21:17:56.664694+0000","mapping_epoch":525,"log_start":"502'2887","ondisk_log_start":"502'2887","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5713,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":189,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451178+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:17:54.451178+0000","last_peered":"2023-09-23T21:17:54.451178+0000","last_clean":"2023-09-23T21:17:54.451178+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:17:54.451178+0000","last_undegraded":"2023-09-23T21:17:54.451178+0000","last_fullsized":"2023-09-23T21:17:54.451178+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":490,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.452598+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:17:54.452598+0000","last_peered":"2023-09-23T21:17:54.452598+0000","last_clean":"2023-09-23T21:17:54.452598+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:17:54.452598+0000","last_undegraded":"2023-09-23T21:17:54.452598+0000","last_fullsized":"2023-09-23T21:17:54.452598+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.1d","version":"561'9014","reported_seq":15112,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.684151+0000","last_change":"2023-09-23T21:17:20.608316+0000","last_active":"2023-09-23T21:17:56.684151+0000","last_peered":"2023-09-23T21:17:56.684151+0000","last_clean":"2023-09-23T21:17:56.684151+0000","last_became_active":"2023-09-23T21:16:03.773235+0000","last_became_peered":"2023-09-23T21:16:03.773235+0000","last_unstale":"2023-09-23T21:17:56.684151+0000","last_undegraded":"2023-09-23T21:17:56.684151+0000","last_fullsized":"2023-09-23T21:17:56.684151+0000","mapping_epoch":528,"log_start":"515'4217","ondisk_log_start":"515'4217","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4797,"log_dups_size":0,"ondisk_log_size":4797,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1212404,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5837,"num_write_kb":187329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2472,"num_bytes_recovered":146276352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":145,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.447015+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:17:54.447015+0000","last_peered":"2023-09-23T21:17:54.447015+0000","last_clean":"2023-09-23T21:17:54.447015+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:17:54.447015+0000","last_undegraded":"2023-09-23T21:17:54.447015+0000","last_fullsized":"2023-09-23T21:17:54.447015+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":213,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451017+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:17:54.451017+0000","last_peered":"2023-09-23T21:17:54.451017+0000","last_clean":"2023-09-23T21:17:54.451017+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:17:54.451017+0000","last_undegraded":"2023-09-23T21:17:54.451017+0000","last_fullsized":"2023-09-23T21:17:54.451017+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1c","version":"561'9075","reported_seq":11204,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.678390+0000","last_change":"2023-09-23T21:16:00.824659+0000","last_active":"2023-09-23T21:17:56.678390+0000","last_peered":"2023-09-23T21:17:56.678390+0000","last_clean":"2023-09-23T21:17:56.678390+0000","last_became_active":"2023-09-23T21:16:00.820619+0000","last_became_peered":"2023-09-23T21:16:00.820619+0000","last_unstale":"2023-09-23T21:17:56.678390+0000","last_undegraded":"2023-09-23T21:17:56.678390+0000","last_fullsized":"2023-09-23T21:17:56.678390+0000","mapping_epoch":525,"log_start":"516'4518","ondisk_log_start":"516'4518","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4557,"log_dups_size":0,"ondisk_log_size":4557,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5891,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":141,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.448418+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:17:54.448418+0000","last_peered":"2023-09-23T21:17:54.448418+0000","last_clean":"2023-09-23T21:17:54.448418+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:17:54.448418+0000","last_undegraded":"2023-09-23T21:17:54.448418+0000","last_fullsized":"2023-09-23T21:17:54.448418+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":149,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449784+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:17:54.449784+0000","last_peered":"2023-09-23T21:17:54.449784+0000","last_clean":"2023-09-23T21:17:54.449784+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:17:54.449784+0000","last_undegraded":"2023-09-23T21:17:54.449784+0000","last_fullsized":"2023-09-23T21:17:54.449784+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1f","version":"561'8931","reported_seq":9054,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.147968+0000","last_change":"2023-09-23T21:17:39.461735+0000","last_active":"2023-09-23T21:17:56.147968+0000","last_peered":"2023-09-23T21:17:56.147968+0000","last_clean":"2023-09-23T21:17:56.147968+0000","last_became_active":"2023-09-23T21:16:51.820010+0000","last_became_peered":"2023-09-23T21:16:51.820010+0000","last_unstale":"2023-09-23T21:17:56.147968+0000","last_undegraded":"2023-09-23T21:17:56.147968+0000","last_fullsized":"2023-09-23T21:17:56.147968+0000","mapping_epoch":545,"log_start":"503'3025","ondisk_log_start":"503'3025","created":525,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5906,"log_dups_size":0,"ondisk_log_size":5906,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5801,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":139,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.446576+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:17:54.446576+0000","last_peered":"2023-09-23T21:17:54.446576+0000","last_clean":"2023-09-23T21:17:54.446576+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:17:54.446576+0000","last_undegraded":"2023-09-23T21:17:54.446576+0000","last_fullsized":"2023-09-23T21:17:54.446576+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":163,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449096+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:17:54.449096+0000","last_peered":"2023-09-23T21:17:54.449096+0000","last_clean":"2023-09-23T21:17:54.449096+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:17:54.449096+0000","last_undegraded":"2023-09-23T21:17:54.449096+0000","last_fullsized":"2023-09-23T21:17:54.449096+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1e","version":"561'8779","reported_seq":8880,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.668443+0000","last_change":"2023-09-23T21:17:20.608819+0000","last_active":"2023-09-23T21:17:56.668443+0000","last_peered":"2023-09-23T21:17:56.668443+0000","last_clean":"2023-09-23T21:17:56.668443+0000","last_became_active":"2023-09-23T21:16:00.844392+0000","last_became_peered":"2023-09-23T21:16:00.844392+0000","last_unstale":"2023-09-23T21:17:56.668443+0000","last_undegraded":"2023-09-23T21:17:56.668443+0000","last_fullsized":"2023-09-23T21:17:56.668443+0000","mapping_epoch":525,"log_start":"506'3234","ondisk_log_start":"506'3234","created":525,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2260992,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5688,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":597,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523084+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:17:45.523084+0000","last_peered":"2023-09-23T21:17:45.523084+0000","last_clean":"2023-09-23T21:17:45.523084+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:17:45.523084+0000","last_undegraded":"2023-09-23T21:17:45.523084+0000","last_fullsized":"2023-09-23T21:17:45.523084+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":213,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.452470+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:17:54.452470+0000","last_peered":"2023-09-23T21:17:54.452470+0000","last_clean":"2023-09-23T21:17:54.452470+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:17:54.452470+0000","last_undegraded":"2023-09-23T21:17:54.452470+0000","last_fullsized":"2023-09-23T21:17:54.452470+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"561'6096","reported_seq":7242,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.605493+0000","last_change":"2023-09-23T21:16:51.863144+0000","last_active":"2023-09-23T21:17:56.605493+0000","last_peered":"2023-09-23T21:17:56.605493+0000","last_clean":"2023-09-23T21:15:36.246625+0000","last_became_active":"2023-09-23T21:16:51.858843+0000","last_became_peered":"2023-09-23T21:16:51.858843+0000","last_unstale":"2023-09-23T21:17:56.605493+0000","last_undegraded":"2023-09-23T21:16:51.799178+0000","last_fullsized":"2023-09-23T21:16:51.792321+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6096,"log_dups_size":0,"ondisk_log_size":6096,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2914,"num_write_kb":91408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":511,"num_bytes_recovered":33488896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":-169}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":143,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.452554+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:17:54.452554+0000","last_peered":"2023-09-23T21:17:54.452554+0000","last_clean":"2023-09-23T21:17:54.452554+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:17:54.452554+0000","last_undegraded":"2023-09-23T21:17:54.452554+0000","last_fullsized":"2023-09-23T21:17:54.452554+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":215,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.452662+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:17:54.452662+0000","last_peered":"2023-09-23T21:17:54.452662+0000","last_clean":"2023-09-23T21:17:54.452662+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:17:54.452662+0000","last_undegraded":"2023-09-23T21:17:54.452662+0000","last_fullsized":"2023-09-23T21:17:54.452662+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"561'6167","reported_seq":9286,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.665389+0000","last_change":"2023-09-23T21:16:51.842306+0000","last_active":"2023-09-23T21:17:56.665389+0000","last_peered":"2023-09-23T21:17:56.665389+0000","last_clean":"2023-09-23T21:15:36.400645+0000","last_became_active":"2023-09-23T21:16:51.834848+0000","last_became_peered":"2023-09-23T21:16:51.834848+0000","last_unstale":"2023-09-23T21:17:56.665389+0000","last_undegraded":"2023-09-23T21:16:51.803926+0000","last_fullsized":"2023-09-23T21:16:51.799106+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3434","last_scrub_stamp":"2023-09-23T21:15:03.500105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:03.500105+0000","objects_scrubbed":1776,"log_size":6167,"log_dups_size":0,"ondisk_log_size":6167,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.62385056100000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2044,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32178176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-161}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":157,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449240+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:17:54.449240+0000","last_peered":"2023-09-23T21:17:54.449240+0000","last_clean":"2023-09-23T21:17:54.449240+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:17:54.449240+0000","last_undegraded":"2023-09-23T21:17:54.449240+0000","last_fullsized":"2023-09-23T21:17:54.449240+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522309+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:17:45.522309+0000","last_peered":"2023-09-23T21:17:45.522309+0000","last_clean":"2023-09-23T21:17:45.522309+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:17:45.522309+0000","last_undegraded":"2023-09-23T21:17:45.522309+0000","last_fullsized":"2023-09-23T21:17:45.522309+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.3","version":"561'6235","reported_seq":8160,"reported_epoch":561,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:56.673198+0000","last_change":"2023-09-23T21:16:01.735614+0000","last_active":"2023-09-23T21:17:56.673198+0000","last_peered":"2023-09-23T21:17:56.673198+0000","last_clean":"2023-09-23T21:15:08.114244+0000","last_became_active":"2023-09-23T21:16:01.731847+0000","last_became_peered":"2023-09-23T21:16:01.731847+0000","last_unstale":"2023-09-23T21:17:56.673198+0000","last_undegraded":"2023-09-23T21:16:01.699708+0000","last_fullsized":"2023-09-23T21:17:56.673198+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3639","last_scrub_stamp":"2023-09-23T21:15:07.323918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:07.323918+0000","objects_scrubbed":1973,"log_size":6235,"log_dups_size":0,"ondisk_log_size":6235,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.48385197600000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":753664,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":143,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1974,"num_write_kb":31968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":879,"num_bytes_recovered":57606144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5,7","objects":11}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":143,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449866+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:17:54.449866+0000","last_peered":"2023-09-23T21:17:54.449866+0000","last_clean":"2023-09-23T21:17:54.449866+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:17:54.449866+0000","last_undegraded":"2023-09-23T21:17:54.449866+0000","last_fullsized":"2023-09-23T21:17:54.449866+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522841+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:17:45.522841+0000","last_peered":"2023-09-23T21:17:45.522841+0000","last_clean":"2023-09-23T21:17:45.522841+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:17:45.522841+0000","last_undegraded":"2023-09-23T21:17:45.522841+0000","last_fullsized":"2023-09-23T21:17:45.522841+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"561'6125","reported_seq":7908,"reported_epoch":561,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T21:17:56.589033+0000","last_change":"2023-09-23T21:17:11.228334+0000","last_active":"2023-09-23T21:17:56.589033+0000","last_peered":"2023-09-23T21:17:56.589033+0000","last_clean":"2023-09-23T21:15:08.126261+0000","last_became_active":"2023-09-23T21:16:01.742399+0000","last_became_peered":"2023-09-23T21:16:01.742399+0000","last_unstale":"2023-09-23T21:17:56.589033+0000","last_undegraded":"2023-09-23T21:17:11.225179+0000","last_fullsized":"2023-09-23T21:17:56.589033+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6125,"log_dups_size":0,"ondisk_log_size":6125,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":133,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2886,"num_write_kb":92800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":54001664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5,6","objects":14}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523214+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:17:45.523214+0000","last_peered":"2023-09-23T21:17:45.523214+0000","last_clean":"2023-09-23T21:17:45.523214+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:17:45.523214+0000","last_undegraded":"2023-09-23T21:17:45.523214+0000","last_fullsized":"2023-09-23T21:17:45.523214+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522806+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:17:45.522806+0000","last_peered":"2023-09-23T21:17:45.522806+0000","last_clean":"2023-09-23T21:17:45.522806+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:17:45.522806+0000","last_undegraded":"2023-09-23T21:17:45.522806+0000","last_fullsized":"2023-09-23T21:17:45.522806+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.5","version":"561'7419","reported_seq":12164,"reported_epoch":561,"state":"active+recovery_wait","last_fresh":"2023-09-23T21:17:56.632818+0000","last_change":"2023-09-23T21:17:45.677588+0000","last_active":"2023-09-23T21:17:56.632818+0000","last_peered":"2023-09-23T21:17:56.632818+0000","last_clean":"2023-09-23T21:15:09.797220+0000","last_became_active":"2023-09-23T21:17:45.633940+0000","last_became_peered":"2023-09-23T21:17:45.633940+0000","last_unstale":"2023-09-23T21:17:56.632818+0000","last_undegraded":"2023-09-23T21:17:56.632818+0000","last_fullsized":"2023-09-23T21:17:56.632818+0000","mapping_epoch":559,"log_start":"509'3620","ondisk_log_start":"509'3620","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3573","last_scrub_stamp":"2023-09-23T21:15:06.423920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:06.423920+0000","objects_scrubbed":1851,"log_size":3799,"log_dups_size":0,"ondisk_log_size":3799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:50.441594+0000","scrub_duration":1.1490737,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3863,"num_write_kb":64384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2258,"num_bytes_recovered":21233664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":["4","3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":229,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.446933+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:17:54.446933+0000","last_peered":"2023-09-23T21:17:54.446933+0000","last_clean":"2023-09-23T21:17:54.446933+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:17:54.446933+0000","last_undegraded":"2023-09-23T21:17:54.446933+0000","last_fullsized":"2023-09-23T21:17:54.446933+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522965+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:17:45.522965+0000","last_peered":"2023-09-23T21:17:45.522965+0000","last_clean":"2023-09-23T21:17:45.522965+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:17:45.522965+0000","last_undegraded":"2023-09-23T21:17:45.522965+0000","last_fullsized":"2023-09-23T21:17:45.522965+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.4","version":"561'7432","reported_seq":9739,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.536031+0000","last_change":"2023-09-23T21:16:51.065805+0000","last_active":"2023-09-23T21:17:56.536031+0000","last_peered":"2023-09-23T21:17:56.536031+0000","last_clean":"2023-09-23T21:15:36.124918+0000","last_became_active":"2023-09-23T21:16:51.051146+0000","last_became_peered":"2023-09-23T21:16:51.051146+0000","last_unstale":"2023-09-23T21:17:56.536031+0000","last_undegraded":"2023-09-23T21:16:50.809981+0000","last_fullsized":"2023-09-23T21:16:50.790363+0000","mapping_epoch":544,"log_start":"502'2908","ondisk_log_start":"502'2908","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3701","last_scrub_stamp":"2023-09-23T21:15:08.301026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.301026+0000","objects_scrubbed":2028,"log_size":4524,"log_dups_size":0,"ondisk_log_size":4524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:23.280602+0000","scrub_duration":0.193991794,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1566,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3732,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-242}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523122+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:17:45.523122+0000","last_peered":"2023-09-23T21:17:45.523122+0000","last_clean":"2023-09-23T21:17:45.523122+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:17:45.523122+0000","last_undegraded":"2023-09-23T21:17:45.523122+0000","last_fullsized":"2023-09-23T21:17:45.523122+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":603,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.531315+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:17:45.531315+0000","last_peered":"2023-09-23T21:17:45.531315+0000","last_clean":"2023-09-23T21:17:45.531315+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:17:45.531315+0000","last_undegraded":"2023-09-23T21:17:45.531315+0000","last_fullsized":"2023-09-23T21:17:45.531315+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"561'7330","reported_seq":7454,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.678836+0000","last_change":"2023-09-23T21:16:51.082314+0000","last_active":"2023-09-23T21:17:56.678836+0000","last_peered":"2023-09-23T21:17:56.678836+0000","last_clean":"2023-09-23T21:15:36.269852+0000","last_became_active":"2023-09-23T21:16:51.064800+0000","last_became_peered":"2023-09-23T21:16:51.064800+0000","last_unstale":"2023-09-23T21:17:56.678836+0000","last_undegraded":"2023-09-23T21:16:50.820535+0000","last_fullsized":"2023-09-23T21:16:50.800952+0000","mapping_epoch":544,"log_start":"508'3295","ondisk_log_start":"508'3295","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4035,"log_dups_size":0,"ondisk_log_size":4035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1769472,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5687,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":-268}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":199,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451173+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:17:54.451173+0000","last_peered":"2023-09-23T21:17:54.451173+0000","last_clean":"2023-09-23T21:17:54.451173+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:17:54.451173+0000","last_undegraded":"2023-09-23T21:17:54.451173+0000","last_fullsized":"2023-09-23T21:17:54.451173+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522810+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:17:45.522810+0000","last_peered":"2023-09-23T21:17:45.522810+0000","last_clean":"2023-09-23T21:17:45.522810+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:17:45.522810+0000","last_undegraded":"2023-09-23T21:17:45.522810+0000","last_fullsized":"2023-09-23T21:17:45.522810+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.6","version":"561'7342","reported_seq":11446,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.642263+0000","last_change":"2023-09-23T21:17:41.511480+0000","last_active":"2023-09-23T21:17:56.642263+0000","last_peered":"2023-09-23T21:17:56.642263+0000","last_clean":"2023-09-23T21:17:56.642263+0000","last_became_active":"2023-09-23T21:17:41.488891+0000","last_became_peered":"2023-09-23T21:17:41.488891+0000","last_unstale":"2023-09-23T21:17:56.642263+0000","last_undegraded":"2023-09-23T21:17:56.642263+0000","last_fullsized":"2023-09-23T21:17:56.642263+0000","mapping_epoch":556,"log_start":"510'3612","ondisk_log_start":"510'3612","created":482,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3613","last_scrub_stamp":"2023-09-23T21:15:08.897536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:08.897536+0000","objects_scrubbed":1945,"log_size":3730,"log_dups_size":0,"ondisk_log_size":3730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.993310+0000","scrub_duration":0.18673705400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3729,"num_write_kb":57088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1945,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":213,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.448448+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:17:54.448448+0000","last_peered":"2023-09-23T21:17:54.448448+0000","last_clean":"2023-09-23T21:17:54.448448+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:17:54.448448+0000","last_undegraded":"2023-09-23T21:17:54.448448+0000","last_fullsized":"2023-09-23T21:17:54.448448+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":301,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451599+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:17:54.451599+0000","last_peered":"2023-09-23T21:17:54.451599+0000","last_clean":"2023-09-23T21:17:54.451599+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:17:54.451599+0000","last_undegraded":"2023-09-23T21:17:54.451599+0000","last_fullsized":"2023-09-23T21:17:54.451599+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.b","version":"561'9153","reported_seq":14220,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.649426+0000","last_change":"2023-09-23T21:17:44.322885+0000","last_active":"2023-09-23T21:17:56.649426+0000","last_peered":"2023-09-23T21:17:56.649426+0000","last_clean":"2023-09-23T21:17:56.649426+0000","last_became_active":"2023-09-23T21:16:01.723118+0000","last_became_peered":"2023-09-23T21:16:01.723118+0000","last_unstale":"2023-09-23T21:17:56.649426+0000","last_undegraded":"2023-09-23T21:17:56.649426+0000","last_fullsized":"2023-09-23T21:17:56.649426+0000","mapping_epoch":526,"log_start":"505'3242","ondisk_log_start":"505'3242","created":482,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2785280,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5917,"num_write_kb":190720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":129957888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":486,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451095+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:17:54.451095+0000","last_peered":"2023-09-23T21:17:54.451095+0000","last_clean":"2023-09-23T21:17:54.451095+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:17:54.451095+0000","last_undegraded":"2023-09-23T21:17:54.451095+0000","last_fullsized":"2023-09-23T21:17:54.451095+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.452339+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:17:54.452339+0000","last_peered":"2023-09-23T21:17:54.452339+0000","last_clean":"2023-09-23T21:17:54.452339+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:17:54.452339+0000","last_undegraded":"2023-09-23T21:17:54.452339+0000","last_fullsized":"2023-09-23T21:17:54.452339+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":111,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451027+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:17:54.451027+0000","last_peered":"2023-09-23T21:17:54.451027+0000","last_clean":"2023-09-23T21:17:54.451027+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:17:54.451027+0000","last_undegraded":"2023-09-23T21:17:54.451027+0000","last_fullsized":"2023-09-23T21:17:54.451027+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"561'9050","reported_seq":15170,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.670546+0000","last_change":"2023-09-23T21:17:15.462587+0000","last_active":"2023-09-23T21:17:56.670546+0000","last_peered":"2023-09-23T21:17:56.670546+0000","last_clean":"2023-09-23T21:17:56.670546+0000","last_became_active":"2023-09-23T21:16:01.764537+0000","last_became_peered":"2023-09-23T21:16:01.764537+0000","last_unstale":"2023-09-23T21:17:56.670546+0000","last_undegraded":"2023-09-23T21:17:56.670546+0000","last_fullsized":"2023-09-23T21:17:56.670546+0000","mapping_epoch":526,"log_start":"515'4386","ondisk_log_start":"515'4386","created":482,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4664,"log_dups_size":0,"ondisk_log_size":4664,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5873,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2487,"num_bytes_recovered":146014208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522442+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:17:45.522442+0000","last_peered":"2023-09-23T21:17:45.522442+0000","last_clean":"2023-09-23T21:17:45.522442+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:17:45.522442+0000","last_undegraded":"2023-09-23T21:17:45.522442+0000","last_fullsized":"2023-09-23T21:17:45.522442+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523072+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:17:45.523072+0000","last_peered":"2023-09-23T21:17:45.523072+0000","last_clean":"2023-09-23T21:17:45.523072+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:17:45.523072+0000","last_undegraded":"2023-09-23T21:17:45.523072+0000","last_fullsized":"2023-09-23T21:17:45.523072+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":275,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523703+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:17:45.523703+0000","last_peered":"2023-09-23T21:17:45.523703+0000","last_clean":"2023-09-23T21:17:45.523703+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:17:45.523703+0000","last_undegraded":"2023-09-23T21:17:45.523703+0000","last_fullsized":"2023-09-23T21:17:45.523703+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"561'8848","reported_seq":8946,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.675027+0000","last_change":"2023-09-23T21:16:00.841315+0000","last_active":"2023-09-23T21:17:56.675027+0000","last_peered":"2023-09-23T21:17:56.675027+0000","last_clean":"2023-09-23T21:17:56.675027+0000","last_became_active":"2023-09-23T21:16:00.829607+0000","last_became_peered":"2023-09-23T21:16:00.829607+0000","last_unstale":"2023-09-23T21:17:56.675027+0000","last_undegraded":"2023-09-23T21:17:56.675027+0000","last_fullsized":"2023-09-23T21:17:56.675027+0000","mapping_epoch":482,"log_start":"509'3625","ondisk_log_start":"509'3625","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":5223,"log_dups_size":0,"ondisk_log_size":5223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5756,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":570,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522781+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:17:45.522781+0000","last_peered":"2023-09-23T21:17:45.522781+0000","last_clean":"2023-09-23T21:17:45.522781+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:17:45.522781+0000","last_undegraded":"2023-09-23T21:17:45.522781+0000","last_fullsized":"2023-09-23T21:17:45.522781+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":307,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.521989+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:17:45.521989+0000","last_peered":"2023-09-23T21:17:45.521989+0000","last_clean":"2023-09-23T21:17:45.521989+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:17:45.521989+0000","last_undegraded":"2023-09-23T21:17:45.521989+0000","last_fullsized":"2023-09-23T21:17:45.521989+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":215,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451671+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:17:54.451671+0000","last_peered":"2023-09-23T21:17:54.451671+0000","last_clean":"2023-09-23T21:17:54.451671+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:17:54.451671+0000","last_undegraded":"2023-09-23T21:17:54.451671+0000","last_fullsized":"2023-09-23T21:17:54.451671+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.c","version":"561'9050","reported_seq":11180,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.668494+0000","last_change":"2023-09-23T21:16:00.748968+0000","last_active":"2023-09-23T21:17:56.668494+0000","last_peered":"2023-09-23T21:17:56.668494+0000","last_clean":"2023-09-23T21:17:56.668494+0000","last_became_active":"2023-09-23T21:16:00.745123+0000","last_became_peered":"2023-09-23T21:16:00.745123+0000","last_unstale":"2023-09-23T21:17:56.668494+0000","last_undegraded":"2023-09-23T21:17:56.668494+0000","last_fullsized":"2023-09-23T21:17:56.668494+0000","mapping_epoch":501,"log_start":"515'4349","ondisk_log_start":"515'4349","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4701,"log_dups_size":0,"ondisk_log_size":4701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":819200,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5865,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":502,"num_bytes_recovered":32931840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":390,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:48.161484+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:17:48.161484+0000","last_peered":"2023-09-23T21:17:48.161484+0000","last_clean":"2023-09-23T21:17:48.161484+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:17:48.161484+0000","last_undegraded":"2023-09-23T21:17:48.161484+0000","last_fullsized":"2023-09-23T21:17:48.161484+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":908,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.447605+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:17:54.447605+0000","last_peered":"2023-09-23T21:17:54.447605+0000","last_clean":"2023-09-23T21:17:54.447605+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:17:54.447605+0000","last_undegraded":"2023-09-23T21:17:54.447605+0000","last_fullsized":"2023-09-23T21:17:54.447605+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":134,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.450607+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:17:54.450607+0000","last_peered":"2023-09-23T21:17:54.450607+0000","last_clean":"2023-09-23T21:17:54.450607+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:17:54.450607+0000","last_undegraded":"2023-09-23T21:17:54.450607+0000","last_fullsized":"2023-09-23T21:17:54.450607+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522744+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:17:45.522744+0000","last_peered":"2023-09-23T21:17:45.522744+0000","last_clean":"2023-09-23T21:17:45.522744+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:17:45.522744+0000","last_undegraded":"2023-09-23T21:17:45.522744+0000","last_fullsized":"2023-09-23T21:17:45.522744+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"561'8883","reported_seq":8996,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.098064+0000","last_change":"2023-09-23T21:16:51.832676+0000","last_active":"2023-09-23T21:17:56.098064+0000","last_peered":"2023-09-23T21:17:56.098064+0000","last_clean":"2023-09-23T21:17:56.098064+0000","last_became_active":"2023-09-23T21:16:51.803067+0000","last_became_peered":"2023-09-23T21:16:51.803067+0000","last_unstale":"2023-09-23T21:17:56.098064+0000","last_undegraded":"2023-09-23T21:17:56.098064+0000","last_fullsized":"2023-09-23T21:17:56.098064+0000","mapping_epoch":545,"log_start":"500'2669","ondisk_log_start":"500'2669","created":482,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2392064,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5752,"num_write_kb":185248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":189,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449444+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:17:54.449444+0000","last_peered":"2023-09-23T21:17:54.449444+0000","last_clean":"2023-09-23T21:17:54.449444+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:17:54.449444+0000","last_undegraded":"2023-09-23T21:17:54.449444+0000","last_fullsized":"2023-09-23T21:17:54.449444+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":58,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.446325+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:17:54.446325+0000","last_peered":"2023-09-23T21:17:54.446325+0000","last_clean":"2023-09-23T21:17:54.446325+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:17:54.446325+0000","last_undegraded":"2023-09-23T21:17:54.446325+0000","last_fullsized":"2023-09-23T21:17:54.446325+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":470,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523825+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:17:45.523825+0000","last_peered":"2023-09-23T21:17:45.523825+0000","last_clean":"2023-09-23T21:17:45.523825+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:17:45.523825+0000","last_undegraded":"2023-09-23T21:17:45.523825+0000","last_fullsized":"2023-09-23T21:17:45.523825+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"561'8825","reported_seq":10961,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.633537+0000","last_change":"2023-09-23T21:16:00.763134+0000","last_active":"2023-09-23T21:17:56.633537+0000","last_peered":"2023-09-23T21:17:56.633537+0000","last_clean":"2023-09-23T21:17:56.633537+0000","last_became_active":"2023-09-23T21:16:00.758875+0000","last_became_peered":"2023-09-23T21:16:00.758875+0000","last_unstale":"2023-09-23T21:17:56.633537+0000","last_undegraded":"2023-09-23T21:17:56.633537+0000","last_fullsized":"2023-09-23T21:17:56.633537+0000","mapping_epoch":498,"log_start":"502'2819","ondisk_log_start":"502'2819","created":482,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5713,"num_write_kb":182816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":505,"num_bytes_recovered":33095680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":159,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.449648+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:17:54.449648+0000","last_peered":"2023-09-23T21:17:54.449648+0000","last_clean":"2023-09-23T21:17:54.449648+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:17:54.449648+0000","last_undegraded":"2023-09-23T21:17:54.449648+0000","last_fullsized":"2023-09-23T21:17:54.449648+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":374,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523773+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:17:45.523773+0000","last_peered":"2023-09-23T21:17:45.523773+0000","last_clean":"2023-09-23T21:17:45.523773+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:17:45.523773+0000","last_undegraded":"2023-09-23T21:17:45.523773+0000","last_fullsized":"2023-09-23T21:17:45.523773+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":267,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451164+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:17:54.451164+0000","last_peered":"2023-09-23T21:17:54.451164+0000","last_clean":"2023-09-23T21:17:54.451164+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:17:54.451164+0000","last_undegraded":"2023-09-23T21:17:54.451164+0000","last_fullsized":"2023-09-23T21:17:54.451164+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"561'7403","reported_seq":7617,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.644312+0000","last_change":"2023-09-23T21:16:51.076997+0000","last_active":"2023-09-23T21:17:56.644312+0000","last_peered":"2023-09-23T21:17:56.644312+0000","last_clean":"2023-09-23T21:15:36.171427+0000","last_became_active":"2023-09-23T21:16:51.057102+0000","last_became_peered":"2023-09-23T21:16:51.057102+0000","last_unstale":"2023-09-23T21:17:56.644312+0000","last_undegraded":"2023-09-23T21:16:50.840109+0000","last_fullsized":"2023-09-23T21:16:50.799660+0000","mapping_epoch":544,"log_start":"515'3690","ondisk_log_start":"515'3690","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"510'3562","last_scrub_stamp":"2023-09-23T21:15:09.914277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:09.914277+0000","objects_scrubbed":1939,"log_size":3713,"log_dups_size":0,"ondisk_log_size":3713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.748037+0000","scrub_duration":0.18890970800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1565,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3841,"num_write_kb":60864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":-315}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":217,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.452643+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:17:54.452643+0000","last_peered":"2023-09-23T21:17:54.452643+0000","last_clean":"2023-09-23T21:17:54.452643+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:17:54.452643+0000","last_undegraded":"2023-09-23T21:17:54.452643+0000","last_fullsized":"2023-09-23T21:17:54.452643+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522684+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:17:45.522684+0000","last_peered":"2023-09-23T21:17:45.522684+0000","last_clean":"2023-09-23T21:17:45.522684+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:17:45.522684+0000","last_undegraded":"2023-09-23T21:17:45.522684+0000","last_fullsized":"2023-09-23T21:17:45.522684+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":242,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.447527+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:17:54.447527+0000","last_peered":"2023-09-23T21:17:54.447527+0000","last_clean":"2023-09-23T21:17:54.447527+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:17:54.447527+0000","last_undegraded":"2023-09-23T21:17:54.447527+0000","last_fullsized":"2023-09-23T21:17:54.447527+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"561'7543","reported_seq":7704,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.652120+0000","last_change":"2023-09-23T21:16:51.063450+0000","last_active":"2023-09-23T21:17:56.652120+0000","last_peered":"2023-09-23T21:17:56.652120+0000","last_clean":"2023-09-23T21:15:36.238755+0000","last_became_active":"2023-09-23T21:16:51.051345+0000","last_became_peered":"2023-09-23T21:16:51.051345+0000","last_unstale":"2023-09-23T21:17:56.652120+0000","last_undegraded":"2023-09-23T21:16:50.807164+0000","last_fullsized":"2023-09-23T21:16:50.788756+0000","mapping_epoch":544,"log_start":"503'3163","ondisk_log_start":"503'3163","created":482,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"509'3502","last_scrub_stamp":"2023-09-23T21:15:04.025929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:15:04.025929+0000","objects_scrubbed":1799,"log_size":4380,"log_dups_size":0,"ondisk_log_size":4380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:15.933191+0000","scrub_duration":0.37441795900000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1557,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4055,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-301}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":467,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523252+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:17:45.523252+0000","last_peered":"2023-09-23T21:17:45.523252+0000","last_clean":"2023-09-23T21:17:45.523252+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:17:45.523252+0000","last_undegraded":"2023-09-23T21:17:45.523252+0000","last_fullsized":"2023-09-23T21:17:45.523252+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":99,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451530+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:17:54.451530+0000","last_peered":"2023-09-23T21:17:54.451530+0000","last_clean":"2023-09-23T21:17:54.451530+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:17:54.451530+0000","last_undegraded":"2023-09-23T21:17:54.451530+0000","last_fullsized":"2023-09-23T21:17:54.451530+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":247,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.453846+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:17:54.453846+0000","last_peered":"2023-09-23T21:17:54.453846+0000","last_clean":"2023-09-23T21:17:54.453846+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:17:54.453846+0000","last_undegraded":"2023-09-23T21:17:54.453846+0000","last_fullsized":"2023-09-23T21:17:54.453846+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.15","version":"561'7559","reported_seq":8604,"reported_epoch":562,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:58.555189+0000","last_change":"2023-09-23T21:17:57.056504+0000","last_active":"2023-09-23T21:17:58.555189+0000","last_peered":"2023-09-23T21:17:58.555189+0000","last_clean":"2023-09-23T21:15:09.797020+0000","last_became_active":"2023-09-23T21:16:51.048602+0000","last_became_peered":"2023-09-23T21:16:51.048602+0000","last_unstale":"2023-09-23T21:17:58.555189+0000","last_undegraded":"2023-09-23T21:16:50.818351+0000","last_fullsized":"2023-09-23T21:16:50.791414+0000","mapping_epoch":544,"log_start":"515'3837","ondisk_log_start":"515'3837","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3722,"log_dups_size":0,"ondisk_log_size":3722,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":986,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5886,"num_write_kb":187328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":454,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5],"avail_no_missing":["4","3","5"],"object_location_counts":[{"shards":"4,5","objects":-986}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":189,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.447062+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:17:54.447062+0000","last_peered":"2023-09-23T21:17:54.447062+0000","last_clean":"2023-09-23T21:17:54.447062+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:17:54.447062+0000","last_undegraded":"2023-09-23T21:17:54.447062+0000","last_fullsized":"2023-09-23T21:17:54.447062+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":624,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523841+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:17:45.523841+0000","last_peered":"2023-09-23T21:17:45.523841+0000","last_clean":"2023-09-23T21:17:45.523841+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:17:45.523841+0000","last_undegraded":"2023-09-23T21:17:45.523841+0000","last_fullsized":"2023-09-23T21:17:45.523841+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.12","version":"561'7407","reported_seq":11464,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.659384+0000","last_change":"2023-09-23T21:16:37.510873+0000","last_active":"2023-09-23T21:17:56.659384+0000","last_peered":"2023-09-23T21:17:56.659384+0000","last_clean":"2023-09-23T21:17:56.659384+0000","last_became_active":"2023-09-23T21:16:37.500341+0000","last_became_peered":"2023-09-23T21:16:37.500341+0000","last_unstale":"2023-09-23T21:17:56.659384+0000","last_undegraded":"2023-09-23T21:17:56.659384+0000","last_fullsized":"2023-09-23T21:17:56.659384+0000","mapping_epoch":538,"log_start":"515'3649","ondisk_log_start":"515'3649","created":507,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3758,"log_dups_size":0,"ondisk_log_size":3758,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5746,"num_write_kb":183552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1970,"num_bytes_recovered":94175232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":266,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.451479+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:17:54.451479+0000","last_peered":"2023-09-23T21:17:54.451479+0000","last_clean":"2023-09-23T21:17:54.451479+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:17:54.451479+0000","last_undegraded":"2023-09-23T21:17:54.451479+0000","last_fullsized":"2023-09-23T21:17:54.451479+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":219,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523164+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:17:45.523164+0000","last_peered":"2023-09-23T21:17:45.523164+0000","last_clean":"2023-09-23T21:17:45.523164+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:17:45.523164+0000","last_undegraded":"2023-09-23T21:17:45.523164+0000","last_fullsized":"2023-09-23T21:17:45.523164+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.13","version":"561'7440","reported_seq":10183,"reported_epoch":561,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:17:56.629776+0000","last_change":"2023-09-23T21:16:50.886093+0000","last_active":"2023-09-23T21:17:56.629776+0000","last_peered":"2023-09-23T21:17:56.629776+0000","last_clean":"2023-09-23T21:15:08.028023+0000","last_became_active":"2023-09-23T21:15:12.856600+0000","last_became_peered":"2023-09-23T21:15:12.856600+0000","last_unstale":"2023-09-23T21:17:56.629776+0000","last_undegraded":"2023-09-23T21:15:12.694807+0000","last_fullsized":"2023-09-23T21:15:12.679688+0000","mapping_epoch":514,"log_start":"515'3734","ondisk_log_start":"515'3734","created":507,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":3706,"log_dups_size":0,"ondisk_log_size":3706,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1867776,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":623,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5757,"num_write_kb":185120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1318,"num_bytes_recovered":61800448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"4,5","objects":-594}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":30,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522695+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:17:45.522695+0000","last_peered":"2023-09-23T21:17:45.522695+0000","last_clean":"2023-09-23T21:17:45.522695+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:17:45.522695+0000","last_undegraded":"2023-09-23T21:17:45.522695+0000","last_fullsized":"2023-09-23T21:17:45.522695+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":206,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522624+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:17:45.522624+0000","last_peered":"2023-09-23T21:17:45.522624+0000","last_clean":"2023-09-23T21:17:45.522624+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:17:45.522624+0000","last_undegraded":"2023-09-23T21:17:45.522624+0000","last_fullsized":"2023-09-23T21:17:45.522624+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.10","version":"561'7271","reported_seq":13145,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.619604+0000","last_change":"2023-09-23T21:15:36.462137+0000","last_active":"2023-09-23T21:17:56.619604+0000","last_peered":"2023-09-23T21:17:56.619604+0000","last_clean":"2023-09-23T21:17:56.619604+0000","last_became_active":"2023-09-23T21:15:36.460043+0000","last_became_peered":"2023-09-23T21:15:36.460043+0000","last_unstale":"2023-09-23T21:17:56.619604+0000","last_undegraded":"2023-09-23T21:17:56.619604+0000","last_fullsized":"2023-09-23T21:17:56.619604+0000","mapping_epoch":518,"log_start":"500'2632","ondisk_log_start":"500'2632","created":507,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4639,"log_dups_size":0,"ondisk_log_size":4639,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5608,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2392,"num_bytes_recovered":156762112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":26,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522658+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:17:45.522658+0000","last_peered":"2023-09-23T21:17:45.522658+0000","last_clean":"2023-09-23T21:17:45.522658+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:17:45.522658+0000","last_undegraded":"2023-09-23T21:17:45.522658+0000","last_fullsized":"2023-09-23T21:17:45.522658+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":421,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.523193+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:17:45.523193+0000","last_peered":"2023-09-23T21:17:45.523193+0000","last_clean":"2023-09-23T21:17:45.523193+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:17:45.523193+0000","last_undegraded":"2023-09-23T21:17:45.523193+0000","last_fullsized":"2023-09-23T21:17:45.523193+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.11","version":"561'7354","reported_seq":11323,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:56.067636+0000","last_change":"2023-09-23T21:16:49.855987+0000","last_active":"2023-09-23T21:17:56.067636+0000","last_peered":"2023-09-23T21:17:56.067636+0000","last_clean":"2023-09-23T21:17:56.067636+0000","last_became_active":"2023-09-23T21:16:15.742634+0000","last_became_peered":"2023-09-23T21:16:15.742634+0000","last_unstale":"2023-09-23T21:17:56.067636+0000","last_undegraded":"2023-09-23T21:17:56.067636+0000","last_fullsized":"2023-09-23T21:17:56.067636+0000","mapping_epoch":531,"log_start":"501'2700","ondisk_log_start":"501'2700","created":507,"last_epoch_clean":544,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:14:11.627614+0000","last_clean_scrub_stamp":"2023-09-23T21:14:11.627614+0000","objects_scrubbed":0,"log_size":4654,"log_dups_size":0,"ondisk_log_size":4654,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2195456,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5718,"num_write_kb":184032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1921,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":517,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-09-23T21:17:45.522803+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:17:45.522803+0000","last_peered":"2023-09-23T21:17:45.522803+0000","last_clean":"2023-09-23T21:17:45.522803+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:17:45.522803+0000","last_undegraded":"2023-09-23T21:17:45.522803+0000","last_fullsized":"2023-09-23T21:17:45.522803+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":229,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-09-23T21:17:54.450468+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:17:54.450468+0000","last_peered":"2023-09-23T21:17:54.450468+0000","last_clean":"2023-09-23T21:17:54.450468+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:17:54.450468+0000","last_undegraded":"2023-09-23T21:17:54.450468+0000","last_fullsized":"2023-09-23T21:17:54.450468+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":36,"stat_sum":{"num_bytes":23937012,"num_objects":365,"num_object_clones":0,"num_object_copies":1095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14251,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":365,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169838,"num_write_kb":4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34556,"num_bytes_recovered":1665368064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1991376896,"data_stored":1991376896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":179612,"ondisk_log_size":179612,"up":108,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":38918,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201871,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":857704,"kb_used_data":64224,"kb_used_omap":0,"kb_used_meta":793472,"kb_avail":92887448,"statfs":{"total":95995035648,"available":95116746752,"internally_reserved":0,"allocated":65765376,"data_stored":64229550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":812515328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4630000000000001,"5min":3.0539999999999998,"15min":2.2519999999999998},"min":{"1min":0.26300000000000001,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":7.726,"5min":141.33199999999999,"15min":141.33199999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.2109999999999999,"5min":3.3530000000000002,"15min":2.3199999999999998},"min":{"1min":0.307,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.3940000000000001,"5min":141.31700000000001,"15min":141.31700000000001},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:17:19 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.1280000000000001,"15min":3.4239999999999999},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.4329999999999998,"5min":16.684000000000001,"15min":114.16500000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.5920000000000001,"5min":2.8799999999999999,"15min":3.867},"min":{"1min":0.307,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.7699999999999996,"5min":117.637,"15min":117.637},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":4.7679999999999998,"15min":4.4850000000000003},"min":{"1min":0.246,"5min":0.224,"15min":0.224},"max":{"1min":5.4160000000000004,"5min":135.529,"15min":183.44200000000001},"last":53.988},{"interface":"front","average":{"1min":1.5740000000000001,"5min":5.6550000000000002,"15min":4.6909999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.2370000000000001,"5min":135.56700000000001,"15min":183.46199999999999},"last":54}]},{"osd":3,"last update":"Sat Sep 23 21:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":7.1909999999999998,"15min":4.1210000000000004},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":7.7960000000000003,"5min":135.66200000000001,"15min":135.66200000000001},"last":0.371},{"interface":"front","average":{"1min":1.613,"5min":7.6120000000000001,"15min":4.2199999999999998},"min":{"1min":0.375,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":5.2779999999999996,"5min":135.47,"15min":135.47},"last":0.32100000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.9430000000000001,"15min":1.2090000000000001},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1829999999999998,"5min":10.989000000000001,"15min":13.739000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":3.5840000000000001,"5min":2.0950000000000002,"15min":1.2},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":20.306999999999999,"5min":20.306999999999999,"15min":20.306999999999999},"last":0.26800000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":2.6389999999999998,"15min":1.6080000000000001},"min":{"1min":0.53100000000000003,"5min":0.29099999999999998,"15min":0.26900000000000002},"max":{"1min":14.105,"5min":14.105,"15min":14.105},"last":0.28499999999999998},{"interface":"front","average":{"1min":3.9630000000000001,"5min":2.5339999999999998,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":17.585999999999999,"5min":17.585999999999999,"15min":17.585999999999999},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.2930000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":9.9969999999999999,"5min":13.369,"15min":143.18700000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.9380000000000002,"5min":1.819,"15min":1.929},"min":{"1min":0.31,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":15.566000000000001,"5min":15.566000000000001,"15min":142.70400000000001},"last":0.36299999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463491,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1444372,"kb_used_data":691852,"kb_used_omap":0,"kb_used_meta":752448,"kb_avail":92300780,"statfs":{"total":95995035648,"available":94515998720,"internally_reserved":0,"allocated":708456448,"data_stored":706933030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":770506752},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":4.5339999999999998,"15min":2.7490000000000001},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":12.659000000000001,"5min":149.61600000000001,"15min":149.61600000000001},"last":6.234},{"interface":"front","average":{"1min":2.0369999999999999,"5min":4.899,"15min":2.835},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":16.021999999999998,"5min":149.303,"15min":149.303},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":2.379,"15min":3.6080000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.22700000000000001},"max":{"1min":16.126999999999999,"5min":25.122,"15min":129.37899999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.986,"5min":1.8420000000000001,"15min":3.387},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":10.977,"5min":25.193999999999999,"15min":129.40100000000001},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":11.577999999999999,"5min":7.6189999999999998,"15min":5.9340000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26800000000000002},"max":{"1min":120.28700000000001,"5min":133.46600000000001,"15min":199.23400000000001},"last":6.1879999999999997},{"interface":"front","average":{"1min":12.118,"5min":7.8120000000000003,"15min":5.9119999999999999},"min":{"1min":0.33100000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":120.414,"5min":133.33000000000001,"15min":199.08000000000001},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":5.6669999999999998,"15min":3.3199999999999998},"min":{"1min":0.38,"5min":0.218,"15min":0.218},"max":{"1min":3.5089999999999999,"5min":137.875,"15min":137.875},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.7250000000000001,"5min":5.391,"15min":3.3650000000000002},"min":{"1min":0.30099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.676,"5min":137.60900000000001,"15min":137.60900000000001},"last":6.298}]},{"osd":4,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":8.9060000000000006,"5min":2.9329999999999998,"15min":1.673},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.22600000000000001},"max":{"1min":55.494,"5min":55.494,"15min":55.494},"last":0.36499999999999999},{"interface":"front","average":{"1min":8.7550000000000008,"5min":2.8100000000000001,"15min":1.4970000000000001},"min":{"1min":0.39900000000000002,"5min":0.28299999999999997,"15min":0.23999999999999999},"max":{"1min":55.305,"5min":55.305,"15min":55.305},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:16:57 2023","interfaces":[{"interface":"back","average":{"1min":15.223000000000001,"5min":4.3630000000000004,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":222.90899999999999,"5min":222.90899999999999,"15min":222.90899999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":15.375,"5min":4.4930000000000003,"15min":2.1739999999999999},"min":{"1min":0.37,"5min":0.32200000000000001,"15min":0.28499999999999998},"max":{"1min":224.655,"5min":224.655,"15min":224.655},"last":0.68799999999999994}]},{"osd":7,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":13.882999999999999,"5min":3.629,"15min":1.883},"min":{"1min":0.39600000000000002,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":165.96799999999999,"5min":165.96799999999999,"15min":165.96799999999999},"last":6.2709999999999999},{"interface":"front","average":{"1min":13.532,"5min":3.859,"15min":1.98},"min":{"1min":0.442,"5min":0.251,"15min":0.251},"max":{"1min":165.82400000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":0.53300000000000003}]}]},{"osd":1,"up_from":0,"seq":1185410973938,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991678,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781569,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1160012,"kb_used_data":46468,"kb_used_omap":7,"kb_used_meta":1113528,"kb_avail":92585140,"statfs":{"total":95995035648,"available":94807183360,"internally_reserved":0,"allocated":47583232,"data_stored":44889198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1140252753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":2.8420000000000001,"15min":2.157},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":8.0079999999999991,"5min":103.021,"15min":103.021},"last":0.191},{"interface":"front","average":{"1min":0.29599999999999999,"5min":3.133,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":0.378,"5min":103.154,"15min":103.154},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:17:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.2000000000000002,"15min":4.0869999999999997},"min":{"1min":0.24099999999999999,"5min":0.224,"15min":0.19},"max":{"1min":2.4409999999999998,"5min":54.521000000000001,"15min":212.12},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.98999999999999999,"5min":2.4820000000000002,"15min":4.1399999999999997},"min":{"1min":0.254,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":8.048,"5min":54.598999999999997,"15min":213.02799999999999},"last":0.25900000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.5670000000000002,"15min":4.0339999999999998},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5190000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.2250000000000001,"15min":4.2519999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.2189999999999994,"5min":82.241,"15min":143.27799999999999},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":13.896000000000001,"5min":4.3010000000000002,"15min":3.1800000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":220.82900000000001,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":13.894,"5min":4.9130000000000003,"15min":3.823},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28299999999999997},"max":{"1min":220.84299999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.541,"5min":5.1059999999999999,"15min":3.48},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":220.63900000000001,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":16.257000000000001,"5min":4.7380000000000004,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":220.31999999999999,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":15.597,"5min":4.6210000000000004,"15min":3.762},"min":{"1min":0.41399999999999998,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":210.74700000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":4.117},{"interface":"front","average":{"1min":14.371,"5min":4.4640000000000004,"15min":3.1259999999999999},"min":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":210.72900000000001,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 21:17:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.4940000000000002,"15min":1.948},"min":{"1min":0.371,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":14.019,"5min":25.260000000000002,"15min":25.260000000000002},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.921,"5min":2.4990000000000001,"15min":2.2850000000000001},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":5.0229999999999997,"5min":12.613,"15min":16.875},"last":0.32700000000000001}]}]},{"osd":3,"up_from":0,"seq":635655160162,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986781,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1026652,"kb_used_data":218324,"kb_used_omap":0,"kb_used_meta":808320,"kb_avail":92718500,"statfs":{"total":95995035648,"available":94943744000,"internally_reserved":0,"allocated":223563776,"data_stored":222032138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":827719680},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":3.3410000000000002,"15min":2.7639999999999998},"min":{"1min":0.28000000000000003,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":9.2200000000000006,"5min":109.23099999999999,"15min":226.751},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.754,"15min":2.5430000000000001},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":6.0919999999999996,"5min":109.16800000000001,"15min":226.43299999999999},"last":0.28599999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":3.802,"15min":5.2999999999999998},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2510000000000003,"5min":119.55800000000001,"15min":127.411},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.8050000000000002,"5min":3.71,"15min":4.5830000000000002},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":10.506,"5min":116.78100000000001,"15min":118.075},"last":0.38200000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":8.7449999999999992,"5min":6.4420000000000002,"15min":6.327},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":85.994,"5min":155.51300000000001,"15min":210.50299999999999},"last":85.994},{"interface":"front","average":{"1min":9.2530000000000001,"5min":7.3230000000000004,"15min":6.4809999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":85.971999999999994,"5min":155.77099999999999,"15min":210.59},"last":85.971999999999994}]},{"osd":3,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":3.2440000000000002,"15min":4.1200000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0650000000000004,"5min":111.22199999999999,"15min":214.64099999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":3.0659999999999998,"5min":4.508,"15min":4.5209999999999999},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":10.529,"5min":111.39400000000001,"15min":214.54900000000001},"last":0.48199999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7829999999999999,"5min":1.7749999999999999,"15min":1.139},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.417,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.311},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.8399999999999999,"15min":1.49},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":14.675000000000001,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":2.6579999999999999,"15min":1.6970000000000001},"min":{"1min":0.35699999999999998,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":41.938000000000002,"5min":41.938000000000002,"15min":41.938000000000002},"last":0.40000000000000002},{"interface":"front","average":{"1min":5.4660000000000002,"5min":2.5390000000000001,"15min":1.635},"min":{"1min":0.33600000000000002,"5min":0.32300000000000001,"15min":0.27200000000000002},"max":{"1min":40.444000000000003,"5min":40.444000000000003,"15min":40.444000000000003},"last":0.34000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":2.2170000000000001,"15min":1.8740000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":11.859,"5min":18.352,"15min":145.721},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.8600000000000001,"15min":1.893},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"max":{"1min":8.5570000000000004,"5min":10.677,"15min":148.45699999999999},"last":0.36899999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725145,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915096,"kb_used_data":44112,"kb_used_omap":0,"kb_used_meta":870976,"kb_avail":92830056,"statfs":{"total":95995035648,"available":95057977344,"internally_reserved":0,"allocated":45170688,"data_stored":43646721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":891879424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.427,"15min":1.71},"min":{"1min":0.34899999999999998,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":4.1260000000000003,"5min":8.6660000000000004,"15min":82.280000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.798,"15min":1.889},"min":{"1min":0.27600000000000002,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.8810000000000002,"5min":16.977,"15min":85.236000000000004},"last":3.1619999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3479999999999999,"15min":4.7039999999999997},"min":{"1min":0.255,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":11.374000000000001,"5min":12.032999999999999,"15min":233.77099999999999},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.3979999999999999,"5min":1.6399999999999999,"15min":4.359},"min":{"1min":0.39600000000000002,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":4.0590000000000002,"5min":12.016,"15min":233.75200000000001},"last":0.81499999999999995}]},{"osd":2,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":1.73,"15min":3.2269999999999999},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.1040000000000001,"5min":10.454000000000001,"15min":231.34700000000001},"last":3.8380000000000001},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.9930000000000001,"15min":3.4220000000000002},"min":{"1min":0.374,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9180000000000001,"5min":10.621,"15min":231.381},"last":3.258}]},{"osd":3,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.403,"15min":3.7440000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":11.42,"5min":11.597,"15min":263.887},"last":1.169},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.0099999999999998,"15min":3.4380000000000002},"min":{"1min":0.44400000000000001,"5min":0.39700000000000002,"15min":0.30099999999999999},"max":{"1min":4.173,"5min":10.638,"15min":263.70100000000002},"last":3.8740000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.7709999999999999,"15min":1.0129999999999999},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":4.0289999999999999,"5min":11.593999999999999,"15min":11.593999999999999},"last":3.8999999999999999},{"interface":"front","average":{"1min":3.3860000000000001,"5min":1.95,"15min":1.1899999999999999},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.26400000000000001},"max":{"1min":12.49,"5min":12.49,"15min":12.49},"last":3.29}]},{"osd":6,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":2.0339999999999998,"15min":1.619},"min":{"1min":0.48499999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":6.984,"5min":8.9130000000000003,"15min":83.451999999999998},"last":3.331},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.73,"15min":1.8859999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":11.696999999999999,"5min":12.725,"15min":83.484999999999999},"last":7.5659999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:17:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1000000000000001,"5min":2.161,"15min":1.419},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":7.4020000000000001,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":3.3580000000000001},{"interface":"front","average":{"1min":2.431,"5min":1.585,"15min":0.995},"min":{"1min":0.45500000000000002,"5min":0.34899999999999998,"15min":0.28899999999999998},"max":{"1min":10.196999999999999,"5min":11.640000000000001,"15min":11.640000000000001},"last":0.84199999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":39780352,"data_stored":39780352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":952303616,"data_stored":952303616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":217645056,"data_stored":217645056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":39256064,"data_stored":39256064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":683147264,"data_stored":683147264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":59244544,"data_stored":59244544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:02.158 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:18:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:01 smithi123 ceph-mon[193903]: from='client.47369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:01 smithi123 ceph-mon[193903]: pgmap v1757: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 436 op/s; 14251/1104 objects degraded (1290.851%); 0 B/s, 170 objects/s recovering 2023-09-23T21:18:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:01 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14251/1167 objects degraded (1221.165%), 14 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:01 smithi123 ceph-mon[193903]: osdmap e564: 8 total, 8 up, 5 in 2023-09-23T21:18:02.246 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:02.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:02.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:02.354 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:02.392 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:02.431 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:02.447 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:02.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:02.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:02.476 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:02.476 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:02.553 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:02.581 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:02.631 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:02.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:02.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:02.740 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:02.769 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e564 with expected crc 2023-09-23T21:18:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e564 with expected crc 2023-09-23T21:18:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e564 with expected crc 2023-09-23T21:18:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e564 with expected crc 2023-09-23T21:18:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e565 with expected crc 2023-09-23T21:18:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e565 with expected crc 2023-09-23T21:18:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: osdmap e565: 8 total, 8 up, 5 in 2023-09-23T21:18:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e565 with expected crc 2023-09-23T21:18:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:02 smithi123 ceph-mon[193903]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: osdmap e565: 8 total, 8 up, 5 in 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[211526]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e564 with expected crc 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: osdmap e565: 8 total, 8 up, 5 in 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:02 smithi121 ceph-mon[219709]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.258 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:03.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:03.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:03.362 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:03.364 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:18:03.406 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:03.436 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:03.471 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:03.488 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:03.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:03.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:03.592 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:03.620 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:03.669 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:03.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:03.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:03.775 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:03.806 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:03.816 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:18:03.819 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[219709]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.819 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2617446223' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:03.819 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:03.819 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[219709]: pgmap v1760: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 466 op/s; 14251/1104 objects degraded (1290.851%); 0 B/s, 211 objects/s recovering 2023-09-23T21:18:03.819 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[211526]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.819 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2617446223' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:03.820 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:03.820 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:03 smithi121 ceph-mon[211526]: pgmap v1760: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 466 op/s; 14251/1104 objects degraded (1290.851%); 0 B/s, 211 objects/s recovering 2023-09-23T21:18:03.963 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:03.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:03.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:03 smithi123 ceph-mon[193903]: failed to encode map e565 with expected crc 2023-09-23T21:18:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2617446223' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:03 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:03 smithi123 ceph-mon[193903]: pgmap v1760: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 466 op/s; 14251/1104 objects degraded (1290.851%); 0 B/s, 211 objects/s recovering 2023-09-23T21:18:04.069 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:04.099 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:04.153 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:04.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:04.257 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:04.288 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:04.339 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:04.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:04.446 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:04.475 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:05.093 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:18:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:18:04] "GET /metrics HTTP/1.1" 200 36711 "" "Prometheus/2.43.0" 2023-09-23T21:18:05.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:05.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[219709]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[219709]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[219709]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:05.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[219709]: osdmap e566: 8 total, 8 up, 5 in 2023-09-23T21:18:05.094 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[219709]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:05.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[211526]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[211526]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[211526]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:05.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[211526]: osdmap e566: 8 total, 8 up, 5 in 2023-09-23T21:18:05.095 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:04 smithi121 ceph-mon[211526]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:04 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:05.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:04 smithi123 ceph-mon[193903]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:04 smithi123 ceph-mon[193903]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:04 smithi123 ceph-mon[193903]: failed to encode map e566 with expected crc 2023-09-23T21:18:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:04 smithi123 ceph-mon[193903]: osdmap e566: 8 total, 8 up, 5 in 2023-09-23T21:18:05.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:04 smithi123 ceph-mon[193903]: failed to encode map e566 with expected crc 2023-09-23T21:18:06.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: osdmap e567: 8 total, 8 up, 5 in 2023-09-23T21:18:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[211526]: pgmap v1763: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 11660/1104 objects degraded (1056.159%); 0 B/s, 833 objects/s recovering 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: osdmap e567: 8 total, 8 up, 5 in 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:05 smithi121 ceph-mon[219709]: pgmap v1763: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 11660/1104 objects degraded (1056.159%); 0 B/s, 833 objects/s recovering 2023-09-23T21:18:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: osdmap e567: 8 total, 8 up, 5 in 2023-09-23T21:18:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: failed to encode map e567 with expected crc 2023-09-23T21:18:06.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:05 smithi123 ceph-mon[193903]: pgmap v1763: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 22 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 11660/1104 objects degraded (1056.159%); 0 B/s, 833 objects/s recovering 2023-09-23T21:18:06.696 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:06.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:06.801 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:06.828 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:06.872 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:06.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:06.979 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:07.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[211526]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[211526]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[211526]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[211526]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[211526]: osdmap e568: 8 total, 8 up, 5 in 2023-09-23T21:18:07.004 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[211526]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11660/1104 objects degraded (1056.159%), 12 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[219709]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[219709]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[219709]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[219709]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[219709]: osdmap e568: 8 total, 8 up, 5 in 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[219709]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.005 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11660/1104 objects degraded (1056.159%), 12 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:07.007 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:06 smithi123 ceph-mon[193903]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:06 smithi123 ceph-mon[193903]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:06 smithi123 ceph-mon[193903]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:06 smithi123 ceph-mon[193903]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:06 smithi123 ceph-mon[193903]: osdmap e568: 8 total, 8 up, 5 in 2023-09-23T21:18:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:06 smithi123 ceph-mon[193903]: failed to encode map e568 with expected crc 2023-09-23T21:18:07.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11660/1104 objects degraded (1056.159%), 12 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:08.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:07 smithi121 ceph-mon[211526]: pgmap v1765: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 22 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 11660/1104 objects degraded (1056.159%); 0 B/s, 722 objects/s recovering 2023-09-23T21:18:08.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:07 smithi121 ceph-mon[219709]: pgmap v1765: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 22 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 11660/1104 objects degraded (1056.159%); 0 B/s, 722 objects/s recovering 2023-09-23T21:18:08.208 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:08.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:07 smithi123 ceph-mon[193903]: pgmap v1765: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 22 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 11660/1104 objects degraded (1056.159%); 0 B/s, 722 objects/s recovering 2023-09-23T21:18:08.311 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:08.339 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:08.384 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:08.491 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:08.518 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:08.561 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:08.597 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:08.671 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:08.697 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:08.712 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:18:08.742 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:08.849 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:08.876 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:08.926 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:08.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:08.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:08 smithi123 ceph-mon[193903]: failed to encode map e569 with expected crc 2023-09-23T21:18:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:08 smithi123 ceph-mon[193903]: failed to encode map e569 with expected crc 2023-09-23T21:18:08.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:08 smithi123 ceph-mon[193903]: failed to encode map e569 with expected crc 2023-09-23T21:18:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:08 smithi123 ceph-mon[193903]: osdmap e569: 8 total, 8 up, 5 in 2023-09-23T21:18:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:08 smithi123 ceph-mon[193903]: failed to encode map e569 with expected crc 2023-09-23T21:18:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:08 smithi123 ceph-mon[193903]: failed to encode map e569 with expected crc 2023-09-23T21:18:08.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:08 smithi123 ceph-mon[193903]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.033 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:09.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[211526]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[211526]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[211526]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[211526]: osdmap e569: 8 total, 8 up, 5 in 2023-09-23T21:18:09.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[211526]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.062 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[211526]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[211526]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[219709]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[219709]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[219709]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[219709]: osdmap e569: 8 total, 8 up, 5 in 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[219709]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[219709]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.063 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:08 smithi121 ceph-mon[219709]: failed to encode map e569 with expected crc 2023-09-23T21:18:09.064 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:09.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:09.217 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:09.245 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.260 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:18:09.261 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11660/1104 objects degraded (1056.159%), 12 pgs degraded, 11 pgs undersized","count":23},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2345,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":570,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":5,"osd_in_since":1695503710,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":95},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":10},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":109,"num_pools":5,"num_objects":368,"data_bytes":23156628,"bytes_used":5391814656,"bytes_avail":474583363584,"bytes_total":479975178240,"degraded_objects":11660,"degraded_total":1104,"degraded_ratio":10.561594202898551,"recovering_objects_per_sec":722,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":3335,"num_bytes_recovered":0,"num_keys_recovered":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T21:17:03.337132+0000","services":{}},"progress_events":{"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true},"d33d2ace-2031-48c2-a8f8-693bca5c747e":{"message":"Global Recovery Event (2m)\n [=======================.....] (remaining: 27s)","progress":0.85321098566055298,"add_to_ceph_s":true}}} 2023-09-23T21:18:09.291 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:09.351 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:18:09.377 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:18:09.397 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:09.403 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:18:09.423 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:18:09.425 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.443 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:18:09.463 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:18:09.479 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:09.482 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:18:09.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:09.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:09.500 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.502 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:18:09.587 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:09.636 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:09.843 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_8' removed 2023-09-23T21:18:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: osdmap e570: 8 total, 8 up, 5 in 2023-09-23T21:18:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2975950497' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/501693725' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1436355879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[211526]: pgmap v1768: 109 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 22 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 7285/1104 objects degraded (659.873%); 0 B/s, 717 objects/s recovering 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: osdmap e570: 8 total, 8 up, 5 in 2023-09-23T21:18:09.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: failed to encode map e570 with expected crc 2023-09-23T21:18:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2975950497' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:18:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/501693725' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1436355879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:09.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:09 smithi121 ceph-mon[219709]: pgmap v1768: 109 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 22 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 7285/1104 objects degraded (659.873%); 0 B/s, 717 objects/s recovering 2023-09-23T21:18:10.031 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:10.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:10.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:10.135 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:10.162 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: failed to encode map e570 with expected crc 2023-09-23T21:18:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: failed to encode map e570 with expected crc 2023-09-23T21:18:10.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: osdmap e570: 8 total, 8 up, 5 in 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: failed to encode map e570 with expected crc 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: failed to encode map e570 with expected crc 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: failed to encode map e570 with expected crc 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: failed to encode map e570 with expected crc 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2975950497' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/501693725' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1436355879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:10.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:09 smithi123 ceph-mon[193903]: pgmap v1768: 109 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 22 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 7285/1104 objects degraded (659.873%); 0 B/s, 717 objects/s recovering 2023-09-23T21:18:10.917 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:18:10.917 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[211526]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[211526]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2975950497' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:18:11.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[211526]: osdmap e571: 8 total, 8 up, 5 in 2023-09-23T21:18:11.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[211526]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[219709]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[219709]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2975950497' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:18:11.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[219709]: osdmap e571: 8 total, 8 up, 5 in 2023-09-23T21:18:11.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:10 smithi121 ceph-mon[219709]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:10 smithi123 ceph-mon[193903]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:10 smithi123 ceph-mon[193903]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2975950497' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:18:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:10 smithi123 ceph-mon[193903]: osdmap e571: 8 total, 8 up, 5 in 2023-09-23T21:18:11.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:10 smithi123 ceph-mon[193903]: failed to encode map e571 with expected crc 2023-09-23T21:18:11.417 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:11.522 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:11.552 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:11.598 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:11.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:11.698 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:11.703 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:11.721 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T21:18:11.721 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T21:18:11.722 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-09-23T21:18:11.722 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_9 16 2023-09-23T21:18:11.737 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:11.784 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:11.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:11.840 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:11 smithi121 ceph-mon[211526]: pgmap v1770: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:11.840 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:11 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7285/1104 objects degraded (659.873%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:11.840 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:11 smithi121 ceph-mon[219709]: pgmap v1770: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:11.841 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:11 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7285/1104 objects degraded (659.873%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:11.889 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:11.917 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:11.961 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:11.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:11.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:12.066 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:12.089 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:11 smithi123 ceph-mon[193903]: pgmap v1770: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:12.089 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:11 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7285/1104 objects degraded (659.873%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T21:18:12.091 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.133 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:12.162 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:18:12.163 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:18:12.163 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:18:12.163 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:18:12.163 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:18:12.239 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:12.267 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.309 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:12.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:12.418 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:12.443 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.485 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:12.594 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:12.620 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:12.661 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:12.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:12.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:12.765 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:12.784 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:12.802 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:13.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:12 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7285/1104 objects degraded (659.873%), 7 pgs degraded, 8 pgs undersized) 2023-09-23T21:18:13.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:13.008 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:13.008 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:12 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7285/1104 objects degraded (659.873%), 7 pgs degraded, 8 pgs undersized) 2023-09-23T21:18:13.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:13.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:13.025 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:13.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:13.129 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:13.156 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:12 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7285/1104 objects degraded (659.873%), 7 pgs degraded, 8 pgs undersized) 2023-09-23T21:18:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:13.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:13.298 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:18:13.385 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:18:13.557 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:13.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:13.563 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:13.661 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:13.693 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:13.743 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:13.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:13.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:13.849 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:13.877 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:13.920 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:13.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:13.935 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:13 smithi121 ceph-mon[211526]: pgmap v1771: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:13.936 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:13 smithi121 ceph-mon[219709]: pgmap v1771: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:13.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:13 smithi123 ceph-mon[193903]: pgmap v1771: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:14.026 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:14.054 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:15.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:18:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:18:14] "GET /metrics HTTP/1.1" 200 34248 "" "Prometheus/2.43.0" 2023-09-23T21:18:15.348 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:15.353 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:15.459 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:15.491 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:15.540 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:15.566 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:15.644 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:15.675 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:15.824 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:15.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:15.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:15.928 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:15.959 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:16.082 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:18:16.083 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:18:16.322 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:16.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:16.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:16.432 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:18:16.454 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:16 smithi121 ceph-mon[211526]: pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:16.454 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2560930257' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:16.455 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:16 smithi121 ceph-mon[219709]: pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:16.455 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2560930257' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:16.458 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:16 smithi123 ceph-mon[193903]: pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:16.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2560930257' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:17.435 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_9' created 2023-09-23T21:18:17.574 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: from='client.47405 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:17.574 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:17.574 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:18:17.574 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.574 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2560930257' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: osdmap e572: 8 total, 8 up, 5 in 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/578036385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: from='client.47405 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.576 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2560930257' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:17.576 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: osdmap e572: 8 total, 8 up, 5 in 2023-09-23T21:18:17.576 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.576 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/578036385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T21:18:17.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: from='client.47405 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:17.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:17.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:18:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2560930257' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: osdmap e572: 8 total, 8 up, 5 in 2023-09-23T21:18:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: failed to encode map e572 with expected crc 2023-09-23T21:18:17.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/578036385' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-09-23T21:18:18.410 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1772,"stamp":"2023-09-23T21:18:15.363409+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":5352230,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":5068792,"kb_used_data":681424,"kb_used_omap":7,"kb_used_meta":4387320,"kb_avail":463656968,"statfs":{"total":479975178240,"available":474784735232,"internally_reserved":0,"allocated":697778176,"data_stored":688776195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4492615761},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":91,"apply_latency_ms":91,"commit_latency_ns":91000000,"apply_latency_ns":91000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-22560756,"num_objects":-344,"num_object_clones":0,"num_object_copies":-1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-11660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-344,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-169875,"num_write_kb":-4952385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-37891,"num_bytes_recovered":-1665368064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.535022"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":173,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834507+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:18:08.834507+0000","last_peered":"2023-09-23T21:18:08.834507+0000","last_clean":"2023-09-23T21:18:08.834507+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:18:08.834507+0000","last_undegraded":"2023-09-23T21:18:08.834507+0000","last_fullsized":"2023-09-23T21:18:08.834507+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840730+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:18:08.840730+0000","last_peered":"2023-09-23T21:18:08.840730+0000","last_clean":"2023-09-23T21:18:08.840730+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:18:08.840730+0000","last_undegraded":"2023-09-23T21:18:08.840730+0000","last_fullsized":"2023-09-23T21:18:08.840730+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":358,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.838258+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:18:08.838258+0000","last_peered":"2023-09-23T21:18:08.838258+0000","last_clean":"2023-09-23T21:18:08.838258+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:18:08.838258+0000","last_undegraded":"2023-09-23T21:18:08.838258+0000","last_fullsized":"2023-09-23T21:18:08.838258+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836159+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:18:08.836159+0000","last_peered":"2023-09-23T21:18:08.836159+0000","last_clean":"2023-09-23T21:18:08.836159+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:18:08.836159+0000","last_undegraded":"2023-09-23T21:18:08.836159+0000","last_fullsized":"2023-09-23T21:18:08.836159+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":231,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.833588+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:18:08.833588+0000","last_peered":"2023-09-23T21:18:08.833588+0000","last_clean":"2023-09-23T21:18:08.833588+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:18:08.833588+0000","last_undegraded":"2023-09-23T21:18:08.833588+0000","last_fullsized":"2023-09-23T21:18:08.833588+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.841288+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:18:08.841288+0000","last_peered":"2023-09-23T21:18:08.841288+0000","last_clean":"2023-09-23T21:18:08.841288+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:18:08.841288+0000","last_undegraded":"2023-09-23T21:18:08.841288+0000","last_fullsized":"2023-09-23T21:18:08.841288+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":165,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.841698+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:18:08.841698+0000","last_peered":"2023-09-23T21:18:08.841698+0000","last_clean":"2023-09-23T21:18:08.841698+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:18:08.841698+0000","last_undegraded":"2023-09-23T21:18:08.841698+0000","last_fullsized":"2023-09-23T21:18:08.841698+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840445+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:18:08.840445+0000","last_peered":"2023-09-23T21:18:08.840445+0000","last_clean":"2023-09-23T21:18:08.840445+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:18:08.840445+0000","last_undegraded":"2023-09-23T21:18:08.840445+0000","last_fullsized":"2023-09-23T21:18:08.840445+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":224,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.841403+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:18:08.841403+0000","last_peered":"2023-09-23T21:18:08.841403+0000","last_clean":"2023-09-23T21:18:08.841403+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:18:08.841403+0000","last_undegraded":"2023-09-23T21:18:08.841403+0000","last_fullsized":"2023-09-23T21:18:08.841403+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":193,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834114+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:18:08.834114+0000","last_peered":"2023-09-23T21:18:08.834114+0000","last_clean":"2023-09-23T21:18:08.834114+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:18:08.834114+0000","last_undegraded":"2023-09-23T21:18:08.834114+0000","last_fullsized":"2023-09-23T21:18:08.834114+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":174,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835498+0000","last_change":"2023-09-23T21:17:11.237055+0000","last_active":"2023-09-23T21:18:08.835498+0000","last_peered":"2023-09-23T21:18:08.835498+0000","last_clean":"2023-09-23T21:18:08.835498+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:18:08.835498+0000","last_undegraded":"2023-09-23T21:18:08.835498+0000","last_fullsized":"2023-09-23T21:18:08.835498+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.839868+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:18:08.839868+0000","last_peered":"2023-09-23T21:18:08.839868+0000","last_clean":"2023-09-23T21:18:08.839868+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:18:08.839868+0000","last_undegraded":"2023-09-23T21:18:08.839868+0000","last_fullsized":"2023-09-23T21:18:08.839868+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":198,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835374+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:18:08.835374+0000","last_peered":"2023-09-23T21:18:08.835374+0000","last_clean":"2023-09-23T21:18:08.835374+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:18:08.835374+0000","last_undegraded":"2023-09-23T21:18:08.835374+0000","last_fullsized":"2023-09-23T21:18:08.835374+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":499,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.837505+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:18:08.837505+0000","last_peered":"2023-09-23T21:18:08.837505+0000","last_clean":"2023-09-23T21:18:08.837505+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:18:08.837505+0000","last_undegraded":"2023-09-23T21:18:08.837505+0000","last_fullsized":"2023-09-23T21:18:08.837505+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":154,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835071+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:18:08.835071+0000","last_peered":"2023-09-23T21:18:08.835071+0000","last_clean":"2023-09-23T21:18:08.835071+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:18:08.835071+0000","last_undegraded":"2023-09-23T21:18:08.835071+0000","last_fullsized":"2023-09-23T21:18:08.835071+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":222,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835270+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:18:08.835270+0000","last_peered":"2023-09-23T21:18:08.835270+0000","last_clean":"2023-09-23T21:18:08.835270+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:18:08.835270+0000","last_undegraded":"2023-09-23T21:18:08.835270+0000","last_fullsized":"2023-09-23T21:18:08.835270+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":150,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834594+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:18:08.834594+0000","last_peered":"2023-09-23T21:18:08.834594+0000","last_clean":"2023-09-23T21:18:08.834594+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:18:08.834594+0000","last_undegraded":"2023-09-23T21:18:08.834594+0000","last_fullsized":"2023-09-23T21:18:08.834594+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835696+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:18:08.835696+0000","last_peered":"2023-09-23T21:18:08.835696+0000","last_clean":"2023-09-23T21:18:08.835696+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:18:08.835696+0000","last_undegraded":"2023-09-23T21:18:08.835696+0000","last_fullsized":"2023-09-23T21:18:08.835696+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":148,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834301+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:18:08.834301+0000","last_peered":"2023-09-23T21:18:08.834301+0000","last_clean":"2023-09-23T21:18:08.834301+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:18:08.834301+0000","last_undegraded":"2023-09-23T21:18:08.834301+0000","last_fullsized":"2023-09-23T21:18:08.834301+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836117+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:18:08.836117+0000","last_peered":"2023-09-23T21:18:08.836117+0000","last_clean":"2023-09-23T21:18:08.836117+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:18:08.836117+0000","last_undegraded":"2023-09-23T21:18:08.836117+0000","last_fullsized":"2023-09-23T21:18:08.836117+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":607,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840791+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:18:08.840791+0000","last_peered":"2023-09-23T21:18:08.840791+0000","last_clean":"2023-09-23T21:18:08.840791+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:18:08.840791+0000","last_undegraded":"2023-09-23T21:18:08.840791+0000","last_fullsized":"2023-09-23T21:18:08.840791+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":222,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836632+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:18:08.836632+0000","last_peered":"2023-09-23T21:18:08.836632+0000","last_clean":"2023-09-23T21:18:08.836632+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:18:08.836632+0000","last_undegraded":"2023-09-23T21:18:08.836632+0000","last_fullsized":"2023-09-23T21:18:08.836632+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":152,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836718+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:18:08.836718+0000","last_peered":"2023-09-23T21:18:08.836718+0000","last_clean":"2023-09-23T21:18:08.836718+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:18:08.836718+0000","last_undegraded":"2023-09-23T21:18:08.836718+0000","last_fullsized":"2023-09-23T21:18:08.836718+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":224,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.838235+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:18:08.838235+0000","last_peered":"2023-09-23T21:18:08.838235+0000","last_clean":"2023-09-23T21:18:08.838235+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:18:08.838235+0000","last_undegraded":"2023-09-23T21:18:08.838235+0000","last_fullsized":"2023-09-23T21:18:08.838235+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":166,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836209+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:18:08.836209+0000","last_peered":"2023-09-23T21:18:08.836209+0000","last_clean":"2023-09-23T21:18:08.836209+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:18:08.836209+0000","last_undegraded":"2023-09-23T21:18:08.836209+0000","last_fullsized":"2023-09-23T21:18:08.836209+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.839187+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:18:08.839187+0000","last_peered":"2023-09-23T21:18:08.839187+0000","last_clean":"2023-09-23T21:18:08.839187+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:18:08.839187+0000","last_undegraded":"2023-09-23T21:18:08.839187+0000","last_fullsized":"2023-09-23T21:18:08.839187+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":152,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835746+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:18:08.835746+0000","last_peered":"2023-09-23T21:18:08.835746+0000","last_clean":"2023-09-23T21:18:08.835746+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:18:08.835746+0000","last_undegraded":"2023-09-23T21:18:08.835746+0000","last_fullsized":"2023-09-23T21:18:08.835746+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840296+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:18:08.840296+0000","last_peered":"2023-09-23T21:18:08.840296+0000","last_clean":"2023-09-23T21:18:08.840296+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:18:08.840296+0000","last_undegraded":"2023-09-23T21:18:08.840296+0000","last_fullsized":"2023-09-23T21:18:08.840296+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840050+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:18:08.840050+0000","last_peered":"2023-09-23T21:18:08.840050+0000","last_clean":"2023-09-23T21:18:08.840050+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:18:08.840050+0000","last_undegraded":"2023-09-23T21:18:08.840050+0000","last_fullsized":"2023-09-23T21:18:08.840050+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840046+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:18:08.840046+0000","last_peered":"2023-09-23T21:18:08.840046+0000","last_clean":"2023-09-23T21:18:08.840046+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:18:08.840046+0000","last_undegraded":"2023-09-23T21:18:08.840046+0000","last_fullsized":"2023-09-23T21:18:08.840046+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":238,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834974+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:18:08.834974+0000","last_peered":"2023-09-23T21:18:08.834974+0000","last_clean":"2023-09-23T21:18:08.834974+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:18:08.834974+0000","last_undegraded":"2023-09-23T21:18:08.834974+0000","last_fullsized":"2023-09-23T21:18:08.834974+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840200+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:18:08.840200+0000","last_peered":"2023-09-23T21:18:08.840200+0000","last_clean":"2023-09-23T21:18:08.840200+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:18:08.840200+0000","last_undegraded":"2023-09-23T21:18:08.840200+0000","last_fullsized":"2023-09-23T21:18:08.840200+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.839947+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:18:08.839947+0000","last_peered":"2023-09-23T21:18:08.839947+0000","last_clean":"2023-09-23T21:18:08.839947+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:18:08.839947+0000","last_undegraded":"2023-09-23T21:18:08.839947+0000","last_fullsized":"2023-09-23T21:18:08.839947+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":613,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834671+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:18:08.834671+0000","last_peered":"2023-09-23T21:18:08.834671+0000","last_clean":"2023-09-23T21:18:08.834671+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:18:08.834671+0000","last_undegraded":"2023-09-23T21:18:08.834671+0000","last_fullsized":"2023-09-23T21:18:08.834671+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":208,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.838065+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:18:08.838065+0000","last_peered":"2023-09-23T21:18:08.838065+0000","last_clean":"2023-09-23T21:18:08.838065+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:18:08.838065+0000","last_undegraded":"2023-09-23T21:18:08.838065+0000","last_fullsized":"2023-09-23T21:18:08.838065+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840442+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:18:08.840442+0000","last_peered":"2023-09-23T21:18:08.840442+0000","last_clean":"2023-09-23T21:18:08.840442+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:18:08.840442+0000","last_undegraded":"2023-09-23T21:18:08.840442+0000","last_fullsized":"2023-09-23T21:18:08.840442+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":222,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836008+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:18:08.836008+0000","last_peered":"2023-09-23T21:18:08.836008+0000","last_clean":"2023-09-23T21:18:08.836008+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:18:08.836008+0000","last_undegraded":"2023-09-23T21:18:08.836008+0000","last_fullsized":"2023-09-23T21:18:08.836008+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":310,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836442+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:18:08.836442+0000","last_peered":"2023-09-23T21:18:08.836442+0000","last_clean":"2023-09-23T21:18:08.836442+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:18:08.836442+0000","last_undegraded":"2023-09-23T21:18:08.836442+0000","last_fullsized":"2023-09-23T21:18:08.836442+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":495,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835326+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:18:08.835326+0000","last_peered":"2023-09-23T21:18:08.835326+0000","last_clean":"2023-09-23T21:18:08.835326+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:18:08.835326+0000","last_undegraded":"2023-09-23T21:18:08.835326+0000","last_fullsized":"2023-09-23T21:18:08.835326+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":246,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840695+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:18:08.840695+0000","last_peered":"2023-09-23T21:18:08.840695+0000","last_clean":"2023-09-23T21:18:08.840695+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:18:08.840695+0000","last_undegraded":"2023-09-23T21:18:08.840695+0000","last_fullsized":"2023-09-23T21:18:08.840695+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.839419+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:18:08.839419+0000","last_peered":"2023-09-23T21:18:08.839419+0000","last_clean":"2023-09-23T21:18:08.839419+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:18:08.839419+0000","last_undegraded":"2023-09-23T21:18:08.839419+0000","last_fullsized":"2023-09-23T21:18:08.839419+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.839784+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:18:08.839784+0000","last_peered":"2023-09-23T21:18:08.839784+0000","last_clean":"2023-09-23T21:18:08.839784+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:18:08.839784+0000","last_undegraded":"2023-09-23T21:18:08.839784+0000","last_fullsized":"2023-09-23T21:18:08.839784+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.839579+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:18:08.839579+0000","last_peered":"2023-09-23T21:18:08.839579+0000","last_clean":"2023-09-23T21:18:08.839579+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:18:08.839579+0000","last_undegraded":"2023-09-23T21:18:08.839579+0000","last_fullsized":"2023-09-23T21:18:08.839579+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":285,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835338+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:18:08.835338+0000","last_peered":"2023-09-23T21:18:08.835338+0000","last_clean":"2023-09-23T21:18:08.835338+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:18:08.835338+0000","last_undegraded":"2023-09-23T21:18:08.835338+0000","last_fullsized":"2023-09-23T21:18:08.835338+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":580,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.833673+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:18:08.833673+0000","last_peered":"2023-09-23T21:18:08.833673+0000","last_clean":"2023-09-23T21:18:08.833673+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:18:08.833673+0000","last_undegraded":"2023-09-23T21:18:08.833673+0000","last_fullsized":"2023-09-23T21:18:08.833673+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":317,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.833139+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:18:08.833139+0000","last_peered":"2023-09-23T21:18:08.833139+0000","last_clean":"2023-09-23T21:18:08.833139+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:18:08.833139+0000","last_undegraded":"2023-09-23T21:18:08.833139+0000","last_fullsized":"2023-09-23T21:18:08.833139+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":224,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836473+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:18:08.836473+0000","last_peered":"2023-09-23T21:18:08.836473+0000","last_clean":"2023-09-23T21:18:08.836473+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:18:08.836473+0000","last_undegraded":"2023-09-23T21:18:08.836473+0000","last_fullsized":"2023-09-23T21:18:08.836473+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":400,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834493+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:18:08.834493+0000","last_peered":"2023-09-23T21:18:08.834493+0000","last_clean":"2023-09-23T21:18:08.834493+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:18:08.834493+0000","last_undegraded":"2023-09-23T21:18:08.834493+0000","last_fullsized":"2023-09-23T21:18:08.834493+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":917,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834720+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:18:08.834720+0000","last_peered":"2023-09-23T21:18:08.834720+0000","last_clean":"2023-09-23T21:18:08.834720+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:18:08.834720+0000","last_undegraded":"2023-09-23T21:18:08.834720+0000","last_fullsized":"2023-09-23T21:18:08.834720+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":143,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.839005+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:18:08.839005+0000","last_peered":"2023-09-23T21:18:08.839005+0000","last_clean":"2023-09-23T21:18:08.839005+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:18:08.839005+0000","last_undegraded":"2023-09-23T21:18:08.839005+0000","last_fullsized":"2023-09-23T21:18:08.839005+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840516+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:18:08.840516+0000","last_peered":"2023-09-23T21:18:08.840516+0000","last_clean":"2023-09-23T21:18:08.840516+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:18:08.840516+0000","last_undegraded":"2023-09-23T21:18:08.840516+0000","last_fullsized":"2023-09-23T21:18:08.840516+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":198,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834801+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:18:08.834801+0000","last_peered":"2023-09-23T21:18:08.834801+0000","last_clean":"2023-09-23T21:18:08.834801+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:18:08.834801+0000","last_undegraded":"2023-09-23T21:18:08.834801+0000","last_fullsized":"2023-09-23T21:18:08.834801+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":67,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834030+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:18:08.834030+0000","last_peered":"2023-09-23T21:18:08.834030+0000","last_clean":"2023-09-23T21:18:08.834030+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:18:08.834030+0000","last_undegraded":"2023-09-23T21:18:08.834030+0000","last_fullsized":"2023-09-23T21:18:08.834030+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":480,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834951+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:18:08.834951+0000","last_peered":"2023-09-23T21:18:08.834951+0000","last_clean":"2023-09-23T21:18:08.834951+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:18:08.834951+0000","last_undegraded":"2023-09-23T21:18:08.834951+0000","last_fullsized":"2023-09-23T21:18:08.834951+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":168,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835592+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:18:08.835592+0000","last_peered":"2023-09-23T21:18:08.835592+0000","last_clean":"2023-09-23T21:18:08.835592+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:18:08.835592+0000","last_undegraded":"2023-09-23T21:18:08.835592+0000","last_fullsized":"2023-09-23T21:18:08.835592+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":384,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835340+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:18:08.835340+0000","last_peered":"2023-09-23T21:18:08.835340+0000","last_clean":"2023-09-23T21:18:08.835340+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:18:08.835340+0000","last_undegraded":"2023-09-23T21:18:08.835340+0000","last_fullsized":"2023-09-23T21:18:08.835340+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":276,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.841688+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:18:08.841688+0000","last_peered":"2023-09-23T21:18:08.841688+0000","last_clean":"2023-09-23T21:18:08.841688+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:18:08.841688+0000","last_undegraded":"2023-09-23T21:18:08.841688+0000","last_fullsized":"2023-09-23T21:18:08.841688+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":226,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.836804+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:18:08.836804+0000","last_peered":"2023-09-23T21:18:08.836804+0000","last_clean":"2023-09-23T21:18:08.836804+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:18:08.836804+0000","last_undegraded":"2023-09-23T21:18:08.836804+0000","last_fullsized":"2023-09-23T21:18:08.836804+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840371+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:18:08.840371+0000","last_peered":"2023-09-23T21:18:08.840371+0000","last_clean":"2023-09-23T21:18:08.840371+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:18:08.840371+0000","last_undegraded":"2023-09-23T21:18:08.840371+0000","last_fullsized":"2023-09-23T21:18:08.840371+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":251,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834645+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:18:08.834645+0000","last_peered":"2023-09-23T21:18:08.834645+0000","last_clean":"2023-09-23T21:18:08.834645+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:18:08.834645+0000","last_undegraded":"2023-09-23T21:18:08.834645+0000","last_fullsized":"2023-09-23T21:18:08.834645+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":477,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.841419+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:18:08.841419+0000","last_peered":"2023-09-23T21:18:08.841419+0000","last_clean":"2023-09-23T21:18:08.841419+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:18:08.841419+0000","last_undegraded":"2023-09-23T21:18:08.841419+0000","last_fullsized":"2023-09-23T21:18:08.841419+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":108,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.838735+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:18:08.838735+0000","last_peered":"2023-09-23T21:18:08.838735+0000","last_clean":"2023-09-23T21:18:08.838735+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:18:08.838735+0000","last_undegraded":"2023-09-23T21:18:08.838735+0000","last_fullsized":"2023-09-23T21:18:08.838735+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":256,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.838765+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:18:08.838765+0000","last_peered":"2023-09-23T21:18:08.838765+0000","last_clean":"2023-09-23T21:18:08.838765+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:18:08.838765+0000","last_undegraded":"2023-09-23T21:18:08.838765+0000","last_fullsized":"2023-09-23T21:18:08.838765+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":198,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835067+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:18:08.835067+0000","last_peered":"2023-09-23T21:18:08.835067+0000","last_clean":"2023-09-23T21:18:08.835067+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:18:08.835067+0000","last_undegraded":"2023-09-23T21:18:08.835067+0000","last_fullsized":"2023-09-23T21:18:08.835067+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":634,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835393+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:18:08.835393+0000","last_peered":"2023-09-23T21:18:08.835393+0000","last_clean":"2023-09-23T21:18:08.835393+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:18:08.835393+0000","last_undegraded":"2023-09-23T21:18:08.835393+0000","last_fullsized":"2023-09-23T21:18:08.835393+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":275,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.837276+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:18:08.837276+0000","last_peered":"2023-09-23T21:18:08.837276+0000","last_clean":"2023-09-23T21:18:08.837276+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:18:08.837276+0000","last_undegraded":"2023-09-23T21:18:08.837276+0000","last_fullsized":"2023-09-23T21:18:08.837276+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":229,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834392+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:18:08.834392+0000","last_peered":"2023-09-23T21:18:08.834392+0000","last_clean":"2023-09-23T21:18:08.834392+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:18:08.834392+0000","last_undegraded":"2023-09-23T21:18:08.834392+0000","last_fullsized":"2023-09-23T21:18:08.834392+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":40,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840438+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:18:08.840438+0000","last_peered":"2023-09-23T21:18:08.840438+0000","last_clean":"2023-09-23T21:18:08.840438+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:18:08.840438+0000","last_undegraded":"2023-09-23T21:18:08.840438+0000","last_fullsized":"2023-09-23T21:18:08.840438+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":216,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.834300+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:18:08.834300+0000","last_peered":"2023-09-23T21:18:08.834300+0000","last_clean":"2023-09-23T21:18:08.834300+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:18:08.834300+0000","last_undegraded":"2023-09-23T21:18:08.834300+0000","last_fullsized":"2023-09-23T21:18:08.834300+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.840291+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:18:08.840291+0000","last_peered":"2023-09-23T21:18:08.840291+0000","last_clean":"2023-09-23T21:18:08.840291+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:18:08.840291+0000","last_undegraded":"2023-09-23T21:18:08.840291+0000","last_fullsized":"2023-09-23T21:18:08.840291+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":431,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.833992+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:18:08.833992+0000","last_peered":"2023-09-23T21:18:08.833992+0000","last_clean":"2023-09-23T21:18:08.833992+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:18:08.833992+0000","last_undegraded":"2023-09-23T21:18:08.833992+0000","last_fullsized":"2023-09-23T21:18:08.833992+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":527,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.835190+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:18:08.835190+0000","last_peered":"2023-09-23T21:18:08.835190+0000","last_clean":"2023-09-23T21:18:08.835190+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:18:08.835190+0000","last_undegraded":"2023-09-23T21:18:08.835190+0000","last_fullsized":"2023-09-23T21:18:08.835190+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":238,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-09-23T21:18:08.837668+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:18:08.837668+0000","last_peered":"2023-09-23T21:18:08.837668+0000","last_clean":"2023-09-23T21:18:08.837668+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:18:08.837668+0000","last_undegraded":"2023-09-23T21:18:08.837668+0000","last_fullsized":"2023-09-23T21:18:08.837668+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":38918,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201874,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":842132,"kb_used_data":35788,"kb_used_omap":0,"kb_used_meta":806336,"kb_avail":92903020,"statfs":{"total":95995035648,"available":95132692480,"internally_reserved":0,"allocated":36646912,"data_stored":35074862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":825688064},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":18,"apply_latency_ms":18,"commit_latency_ns":18000000,"apply_latency_ns":18000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":2.7679999999999998,"15min":2.2040000000000002},"min":{"1min":0.246,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":6.9470000000000001,"5min":141.33199999999999,"15min":141.33199999999999},"last":0.497},{"interface":"front","average":{"1min":1.1990000000000001,"5min":3.0579999999999998,"15min":2.2519999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":5.1340000000000003,"5min":141.31700000000001,"15min":141.31700000000001},"last":0.52000000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:17:19 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":2.1280000000000001,"15min":3.4239999999999999},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.23699999999999999},"max":{"1min":5.4329999999999998,"5min":16.684000000000001,"15min":114.16500000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.5920000000000001,"5min":2.8799999999999999,"15min":3.867},"min":{"1min":0.307,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.7699999999999996,"5min":117.637,"15min":117.637},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":3.71,"5min":3.8260000000000001,"15min":4.5519999999999996},"min":{"1min":0.248,"5min":0.224,"15min":0.224},"max":{"1min":53.988,"5min":135.529,"15min":183.44200000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":4.0659999999999998,"5min":4.7190000000000003,"15min":4.7750000000000004},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":54,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":4.9870000000000001,"15min":3.9319999999999999},"min":{"1min":0.33800000000000002,"5min":0.30499999999999999,"15min":0.26700000000000002},"max":{"1min":7.0099999999999998,"5min":135.66200000000001,"15min":135.66200000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.484,"5min":5.3570000000000002,"15min":4.056},"min":{"1min":0.32100000000000001,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":7.4729999999999999,"5min":135.47,"15min":135.47},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.9430000000000001,"15min":1.2090000000000001},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1829999999999998,"5min":10.989000000000001,"15min":13.739000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":3.5840000000000001,"5min":2.0950000000000002,"15min":1.2},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":20.306999999999999,"5min":20.306999999999999,"15min":20.306999999999999},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":2.6389999999999998,"15min":1.6080000000000001},"min":{"1min":0.53100000000000003,"5min":0.29099999999999998,"15min":0.26900000000000002},"max":{"1min":14.105,"5min":14.105,"15min":14.105},"last":0.40000000000000002},{"interface":"front","average":{"1min":3.9630000000000001,"5min":2.5339999999999998,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":17.585999999999999,"5min":17.585999999999999,"15min":17.585999999999999},"last":0.443}]},{"osd":6,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.2930000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":9.9969999999999999,"5min":13.369,"15min":143.18700000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.9380000000000002,"5min":1.819,"15min":1.929},"min":{"1min":0.31,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":15.566000000000001,"5min":15.566000000000001,"15min":142.70400000000001},"last":0.50700000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463494,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173452,"kb_used_data":395204,"kb_used_omap":0,"kb_used_meta":778240,"kb_avail":92571700,"statfs":{"total":95995035648,"available":94793420800,"internally_reserved":0,"allocated":404688896,"data_stored":403125329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":796917760},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.7549999999999999,"15min":2.7610000000000001},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":7.4400000000000004,"5min":121.762,"15min":149.61600000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.8809999999999998,"5min":3.4249999999999998,"15min":2.9369999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":9.7449999999999992,"5min":121.73099999999999,"15min":149.303},"last":0.41199999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":2.379,"15min":3.6080000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.22700000000000001},"max":{"1min":16.126999999999999,"5min":25.122,"15min":129.37899999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.986,"5min":1.8420000000000001,"15min":3.387},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":10.977,"5min":25.193999999999999,"15min":129.40100000000001},"last":0.61899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":15.675000000000001,"5min":8.2889999999999997,"15min":6.7590000000000003},"min":{"1min":0.36499999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":181.846,"5min":181.846,"15min":199.23400000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":17.042000000000002,"5min":9.0250000000000004,"15min":6.8579999999999997},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":182.14500000000001,"5min":182.14500000000001,"15min":199.08000000000001},"last":0.56999999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":3.1150000000000002,"15min":3.2389999999999999},"min":{"1min":0.33200000000000002,"5min":0.218,"15min":0.218},"max":{"1min":10.734999999999999,"5min":126.649,"15min":137.875},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.367,"5min":3.2429999999999999,"15min":3.3170000000000002},"min":{"1min":0.34799999999999998,"5min":0.30099999999999999,"15min":0.29799999999999999},"max":{"1min":7.4660000000000002,"5min":125.324,"15min":137.60900000000001},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6299999999999999,"5min":3.5640000000000001,"15min":1.8879999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":9.6720000000000006,"5min":55.494,"15min":55.494},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.1669999999999998,"5min":3.359,"15min":1.669},"min":{"1min":0.39000000000000001,"5min":0.29399999999999998,"15min":0.23999999999999999},"max":{"1min":11.228,"5min":55.305,"15min":55.305},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":4.9290000000000003,"5min":5.2089999999999996,"15min":2.4329999999999998},"min":{"1min":0.32500000000000001,"5min":0.32300000000000001,"15min":0.27300000000000002},"max":{"1min":12.975,"5min":222.90899999999999,"15min":222.90899999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":3.9239999999999999,"5min":5.1970000000000001,"15min":2.3740000000000001},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.28499999999999998},"max":{"1min":11.199999999999999,"5min":224.655,"15min":224.655},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":13.882999999999999,"5min":3.629,"15min":1.883},"min":{"1min":0.39600000000000002,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":165.96799999999999,"5min":165.96799999999999,"15min":165.96799999999999},"last":0.499},{"interface":"front","average":{"1min":13.532,"5min":3.859,"15min":1.98},"min":{"1min":0.442,"5min":0.251,"15min":0.251},"max":{"1min":165.82400000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":0.58499999999999996}]}]},{"osd":1,"up_from":0,"seq":1185410973941,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991681,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781572,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152580,"kb_used_data":39036,"kb_used_omap":7,"kb_used_meta":1113528,"kb_avail":92592572,"statfs":{"total":95995035648,"available":94814793728,"internally_reserved":0,"allocated":39972864,"data_stored":37234936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1140252753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.5069999999999999,"15min":2.0710000000000002},"min":{"1min":0.17899999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":4.0830000000000002,"5min":25.297000000000001,"15min":103.021},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.669,"15min":2.6070000000000002},"min":{"1min":0.25700000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":4.5510000000000002,"5min":18.879999999999999,"15min":103.154},"last":0.54700000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:17:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.2000000000000002,"15min":4.0869999999999997},"min":{"1min":0.24099999999999999,"5min":0.224,"15min":0.19},"max":{"1min":2.4409999999999998,"5min":54.521000000000001,"15min":212.12},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.98999999999999999,"5min":2.4820000000000002,"15min":4.1399999999999997},"min":{"1min":0.254,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":8.048,"5min":54.598999999999997,"15min":213.02799999999999},"last":0.48099999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":2.5670000000000002,"15min":4.0339999999999998},"min":{"1min":0.251,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5190000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.622},{"interface":"front","average":{"1min":0.93799999999999994,"5min":3.2250000000000001,"15min":4.2519999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":8.2189999999999994,"5min":82.241,"15min":143.27799999999999},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":4.3300000000000001,"15min":3.198},"min":{"1min":0.32100000000000001,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":7.0330000000000004,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.744,"5min":4.8300000000000001,"15min":3.7879999999999998},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.29499999999999998},"max":{"1min":7.0449999999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":5.0979999999999999,"15min":3.448},"min":{"1min":0.31,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.4349999999999996,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.625,"5min":4.7199999999999998,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.34399999999999997,"15min":0.29399999999999998},"max":{"1min":4.1440000000000001,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":4.7480000000000002,"15min":3.7970000000000002},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":5.3220000000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.3380000000000001,"5min":4.4829999999999997,"15min":3.1720000000000002},"min":{"1min":0.48399999999999999,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":7.1619999999999999,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.56599999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":2.4830000000000001,"15min":1.9530000000000001},"min":{"1min":0.373,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":4.2640000000000002,"5min":25.260000000000002,"15min":25.260000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.4079999999999999,"5min":2.5419999999999998,"15min":2.2610000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.5289999999999999,"5min":12.613,"15min":16.875},"last":0.502}]}]},{"osd":3,"up_from":0,"seq":635655160165,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986784,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":991696,"kb_used_data":178568,"kb_used_omap":0,"kb_used_meta":813120,"kb_avail":92753456,"statfs":{"total":95995035648,"available":94979538944,"internally_reserved":0,"allocated":182853632,"data_stored":181285387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":832634880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":21,"apply_latency_ms":21,"commit_latency_ns":21000000,"apply_latency_ns":21000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":3.3410000000000002,"15min":2.7639999999999998},"min":{"1min":0.28000000000000003,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":9.2200000000000006,"5min":109.23099999999999,"15min":226.751},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.754,"15min":2.5430000000000001},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":6.0919999999999996,"5min":109.16800000000001,"15min":226.43299999999999},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":3.802,"15min":5.2999999999999998},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2510000000000003,"5min":119.55800000000001,"15min":127.411},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.8050000000000002,"5min":3.71,"15min":4.5830000000000002},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":10.506,"5min":116.78100000000001,"15min":118.075},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":8.7449999999999992,"5min":6.4420000000000002,"15min":6.327},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":85.994,"5min":155.51300000000001,"15min":210.50299999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":9.2530000000000001,"5min":7.3230000000000004,"15min":6.4809999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":85.971999999999994,"5min":155.77099999999999,"15min":210.59},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":3.2440000000000002,"15min":4.1200000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0650000000000004,"5min":111.22199999999999,"15min":214.64099999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":3.0659999999999998,"5min":4.508,"15min":4.5209999999999999},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":10.529,"5min":111.39400000000001,"15min":214.54900000000001},"last":0.502}]},{"osd":5,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7829999999999999,"5min":1.7749999999999999,"15min":1.139},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.417,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.8399999999999999,"15min":1.49},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":14.675000000000001,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":2.6579999999999999,"15min":1.6970000000000001},"min":{"1min":0.35699999999999998,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":41.938000000000002,"5min":41.938000000000002,"15min":41.938000000000002},"last":0.442},{"interface":"front","average":{"1min":5.4660000000000002,"5min":2.5390000000000001,"15min":1.635},"min":{"1min":0.33600000000000002,"5min":0.32300000000000001,"15min":0.27200000000000002},"max":{"1min":40.444000000000003,"5min":40.444000000000003,"15min":40.444000000000003},"last":0.54500000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":2.2170000000000001,"15min":1.8740000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":11.859,"5min":18.352,"15min":145.721},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.8600000000000001,"15min":1.893},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"max":{"1min":8.5570000000000004,"5min":10.677,"15min":148.45699999999999},"last":0.49399999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725148,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908932,"kb_used_data":32828,"kb_used_omap":0,"kb_used_meta":876096,"kb_avail":92836220,"statfs":{"total":95995035648,"available":95064289280,"internally_reserved":0,"allocated":33615872,"data_stored":32055681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":897122304},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":37,"apply_latency_ms":37,"commit_latency_ns":37000000,"apply_latency_ns":37000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.427,"15min":1.71},"min":{"1min":0.34899999999999998,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":4.1260000000000003,"5min":8.6660000000000004,"15min":82.280000000000001},"last":0.436},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.798,"15min":1.889},"min":{"1min":0.27600000000000002,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.8810000000000002,"5min":16.977,"15min":85.236000000000004},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3479999999999999,"15min":4.7039999999999997},"min":{"1min":0.255,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":11.374000000000001,"5min":12.032999999999999,"15min":233.77099999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.3979999999999999,"5min":1.6399999999999999,"15min":4.359},"min":{"1min":0.39600000000000002,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":4.0590000000000002,"5min":12.016,"15min":233.75200000000001},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":1.73,"15min":3.2269999999999999},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.1040000000000001,"5min":10.454000000000001,"15min":231.34700000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.9930000000000001,"15min":3.4220000000000002},"min":{"1min":0.374,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9180000000000001,"5min":10.621,"15min":231.381},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.403,"15min":3.7440000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":11.42,"5min":11.597,"15min":263.887},"last":0.38},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.0099999999999998,"15min":3.4380000000000002},"min":{"1min":0.44400000000000001,"5min":0.39700000000000002,"15min":0.30099999999999999},"max":{"1min":4.173,"5min":10.638,"15min":263.70100000000002},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.7709999999999999,"15min":1.0129999999999999},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":4.0289999999999999,"5min":11.593999999999999,"15min":11.593999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":3.3860000000000001,"5min":1.95,"15min":1.1899999999999999},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.26400000000000001},"max":{"1min":12.49,"5min":12.49,"15min":12.49},"last":0.53500000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":2.0339999999999998,"15min":1.619},"min":{"1min":0.48499999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":6.984,"5min":8.9130000000000003,"15min":83.451999999999998},"last":0.56599999999999995},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.73,"15min":1.8859999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":11.696999999999999,"5min":12.725,"15min":83.484999999999999},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:17:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1000000000000001,"5min":2.161,"15min":1.419},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":7.4020000000000001,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":0.503},{"interface":"front","average":{"1min":2.431,"5min":1.585,"15min":0.995},"min":{"1min":0.45500000000000002,"5min":0.34899999999999998,"15min":0.28899999999999998},"max":{"1min":10.196999999999999,"5min":11.640000000000001,"15min":11.640000000000001},"last":0.58199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:18.453 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:18.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:18.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:18.557 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:18.586 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:18.633 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:18.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:18.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[211526]: pgmap v1774: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[211526]: failed to encode map e573 with expected crc 2023-09-23T21:18:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/578036385' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T21:18:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[211526]: osdmap e573: 8 total, 8 up, 5 in 2023-09-23T21:18:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[219709]: pgmap v1774: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:18.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[219709]: failed to encode map e573 with expected crc 2023-09-23T21:18:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/578036385' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T21:18:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[219709]: osdmap e573: 8 total, 8 up, 5 in 2023-09-23T21:18:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:18 smithi123 ceph-mon[193903]: pgmap v1774: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:18 smithi123 ceph-mon[193903]: failed to encode map e573 with expected crc 2023-09-23T21:18:18.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/578036385' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-09-23T21:18:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:18 smithi123 ceph-mon[193903]: osdmap e573: 8 total, 8 up, 5 in 2023-09-23T21:18:18.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:18.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:18.765 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:18.807 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:18.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:18.915 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:18.945 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.041 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:19.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:19.050 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-09-23T21:18:19.144 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:19.174 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.321 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:19.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:19.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:19.361 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:18:19.381 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:18:19.403 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:18:19.424 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:18:19.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:19.450 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:18:19.461 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:18:19.496 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:18:19.508 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:19.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:19.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:19.516 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:18:19.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.517 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.518 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:18:19.612 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:19.639 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.642 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:19.642 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:19.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: osdmap e574: 8 total, 8 up, 5 in 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3615512418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[211526]: pgmap v1777: 89 pgs: 5 creating+peering, 4 unknown, 80 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:19.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: osdmap e574: 8 total, 8 up, 5 in 2023-09-23T21:18:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3615512418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:19 smithi121 ceph-mon[219709]: pgmap v1777: 89 pgs: 5 creating+peering, 4 unknown, 80 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:19.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: osdmap e574: 8 total, 8 up, 5 in 2023-09-23T21:18:19.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: failed to encode map e574 with expected crc 2023-09-23T21:18:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3615512418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:19.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:19 smithi123 ceph-mon[193903]: pgmap v1777: 89 pgs: 5 creating+peering, 4 unknown, 80 active+clean; 582 KiB data, 4.7 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:19.789 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:19.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:19.892 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:19.924 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:19.975 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:19.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:20.081 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:20.106 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:20.654 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:20.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:20.700 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[211526]: failed to encode map e575 with expected crc 2023-09-23T21:18:20.701 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[211526]: osdmap e575: 8 total, 8 up, 5 in 2023-09-23T21:18:20.701 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:18:20.701 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[211526]: failed to encode map e575 with expected crc 2023-09-23T21:18:20.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[219709]: failed to encode map e575 with expected crc 2023-09-23T21:18:20.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[219709]: osdmap e575: 8 total, 8 up, 5 in 2023-09-23T21:18:20.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:18:20.701 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:20 smithi121 ceph-mon[219709]: failed to encode map e575 with expected crc 2023-09-23T21:18:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:20 smithi123 ceph-mon[193903]: failed to encode map e575 with expected crc 2023-09-23T21:18:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:20 smithi123 ceph-mon[193903]: osdmap e575: 8 total, 8 up, 5 in 2023-09-23T21:18:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:20 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:18:20.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:20 smithi123 ceph-mon[193903]: failed to encode map e575 with expected crc 2023-09-23T21:18:20.758 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:20.785 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:20.851 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:18:20.980 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:18:21.010 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:21.029 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:21.133 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:21.162 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:21.204 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:21.307 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:21.339 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:21.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:21.417 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:21.496 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:21.530 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:21 smithi121 ceph-mon[211526]: failed to encode map e575 with expected crc 2023-09-23T21:18:21.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:21 smithi121 ceph-mon[211526]: pgmap v1779: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:21.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:21 smithi121 ceph-mon[219709]: failed to encode map e575 with expected crc 2023-09-23T21:18:21.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:21 smithi121 ceph-mon[219709]: pgmap v1779: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:21.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:21 smithi123 ceph-mon[193903]: failed to encode map e575 with expected crc 2023-09-23T21:18:21.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:21 smithi123 ceph-mon[193903]: pgmap v1779: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:21.736 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:21.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:21.848 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:21.876 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:22.021 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:22.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:22.125 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:22.153 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:22.350 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:22.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:22.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:22.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:22.482 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:22.562 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:22.583 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:22.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:22.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:22.619 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:22.687 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:22.717 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:22.720 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:18:23.031 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:23.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:23.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:23.093 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:23.138 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:23.165 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:23.372 INFO:teuthology.orchestra.run.smithi121.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-09-23T21:18:23.614 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2956335379' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:18:23.615 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2956335379' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:18:23.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2956335379' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:18:24.480 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:24.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:24.584 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:24.612 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:24.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[211526]: pgmap v1780: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[211526]: failed to encode map e576 with expected crc 2023-09-23T21:18:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[211526]: failed to encode map e576 with expected crc 2023-09-23T21:18:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2956335379' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:18:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[211526]: osdmap e576: 8 total, 8 up, 5 in 2023-09-23T21:18:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:24.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:24.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[219709]: pgmap v1780: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[219709]: failed to encode map e576 with expected crc 2023-09-23T21:18:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[219709]: failed to encode map e576 with expected crc 2023-09-23T21:18:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2956335379' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:18:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[219709]: osdmap e576: 8 total, 8 up, 5 in 2023-09-23T21:18:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:24.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:24 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:24.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:24 smithi123 ceph-mon[193903]: pgmap v1780: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:24.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:24 smithi123 ceph-mon[193903]: failed to encode map e576 with expected crc 2023-09-23T21:18:24.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:24 smithi123 ceph-mon[193903]: failed to encode map e576 with expected crc 2023-09-23T21:18:24.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2956335379' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T21:18:24.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:24 smithi123 ceph-mon[193903]: osdmap e576: 8 total, 8 up, 5 in 2023-09-23T21:18:24.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:24 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:24.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:24 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:25.043 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:18:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:18:24] "GET /metrics HTTP/1.1" 200 34248 "" "Prometheus/2.43.0" 2023-09-23T21:18:25.415 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:25.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:25.519 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:25.547 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:25.598 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:25.693 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:25.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:25.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:25.797 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:25.833 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:25.884 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:25.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:25.992 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:26.021 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:26.065 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:26.067 DEBUG:teuthology.orchestra.run.smithi121:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-09-23T21:18:26.069 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T21:18:26.069 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T21:18:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:26.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:26.173 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:26.201 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:26.243 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:26.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:26.349 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:26.375 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:26.381 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:26 smithi121 ceph-mon[219709]: pgmap v1782: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:26.381 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1538604484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:26.381 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:26 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:26.387 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:18:26.419 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:26.421 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:hints = 1 2023-09-23T21:18:26.421 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T21:18:26.421 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object prefix: benchmark_data_smithi121_684892 2023-09-23T21:18:26.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:18:26.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 0 0 0 0 0 0 - 0 2023-09-23T21:18:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:26.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:26 smithi123 ceph-mon[193903]: pgmap v1782: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1538604484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:26 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:26.527 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:26.582 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:26.636 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:26 smithi121 ceph-mon[211526]: pgmap v1782: 89 pgs: 89 active+clean; 582 KiB data, 4.6 GiB used, 442 GiB / 447 GiB avail 2023-09-23T21:18:26.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1538604484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:26.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:26 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:18:26.678 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:26.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:26.791 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:26.824 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:27.101 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:18:27.102 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:18:27.383 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:27.383 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.383 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.383 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:27.383 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: osdmap e577: 8 total, 8 up, 5 in 2023-09-23T21:18:27.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[211526]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 1 16 2768 2752 171.968 172 0.00656432 0.00579531 2023-09-23T21:18:27.537 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:27.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:27.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: osdmap e577: 8 total, 8 up, 5 in 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.639 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:27 smithi121 ceph-mon[219709]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.644 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:27.682 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:18:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: osdmap e577: 8 total, 8 up, 5 in 2023-09-23T21:18:27.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: failed to encode map e577 with expected crc 2023-09-23T21:18:27.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:27 smithi123 ceph-mon[193903]: failed to encode map e577 with expected crc 2023-09-23T21:18:28.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 2 16 5630 5614 175.414 178.875 0.00616157 0.00567658 2023-09-23T21:18:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:28 smithi121 ceph-mon[219709]: pgmap v1784: 89 pgs: 89 active+clean; 582 KiB data, 4.5 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:18:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:28 smithi121 ceph-mon[211526]: pgmap v1784: 89 pgs: 89 active+clean; 582 KiB data, 4.5 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:18:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:28.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:28 smithi123 ceph-mon[193903]: pgmap v1784: 89 pgs: 89 active+clean; 582 KiB data, 4.5 GiB used, 443 GiB / 447 GiB avail 2023-09-23T21:18:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:28.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:28.843 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:28.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:28.948 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:28.981 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:18:29.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 3 16 8351 8335 173.625 170.062 0.00440425 0.0057521 2023-09-23T21:18:29.506 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:18:29.642 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:29.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:29.748 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:29.786 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:29.913 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:29.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:30.022 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:30.081 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.172 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:30.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:30.264 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:18:30.266 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:18:30.279 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:30.329 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.346 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:30.351 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:30.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 4 16 10822 10806 168.826 154.438 0.00496325 0.00591471 2023-09-23T21:18:30.435 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:30.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:30.478 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:18:30.504 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:30 smithi121 ceph-mon[211526]: pgmap v1785: 89 pgs: 89 active+clean; 259 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 32 MiB/s wr, 516 op/s 2023-09-23T21:18:30.505 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:30 smithi121 ceph-mon[219709]: pgmap v1785: 89 pgs: 89 active+clean; 259 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 32 MiB/s wr, 516 op/s 2023-09-23T21:18:30.507 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:18:30.533 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:18:30.543 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:30.556 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:18:30.582 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:18:30.595 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:30 smithi123 ceph-mon[193903]: pgmap v1785: 89 pgs: 89 active+clean; 259 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 32 MiB/s wr, 516 op/s 2023-09-23T21:18:30.607 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.610 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:18:30.635 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:18:30.659 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.661 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:18:30.716 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:30.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:30.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:30.828 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:30.882 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:30.987 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:31.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:31.104 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:31.139 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:31.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 5 16 13230 13214 165.158 150.5 0.0040401 0.0060485 2023-09-23T21:18:31.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:31 smithi121 ceph-mon[211526]: from='client.37470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:31.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/156469139' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:31 smithi121 ceph-mon[219709]: from='client.37470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/156469139' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:31 smithi123 ceph-mon[193903]: from='client.37470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:31.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/156469139' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:32.107 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:32.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:32.125 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1785,"stamp":"2023-09-23T21:18:29.366816+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":271390624,"num_objects":4156,"num_object_clones":0,"num_object_copies":12468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4156,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":4432,"num_write_kb":266893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":5352230,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4348,"ondisk_log_size":4348,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":275,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":5369644,"kb_used_data":958724,"kb_used_omap":7,"kb_used_meta":4410872,"kb_avail":463356116,"statfs":{"total":479975178240,"available":474476662784,"internally_reserved":0,"allocated":981733376,"data_stored":972713153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4516733009},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":270794752,"num_objects":4132,"num_object_clones":0,"num_object_copies":12396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4132,"num_write_kb":264448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001596"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":179,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609378+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:18:23.609378+0000","last_peered":"2023-09-23T21:18:23.609378+0000","last_clean":"2023-09-23T21:18:23.609378+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:18:23.609378+0000","last_undegraded":"2023-09-23T21:18:23.609378+0000","last_fullsized":"2023-09-23T21:18:23.609378+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324862+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:18:20.324862+0000","last_peered":"2023-09-23T21:18:20.324862+0000","last_clean":"2023-09-23T21:18:20.324862+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:18:20.324862+0000","last_undegraded":"2023-09-23T21:18:20.324862+0000","last_fullsized":"2023-09-23T21:18:20.324862+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":364,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609374+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:18:23.609374+0000","last_peered":"2023-09-23T21:18:23.609374+0000","last_clean":"2023-09-23T21:18:23.609374+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:18:23.609374+0000","last_undegraded":"2023-09-23T21:18:23.609374+0000","last_fullsized":"2023-09-23T21:18:23.609374+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":224,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370828+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:18:23.370828+0000","last_peered":"2023-09-23T21:18:23.370828+0000","last_clean":"2023-09-23T21:18:23.370828+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:18:23.370828+0000","last_undegraded":"2023-09-23T21:18:23.370828+0000","last_fullsized":"2023-09-23T21:18:23.370828+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609278+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:18:23.609278+0000","last_peered":"2023-09-23T21:18:23.609278+0000","last_clean":"2023-09-23T21:18:23.609278+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:18:23.609278+0000","last_undegraded":"2023-09-23T21:18:23.609278+0000","last_fullsized":"2023-09-23T21:18:23.609278+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.325116+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:18:20.325116+0000","last_peered":"2023-09-23T21:18:20.325116+0000","last_clean":"2023-09-23T21:18:20.325116+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:18:20.325116+0000","last_undegraded":"2023-09-23T21:18:20.325116+0000","last_fullsized":"2023-09-23T21:18:20.325116+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":171,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371292+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:18:23.371292+0000","last_peered":"2023-09-23T21:18:23.371292+0000","last_clean":"2023-09-23T21:18:23.371292+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:18:23.371292+0000","last_undegraded":"2023-09-23T21:18:23.371292+0000","last_fullsized":"2023-09-23T21:18:23.371292+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324046+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:18:20.324046+0000","last_peered":"2023-09-23T21:18:20.324046+0000","last_clean":"2023-09-23T21:18:20.324046+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:18:20.324046+0000","last_undegraded":"2023-09-23T21:18:20.324046+0000","last_fullsized":"2023-09-23T21:18:20.324046+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370529+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:18:23.370529+0000","last_peered":"2023-09-23T21:18:23.370529+0000","last_clean":"2023-09-23T21:18:23.370529+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:18:23.370529+0000","last_undegraded":"2023-09-23T21:18:23.370529+0000","last_fullsized":"2023-09-23T21:18:23.370529+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":199,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370226+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:18:23.370226+0000","last_peered":"2023-09-23T21:18:23.370226+0000","last_clean":"2023-09-23T21:18:23.370226+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:18:23.370226+0000","last_undegraded":"2023-09-23T21:18:23.370226+0000","last_fullsized":"2023-09-23T21:18:23.370226+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":180,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371083+0000","last_change":"2023-09-23T21:17:11.237055+0000","last_active":"2023-09-23T21:18:23.371083+0000","last_peered":"2023-09-23T21:18:23.371083+0000","last_clean":"2023-09-23T21:18:23.371083+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:18:23.371083+0000","last_undegraded":"2023-09-23T21:18:23.371083+0000","last_fullsized":"2023-09-23T21:18:23.371083+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324702+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:18:20.324702+0000","last_peered":"2023-09-23T21:18:20.324702+0000","last_clean":"2023-09-23T21:18:20.324702+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:18:20.324702+0000","last_undegraded":"2023-09-23T21:18:20.324702+0000","last_fullsized":"2023-09-23T21:18:20.324702+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":204,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370931+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:18:23.370931+0000","last_peered":"2023-09-23T21:18:23.370931+0000","last_clean":"2023-09-23T21:18:23.370931+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:18:23.370931+0000","last_undegraded":"2023-09-23T21:18:23.370931+0000","last_fullsized":"2023-09-23T21:18:23.370931+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":505,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609667+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:18:23.609667+0000","last_peered":"2023-09-23T21:18:23.609667+0000","last_clean":"2023-09-23T21:18:23.609667+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:18:23.609667+0000","last_undegraded":"2023-09-23T21:18:23.609667+0000","last_fullsized":"2023-09-23T21:18:23.609667+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":160,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371529+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:18:23.371529+0000","last_peered":"2023-09-23T21:18:23.371529+0000","last_clean":"2023-09-23T21:18:23.371529+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:18:23.371529+0000","last_undegraded":"2023-09-23T21:18:23.371529+0000","last_fullsized":"2023-09-23T21:18:23.371529+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":228,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370773+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:18:23.370773+0000","last_peered":"2023-09-23T21:18:23.370773+0000","last_clean":"2023-09-23T21:18:23.370773+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:18:23.370773+0000","last_undegraded":"2023-09-23T21:18:23.370773+0000","last_fullsized":"2023-09-23T21:18:23.370773+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":156,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370496+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:18:23.370496+0000","last_peered":"2023-09-23T21:18:23.370496+0000","last_clean":"2023-09-23T21:18:23.370496+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:18:23.370496+0000","last_undegraded":"2023-09-23T21:18:23.370496+0000","last_fullsized":"2023-09-23T21:18:23.370496+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":164,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371205+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:18:23.371205+0000","last_peered":"2023-09-23T21:18:23.371205+0000","last_clean":"2023-09-23T21:18:23.371205+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:18:23.371205+0000","last_undegraded":"2023-09-23T21:18:23.371205+0000","last_fullsized":"2023-09-23T21:18:23.371205+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370362+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:18:23.370362+0000","last_peered":"2023-09-23T21:18:23.370362+0000","last_clean":"2023-09-23T21:18:23.370362+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:18:23.370362+0000","last_undegraded":"2023-09-23T21:18:23.370362+0000","last_fullsized":"2023-09-23T21:18:23.370362+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":178,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370751+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:18:23.370751+0000","last_peered":"2023-09-23T21:18:23.370751+0000","last_clean":"2023-09-23T21:18:23.370751+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:18:23.370751+0000","last_undegraded":"2023-09-23T21:18:23.370751+0000","last_fullsized":"2023-09-23T21:18:23.370751+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":612,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324190+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:18:20.324190+0000","last_peered":"2023-09-23T21:18:20.324190+0000","last_clean":"2023-09-23T21:18:20.324190+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:18:20.324190+0000","last_undegraded":"2023-09-23T21:18:20.324190+0000","last_fullsized":"2023-09-23T21:18:20.324190+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":228,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371239+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:18:23.371239+0000","last_peered":"2023-09-23T21:18:23.371239+0000","last_clean":"2023-09-23T21:18:23.371239+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:18:23.371239+0000","last_undegraded":"2023-09-23T21:18:23.371239+0000","last_fullsized":"2023-09-23T21:18:23.371239+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":11,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324426+0000","last_change":"2023-09-23T21:18:18.461141+0000","last_active":"2023-09-23T21:18:20.324426+0000","last_peered":"2023-09-23T21:18:20.324426+0000","last_clean":"2023-09-23T21:18:20.324426+0000","last_became_active":"2023-09-23T21:18:18.460862+0000","last_became_peered":"2023-09-23T21:18:18.460862+0000","last_unstale":"2023-09-23T21:18:20.324426+0000","last_undegraded":"2023-09-23T21:18:20.324426+0000","last_fullsized":"2023-09-23T21:18:20.324426+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:32.796044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":158,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371335+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:18:23.371335+0000","last_peered":"2023-09-23T21:18:23.371335+0000","last_clean":"2023-09-23T21:18:23.371335+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:18:23.371335+0000","last_undegraded":"2023-09-23T21:18:23.371335+0000","last_fullsized":"2023-09-23T21:18:23.371335+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.369838+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:18:23.369838+0000","last_peered":"2023-09-23T21:18:23.369838+0000","last_clean":"2023-09-23T21:18:23.369838+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:18:23.369838+0000","last_undegraded":"2023-09-23T21:18:23.369838+0000","last_fullsized":"2023-09-23T21:18:23.369838+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":11,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324427+0000","last_change":"2023-09-23T21:18:18.461922+0000","last_active":"2023-09-23T21:18:20.324427+0000","last_peered":"2023-09-23T21:18:20.324427+0000","last_clean":"2023-09-23T21:18:20.324427+0000","last_became_active":"2023-09-23T21:18:18.461627+0000","last_became_peered":"2023-09-23T21:18:18.461627+0000","last_unstale":"2023-09-23T21:18:20.324427+0000","last_undegraded":"2023-09-23T21:18:20.324427+0000","last_fullsized":"2023-09-23T21:18:20.324427+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:17.785201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":172,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370917+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:18:23.370917+0000","last_peered":"2023-09-23T21:18:23.370917+0000","last_clean":"2023-09-23T21:18:23.370917+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:18:23.370917+0000","last_undegraded":"2023-09-23T21:18:23.370917+0000","last_fullsized":"2023-09-23T21:18:23.370917+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324236+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:18:20.324236+0000","last_peered":"2023-09-23T21:18:20.324236+0000","last_clean":"2023-09-23T21:18:20.324236+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:18:20.324236+0000","last_undegraded":"2023-09-23T21:18:20.324236+0000","last_fullsized":"2023-09-23T21:18:20.324236+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"577'311","reported_seq":324,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.156542+0000","last_change":"2023-09-23T21:18:18.458070+0000","last_active":"2023-09-23T21:18:28.156542+0000","last_peered":"2023-09-23T21:18:28.156542+0000","last_clean":"2023-09-23T21:18:28.156542+0000","last_became_active":"2023-09-23T21:18:18.457766+0000","last_became_peered":"2023-09-23T21:18:18.457766+0000","last_unstale":"2023-09-23T21:18:28.156542+0000","last_undegraded":"2023-09-23T21:18:28.156542+0000","last_fullsized":"2023-09-23T21:18:28.156542+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":311,"log_dups_size":0,"ondisk_log_size":311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:50.906117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20381696,"num_objects":311,"num_object_clones":0,"num_object_copies":933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":311,"num_write_kb":19904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":158,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371250+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:18:23.371250+0000","last_peered":"2023-09-23T21:18:23.371250+0000","last_clean":"2023-09-23T21:18:23.371250+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:18:23.371250+0000","last_undegraded":"2023-09-23T21:18:23.371250+0000","last_fullsized":"2023-09-23T21:18:23.371250+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324905+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:18:20.324905+0000","last_peered":"2023-09-23T21:18:20.324905+0000","last_clean":"2023-09-23T21:18:20.324905+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:18:20.324905+0000","last_undegraded":"2023-09-23T21:18:20.324905+0000","last_fullsized":"2023-09-23T21:18:20.324905+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"577'294","reported_seq":307,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.157568+0000","last_change":"2023-09-23T21:18:18.453879+0000","last_active":"2023-09-23T21:18:28.157568+0000","last_peered":"2023-09-23T21:18:28.157568+0000","last_clean":"2023-09-23T21:18:28.157568+0000","last_became_active":"2023-09-23T21:18:18.453619+0000","last_became_peered":"2023-09-23T21:18:18.453619+0000","last_unstale":"2023-09-23T21:18:28.157568+0000","last_undegraded":"2023-09-23T21:18:28.157568+0000","last_fullsized":"2023-09-23T21:18:28.157568+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":294,"log_dups_size":0,"ondisk_log_size":294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:41.497559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267584,"num_objects":294,"num_object_clones":0,"num_object_copies":882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":294,"num_write_kb":18816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324747+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:18:20.324747+0000","last_peered":"2023-09-23T21:18:20.324747+0000","last_clean":"2023-09-23T21:18:20.324747+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:18:20.324747+0000","last_undegraded":"2023-09-23T21:18:20.324747+0000","last_fullsized":"2023-09-23T21:18:20.324747+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324459+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:18:20.324459+0000","last_peered":"2023-09-23T21:18:20.324459+0000","last_clean":"2023-09-23T21:18:20.324459+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:18:20.324459+0000","last_undegraded":"2023-09-23T21:18:20.324459+0000","last_fullsized":"2023-09-23T21:18:20.324459+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"577'288","reported_seq":301,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.159806+0000","last_change":"2023-09-23T21:18:18.453978+0000","last_active":"2023-09-23T21:18:28.159806+0000","last_peered":"2023-09-23T21:18:28.159806+0000","last_clean":"2023-09-23T21:18:28.159806+0000","last_became_active":"2023-09-23T21:18:18.453696+0000","last_became_peered":"2023-09-23T21:18:18.453696+0000","last_unstale":"2023-09-23T21:18:28.159806+0000","last_undegraded":"2023-09-23T21:18:28.159806+0000","last_fullsized":"2023-09-23T21:18:28.159806+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":288,"log_dups_size":0,"ondisk_log_size":288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:03.250012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18874368,"num_objects":288,"num_object_clones":0,"num_object_copies":864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":288,"num_write_kb":18432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":244,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371454+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:18:23.371454+0000","last_peered":"2023-09-23T21:18:23.371454+0000","last_clean":"2023-09-23T21:18:23.371454+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:18:23.371454+0000","last_undegraded":"2023-09-23T21:18:23.371454+0000","last_fullsized":"2023-09-23T21:18:23.371454+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324909+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:18:20.324909+0000","last_peered":"2023-09-23T21:18:20.324909+0000","last_clean":"2023-09-23T21:18:20.324909+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:18:20.324909+0000","last_undegraded":"2023-09-23T21:18:20.324909+0000","last_fullsized":"2023-09-23T21:18:20.324909+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"577'375","reported_seq":385,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.576885+0000","last_change":"2023-09-23T21:18:18.457504+0000","last_active":"2023-09-23T21:18:28.576885+0000","last_peered":"2023-09-23T21:18:28.576885+0000","last_clean":"2023-09-23T21:18:28.576885+0000","last_became_active":"2023-09-23T21:18:18.456595+0000","last_became_peered":"2023-09-23T21:18:18.456595+0000","last_unstale":"2023-09-23T21:18:28.576885+0000","last_undegraded":"2023-09-23T21:18:28.576885+0000","last_fullsized":"2023-09-23T21:18:28.576885+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":375,"log_dups_size":0,"ondisk_log_size":375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:59.367182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24576000,"num_objects":375,"num_object_clones":0,"num_object_copies":1125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":375,"num_write_kb":24000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324695+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:18:20.324695+0000","last_peered":"2023-09-23T21:18:20.324695+0000","last_clean":"2023-09-23T21:18:20.324695+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:18:20.324695+0000","last_undegraded":"2023-09-23T21:18:20.324695+0000","last_fullsized":"2023-09-23T21:18:20.324695+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":619,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609817+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:18:23.609817+0000","last_peered":"2023-09-23T21:18:23.609817+0000","last_clean":"2023-09-23T21:18:23.609817+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:18:23.609817+0000","last_undegraded":"2023-09-23T21:18:23.609817+0000","last_fullsized":"2023-09-23T21:18:23.609817+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":11,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324557+0000","last_change":"2023-09-23T21:18:18.460279+0000","last_active":"2023-09-23T21:18:20.324557+0000","last_peered":"2023-09-23T21:18:20.324557+0000","last_clean":"2023-09-23T21:18:20.324557+0000","last_became_active":"2023-09-23T21:18:18.459980+0000","last_became_peered":"2023-09-23T21:18:18.459980+0000","last_unstale":"2023-09-23T21:18:20.324557+0000","last_undegraded":"2023-09-23T21:18:20.324557+0000","last_fullsized":"2023-09-23T21:18:20.324557+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:13.583349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":214,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609531+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:18:23.609531+0000","last_peered":"2023-09-23T21:18:23.609531+0000","last_clean":"2023-09-23T21:18:23.609531+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:18:23.609531+0000","last_undegraded":"2023-09-23T21:18:23.609531+0000","last_fullsized":"2023-09-23T21:18:23.609531+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324954+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:18:20.324954+0000","last_peered":"2023-09-23T21:18:20.324954+0000","last_clean":"2023-09-23T21:18:20.324954+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:18:20.324954+0000","last_undegraded":"2023-09-23T21:18:20.324954+0000","last_fullsized":"2023-09-23T21:18:20.324954+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"577'385","reported_seq":396,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.552251+0000","last_change":"2023-09-23T21:18:18.457562+0000","last_active":"2023-09-23T21:18:28.552251+0000","last_peered":"2023-09-23T21:18:28.552251+0000","last_clean":"2023-09-23T21:18:28.552251+0000","last_became_active":"2023-09-23T21:18:18.457231+0000","last_became_peered":"2023-09-23T21:18:18.457231+0000","last_unstale":"2023-09-23T21:18:28.552251+0000","last_undegraded":"2023-09-23T21:18:28.552251+0000","last_fullsized":"2023-09-23T21:18:28.552251+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":385,"log_dups_size":0,"ondisk_log_size":385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:26.541539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25231360,"num_objects":385,"num_object_clones":0,"num_object_copies":1155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":385,"num_write_kb":24640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":228,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370932+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:18:23.370932+0000","last_peered":"2023-09-23T21:18:23.370932+0000","last_clean":"2023-09-23T21:18:23.370932+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:18:23.370932+0000","last_undegraded":"2023-09-23T21:18:23.370932+0000","last_fullsized":"2023-09-23T21:18:23.370932+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":316,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370961+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:18:23.370961+0000","last_peered":"2023-09-23T21:18:23.370961+0000","last_clean":"2023-09-23T21:18:23.370961+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:18:23.370961+0000","last_undegraded":"2023-09-23T21:18:23.370961+0000","last_fullsized":"2023-09-23T21:18:23.370961+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.8","version":"577'308","reported_seq":321,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.149750+0000","last_change":"2023-09-23T21:18:18.458504+0000","last_active":"2023-09-23T21:18:28.149750+0000","last_peered":"2023-09-23T21:18:28.149750+0000","last_clean":"2023-09-23T21:18:28.149750+0000","last_became_active":"2023-09-23T21:18:18.458198+0000","last_became_peered":"2023-09-23T21:18:18.458198+0000","last_unstale":"2023-09-23T21:18:28.149750+0000","last_undegraded":"2023-09-23T21:18:28.149750+0000","last_fullsized":"2023-09-23T21:18:28.149750+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":308,"log_dups_size":0,"ondisk_log_size":308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:41.150311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20185088,"num_objects":308,"num_object_clones":0,"num_object_copies":924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":308,"num_write_kb":19712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":501,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370852+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:18:23.370852+0000","last_peered":"2023-09-23T21:18:23.370852+0000","last_clean":"2023-09-23T21:18:23.370852+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:18:23.370852+0000","last_undegraded":"2023-09-23T21:18:23.370852+0000","last_fullsized":"2023-09-23T21:18:23.370852+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":252,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370307+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:18:23.370307+0000","last_peered":"2023-09-23T21:18:23.370307+0000","last_clean":"2023-09-23T21:18:23.370307+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:18:23.370307+0000","last_undegraded":"2023-09-23T21:18:23.370307+0000","last_fullsized":"2023-09-23T21:18:23.370307+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":126,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370422+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:18:23.370422+0000","last_peered":"2023-09-23T21:18:23.370422+0000","last_clean":"2023-09-23T21:18:23.370422+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:18:23.370422+0000","last_undegraded":"2023-09-23T21:18:23.370422+0000","last_fullsized":"2023-09-23T21:18:23.370422+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"577'367","reported_seq":379,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.552242+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:18:28.552242+0000","last_peered":"2023-09-23T21:18:28.552242+0000","last_clean":"2023-09-23T21:18:28.552242+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:18:28.552242+0000","last_undegraded":"2023-09-23T21:18:28.552242+0000","last_fullsized":"2023-09-23T21:18:28.552242+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":367,"log_dups_size":0,"ondisk_log_size":367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24051712,"num_objects":367,"num_object_clones":0,"num_object_copies":1101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":367,"num_write_kb":23488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324464+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:18:20.324464+0000","last_peered":"2023-09-23T21:18:20.324464+0000","last_clean":"2023-09-23T21:18:20.324464+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:18:20.324464+0000","last_undegraded":"2023-09-23T21:18:20.324464+0000","last_fullsized":"2023-09-23T21:18:20.324464+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324522+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:18:20.324522+0000","last_peered":"2023-09-23T21:18:20.324522+0000","last_clean":"2023-09-23T21:18:20.324522+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:18:20.324522+0000","last_undegraded":"2023-09-23T21:18:20.324522+0000","last_fullsized":"2023-09-23T21:18:20.324522+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":291,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609579+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:18:23.609579+0000","last_peered":"2023-09-23T21:18:23.609579+0000","last_clean":"2023-09-23T21:18:23.609579+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:18:23.609579+0000","last_undegraded":"2023-09-23T21:18:23.609579+0000","last_fullsized":"2023-09-23T21:18:23.609579+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"0'0","reported_seq":11,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324534+0000","last_change":"2023-09-23T21:18:18.460402+0000","last_active":"2023-09-23T21:18:20.324534+0000","last_peered":"2023-09-23T21:18:20.324534+0000","last_clean":"2023-09-23T21:18:20.324534+0000","last_became_active":"2023-09-23T21:18:18.459784+0000","last_became_peered":"2023-09-23T21:18:18.459784+0000","last_unstale":"2023-09-23T21:18:20.324534+0000","last_undegraded":"2023-09-23T21:18:20.324534+0000","last_fullsized":"2023-09-23T21:18:20.324534+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:40.883508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":586,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609763+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:18:23.609763+0000","last_peered":"2023-09-23T21:18:23.609763+0000","last_clean":"2023-09-23T21:18:23.609763+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:18:23.609763+0000","last_undegraded":"2023-09-23T21:18:23.609763+0000","last_fullsized":"2023-09-23T21:18:23.609763+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":323,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609274+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:18:23.609274+0000","last_peered":"2023-09-23T21:18:23.609274+0000","last_clean":"2023-09-23T21:18:23.609274+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:18:23.609274+0000","last_undegraded":"2023-09-23T21:18:23.609274+0000","last_fullsized":"2023-09-23T21:18:23.609274+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371042+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:18:23.371042+0000","last_peered":"2023-09-23T21:18:23.371042+0000","last_clean":"2023-09-23T21:18:23.371042+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:18:23.371042+0000","last_undegraded":"2023-09-23T21:18:23.371042+0000","last_fullsized":"2023-09-23T21:18:23.371042+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.f","version":"577'374","reported_seq":386,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.459871+0000","last_change":"2023-09-23T21:18:18.457700+0000","last_active":"2023-09-23T21:18:28.459871+0000","last_peered":"2023-09-23T21:18:28.459871+0000","last_clean":"2023-09-23T21:18:28.459871+0000","last_became_active":"2023-09-23T21:18:18.457261+0000","last_became_peered":"2023-09-23T21:18:18.457261+0000","last_unstale":"2023-09-23T21:18:28.459871+0000","last_undegraded":"2023-09-23T21:18:28.459871+0000","last_fullsized":"2023-09-23T21:18:28.459871+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":374,"log_dups_size":0,"ondisk_log_size":374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:52.984711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510464,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":374,"num_write_kb":23936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":406,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609867+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:18:23.609867+0000","last_peered":"2023-09-23T21:18:23.609867+0000","last_clean":"2023-09-23T21:18:23.609867+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:18:23.609867+0000","last_undegraded":"2023-09-23T21:18:23.609867+0000","last_fullsized":"2023-09-23T21:18:23.609867+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":923,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370762+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:18:23.370762+0000","last_peered":"2023-09-23T21:18:23.370762+0000","last_clean":"2023-09-23T21:18:23.370762+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:18:23.370762+0000","last_undegraded":"2023-09-23T21:18:23.370762+0000","last_fullsized":"2023-09-23T21:18:23.370762+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":149,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370150+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:18:23.370150+0000","last_peered":"2023-09-23T21:18:23.370150+0000","last_clean":"2023-09-23T21:18:23.370150+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:18:23.370150+0000","last_undegraded":"2023-09-23T21:18:23.370150+0000","last_fullsized":"2023-09-23T21:18:23.370150+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324708+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:18:20.324708+0000","last_peered":"2023-09-23T21:18:20.324708+0000","last_clean":"2023-09-23T21:18:20.324708+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:18:20.324708+0000","last_undegraded":"2023-09-23T21:18:20.324708+0000","last_fullsized":"2023-09-23T21:18:20.324708+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"577'379","reported_seq":391,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.555505+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:18:28.555505+0000","last_peered":"2023-09-23T21:18:28.555505+0000","last_clean":"2023-09-23T21:18:28.555505+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:18:28.555505+0000","last_undegraded":"2023-09-23T21:18:28.555505+0000","last_fullsized":"2023-09-23T21:18:28.555505+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":379,"log_dups_size":0,"ondisk_log_size":379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24838144,"num_objects":379,"num_object_clones":0,"num_object_copies":1137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":379,"num_write_kb":24256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":204,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370845+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:18:23.370845+0000","last_peered":"2023-09-23T21:18:23.370845+0000","last_clean":"2023-09-23T21:18:23.370845+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:18:23.370845+0000","last_undegraded":"2023-09-23T21:18:23.370845+0000","last_fullsized":"2023-09-23T21:18:23.370845+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370142+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:18:23.370142+0000","last_peered":"2023-09-23T21:18:23.370142+0000","last_clean":"2023-09-23T21:18:23.370142+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:18:23.370142+0000","last_undegraded":"2023-09-23T21:18:23.370142+0000","last_fullsized":"2023-09-23T21:18:23.370142+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":486,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609700+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:18:23.609700+0000","last_peered":"2023-09-23T21:18:23.609700+0000","last_clean":"2023-09-23T21:18:23.609700+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:18:23.609700+0000","last_undegraded":"2023-09-23T21:18:23.609700+0000","last_fullsized":"2023-09-23T21:18:23.609700+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"577'383","reported_seq":395,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.548085+0000","last_change":"2023-09-23T21:18:18.454773+0000","last_active":"2023-09-23T21:18:28.548085+0000","last_peered":"2023-09-23T21:18:28.548085+0000","last_clean":"2023-09-23T21:18:28.548085+0000","last_became_active":"2023-09-23T21:18:18.454490+0000","last_became_peered":"2023-09-23T21:18:18.454490+0000","last_unstale":"2023-09-23T21:18:28.548085+0000","last_undegraded":"2023-09-23T21:18:28.548085+0000","last_fullsized":"2023-09-23T21:18:28.548085+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":383,"log_dups_size":0,"ondisk_log_size":383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:37.641550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25100288,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":383,"num_write_kb":24512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":174,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371163+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:18:23.371163+0000","last_peered":"2023-09-23T21:18:23.371163+0000","last_clean":"2023-09-23T21:18:23.371163+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:18:23.371163+0000","last_undegraded":"2023-09-23T21:18:23.371163+0000","last_fullsized":"2023-09-23T21:18:23.371163+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":390,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609628+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:18:23.609628+0000","last_peered":"2023-09-23T21:18:23.609628+0000","last_clean":"2023-09-23T21:18:23.609628+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:18:23.609628+0000","last_undegraded":"2023-09-23T21:18:23.609628+0000","last_fullsized":"2023-09-23T21:18:23.609628+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":282,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370554+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:18:23.370554+0000","last_peered":"2023-09-23T21:18:23.370554+0000","last_clean":"2023-09-23T21:18:23.370554+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:18:23.370554+0000","last_undegraded":"2023-09-23T21:18:23.370554+0000","last_fullsized":"2023-09-23T21:18:23.370554+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"577'362","reported_seq":374,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.548751+0000","last_change":"2023-09-23T21:18:18.459893+0000","last_active":"2023-09-23T21:18:28.548751+0000","last_peered":"2023-09-23T21:18:28.548751+0000","last_clean":"2023-09-23T21:18:28.548751+0000","last_became_active":"2023-09-23T21:18:18.459600+0000","last_became_peered":"2023-09-23T21:18:18.459600+0000","last_unstale":"2023-09-23T21:18:28.548751+0000","last_undegraded":"2023-09-23T21:18:28.548751+0000","last_fullsized":"2023-09-23T21:18:28.548751+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":362,"log_dups_size":0,"ondisk_log_size":362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:01.769949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23724032,"num_objects":362,"num_object_clones":0,"num_object_copies":1086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":362,"num_write_kb":23168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":232,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371437+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:18:23.371437+0000","last_peered":"2023-09-23T21:18:23.371437+0000","last_clean":"2023-09-23T21:18:23.371437+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:18:23.371437+0000","last_undegraded":"2023-09-23T21:18:23.371437+0000","last_fullsized":"2023-09-23T21:18:23.371437+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324935+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:18:20.324935+0000","last_peered":"2023-09-23T21:18:20.324935+0000","last_clean":"2023-09-23T21:18:20.324935+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:18:20.324935+0000","last_undegraded":"2023-09-23T21:18:20.324935+0000","last_fullsized":"2023-09-23T21:18:20.324935+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":257,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370673+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:18:23.370673+0000","last_peered":"2023-09-23T21:18:23.370673+0000","last_clean":"2023-09-23T21:18:23.370673+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:18:23.370673+0000","last_undegraded":"2023-09-23T21:18:23.370673+0000","last_fullsized":"2023-09-23T21:18:23.370673+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"577'306","reported_seq":319,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:28.144128+0000","last_change":"2023-09-23T21:18:18.456335+0000","last_active":"2023-09-23T21:18:28.144128+0000","last_peered":"2023-09-23T21:18:28.144128+0000","last_clean":"2023-09-23T21:18:28.144128+0000","last_became_active":"2023-09-23T21:18:18.455992+0000","last_became_peered":"2023-09-23T21:18:18.455992+0000","last_unstale":"2023-09-23T21:18:28.144128+0000","last_undegraded":"2023-09-23T21:18:28.144128+0000","last_fullsized":"2023-09-23T21:18:28.144128+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:00.021252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20054016,"num_objects":306,"num_object_clones":0,"num_object_copies":918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":306,"num_write_kb":19584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":482,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.323908+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:18:20.323908+0000","last_peered":"2023-09-23T21:18:20.323908+0000","last_clean":"2023-09-23T21:18:20.323908+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:18:20.323908+0000","last_undegraded":"2023-09-23T21:18:20.323908+0000","last_fullsized":"2023-09-23T21:18:20.323908+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":114,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610199+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:18:23.610199+0000","last_peered":"2023-09-23T21:18:23.610199+0000","last_clean":"2023-09-23T21:18:23.610199+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:18:23.610199+0000","last_undegraded":"2023-09-23T21:18:23.610199+0000","last_fullsized":"2023-09-23T21:18:23.610199+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":262,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.369807+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:18:23.369807+0000","last_peered":"2023-09-23T21:18:23.369807+0000","last_clean":"2023-09-23T21:18:23.369807+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:18:23.369807+0000","last_undegraded":"2023-09-23T21:18:23.369807+0000","last_fullsized":"2023-09-23T21:18:23.369807+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":204,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371574+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:18:23.371574+0000","last_peered":"2023-09-23T21:18:23.371574+0000","last_clean":"2023-09-23T21:18:23.371574+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:18:23.371574+0000","last_undegraded":"2023-09-23T21:18:23.371574+0000","last_fullsized":"2023-09-23T21:18:23.371574+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":640,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609675+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:18:23.609675+0000","last_peered":"2023-09-23T21:18:23.609675+0000","last_clean":"2023-09-23T21:18:23.609675+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:18:23.609675+0000","last_undegraded":"2023-09-23T21:18:23.609675+0000","last_fullsized":"2023-09-23T21:18:23.609675+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":281,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609506+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:18:23.609506+0000","last_peered":"2023-09-23T21:18:23.609506+0000","last_clean":"2023-09-23T21:18:23.609506+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:18:23.609506+0000","last_undegraded":"2023-09-23T21:18:23.609506+0000","last_fullsized":"2023-09-23T21:18:23.609506+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":235,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609824+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:18:23.609824+0000","last_peered":"2023-09-23T21:18:23.609824+0000","last_clean":"2023-09-23T21:18:23.609824+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:18:23.609824+0000","last_undegraded":"2023-09-23T21:18:23.609824+0000","last_fullsized":"2023-09-23T21:18:23.609824+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":45,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324663+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:18:20.324663+0000","last_peered":"2023-09-23T21:18:20.324663+0000","last_clean":"2023-09-23T21:18:20.324663+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:18:20.324663+0000","last_undegraded":"2023-09-23T21:18:20.324663+0000","last_fullsized":"2023-09-23T21:18:20.324663+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":222,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609315+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:18:23.609315+0000","last_peered":"2023-09-23T21:18:23.609315+0000","last_clean":"2023-09-23T21:18:23.609315+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:18:23.609315+0000","last_undegraded":"2023-09-23T21:18:23.609315+0000","last_fullsized":"2023-09-23T21:18:23.609315+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":41,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T21:18:20.324886+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:18:20.324886+0000","last_peered":"2023-09-23T21:18:20.324886+0000","last_clean":"2023-09-23T21:18:20.324886+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:18:20.324886+0000","last_undegraded":"2023-09-23T21:18:20.324886+0000","last_fullsized":"2023-09-23T21:18:20.324886+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":437,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609288+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:18:23.609288+0000","last_peered":"2023-09-23T21:18:23.609288+0000","last_clean":"2023-09-23T21:18:23.609288+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:18:23.609288+0000","last_undegraded":"2023-09-23T21:18:23.609288+0000","last_fullsized":"2023-09-23T21:18:23.609288+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":534,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.425594+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:18:26.425594+0000","last_peered":"2023-09-23T21:18:26.425594+0000","last_clean":"2023-09-23T21:18:26.425594+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:18:26.425594+0000","last_undegraded":"2023-09-23T21:18:26.425594+0000","last_fullsized":"2023-09-23T21:18:26.425594+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":244,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609063+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:18:23.609063+0000","last_peered":"2023-09-23T21:18:23.609063+0000","last_clean":"2023-09-23T21:18:23.609063+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:18:23.609063+0000","last_undegraded":"2023-09-23T21:18:23.609063+0000","last_fullsized":"2023-09-23T21:18:23.609063+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":270794752,"num_objects":4132,"num_object_clones":0,"num_object_copies":12396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4132,"num_write_kb":264448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":851181568,"data_stored":851181568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4132,"ondisk_log_size":4132,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":38918,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201877,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927620,"kb_used_data":114876,"kb_used_omap":0,"kb_used_meta":812736,"kb_avail":92817532,"statfs":{"total":95995035648,"available":95045152768,"internally_reserved":0,"allocated":117633024,"data_stored":116059528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":832241664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":2.7679999999999998,"15min":2.2040000000000002},"min":{"1min":0.246,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":6.9470000000000001,"5min":141.33199999999999,"15min":141.33199999999999},"last":0.749},{"interface":"front","average":{"1min":1.1990000000000001,"5min":3.0579999999999998,"15min":2.2519999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":5.1340000000000003,"5min":141.31700000000001,"15min":141.31700000000001},"last":0.499}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.7749999999999999,"15min":3.3639999999999999},"min":{"1min":0.29699999999999999,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":5.1100000000000003,"5min":10.407999999999999,"15min":114.16500000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.4359999999999999,"15min":3.8039999999999998},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.22900000000000001},"max":{"1min":6.9820000000000002,"5min":8.8650000000000002,"15min":117.637},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":3.71,"5min":3.8260000000000001,"15min":4.5519999999999996},"min":{"1min":0.248,"5min":0.224,"15min":0.224},"max":{"1min":53.988,"5min":135.529,"15min":183.44200000000001},"last":0.437},{"interface":"front","average":{"1min":4.0659999999999998,"5min":4.7190000000000003,"15min":4.7750000000000004},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":54,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.57299999999999995}]},{"osd":3,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":4.9870000000000001,"15min":3.9319999999999999},"min":{"1min":0.33800000000000002,"5min":0.30499999999999999,"15min":0.26700000000000002},"max":{"1min":7.0099999999999998,"5min":135.66200000000001,"15min":135.66200000000001},"last":0.874},{"interface":"front","average":{"1min":1.484,"5min":5.3570000000000002,"15min":4.056},"min":{"1min":0.32100000000000001,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":7.4729999999999999,"5min":135.47,"15min":135.47},"last":0.61599999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.9430000000000001,"15min":1.2090000000000001},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1829999999999998,"5min":10.989000000000001,"15min":13.739000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.5840000000000001,"5min":2.0950000000000002,"15min":1.2},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":20.306999999999999,"5min":20.306999999999999,"15min":20.306999999999999},"last":0.95799999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":2.6389999999999998,"15min":1.6080000000000001},"min":{"1min":0.53100000000000003,"5min":0.29099999999999998,"15min":0.26900000000000002},"max":{"1min":14.105,"5min":14.105,"15min":14.105},"last":0.84399999999999997},{"interface":"front","average":{"1min":3.9630000000000001,"5min":2.5339999999999998,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":17.585999999999999,"5min":17.585999999999999,"15min":17.585999999999999},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.2930000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":9.9969999999999999,"5min":13.369,"15min":143.18700000000001},"last":2.1160000000000001},{"interface":"front","average":{"1min":2.9380000000000002,"5min":1.819,"15min":1.929},"min":{"1min":0.31,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":15.566000000000001,"5min":15.566000000000001,"15min":142.70400000000001},"last":0.65300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463497,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":958836,"kb_used_data":180204,"kb_used_omap":0,"kb_used_meta":778624,"kb_avail":92786316,"statfs":{"total":95995035648,"available":95013187584,"internally_reserved":0,"allocated":184528896,"data_stored":182964060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":797310976},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.7549999999999999,"15min":2.7610000000000001},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":7.4400000000000004,"5min":121.762,"15min":149.61600000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.8809999999999998,"5min":3.4249999999999998,"15min":2.9369999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":9.7449999999999992,"5min":121.73099999999999,"15min":149.303},"last":0.501}]},{"osd":1,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":2.379,"15min":3.6080000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.22700000000000001},"max":{"1min":16.126999999999999,"5min":25.122,"15min":129.37899999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.986,"5min":1.8420000000000001,"15min":3.387},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":10.977,"5min":25.193999999999999,"15min":129.40100000000001},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":15.675000000000001,"5min":8.2889999999999997,"15min":6.7590000000000003},"min":{"1min":0.36499999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":181.846,"5min":181.846,"15min":199.23400000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":17.042000000000002,"5min":9.0250000000000004,"15min":6.8579999999999997},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":182.14500000000001,"5min":182.14500000000001,"15min":199.08000000000001},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":3.1150000000000002,"15min":3.2389999999999999},"min":{"1min":0.33200000000000002,"5min":0.218,"15min":0.218},"max":{"1min":10.734999999999999,"5min":126.649,"15min":137.875},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.367,"5min":3.2429999999999999,"15min":3.3170000000000002},"min":{"1min":0.34799999999999998,"5min":0.30099999999999999,"15min":0.29799999999999999},"max":{"1min":7.4660000000000002,"5min":125.324,"15min":137.60900000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6299999999999999,"5min":3.5640000000000001,"15min":1.8879999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":9.6720000000000006,"5min":55.494,"15min":55.494},"last":0.32300000000000001},{"interface":"front","average":{"1min":3.1669999999999998,"5min":3.359,"15min":1.669},"min":{"1min":0.39000000000000001,"5min":0.29399999999999998,"15min":0.23999999999999999},"max":{"1min":11.228,"5min":55.305,"15min":55.305},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":4.9290000000000003,"5min":5.2089999999999996,"15min":2.4329999999999998},"min":{"1min":0.32500000000000001,"5min":0.32300000000000001,"15min":0.27300000000000002},"max":{"1min":12.975,"5min":222.90899999999999,"15min":222.90899999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.9239999999999999,"5min":5.1970000000000001,"15min":2.3740000000000001},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.28499999999999998},"max":{"1min":11.199999999999999,"5min":224.655,"15min":224.655},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:17:25 2023","interfaces":[{"interface":"back","average":{"1min":13.882999999999999,"5min":3.629,"15min":1.883},"min":{"1min":0.39600000000000002,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":165.96799999999999,"5min":165.96799999999999,"15min":165.96799999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":13.532,"5min":3.859,"15min":1.98},"min":{"1min":0.442,"5min":0.251,"15min":0.251},"max":{"1min":165.82400000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":0.46000000000000002}]}]},{"osd":1,"up_from":0,"seq":1185410973944,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991684,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781575,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1219840,"kb_used_data":102200,"kb_used_omap":7,"kb_used_meta":1117624,"kb_avail":92525312,"statfs":{"total":95995035648,"available":94745919488,"internally_reserved":0,"allocated":104652800,"data_stored":101902237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1144447057},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.5069999999999999,"15min":2.0710000000000002},"min":{"1min":0.17899999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":4.0830000000000002,"5min":25.297000000000001,"15min":103.021},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.669,"15min":2.6070000000000002},"min":{"1min":0.25700000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":4.5510000000000002,"5min":18.879999999999999,"15min":103.154},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:17:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.2000000000000002,"15min":4.0869999999999997},"min":{"1min":0.24099999999999999,"5min":0.224,"15min":0.19},"max":{"1min":2.4409999999999998,"5min":54.521000000000001,"15min":212.12},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.98999999999999999,"5min":2.4820000000000002,"15min":4.1399999999999997},"min":{"1min":0.254,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":8.048,"5min":54.598999999999997,"15min":213.02799999999999},"last":0.54800000000000004}]},{"osd":3,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.984,"15min":3.5710000000000002},"min":{"1min":0.30199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5760000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.5379999999999998,"15min":3.819},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.249},"max":{"1min":4.109,"5min":82.241,"15min":143.27799999999999},"last":0.57399999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":4.3300000000000001,"15min":3.198},"min":{"1min":0.32100000000000001,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":7.0330000000000004,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.744,"5min":4.8300000000000001,"15min":3.7879999999999998},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.29499999999999998},"max":{"1min":7.0449999999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":5.0979999999999999,"15min":3.448},"min":{"1min":0.31,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.4349999999999996,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.625,"5min":4.7199999999999998,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.34399999999999997,"15min":0.29399999999999998},"max":{"1min":4.1440000000000001,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.75600000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":4.7480000000000002,"15min":3.7970000000000002},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":5.3220000000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":0.80200000000000005},{"interface":"front","average":{"1min":2.3380000000000001,"5min":4.4829999999999997,"15min":3.1720000000000002},"min":{"1min":0.48399999999999999,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":7.1619999999999999,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.68899999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":2.4830000000000001,"15min":1.9530000000000001},"min":{"1min":0.373,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":4.2640000000000002,"5min":25.260000000000002,"15min":25.260000000000002},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.4079999999999999,"5min":2.5419999999999998,"15min":2.2610000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.5289999999999999,"5min":12.613,"15min":16.875},"last":0.77900000000000003}]}]},{"osd":3,"up_from":0,"seq":635655160168,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986787,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143808,"kb_used_data":318008,"kb_used_omap":0,"kb_used_meta":825792,"kb_avail":92601344,"statfs":{"total":95995035648,"available":94823776256,"internally_reserved":0,"allocated":325640192,"data_stored":324070501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":845611008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":3.3410000000000002,"15min":2.7639999999999998},"min":{"1min":0.28000000000000003,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":9.2200000000000006,"5min":109.23099999999999,"15min":226.751},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.754,"15min":2.5430000000000001},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":6.0919999999999996,"5min":109.16800000000001,"15min":226.43299999999999},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":3.802,"15min":5.2999999999999998},"min":{"1min":0.313,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2510000000000003,"5min":119.55800000000001,"15min":127.411},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.8050000000000002,"5min":3.71,"15min":4.5830000000000002},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":10.506,"5min":116.78100000000001,"15min":118.075},"last":0.59499999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":8.7449999999999992,"5min":6.4420000000000002,"15min":6.327},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":85.994,"5min":155.51300000000001,"15min":210.50299999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":9.2530000000000001,"5min":7.3230000000000004,"15min":6.4809999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":85.971999999999994,"5min":155.77099999999999,"15min":210.59},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 21:17:27 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":3.2440000000000002,"15min":4.1200000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.0650000000000004,"5min":111.22199999999999,"15min":214.64099999999999},"last":0.5},{"interface":"front","average":{"1min":3.0659999999999998,"5min":4.508,"15min":4.5209999999999999},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":10.529,"5min":111.39400000000001,"15min":214.54900000000001},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7829999999999999,"5min":1.7749999999999999,"15min":1.139},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.417,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.8399999999999999,"15min":1.49},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":14.675000000000001,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.61599999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":2.6579999999999999,"15min":1.6970000000000001},"min":{"1min":0.35699999999999998,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":41.938000000000002,"5min":41.938000000000002,"15min":41.938000000000002},"last":0.60499999999999998},{"interface":"front","average":{"1min":5.4660000000000002,"5min":2.5390000000000001,"15min":1.635},"min":{"1min":0.33600000000000002,"5min":0.32300000000000001,"15min":0.27200000000000002},"max":{"1min":40.444000000000003,"5min":40.444000000000003,"15min":40.444000000000003},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":2.2170000000000001,"15min":1.8740000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":11.859,"5min":18.352,"15min":145.721},"last":0.63400000000000001},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.8600000000000001,"15min":1.893},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"max":{"1min":8.5570000000000004,"5min":10.677,"15min":148.45699999999999},"last":0.57799999999999996}]}]},{"osd":5,"up_from":39,"seq":167503725151,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1119540,"kb_used_data":243436,"kb_used_omap":0,"kb_used_meta":876096,"kb_avail":92625612,"statfs":{"total":95995035648,"available":94848626688,"internally_reserved":0,"allocated":249278464,"data_stored":247716827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":897122304},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.427,"15min":1.71},"min":{"1min":0.34899999999999998,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":4.1260000000000003,"5min":8.6660000000000004,"15min":82.280000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.798,"15min":1.889},"min":{"1min":0.27600000000000002,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.8810000000000002,"5min":16.977,"15min":85.236000000000004},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3479999999999999,"15min":4.7039999999999997},"min":{"1min":0.255,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":11.374000000000001,"5min":12.032999999999999,"15min":233.77099999999999},"last":3.601},{"interface":"front","average":{"1min":1.3979999999999999,"5min":1.6399999999999999,"15min":4.359},"min":{"1min":0.39600000000000002,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":4.0590000000000002,"5min":12.016,"15min":233.75200000000001},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":1.73,"15min":3.2269999999999999},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.1040000000000001,"5min":10.454000000000001,"15min":231.34700000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.9930000000000001,"15min":3.4220000000000002},"min":{"1min":0.374,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9180000000000001,"5min":10.621,"15min":231.381},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.403,"15min":3.7440000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":11.42,"5min":11.597,"15min":263.887},"last":3.573},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.0099999999999998,"15min":3.4380000000000002},"min":{"1min":0.44400000000000001,"5min":0.39700000000000002,"15min":0.30099999999999999},"max":{"1min":4.173,"5min":10.638,"15min":263.70100000000002},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.7709999999999999,"15min":1.0129999999999999},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":4.0289999999999999,"5min":11.593999999999999,"15min":11.593999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":3.3860000000000001,"5min":1.95,"15min":1.1899999999999999},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.26400000000000001},"max":{"1min":12.49,"5min":12.49,"15min":12.49},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":2.0339999999999998,"15min":1.619},"min":{"1min":0.48499999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":6.984,"5min":8.9130000000000003,"15min":83.451999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.73,"15min":1.8859999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":11.696999999999999,"5min":12.725,"15min":83.484999999999999},"last":3.6349999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:17:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1000000000000001,"5min":2.161,"15min":1.419},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":7.4020000000000001,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.431,"5min":1.585,"15min":0.995},"min":{"1min":0.45500000000000002,"5min":0.34899999999999998,"15min":0.28899999999999998},"max":{"1min":10.196999999999999,"5min":11.640000000000001,"15min":11.640000000000001},"last":0.60899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":158531584,"data_stored":158531584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":245497856,"data_stored":245497856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":301793280,"data_stored":301793280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":145358848,"data_stored":145358848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:32.131 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:18:32.131 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:18:32.131 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T21:18:32.132 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:18:32.212 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:32.269 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:32.387 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:32.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:32.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:32.409 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:32 smithi121 ceph-mon[211526]: pgmap v1786: 89 pgs: 89 active+clean; 457 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T21:18:32.409 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:32 smithi121 ceph-mon[219709]: pgmap v1786: 89 pgs: 89 active+clean; 457 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T21:18:32.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 6 16 15299 15283 159.182 129.312 0.00602885 0.00627494 2023-09-23T21:18:32.497 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:32.535 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:32.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:32 smithi123 ceph-mon[193903]: pgmap v1786: 89 pgs: 89 active+clean; 457 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T21:18:32.794 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:32.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:32.898 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:32.974 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:33.077 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:33.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:33.191 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:33.217 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:33.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 7 16 17168 17152 153.128 116.812 0.00675194 0.00652442 2023-09-23T21:18:33.658 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:33.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:33.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:33.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:33.809 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:33.912 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:33.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:33.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:34.017 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:34.080 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:34.192 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:34.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:34.302 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:34.328 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:34.399 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:34.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:34 smithi121 ceph-mon[211526]: pgmap v1787: 89 pgs: 89 active+clean; 690 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T21:18:34.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:34.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:34.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 8 16 18940 18924 147.83 110.75 0.0072893 0.0067578 2023-09-23T21:18:34.518 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:34.579 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:34 smithi123 ceph-mon[193903]: pgmap v1787: 89 pgs: 89 active+clean; 690 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T21:18:34.583 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:34 smithi121 ceph-mon[219709]: pgmap v1787: 89 pgs: 89 active+clean; 690 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T21:18:34.670 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:18:34.697 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:34.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:34.807 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:34.846 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:34.930 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:35.070 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:35.072 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:18:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:18:34] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.43.0" 2023-09-23T21:18:35.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:35.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:35.087 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:18:35.087 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:18:35.087 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:18:35.087 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:18:35.087 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:18:35.176 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:35.240 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:35.353 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:35.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:35.422 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 9 16 20851 20835 144.674 119.438 0.0058697 0.00690494 2023-09-23T21:18:35.463 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:35.503 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:36.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 10 16 22402 22386 139.9 96.9375 0.00606906 0.00714171 2023-09-23T21:18:36.526 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:36.527 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:18:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:36.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:36.630 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:36.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:36 smithi121 ceph-mon[211526]: pgmap v1788: 89 pgs: 89 active+clean; 1008 MiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T21:18:36.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:36 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:36.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:36 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:36.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:36 smithi121 ceph-mon[219709]: pgmap v1788: 89 pgs: 89 active+clean; 1008 MiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T21:18:36.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:36 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:36.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:36 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:36.678 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:36 smithi123 ceph-mon[193903]: pgmap v1788: 89 pgs: 89 active+clean; 1008 MiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T21:18:36.678 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:36 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:36.678 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:36 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:36.688 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:36.814 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:36.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:36.928 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:36.970 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:37.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 11 15 23970 23955 136.096 98.0625 0.00471211 0.00734181 2023-09-23T21:18:37.640 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:37.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:37.733 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:37.744 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:37.802 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:37.897 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:37.902 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:37.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:37.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4145443968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:37.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[211526]: pgmap v1789: 89 pgs: 89 active+clean; 1.1 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T21:18:37.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:37.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:37.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4145443968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:37.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:37 smithi121 ceph-mon[219709]: pgmap v1789: 89 pgs: 89 active+clean; 1.1 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T21:18:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:37.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:37.910 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:18:37.914 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:38.005 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:38.038 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:38.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:37 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:38.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:37 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:38.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4145443968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:38.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:37 smithi123 ceph-mon[193903]: pgmap v1789: 89 pgs: 89 active+clean; 1.1 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T21:18:38.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 12 16 25801 25785 134.284 114.375 0.0122353 0.00744161 2023-09-23T21:18:38.830 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:18:38.831 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:18:39.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:39.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4145443968' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:39.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[211526]: osdmap e578: 8 total, 8 up, 5 in 2023-09-23T21:18:39.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[211526]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[211526]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.127 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[211526]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:39.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4145443968' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:39.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[219709]: osdmap e578: 8 total, 8 up, 5 in 2023-09-23T21:18:39.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[219709]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[219709]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.128 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:38 smithi121 ceph-mon[219709]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:38 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4145443968' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:38 smithi123 ceph-mon[193903]: osdmap e578: 8 total, 8 up, 5 in 2023-09-23T21:18:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:38 smithi123 ceph-mon[193903]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:38 smithi123 ceph-mon[193903]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:38 smithi123 ceph-mon[193903]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 13 16 27184 27168 130.603 86.4375 0.00737769 0.00765236 2023-09-23T21:18:39.704 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1790,"stamp":"2023-09-23T21:18:37.895494+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1210587040,"num_objects":18487,"num_object_clones":0,"num_object_copies":55461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18487,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":18763,"num_write_kb":1184077,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":5352230,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18679,"ondisk_log_size":18679,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8084212,"kb_used_data":3632332,"kb_used_omap":7,"kb_used_meta":4451768,"kb_avail":460641548,"statfs":{"total":479975178240,"available":471696945152,"internally_reserved":0,"allocated":3719507968,"data_stored":3710487568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":4558610513},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,3,11,6],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1209991168,"num_objects":18463,"num_object_clones":0,"num_object_copies":55389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18463,"num_write_kb":1181632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.528986"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":179,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609378+0000","last_change":"2023-09-23T21:14:26.676796+0000","last_active":"2023-09-23T21:18:23.609378+0000","last_peered":"2023-09-23T21:18:23.609378+0000","last_clean":"2023-09-23T21:18:23.609378+0000","last_became_active":"2023-09-23T21:14:26.676452+0000","last_became_peered":"2023-09-23T21:14:26.676452+0000","last_unstale":"2023-09-23T21:18:23.609378+0000","last_undegraded":"2023-09-23T21:18:23.609378+0000","last_fullsized":"2023-09-23T21:18:23.609378+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:36.913203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610840+0000","last_change":"2023-09-23T21:16:51.045558+0000","last_active":"2023-09-23T21:18:23.610840+0000","last_peered":"2023-09-23T21:18:23.610840+0000","last_clean":"2023-09-23T21:18:23.610840+0000","last_became_active":"2023-09-23T21:16:51.039791+0000","last_became_peered":"2023-09-23T21:16:51.039791+0000","last_unstale":"2023-09-23T21:18:23.610840+0000","last_undegraded":"2023-09-23T21:18:23.610840+0000","last_fullsized":"2023-09-23T21:18:23.610840+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:16.411198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":364,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609374+0000","last_change":"2023-09-23T21:15:11.713107+0000","last_active":"2023-09-23T21:18:23.609374+0000","last_peered":"2023-09-23T21:18:23.609374+0000","last_clean":"2023-09-23T21:18:23.609374+0000","last_became_active":"2023-09-23T21:15:11.712787+0000","last_became_peered":"2023-09-23T21:15:11.712787+0000","last_unstale":"2023-09-23T21:18:23.609374+0000","last_undegraded":"2023-09-23T21:18:23.609374+0000","last_fullsized":"2023-09-23T21:18:23.609374+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:41.516408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":224,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370828+0000","last_change":"2023-09-23T21:15:11.697718+0000","last_active":"2023-09-23T21:18:23.370828+0000","last_peered":"2023-09-23T21:18:23.370828+0000","last_clean":"2023-09-23T21:18:23.370828+0000","last_became_active":"2023-09-23T21:15:11.697430+0000","last_became_peered":"2023-09-23T21:15:11.697430+0000","last_unstale":"2023-09-23T21:18:23.370828+0000","last_undegraded":"2023-09-23T21:18:23.370828+0000","last_fullsized":"2023-09-23T21:18:23.370828+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:21.661820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609278+0000","last_change":"2023-09-23T21:15:11.975973+0000","last_active":"2023-09-23T21:18:23.609278+0000","last_peered":"2023-09-23T21:18:23.609278+0000","last_clean":"2023-09-23T21:18:23.609278+0000","last_became_active":"2023-09-23T21:15:11.975668+0000","last_became_peered":"2023-09-23T21:15:11.975668+0000","last_unstale":"2023-09-23T21:18:23.609278+0000","last_undegraded":"2023-09-23T21:18:23.609278+0000","last_fullsized":"2023-09-23T21:18:23.609278+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:31.362561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610499+0000","last_change":"2023-09-23T21:16:51.016443+0000","last_active":"2023-09-23T21:18:23.610499+0000","last_peered":"2023-09-23T21:18:23.610499+0000","last_clean":"2023-09-23T21:18:23.610499+0000","last_became_active":"2023-09-23T21:16:51.015926+0000","last_became_peered":"2023-09-23T21:16:51.015926+0000","last_unstale":"2023-09-23T21:18:23.610499+0000","last_undegraded":"2023-09-23T21:18:23.610499+0000","last_fullsized":"2023-09-23T21:18:23.610499+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:24.914704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":171,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371292+0000","last_change":"2023-09-23T21:16:51.027614+0000","last_active":"2023-09-23T21:18:23.371292+0000","last_peered":"2023-09-23T21:18:23.371292+0000","last_clean":"2023-09-23T21:18:23.371292+0000","last_became_active":"2023-09-23T21:16:51.027041+0000","last_became_peered":"2023-09-23T21:16:51.027041+0000","last_unstale":"2023-09-23T21:18:23.371292+0000","last_undegraded":"2023-09-23T21:18:23.371292+0000","last_fullsized":"2023-09-23T21:18:23.371292+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:50.561331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609286+0000","last_change":"2023-09-23T21:16:51.050348+0000","last_active":"2023-09-23T21:18:23.609286+0000","last_peered":"2023-09-23T21:18:23.609286+0000","last_clean":"2023-09-23T21:18:23.609286+0000","last_became_active":"2023-09-23T21:16:51.043835+0000","last_became_peered":"2023-09-23T21:16:51.043835+0000","last_unstale":"2023-09-23T21:18:23.609286+0000","last_undegraded":"2023-09-23T21:18:23.609286+0000","last_fullsized":"2023-09-23T21:18:23.609286+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:11.612746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370529+0000","last_change":"2023-09-23T21:15:11.704624+0000","last_active":"2023-09-23T21:18:23.370529+0000","last_peered":"2023-09-23T21:18:23.370529+0000","last_clean":"2023-09-23T21:18:23.370529+0000","last_became_active":"2023-09-23T21:15:11.704338+0000","last_became_peered":"2023-09-23T21:15:11.704338+0000","last_unstale":"2023-09-23T21:18:23.370529+0000","last_undegraded":"2023-09-23T21:18:23.370529+0000","last_fullsized":"2023-09-23T21:18:23.370529+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:28.106212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":199,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370226+0000","last_change":"2023-09-23T21:16:50.887221+0000","last_active":"2023-09-23T21:18:23.370226+0000","last_peered":"2023-09-23T21:18:23.370226+0000","last_clean":"2023-09-23T21:18:23.370226+0000","last_became_active":"2023-09-23T21:16:50.886883+0000","last_became_peered":"2023-09-23T21:16:50.886883+0000","last_unstale":"2023-09-23T21:18:23.370226+0000","last_undegraded":"2023-09-23T21:18:23.370226+0000","last_fullsized":"2023-09-23T21:18:23.370226+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:32.107824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":180,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371083+0000","last_change":"2023-09-23T21:17:11.237055+0000","last_active":"2023-09-23T21:18:23.371083+0000","last_peered":"2023-09-23T21:18:23.371083+0000","last_clean":"2023-09-23T21:18:23.371083+0000","last_became_active":"2023-09-23T21:16:50.974132+0000","last_became_peered":"2023-09-23T21:16:50.974132+0000","last_unstale":"2023-09-23T21:18:23.371083+0000","last_undegraded":"2023-09-23T21:18:23.371083+0000","last_fullsized":"2023-09-23T21:18:23.371083+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:43.439355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610328+0000","last_change":"2023-09-23T21:16:50.901436+0000","last_active":"2023-09-23T21:18:23.610328+0000","last_peered":"2023-09-23T21:18:23.610328+0000","last_clean":"2023-09-23T21:18:23.610328+0000","last_became_active":"2023-09-23T21:16:50.897093+0000","last_became_peered":"2023-09-23T21:16:50.897093+0000","last_unstale":"2023-09-23T21:18:23.610328+0000","last_undegraded":"2023-09-23T21:18:23.610328+0000","last_fullsized":"2023-09-23T21:18:23.610328+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:30.724334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":204,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370931+0000","last_change":"2023-09-23T21:16:51.074886+0000","last_active":"2023-09-23T21:18:23.370931+0000","last_peered":"2023-09-23T21:18:23.370931+0000","last_clean":"2023-09-23T21:18:23.370931+0000","last_became_active":"2023-09-23T21:16:50.863417+0000","last_became_peered":"2023-09-23T21:16:50.863417+0000","last_unstale":"2023-09-23T21:18:23.370931+0000","last_undegraded":"2023-09-23T21:18:23.370931+0000","last_fullsized":"2023-09-23T21:18:23.370931+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:16.305195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":505,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609667+0000","last_change":"2023-09-23T21:14:26.690041+0000","last_active":"2023-09-23T21:18:23.609667+0000","last_peered":"2023-09-23T21:18:23.609667+0000","last_clean":"2023-09-23T21:18:23.609667+0000","last_became_active":"2023-09-23T21:14:26.689490+0000","last_became_peered":"2023-09-23T21:14:26.689490+0000","last_unstale":"2023-09-23T21:18:23.609667+0000","last_undegraded":"2023-09-23T21:18:23.609667+0000","last_fullsized":"2023-09-23T21:18:23.609667+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:35.305121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":160,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371529+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:18:23.371529+0000","last_peered":"2023-09-23T21:18:23.371529+0000","last_clean":"2023-09-23T21:18:23.371529+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:18:23.371529+0000","last_undegraded":"2023-09-23T21:18:23.371529+0000","last_fullsized":"2023-09-23T21:18:23.371529+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":228,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370773+0000","last_change":"2023-09-23T21:15:11.697155+0000","last_active":"2023-09-23T21:18:23.370773+0000","last_peered":"2023-09-23T21:18:23.370773+0000","last_clean":"2023-09-23T21:18:23.370773+0000","last_became_active":"2023-09-23T21:15:11.696830+0000","last_became_peered":"2023-09-23T21:15:11.696830+0000","last_unstale":"2023-09-23T21:18:23.370773+0000","last_undegraded":"2023-09-23T21:18:23.370773+0000","last_fullsized":"2023-09-23T21:18:23.370773+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:28.152889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":156,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370496+0000","last_change":"2023-09-23T21:15:11.697537+0000","last_active":"2023-09-23T21:18:23.370496+0000","last_peered":"2023-09-23T21:18:23.370496+0000","last_clean":"2023-09-23T21:18:23.370496+0000","last_became_active":"2023-09-23T21:15:11.697190+0000","last_became_peered":"2023-09-23T21:15:11.697190+0000","last_unstale":"2023-09-23T21:18:23.370496+0000","last_undegraded":"2023-09-23T21:18:23.370496+0000","last_fullsized":"2023-09-23T21:18:23.370496+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:38.950414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":164,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371205+0000","last_change":"2023-09-23T21:15:11.700125+0000","last_active":"2023-09-23T21:18:23.371205+0000","last_peered":"2023-09-23T21:18:23.371205+0000","last_clean":"2023-09-23T21:18:23.371205+0000","last_became_active":"2023-09-23T21:15:11.699337+0000","last_became_peered":"2023-09-23T21:15:11.699337+0000","last_unstale":"2023-09-23T21:18:23.371205+0000","last_undegraded":"2023-09-23T21:18:23.371205+0000","last_fullsized":"2023-09-23T21:18:23.371205+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:32.592808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370362+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:18:23.370362+0000","last_peered":"2023-09-23T21:18:23.370362+0000","last_clean":"2023-09-23T21:18:23.370362+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:18:23.370362+0000","last_undegraded":"2023-09-23T21:18:23.370362+0000","last_fullsized":"2023-09-23T21:18:23.370362+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":178,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370751+0000","last_change":"2023-09-23T21:15:11.702284+0000","last_active":"2023-09-23T21:18:23.370751+0000","last_peered":"2023-09-23T21:18:23.370751+0000","last_clean":"2023-09-23T21:18:23.370751+0000","last_became_active":"2023-09-23T21:15:11.701962+0000","last_became_peered":"2023-09-23T21:15:11.701962+0000","last_unstale":"2023-09-23T21:18:23.370751+0000","last_undegraded":"2023-09-23T21:18:23.370751+0000","last_fullsized":"2023-09-23T21:18:23.370751+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:53.005225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":613,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610074+0000","last_change":"2023-09-23T21:15:12.158109+0000","last_active":"2023-09-23T21:18:23.610074+0000","last_peered":"2023-09-23T21:18:23.610074+0000","last_clean":"2023-09-23T21:18:23.610074+0000","last_became_active":"2023-09-23T21:15:11.980987+0000","last_became_peered":"2023-09-23T21:15:11.980987+0000","last_unstale":"2023-09-23T21:18:23.610074+0000","last_undegraded":"2023-09-23T21:18:23.610074+0000","last_fullsized":"2023-09-23T21:18:23.610074+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.536151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":228,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371239+0000","last_change":"2023-09-23T21:15:11.702245+0000","last_active":"2023-09-23T21:18:23.371239+0000","last_peered":"2023-09-23T21:18:23.371239+0000","last_clean":"2023-09-23T21:18:23.371239+0000","last_became_active":"2023-09-23T21:15:11.701893+0000","last_became_peered":"2023-09-23T21:15:11.701893+0000","last_unstale":"2023-09-23T21:18:23.371239+0000","last_undegraded":"2023-09-23T21:18:23.371239+0000","last_fullsized":"2023-09-23T21:18:23.371239+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:13.770378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.2","version":"577'1424","reported_seq":1436,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:36.270422+0000","last_change":"2023-09-23T21:18:18.461141+0000","last_active":"2023-09-23T21:18:36.270422+0000","last_peered":"2023-09-23T21:18:36.270422+0000","last_clean":"2023-09-23T21:18:36.270422+0000","last_became_active":"2023-09-23T21:18:18.460862+0000","last_became_peered":"2023-09-23T21:18:18.460862+0000","last_unstale":"2023-09-23T21:18:36.270422+0000","last_undegraded":"2023-09-23T21:18:36.270422+0000","last_fullsized":"2023-09-23T21:18:36.270422+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1424,"log_dups_size":0,"ondisk_log_size":1424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:32.796044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93323264,"num_objects":1424,"num_object_clones":0,"num_object_copies":4272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1424,"num_write_kb":91136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":158,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371335+0000","last_change":"2023-09-23T21:14:26.813323+0000","last_active":"2023-09-23T21:18:23.371335+0000","last_peered":"2023-09-23T21:18:23.371335+0000","last_clean":"2023-09-23T21:18:23.371335+0000","last_became_active":"2023-09-23T21:14:26.689591+0000","last_became_peered":"2023-09-23T21:14:26.689591+0000","last_unstale":"2023-09-23T21:18:23.371335+0000","last_undegraded":"2023-09-23T21:18:23.371335+0000","last_fullsized":"2023-09-23T21:18:23.371335+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:47.449684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.369838+0000","last_change":"2023-09-23T21:16:50.852036+0000","last_active":"2023-09-23T21:18:23.369838+0000","last_peered":"2023-09-23T21:18:23.369838+0000","last_clean":"2023-09-23T21:18:23.369838+0000","last_became_active":"2023-09-23T21:16:50.851456+0000","last_became_peered":"2023-09-23T21:16:50.851456+0000","last_unstale":"2023-09-23T21:18:23.369838+0000","last_undegraded":"2023-09-23T21:18:23.369838+0000","last_fullsized":"2023-09-23T21:18:23.369838+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:55.506494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"577'1352","reported_seq":1362,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:36.265771+0000","last_change":"2023-09-23T21:18:18.461922+0000","last_active":"2023-09-23T21:18:36.265771+0000","last_peered":"2023-09-23T21:18:36.265771+0000","last_clean":"2023-09-23T21:18:36.265771+0000","last_became_active":"2023-09-23T21:18:18.461627+0000","last_became_peered":"2023-09-23T21:18:18.461627+0000","last_unstale":"2023-09-23T21:18:36.265771+0000","last_undegraded":"2023-09-23T21:18:36.265771+0000","last_fullsized":"2023-09-23T21:18:36.265771+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1352,"log_dups_size":0,"ondisk_log_size":1352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:17.785201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88604672,"num_objects":1352,"num_object_clones":0,"num_object_copies":4056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1352,"num_write_kb":86528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":172,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370917+0000","last_change":"2023-09-23T21:14:26.816105+0000","last_active":"2023-09-23T21:18:23.370917+0000","last_peered":"2023-09-23T21:18:23.370917+0000","last_clean":"2023-09-23T21:18:23.370917+0000","last_became_active":"2023-09-23T21:14:26.691490+0000","last_became_peered":"2023-09-23T21:14:26.691490+0000","last_unstale":"2023-09-23T21:18:23.370917+0000","last_undegraded":"2023-09-23T21:18:23.370917+0000","last_fullsized":"2023-09-23T21:18:23.370917+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:23.095470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.611040+0000","last_change":"2023-09-23T21:16:50.869061+0000","last_active":"2023-09-23T21:18:23.611040+0000","last_peered":"2023-09-23T21:18:23.611040+0000","last_clean":"2023-09-23T21:18:23.611040+0000","last_became_active":"2023-09-23T21:16:50.868604+0000","last_became_peered":"2023-09-23T21:16:50.868604+0000","last_unstale":"2023-09-23T21:18:23.611040+0000","last_undegraded":"2023-09-23T21:18:23.611040+0000","last_fullsized":"2023-09-23T21:18:23.611040+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:40.662118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"577'1055","reported_seq":1068,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.090305+0000","last_change":"2023-09-23T21:18:18.458070+0000","last_active":"2023-09-23T21:18:33.090305+0000","last_peered":"2023-09-23T21:18:33.090305+0000","last_clean":"2023-09-23T21:18:33.090305+0000","last_became_active":"2023-09-23T21:18:18.457766+0000","last_became_peered":"2023-09-23T21:18:18.457766+0000","last_unstale":"2023-09-23T21:18:33.090305+0000","last_undegraded":"2023-09-23T21:18:33.090305+0000","last_fullsized":"2023-09-23T21:18:33.090305+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1055,"log_dups_size":0,"ondisk_log_size":1055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:50.906117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1055,"num_write_kb":67520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":158,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371250+0000","last_change":"2023-09-23T21:10:15.587348+0000","last_active":"2023-09-23T21:18:23.371250+0000","last_peered":"2023-09-23T21:18:23.371250+0000","last_clean":"2023-09-23T21:18:23.371250+0000","last_became_active":"2023-09-23T21:10:15.579164+0000","last_became_peered":"2023-09-23T21:10:15.579164+0000","last_unstale":"2023-09-23T21:18:23.371250+0000","last_undegraded":"2023-09-23T21:18:23.371250+0000","last_fullsized":"2023-09-23T21:18:23.371250+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:47.861254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610647+0000","last_change":"2023-09-23T21:16:50.912921+0000","last_active":"2023-09-23T21:18:23.610647+0000","last_peered":"2023-09-23T21:18:23.610647+0000","last_clean":"2023-09-23T21:18:23.610647+0000","last_became_active":"2023-09-23T21:16:50.900656+0000","last_became_peered":"2023-09-23T21:16:50.900656+0000","last_unstale":"2023-09-23T21:18:23.610647+0000","last_undegraded":"2023-09-23T21:18:23.610647+0000","last_fullsized":"2023-09-23T21:18:23.610647+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:09.750909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"577'1018","reported_seq":1031,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.064257+0000","last_change":"2023-09-23T21:18:18.453879+0000","last_active":"2023-09-23T21:18:33.064257+0000","last_peered":"2023-09-23T21:18:33.064257+0000","last_clean":"2023-09-23T21:18:33.064257+0000","last_became_active":"2023-09-23T21:18:18.453619+0000","last_became_peered":"2023-09-23T21:18:18.453619+0000","last_unstale":"2023-09-23T21:18:33.064257+0000","last_undegraded":"2023-09-23T21:18:33.064257+0000","last_fullsized":"2023-09-23T21:18:33.064257+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1018,"log_dups_size":0,"ondisk_log_size":1018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:41.497559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66715648,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1018,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610839+0000","last_change":"2023-09-23T21:16:51.045338+0000","last_active":"2023-09-23T21:18:23.610839+0000","last_peered":"2023-09-23T21:18:23.610839+0000","last_clean":"2023-09-23T21:18:23.610839+0000","last_became_active":"2023-09-23T21:16:51.044428+0000","last_became_peered":"2023-09-23T21:16:51.044428+0000","last_unstale":"2023-09-23T21:18:23.610839+0000","last_undegraded":"2023-09-23T21:18:23.610839+0000","last_fullsized":"2023-09-23T21:18:23.610839+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:50.447405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610302+0000","last_change":"2023-09-23T21:16:50.895016+0000","last_active":"2023-09-23T21:18:23.610302+0000","last_peered":"2023-09-23T21:18:23.610302+0000","last_clean":"2023-09-23T21:18:23.610302+0000","last_became_active":"2023-09-23T21:16:50.894279+0000","last_became_peered":"2023-09-23T21:16:50.894279+0000","last_unstale":"2023-09-23T21:18:23.610302+0000","last_undegraded":"2023-09-23T21:18:23.610302+0000","last_fullsized":"2023-09-23T21:18:23.610302+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:26.092339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"577'1057","reported_seq":1070,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.075218+0000","last_change":"2023-09-23T21:18:18.453978+0000","last_active":"2023-09-23T21:18:33.075218+0000","last_peered":"2023-09-23T21:18:33.075218+0000","last_clean":"2023-09-23T21:18:33.075218+0000","last_became_active":"2023-09-23T21:18:18.453696+0000","last_became_peered":"2023-09-23T21:18:18.453696+0000","last_unstale":"2023-09-23T21:18:33.075218+0000","last_undegraded":"2023-09-23T21:18:33.075218+0000","last_fullsized":"2023-09-23T21:18:33.075218+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1057,"log_dups_size":0,"ondisk_log_size":1057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:03.250012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69271552,"num_objects":1057,"num_object_clones":0,"num_object_copies":3171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1057,"num_write_kb":67648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":244,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371454+0000","last_change":"2023-09-23T21:15:11.702815+0000","last_active":"2023-09-23T21:18:23.371454+0000","last_peered":"2023-09-23T21:18:23.371454+0000","last_clean":"2023-09-23T21:18:23.371454+0000","last_became_active":"2023-09-23T21:15:11.691115+0000","last_became_peered":"2023-09-23T21:15:11.691115+0000","last_unstale":"2023-09-23T21:18:23.371454+0000","last_undegraded":"2023-09-23T21:18:23.371454+0000","last_fullsized":"2023-09-23T21:18:23.371454+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:42.023008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610921+0000","last_change":"2023-09-23T21:16:51.045560+0000","last_active":"2023-09-23T21:18:23.610921+0000","last_peered":"2023-09-23T21:18:23.610921+0000","last_clean":"2023-09-23T21:18:23.610921+0000","last_became_active":"2023-09-23T21:16:51.044840+0000","last_became_peered":"2023-09-23T21:16:51.044840+0000","last_unstale":"2023-09-23T21:18:23.610921+0000","last_undegraded":"2023-09-23T21:18:23.610921+0000","last_fullsized":"2023-09-23T21:18:23.610921+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:32.627870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"577'1114","reported_seq":1126,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.577808+0000","last_change":"2023-09-23T21:18:18.457504+0000","last_active":"2023-09-23T21:18:33.577808+0000","last_peered":"2023-09-23T21:18:33.577808+0000","last_clean":"2023-09-23T21:18:33.577808+0000","last_became_active":"2023-09-23T21:18:18.456595+0000","last_became_peered":"2023-09-23T21:18:18.456595+0000","last_unstale":"2023-09-23T21:18:33.577808+0000","last_undegraded":"2023-09-23T21:18:33.577808+0000","last_fullsized":"2023-09-23T21:18:33.577808+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1114,"log_dups_size":0,"ondisk_log_size":1114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:59.367182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73007104,"num_objects":1114,"num_object_clones":0,"num_object_copies":3342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1114,"num_write_kb":71296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610423+0000","last_change":"2023-09-23T21:16:50.900974+0000","last_active":"2023-09-23T21:18:23.610423+0000","last_peered":"2023-09-23T21:18:23.610423+0000","last_clean":"2023-09-23T21:18:23.610423+0000","last_became_active":"2023-09-23T21:16:50.896664+0000","last_became_peered":"2023-09-23T21:16:50.896664+0000","last_unstale":"2023-09-23T21:18:23.610423+0000","last_undegraded":"2023-09-23T21:18:23.610423+0000","last_fullsized":"2023-09-23T21:18:23.610423+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:55.094439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":619,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609817+0000","last_change":"2023-09-23T21:16:50.974705+0000","last_active":"2023-09-23T21:18:23.609817+0000","last_peered":"2023-09-23T21:18:23.609817+0000","last_clean":"2023-09-23T21:18:23.609817+0000","last_became_active":"2023-09-23T21:16:50.974417+0000","last_became_peered":"2023-09-23T21:16:50.974417+0000","last_unstale":"2023-09-23T21:18:23.609817+0000","last_undegraded":"2023-09-23T21:18:23.609817+0000","last_fullsized":"2023-09-23T21:18:23.609817+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:41.603340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"577'1389","reported_seq":1401,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:36.262378+0000","last_change":"2023-09-23T21:18:18.460279+0000","last_active":"2023-09-23T21:18:36.262378+0000","last_peered":"2023-09-23T21:18:36.262378+0000","last_clean":"2023-09-23T21:18:36.262378+0000","last_became_active":"2023-09-23T21:18:18.459980+0000","last_became_peered":"2023-09-23T21:18:18.459980+0000","last_unstale":"2023-09-23T21:18:36.262378+0000","last_undegraded":"2023-09-23T21:18:36.262378+0000","last_fullsized":"2023-09-23T21:18:36.262378+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1389,"log_dups_size":0,"ondisk_log_size":1389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:13.583349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91029504,"num_objects":1389,"num_object_clones":0,"num_object_copies":4167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1389,"num_write_kb":88896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":214,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609531+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:18:23.609531+0000","last_peered":"2023-09-23T21:18:23.609531+0000","last_clean":"2023-09-23T21:18:23.609531+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:18:23.609531+0000","last_undegraded":"2023-09-23T21:18:23.609531+0000","last_fullsized":"2023-09-23T21:18:23.609531+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610594+0000","last_change":"2023-09-23T21:16:50.893927+0000","last_active":"2023-09-23T21:18:23.610594+0000","last_peered":"2023-09-23T21:18:23.610594+0000","last_clean":"2023-09-23T21:18:23.610594+0000","last_became_active":"2023-09-23T21:16:50.893600+0000","last_became_peered":"2023-09-23T21:16:50.893600+0000","last_unstale":"2023-09-23T21:18:23.610594+0000","last_undegraded":"2023-09-23T21:18:23.610594+0000","last_fullsized":"2023-09-23T21:18:23.610594+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:00.375853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"577'1140","reported_seq":1152,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.532690+0000","last_change":"2023-09-23T21:18:18.457562+0000","last_active":"2023-09-23T21:18:33.532690+0000","last_peered":"2023-09-23T21:18:33.532690+0000","last_clean":"2023-09-23T21:18:33.532690+0000","last_became_active":"2023-09-23T21:18:18.457231+0000","last_became_peered":"2023-09-23T21:18:18.457231+0000","last_unstale":"2023-09-23T21:18:33.532690+0000","last_undegraded":"2023-09-23T21:18:33.532690+0000","last_fullsized":"2023-09-23T21:18:33.532690+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1140,"log_dups_size":0,"ondisk_log_size":1140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:26.541539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74711040,"num_objects":1140,"num_object_clones":0,"num_object_copies":3420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1140,"num_write_kb":72960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":228,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370932+0000","last_change":"2023-09-23T21:15:11.698069+0000","last_active":"2023-09-23T21:18:23.370932+0000","last_peered":"2023-09-23T21:18:23.370932+0000","last_clean":"2023-09-23T21:18:23.370932+0000","last_became_active":"2023-09-23T21:15:11.697644+0000","last_became_peered":"2023-09-23T21:15:11.697644+0000","last_unstale":"2023-09-23T21:18:23.370932+0000","last_undegraded":"2023-09-23T21:18:23.370932+0000","last_fullsized":"2023-09-23T21:18:23.370932+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:28.405815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":316,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370961+0000","last_change":"2023-09-23T21:15:11.697982+0000","last_active":"2023-09-23T21:18:23.370961+0000","last_peered":"2023-09-23T21:18:23.370961+0000","last_clean":"2023-09-23T21:18:23.370961+0000","last_became_active":"2023-09-23T21:15:11.697488+0000","last_became_peered":"2023-09-23T21:15:11.697488+0000","last_unstale":"2023-09-23T21:18:23.370961+0000","last_undegraded":"2023-09-23T21:18:23.370961+0000","last_fullsized":"2023-09-23T21:18:23.370961+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:49.086145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.8","version":"577'1058","reported_seq":1071,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.068631+0000","last_change":"2023-09-23T21:18:18.458504+0000","last_active":"2023-09-23T21:18:33.068631+0000","last_peered":"2023-09-23T21:18:33.068631+0000","last_clean":"2023-09-23T21:18:33.068631+0000","last_became_active":"2023-09-23T21:18:18.458198+0000","last_became_peered":"2023-09-23T21:18:18.458198+0000","last_unstale":"2023-09-23T21:18:33.068631+0000","last_undegraded":"2023-09-23T21:18:33.068631+0000","last_fullsized":"2023-09-23T21:18:33.068631+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1058,"log_dups_size":0,"ondisk_log_size":1058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:41.150311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69337088,"num_objects":1058,"num_object_clones":0,"num_object_copies":3174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1058,"num_write_kb":67712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":501,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370852+0000","last_change":"2023-09-23T21:15:11.688770+0000","last_active":"2023-09-23T21:18:23.370852+0000","last_peered":"2023-09-23T21:18:23.370852+0000","last_clean":"2023-09-23T21:18:23.370852+0000","last_became_active":"2023-09-23T21:15:11.688443+0000","last_became_peered":"2023-09-23T21:15:11.688443+0000","last_unstale":"2023-09-23T21:18:23.370852+0000","last_undegraded":"2023-09-23T21:18:23.370852+0000","last_fullsized":"2023-09-23T21:18:23.370852+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:19.576995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":252,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370307+0000","last_change":"2023-09-23T21:15:11.704578+0000","last_active":"2023-09-23T21:18:23.370307+0000","last_peered":"2023-09-23T21:18:23.370307+0000","last_clean":"2023-09-23T21:18:23.370307+0000","last_became_active":"2023-09-23T21:15:11.703742+0000","last_became_peered":"2023-09-23T21:15:11.703742+0000","last_unstale":"2023-09-23T21:18:23.370307+0000","last_undegraded":"2023-09-23T21:18:23.370307+0000","last_fullsized":"2023-09-23T21:18:23.370307+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.622443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":126,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370422+0000","last_change":"2023-09-23T21:14:26.692363+0000","last_active":"2023-09-23T21:18:23.370422+0000","last_peered":"2023-09-23T21:18:23.370422+0000","last_clean":"2023-09-23T21:18:23.370422+0000","last_became_active":"2023-09-23T21:14:26.692065+0000","last_became_peered":"2023-09-23T21:14:26.692065+0000","last_unstale":"2023-09-23T21:18:23.370422+0000","last_undegraded":"2023-09-23T21:18:23.370422+0000","last_fullsized":"2023-09-23T21:18:23.370422+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:43.037332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"577'1065","reported_seq":1077,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.543061+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:18:33.543061+0000","last_peered":"2023-09-23T21:18:33.543061+0000","last_clean":"2023-09-23T21:18:33.543061+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:18:33.543061+0000","last_undegraded":"2023-09-23T21:18:33.543061+0000","last_fullsized":"2023-09-23T21:18:33.543061+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1065,"log_dups_size":0,"ondisk_log_size":1065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69795840,"num_objects":1065,"num_object_clones":0,"num_object_copies":3195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1065,"num_write_kb":68160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610306+0000","last_change":"2023-09-23T21:16:50.913216+0000","last_active":"2023-09-23T21:18:23.610306+0000","last_peered":"2023-09-23T21:18:23.610306+0000","last_clean":"2023-09-23T21:18:23.610306+0000","last_became_active":"2023-09-23T21:16:50.900188+0000","last_became_peered":"2023-09-23T21:16:50.900188+0000","last_unstale":"2023-09-23T21:18:23.610306+0000","last_undegraded":"2023-09-23T21:18:23.610306+0000","last_fullsized":"2023-09-23T21:18:23.610306+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:20.572937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610383+0000","last_change":"2023-09-23T21:16:50.895186+0000","last_active":"2023-09-23T21:18:23.610383+0000","last_peered":"2023-09-23T21:18:23.610383+0000","last_clean":"2023-09-23T21:18:23.610383+0000","last_became_active":"2023-09-23T21:16:50.894701+0000","last_became_peered":"2023-09-23T21:16:50.894701+0000","last_unstale":"2023-09-23T21:18:23.610383+0000","last_undegraded":"2023-09-23T21:18:23.610383+0000","last_fullsized":"2023-09-23T21:18:23.610383+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:55.539982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":291,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609579+0000","last_change":"2023-09-23T21:16:51.062070+0000","last_active":"2023-09-23T21:18:23.609579+0000","last_peered":"2023-09-23T21:18:23.609579+0000","last_clean":"2023-09-23T21:18:23.609579+0000","last_became_active":"2023-09-23T21:16:51.050724+0000","last_became_peered":"2023-09-23T21:16:51.050724+0000","last_unstale":"2023-09-23T21:18:23.609579+0000","last_undegraded":"2023-09-23T21:18:23.609579+0000","last_fullsized":"2023-09-23T21:18:23.609579+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:09.413391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"577'1342","reported_seq":1354,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:36.221504+0000","last_change":"2023-09-23T21:18:18.460402+0000","last_active":"2023-09-23T21:18:36.221504+0000","last_peered":"2023-09-23T21:18:36.221504+0000","last_clean":"2023-09-23T21:18:36.221504+0000","last_became_active":"2023-09-23T21:18:18.459784+0000","last_became_peered":"2023-09-23T21:18:18.459784+0000","last_unstale":"2023-09-23T21:18:36.221504+0000","last_undegraded":"2023-09-23T21:18:36.221504+0000","last_fullsized":"2023-09-23T21:18:36.221504+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1342,"log_dups_size":0,"ondisk_log_size":1342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:40.883508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87949312,"num_objects":1342,"num_object_clones":0,"num_object_copies":4026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1342,"num_write_kb":85888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":586,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609763+0000","last_change":"2023-09-23T21:15:11.714901+0000","last_active":"2023-09-23T21:18:23.609763+0000","last_peered":"2023-09-23T21:18:23.609763+0000","last_clean":"2023-09-23T21:18:23.609763+0000","last_became_active":"2023-09-23T21:15:11.714477+0000","last_became_peered":"2023-09-23T21:15:11.714477+0000","last_unstale":"2023-09-23T21:18:23.609763+0000","last_undegraded":"2023-09-23T21:18:23.609763+0000","last_fullsized":"2023-09-23T21:18:23.609763+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:48.643744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":324,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.424956+0000","last_change":"2023-09-23T21:16:50.847594+0000","last_active":"2023-09-23T21:18:26.424956+0000","last_peered":"2023-09-23T21:18:26.424956+0000","last_clean":"2023-09-23T21:18:26.424956+0000","last_became_active":"2023-09-23T21:16:50.846880+0000","last_became_peered":"2023-09-23T21:16:50.846880+0000","last_unstale":"2023-09-23T21:18:26.424956+0000","last_undegraded":"2023-09-23T21:18:26.424956+0000","last_fullsized":"2023-09-23T21:18:26.424956+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:38.525065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371042+0000","last_change":"2023-09-23T21:15:11.697794+0000","last_active":"2023-09-23T21:18:23.371042+0000","last_peered":"2023-09-23T21:18:23.371042+0000","last_clean":"2023-09-23T21:18:23.371042+0000","last_became_active":"2023-09-23T21:15:11.697029+0000","last_became_peered":"2023-09-23T21:15:11.697029+0000","last_unstale":"2023-09-23T21:18:23.371042+0000","last_undegraded":"2023-09-23T21:18:23.371042+0000","last_fullsized":"2023-09-23T21:18:23.371042+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:44.776867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.f","version":"577'1119","reported_seq":1131,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.472994+0000","last_change":"2023-09-23T21:18:18.457700+0000","last_active":"2023-09-23T21:18:33.472994+0000","last_peered":"2023-09-23T21:18:33.472994+0000","last_clean":"2023-09-23T21:18:33.472994+0000","last_became_active":"2023-09-23T21:18:18.457261+0000","last_became_peered":"2023-09-23T21:18:18.457261+0000","last_unstale":"2023-09-23T21:18:33.472994+0000","last_undegraded":"2023-09-23T21:18:33.472994+0000","last_fullsized":"2023-09-23T21:18:33.472994+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1119,"log_dups_size":0,"ondisk_log_size":1119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:52.984711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73334784,"num_objects":1119,"num_object_clones":0,"num_object_copies":3357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1119,"num_write_kb":71616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":406,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609867+0000","last_change":"2023-09-23T21:10:15.586382+0000","last_active":"2023-09-23T21:18:23.609867+0000","last_peered":"2023-09-23T21:18:23.609867+0000","last_clean":"2023-09-23T21:18:23.609867+0000","last_became_active":"2023-09-23T21:10:15.586013+0000","last_became_peered":"2023-09-23T21:10:15.586013+0000","last_unstale":"2023-09-23T21:18:23.609867+0000","last_undegraded":"2023-09-23T21:18:23.609867+0000","last_fullsized":"2023-09-23T21:18:23.609867+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:04.207790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":923,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370762+0000","last_change":"2023-09-23T21:16:51.805778+0000","last_active":"2023-09-23T21:18:23.370762+0000","last_peered":"2023-09-23T21:18:23.370762+0000","last_clean":"2023-09-23T21:18:23.370762+0000","last_became_active":"2023-09-23T21:16:51.805423+0000","last_became_peered":"2023-09-23T21:16:51.805423+0000","last_unstale":"2023-09-23T21:18:23.370762+0000","last_undegraded":"2023-09-23T21:18:23.370762+0000","last_fullsized":"2023-09-23T21:18:23.370762+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:12.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":149,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370150+0000","last_change":"2023-09-23T21:15:11.710191+0000","last_active":"2023-09-23T21:18:23.370150+0000","last_peered":"2023-09-23T21:18:23.370150+0000","last_clean":"2023-09-23T21:18:23.370150+0000","last_became_active":"2023-09-23T21:15:11.709899+0000","last_became_peered":"2023-09-23T21:15:11.709899+0000","last_unstale":"2023-09-23T21:18:23.370150+0000","last_undegraded":"2023-09-23T21:18:23.370150+0000","last_fullsized":"2023-09-23T21:18:23.370150+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:17.084005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610672+0000","last_change":"2023-09-23T21:16:51.045766+0000","last_active":"2023-09-23T21:18:23.610672+0000","last_peered":"2023-09-23T21:18:23.610672+0000","last_clean":"2023-09-23T21:18:23.610672+0000","last_became_active":"2023-09-23T21:16:51.040379+0000","last_became_peered":"2023-09-23T21:16:51.040379+0000","last_unstale":"2023-09-23T21:18:23.610672+0000","last_undegraded":"2023-09-23T21:18:23.610672+0000","last_fullsized":"2023-09-23T21:18:23.610672+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:30.180934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"577'1103","reported_seq":1114,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.556040+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:18:33.556040+0000","last_peered":"2023-09-23T21:18:33.556040+0000","last_clean":"2023-09-23T21:18:33.556040+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:18:33.556040+0000","last_undegraded":"2023-09-23T21:18:33.556040+0000","last_fullsized":"2023-09-23T21:18:33.556040+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1103,"log_dups_size":0,"ondisk_log_size":1103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72286208,"num_objects":1103,"num_object_clones":0,"num_object_copies":3309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1103,"num_write_kb":70592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":204,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370845+0000","last_change":"2023-09-23T21:16:50.810705+0000","last_active":"2023-09-23T21:18:23.370845+0000","last_peered":"2023-09-23T21:18:23.370845+0000","last_clean":"2023-09-23T21:18:23.370845+0000","last_became_active":"2023-09-23T21:16:50.808807+0000","last_became_peered":"2023-09-23T21:16:50.808807+0000","last_unstale":"2023-09-23T21:18:23.370845+0000","last_undegraded":"2023-09-23T21:18:23.370845+0000","last_fullsized":"2023-09-23T21:18:23.370845+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:50.217739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":73,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370142+0000","last_change":"2023-09-23T21:15:11.697221+0000","last_active":"2023-09-23T21:18:23.370142+0000","last_peered":"2023-09-23T21:18:23.370142+0000","last_clean":"2023-09-23T21:18:23.370142+0000","last_became_active":"2023-09-23T21:15:11.696469+0000","last_became_peered":"2023-09-23T21:15:11.696469+0000","last_unstale":"2023-09-23T21:18:23.370142+0000","last_undegraded":"2023-09-23T21:18:23.370142+0000","last_fullsized":"2023-09-23T21:18:23.370142+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:19.441180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":486,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609700+0000","last_change":"2023-09-23T21:14:26.690171+0000","last_active":"2023-09-23T21:18:23.609700+0000","last_peered":"2023-09-23T21:18:23.609700+0000","last_clean":"2023-09-23T21:18:23.609700+0000","last_became_active":"2023-09-23T21:14:26.689772+0000","last_became_peered":"2023-09-23T21:14:26.689772+0000","last_unstale":"2023-09-23T21:18:23.609700+0000","last_undegraded":"2023-09-23T21:18:23.609700+0000","last_fullsized":"2023-09-23T21:18:23.609700+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.047353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"577'1096","reported_seq":1108,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.559766+0000","last_change":"2023-09-23T21:18:18.454773+0000","last_active":"2023-09-23T21:18:33.559766+0000","last_peered":"2023-09-23T21:18:33.559766+0000","last_clean":"2023-09-23T21:18:33.559766+0000","last_became_active":"2023-09-23T21:18:18.454490+0000","last_became_peered":"2023-09-23T21:18:18.454490+0000","last_unstale":"2023-09-23T21:18:33.559766+0000","last_undegraded":"2023-09-23T21:18:33.559766+0000","last_fullsized":"2023-09-23T21:18:33.559766+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1096,"log_dups_size":0,"ondisk_log_size":1096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:37.641550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71827456,"num_objects":1096,"num_object_clones":0,"num_object_copies":3288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1096,"num_write_kb":70144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":174,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371163+0000","last_change":"2023-09-23T21:14:26.689830+0000","last_active":"2023-09-23T21:18:23.371163+0000","last_peered":"2023-09-23T21:18:23.371163+0000","last_clean":"2023-09-23T21:18:23.371163+0000","last_became_active":"2023-09-23T21:14:26.689563+0000","last_became_peered":"2023-09-23T21:14:26.689563+0000","last_unstale":"2023-09-23T21:18:23.371163+0000","last_undegraded":"2023-09-23T21:18:23.371163+0000","last_fullsized":"2023-09-23T21:18:23.371163+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:35.354265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":390,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609628+0000","last_change":"2023-09-23T21:16:51.062259+0000","last_active":"2023-09-23T21:18:23.609628+0000","last_peered":"2023-09-23T21:18:23.609628+0000","last_clean":"2023-09-23T21:18:23.609628+0000","last_became_active":"2023-09-23T21:16:51.051186+0000","last_became_peered":"2023-09-23T21:16:51.051186+0000","last_unstale":"2023-09-23T21:18:23.609628+0000","last_undegraded":"2023-09-23T21:18:23.609628+0000","last_fullsized":"2023-09-23T21:18:23.609628+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:01.408765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":282,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370554+0000","last_change":"2023-09-23T21:14:26.684764+0000","last_active":"2023-09-23T21:18:23.370554+0000","last_peered":"2023-09-23T21:18:23.370554+0000","last_clean":"2023-09-23T21:18:23.370554+0000","last_became_active":"2023-09-23T21:14:26.684488+0000","last_became_peered":"2023-09-23T21:14:26.684488+0000","last_unstale":"2023-09-23T21:18:23.370554+0000","last_undegraded":"2023-09-23T21:18:23.370554+0000","last_fullsized":"2023-09-23T21:18:23.370554+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:32.411427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"577'1074","reported_seq":1086,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.547291+0000","last_change":"2023-09-23T21:18:18.459893+0000","last_active":"2023-09-23T21:18:33.547291+0000","last_peered":"2023-09-23T21:18:33.547291+0000","last_clean":"2023-09-23T21:18:33.547291+0000","last_became_active":"2023-09-23T21:18:18.459600+0000","last_became_peered":"2023-09-23T21:18:18.459600+0000","last_unstale":"2023-09-23T21:18:33.547291+0000","last_undegraded":"2023-09-23T21:18:33.547291+0000","last_fullsized":"2023-09-23T21:18:33.547291+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1074,"log_dups_size":0,"ondisk_log_size":1074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:01.769949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70385664,"num_objects":1074,"num_object_clones":0,"num_object_copies":3222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1074,"num_write_kb":68736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":232,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371437+0000","last_change":"2023-09-23T21:15:11.716522+0000","last_active":"2023-09-23T21:18:23.371437+0000","last_peered":"2023-09-23T21:18:23.371437+0000","last_clean":"2023-09-23T21:18:23.371437+0000","last_became_active":"2023-09-23T21:15:11.697736+0000","last_became_peered":"2023-09-23T21:15:11.697736+0000","last_unstale":"2023-09-23T21:18:23.371437+0000","last_undegraded":"2023-09-23T21:18:23.371437+0000","last_fullsized":"2023-09-23T21:18:23.371437+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:23.790154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610517+0000","last_change":"2023-09-23T21:16:50.872563+0000","last_active":"2023-09-23T21:18:23.610517+0000","last_peered":"2023-09-23T21:18:23.610517+0000","last_clean":"2023-09-23T21:18:23.610517+0000","last_became_active":"2023-09-23T21:16:50.872087+0000","last_became_peered":"2023-09-23T21:16:50.872087+0000","last_unstale":"2023-09-23T21:18:23.610517+0000","last_undegraded":"2023-09-23T21:18:23.610517+0000","last_fullsized":"2023-09-23T21:18:23.610517+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:13.888330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":257,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.370673+0000","last_change":"2023-09-23T21:16:50.810517+0000","last_active":"2023-09-23T21:18:23.370673+0000","last_peered":"2023-09-23T21:18:23.370673+0000","last_clean":"2023-09-23T21:18:23.370673+0000","last_became_active":"2023-09-23T21:16:50.808423+0000","last_became_peered":"2023-09-23T21:16:50.808423+0000","last_unstale":"2023-09-23T21:18:23.370673+0000","last_undegraded":"2023-09-23T21:18:23.370673+0000","last_fullsized":"2023-09-23T21:18:23.370673+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:08.974959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"577'1057","reported_seq":1070,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:33.078184+0000","last_change":"2023-09-23T21:18:18.456335+0000","last_active":"2023-09-23T21:18:33.078184+0000","last_peered":"2023-09-23T21:18:33.078184+0000","last_clean":"2023-09-23T21:18:33.078184+0000","last_became_active":"2023-09-23T21:18:18.455992+0000","last_became_peered":"2023-09-23T21:18:18.455992+0000","last_unstale":"2023-09-23T21:18:33.078184+0000","last_undegraded":"2023-09-23T21:18:33.078184+0000","last_fullsized":"2023-09-23T21:18:33.078184+0000","mapping_epoch":573,"log_start":"0'0","ondisk_log_start":"0'0","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":1057,"log_dups_size":0,"ondisk_log_size":1057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:00.021252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69271552,"num_objects":1057,"num_object_clones":0,"num_object_copies":3171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1057,"num_write_kb":67648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":483,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610574+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:18:23.610574+0000","last_peered":"2023-09-23T21:18:23.610574+0000","last_clean":"2023-09-23T21:18:23.610574+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:18:23.610574+0000","last_undegraded":"2023-09-23T21:18:23.610574+0000","last_fullsized":"2023-09-23T21:18:23.610574+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":114,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610199+0000","last_change":"2023-09-23T21:14:26.814871+0000","last_active":"2023-09-23T21:18:23.610199+0000","last_peered":"2023-09-23T21:18:23.610199+0000","last_clean":"2023-09-23T21:18:23.610199+0000","last_became_active":"2023-09-23T21:14:26.690057+0000","last_became_peered":"2023-09-23T21:14:26.690057+0000","last_unstale":"2023-09-23T21:18:23.610199+0000","last_undegraded":"2023-09-23T21:18:23.610199+0000","last_fullsized":"2023-09-23T21:18:23.610199+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:43.059406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":262,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.369807+0000","last_change":"2023-09-23T21:05:24.116212+0000","last_active":"2023-09-23T21:18:23.369807+0000","last_peered":"2023-09-23T21:18:23.369807+0000","last_clean":"2023-09-23T21:18:23.369807+0000","last_became_active":"2023-09-23T21:05:24.115877+0000","last_became_peered":"2023-09-23T21:05:24.115877+0000","last_unstale":"2023-09-23T21:18:23.369807+0000","last_undegraded":"2023-09-23T21:18:23.369807+0000","last_fullsized":"2023-09-23T21:18:23.369807+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:34.427549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":204,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.371574+0000","last_change":"2023-09-23T21:15:11.699292+0000","last_active":"2023-09-23T21:18:23.371574+0000","last_peered":"2023-09-23T21:18:23.371574+0000","last_clean":"2023-09-23T21:18:23.371574+0000","last_became_active":"2023-09-23T21:15:11.698942+0000","last_became_peered":"2023-09-23T21:15:11.698942+0000","last_unstale":"2023-09-23T21:18:23.371574+0000","last_undegraded":"2023-09-23T21:18:23.371574+0000","last_fullsized":"2023-09-23T21:18:23.371574+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:18.939214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":640,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609675+0000","last_change":"2023-09-23T21:15:11.690914+0000","last_active":"2023-09-23T21:18:23.609675+0000","last_peered":"2023-09-23T21:18:23.609675+0000","last_clean":"2023-09-23T21:18:23.609675+0000","last_became_active":"2023-09-23T21:15:11.690720+0000","last_became_peered":"2023-09-23T21:15:11.690720+0000","last_unstale":"2023-09-23T21:18:23.609675+0000","last_undegraded":"2023-09-23T21:18:23.609675+0000","last_fullsized":"2023-09-23T21:18:23.609675+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:35.630598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,5],"acting":[2,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":281,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609506+0000","last_change":"2023-09-23T21:16:51.062606+0000","last_active":"2023-09-23T21:18:23.609506+0000","last_peered":"2023-09-23T21:18:23.609506+0000","last_clean":"2023-09-23T21:18:23.609506+0000","last_became_active":"2023-09-23T21:16:50.847918+0000","last_became_peered":"2023-09-23T21:16:50.847918+0000","last_unstale":"2023-09-23T21:18:23.609506+0000","last_undegraded":"2023-09-23T21:18:23.609506+0000","last_fullsized":"2023-09-23T21:18:23.609506+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:21.132295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":235,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609824+0000","last_change":"2023-09-23T21:15:11.694832+0000","last_active":"2023-09-23T21:18:23.609824+0000","last_peered":"2023-09-23T21:18:23.609824+0000","last_clean":"2023-09-23T21:18:23.609824+0000","last_became_active":"2023-09-23T21:15:11.694554+0000","last_became_peered":"2023-09-23T21:15:11.694554+0000","last_unstale":"2023-09-23T21:18:23.609824+0000","last_undegraded":"2023-09-23T21:18:23.609824+0000","last_fullsized":"2023-09-23T21:18:23.609824+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:54.810925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":46,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610693+0000","last_change":"2023-09-23T21:17:11.215174+0000","last_active":"2023-09-23T21:18:23.610693+0000","last_peered":"2023-09-23T21:18:23.610693+0000","last_clean":"2023-09-23T21:18:23.610693+0000","last_became_active":"2023-09-23T21:16:51.043790+0000","last_became_peered":"2023-09-23T21:16:51.043790+0000","last_unstale":"2023-09-23T21:18:23.610693+0000","last_undegraded":"2023-09-23T21:18:23.610693+0000","last_fullsized":"2023-09-23T21:18:23.610693+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:58.360757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":222,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609315+0000","last_change":"2023-09-23T21:15:11.709231+0000","last_active":"2023-09-23T21:18:23.609315+0000","last_peered":"2023-09-23T21:18:23.609315+0000","last_clean":"2023-09-23T21:18:23.609315+0000","last_became_active":"2023-09-23T21:15:11.708933+0000","last_became_peered":"2023-09-23T21:15:11.708933+0000","last_unstale":"2023-09-23T21:18:23.609315+0000","last_undegraded":"2023-09-23T21:18:23.609315+0000","last_fullsized":"2023-09-23T21:18:23.609315+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:08.697760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":42,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.610428+0000","last_change":"2023-09-23T21:16:50.877684+0000","last_active":"2023-09-23T21:18:23.610428+0000","last_peered":"2023-09-23T21:18:23.610428+0000","last_clean":"2023-09-23T21:18:23.610428+0000","last_became_active":"2023-09-23T21:16:50.877246+0000","last_became_peered":"2023-09-23T21:16:50.877246+0000","last_unstale":"2023-09-23T21:18:23.610428+0000","last_undegraded":"2023-09-23T21:18:23.610428+0000","last_fullsized":"2023-09-23T21:18:23.610428+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:27.490203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":437,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609288+0000","last_change":"2023-09-23T21:16:50.847675+0000","last_active":"2023-09-23T21:18:23.609288+0000","last_peered":"2023-09-23T21:18:23.609288+0000","last_clean":"2023-09-23T21:18:23.609288+0000","last_became_active":"2023-09-23T21:16:50.847372+0000","last_became_peered":"2023-09-23T21:16:50.847372+0000","last_unstale":"2023-09-23T21:18:23.609288+0000","last_undegraded":"2023-09-23T21:18:23.609288+0000","last_fullsized":"2023-09-23T21:18:23.609288+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:25.888350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":534,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T21:18:26.425594+0000","last_change":"2023-09-23T21:16:51.086680+0000","last_active":"2023-09-23T21:18:26.425594+0000","last_peered":"2023-09-23T21:18:26.425594+0000","last_clean":"2023-09-23T21:18:26.425594+0000","last_became_active":"2023-09-23T21:16:51.020142+0000","last_became_peered":"2023-09-23T21:16:51.020142+0000","last_unstale":"2023-09-23T21:18:26.425594+0000","last_undegraded":"2023-09-23T21:18:26.425594+0000","last_fullsized":"2023-09-23T21:18:26.425594+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:09.944182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":244,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T21:18:23.609063+0000","last_change":"2023-09-23T21:16:51.051288+0000","last_active":"2023-09-23T21:18:23.609063+0000","last_peered":"2023-09-23T21:18:23.609063+0000","last_clean":"2023-09-23T21:18:23.609063+0000","last_became_active":"2023-09-23T21:16:51.050897+0000","last_became_peered":"2023-09-23T21:16:51.050897+0000","last_unstale":"2023-09-23T21:18:23.609063+0000","last_undegraded":"2023-09-23T21:18:23.609063+0000","last_fullsized":"2023-09-23T21:18:23.609063+0000","mapping_epoch":544,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:54.067361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1209991168,"num_objects":18463,"num_object_clones":0,"num_object_copies":55389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18463,"num_write_kb":1181632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3636264960,"data_stored":3636264960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18463,"ondisk_log_size":18463,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":38918,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201878,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1233604,"kb_used_data":416252,"kb_used_omap":0,"kb_used_meta":817344,"kb_avail":92511548,"statfs":{"total":95995035648,"available":94731825152,"internally_reserved":0,"allocated":426242048,"data_stored":424668552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":836960256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":2.7679999999999998,"15min":2.2040000000000002},"min":{"1min":0.246,"5min":0.218,"15min":0.20699999999999999},"max":{"1min":6.9470000000000001,"5min":141.33199999999999,"15min":141.33199999999999},"last":0.315},{"interface":"front","average":{"1min":1.1990000000000001,"5min":3.0579999999999998,"15min":2.2519999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":5.1340000000000003,"5min":141.31700000000001,"15min":141.31700000000001},"last":0.498}]},{"osd":1,"last update":"Sat Sep 23 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.7749999999999999,"15min":3.3639999999999999},"min":{"1min":0.29699999999999999,"5min":0.26100000000000001,"15min":0.23699999999999999},"max":{"1min":5.1100000000000003,"5min":10.407999999999999,"15min":114.16500000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.4359999999999999,"15min":3.8039999999999998},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.22900000000000001},"max":{"1min":6.9820000000000002,"5min":8.8650000000000002,"15min":117.637},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":3.71,"5min":3.8260000000000001,"15min":4.5519999999999996},"min":{"1min":0.248,"5min":0.224,"15min":0.224},"max":{"1min":53.988,"5min":135.529,"15min":183.44200000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":4.0659999999999998,"5min":4.7190000000000003,"15min":4.7750000000000004},"min":{"1min":0.27500000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":54,"5min":135.56700000000001,"15min":183.46199999999999},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:18:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":4.9870000000000001,"15min":3.9319999999999999},"min":{"1min":0.33800000000000002,"5min":0.30499999999999999,"15min":0.26700000000000002},"max":{"1min":7.0099999999999998,"5min":135.66200000000001,"15min":135.66200000000001},"last":0.375},{"interface":"front","average":{"1min":1.484,"5min":5.3570000000000002,"15min":4.056},"min":{"1min":0.32100000000000001,"5min":0.30399999999999999,"15min":0.28499999999999998},"max":{"1min":7.4729999999999999,"5min":135.47,"15min":135.47},"last":0.47799999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.9430000000000001,"15min":1.2090000000000001},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.1829999999999998,"5min":10.989000000000001,"15min":13.739000000000001},"last":1.6799999999999999},{"interface":"front","average":{"1min":3.5840000000000001,"5min":2.0950000000000002,"15min":1.2},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":20.306999999999999,"5min":20.306999999999999,"15min":20.306999999999999},"last":1.6080000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":2.6389999999999998,"15min":1.6080000000000001},"min":{"1min":0.53100000000000003,"5min":0.29099999999999998,"15min":0.26900000000000002},"max":{"1min":14.105,"5min":14.105,"15min":14.105},"last":1.0620000000000001},{"interface":"front","average":{"1min":3.9630000000000001,"5min":2.5339999999999998,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":17.585999999999999,"5min":17.585999999999999,"15min":17.585999999999999},"last":0.86199999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:17:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":2.2930000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.313},"max":{"1min":9.9969999999999999,"5min":13.369,"15min":143.18700000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":2.9380000000000002,"5min":1.819,"15min":1.929},"min":{"1min":0.31,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":15.566000000000001,"5min":15.566000000000001,"15min":142.70400000000001},"last":0.59099999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463499,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1933756,"kb_used_data":1150772,"kb_used_omap":0,"kb_used_meta":782976,"kb_avail":91811396,"statfs":{"total":95995035648,"available":94014869504,"internally_reserved":0,"allocated":1178390528,"data_stored":1176825515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":801767424},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,7],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":2.7549999999999999,"15min":2.7610000000000001},"min":{"1min":0.245,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":7.4400000000000004,"5min":121.762,"15min":149.61600000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.8809999999999998,"5min":3.4249999999999998,"15min":2.9369999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":9.7449999999999992,"5min":121.73099999999999,"15min":149.303},"last":2.6030000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:18:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":2.0369999999999999,"15min":3.5880000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.22700000000000001},"max":{"1min":8.5839999999999996,"5min":16.126999999999999,"15min":129.37899999999999},"last":2.657},{"interface":"front","average":{"1min":1.391,"5min":1.6479999999999999,"15min":3.3679999999999999},"min":{"1min":0.29499999999999998,"5min":0.27200000000000002,"15min":0.23100000000000001},"max":{"1min":6.6749999999999998,"5min":13.292,"15min":129.40100000000001},"last":0.66300000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":15.675000000000001,"5min":8.2889999999999997,"15min":6.7590000000000003},"min":{"1min":0.36499999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":181.846,"5min":181.846,"15min":199.23400000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":17.042000000000002,"5min":9.0250000000000004,"15min":6.8579999999999997},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":182.14500000000001,"5min":182.14500000000001,"15min":199.08000000000001},"last":2.6309999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":3.1150000000000002,"15min":3.2389999999999999},"min":{"1min":0.33200000000000002,"5min":0.218,"15min":0.218},"max":{"1min":10.734999999999999,"5min":126.649,"15min":137.875},"last":0.69799999999999995},{"interface":"front","average":{"1min":2.367,"5min":3.2429999999999999,"15min":3.3170000000000002},"min":{"1min":0.34799999999999998,"5min":0.30099999999999999,"15min":0.29799999999999999},"max":{"1min":7.4660000000000002,"5min":125.324,"15min":137.60900000000001},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":3.6299999999999999,"5min":3.5640000000000001,"15min":1.8879999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":9.6720000000000006,"5min":55.494,"15min":55.494},"last":2.5630000000000002},{"interface":"front","average":{"1min":3.1669999999999998,"5min":3.359,"15min":1.669},"min":{"1min":0.39000000000000001,"5min":0.29399999999999998,"15min":0.23999999999999999},"max":{"1min":11.228,"5min":55.305,"15min":55.305},"last":1.137}]},{"osd":5,"last update":"Sat Sep 23 21:17:58 2023","interfaces":[{"interface":"back","average":{"1min":4.9290000000000003,"5min":5.2089999999999996,"15min":2.4329999999999998},"min":{"1min":0.32500000000000001,"5min":0.32300000000000001,"15min":0.27300000000000002},"max":{"1min":12.975,"5min":222.90899999999999,"15min":222.90899999999999},"last":5.3550000000000004},{"interface":"front","average":{"1min":3.9239999999999999,"5min":5.1970000000000001,"15min":2.3740000000000001},"min":{"1min":0.42199999999999999,"5min":0.36099999999999999,"15min":0.28499999999999998},"max":{"1min":11.199999999999999,"5min":224.655,"15min":224.655},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:18:29 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":3.7909999999999999,"15min":1.925},"min":{"1min":0.40000000000000002,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":6.2709999999999999,"5min":165.96799999999999,"15min":165.96799999999999},"last":0.438},{"interface":"front","average":{"1min":1.915,"5min":4.0679999999999996,"15min":2.056},"min":{"1min":0.35499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":8.8849999999999998,"5min":165.82400000000001,"15min":165.82400000000001},"last":0.59299999999999997}]}]},{"osd":1,"up_from":0,"seq":1185410973946,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":395136991685,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781576,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1621952,"kb_used_data":496120,"kb_used_omap":7,"kb_used_meta":1125816,"kb_avail":92123200,"statfs":{"total":95995035648,"available":94334156800,"internally_reserved":0,"allocated":508026880,"data_stored":505276317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1152835665},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.5069999999999999,"15min":2.0710000000000002},"min":{"1min":0.17899999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":4.0830000000000002,"5min":25.297000000000001,"15min":103.021},"last":0.216},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.669,"15min":2.6070000000000002},"min":{"1min":0.25700000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":4.5510000000000002,"5min":18.879999999999999,"15min":103.154},"last":0.23999999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:18:29 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.1639999999999999,"15min":3.9860000000000002},"min":{"1min":0.248,"5min":0.224,"15min":0.19},"max":{"1min":3.8929999999999998,"5min":12.500999999999999,"15min":212.12},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.72899999999999998,"5min":1.4390000000000001,"15min":3.9849999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":3.8639999999999999,"5min":25.155000000000001,"15min":213.02799999999999},"last":0.313}]},{"osd":3,"last update":"Sat Sep 23 21:18:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.984,"15min":3.5710000000000002},"min":{"1min":0.30199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5760000000000001,"5min":81.653999999999996,"15min":143.50800000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.5379999999999998,"15min":3.819},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.249},"max":{"1min":4.109,"5min":82.241,"15min":143.27799999999999},"last":0.28399999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":4.3300000000000001,"15min":3.198},"min":{"1min":0.32100000000000001,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":7.0330000000000004,"5min":220.82900000000001,"15min":225.80799999999999},"last":0.37},{"interface":"front","average":{"1min":1.744,"5min":4.8300000000000001,"15min":3.7879999999999998},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.29499999999999998},"max":{"1min":7.0449999999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":5.0979999999999999,"15min":3.448},"min":{"1min":0.31,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.4349999999999996,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.625,"5min":4.7199999999999998,"15min":3.4510000000000001},"min":{"1min":0.42199999999999999,"5min":0.34399999999999997,"15min":0.29399999999999998},"max":{"1min":4.1440000000000001,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.35399999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":4.7480000000000002,"15min":3.7970000000000002},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":5.3220000000000001,"5min":210.74700000000001,"15min":210.74700000000001},"last":0.378},{"interface":"front","average":{"1min":2.3380000000000001,"5min":4.4829999999999997,"15min":3.1720000000000002},"min":{"1min":0.48399999999999999,"5min":0.36399999999999999,"15min":0.33800000000000002},"max":{"1min":7.1619999999999999,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.39200000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:18:02 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":2.4830000000000001,"15min":1.9530000000000001},"min":{"1min":0.373,"5min":0.371,"15min":0.36399999999999999},"max":{"1min":4.2640000000000002,"5min":25.260000000000002,"15min":25.260000000000002},"last":0.76800000000000002},{"interface":"front","average":{"1min":2.4079999999999999,"5min":2.5419999999999998,"15min":2.2610000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.5289999999999999,"5min":12.613,"15min":16.875},"last":0.502}]}]},{"osd":3,"up_from":0,"seq":635655160170,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":31,"seq":133143986788,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1593024,"kb_used_data":756216,"kb_used_omap":0,"kb_used_meta":836736,"kb_avail":92152128,"statfs":{"total":95995035648,"available":94363779072,"internally_reserved":0,"allocated":774365184,"data_stored":772795493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":856817664},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":3.3410000000000002,"15min":2.7639999999999998},"min":{"1min":0.28000000000000003,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":9.2200000000000006,"5min":109.23099999999999,"15min":226.751},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.754,"15min":2.5430000000000001},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":6.0919999999999996,"5min":109.16800000000001,"15min":226.43299999999999},"last":0.81699999999999995}]},{"osd":1,"last update":"Sat Sep 23 21:18:31 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":2.915,"15min":4.9009999999999998},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.1280000000000001,"5min":119.55800000000001,"15min":127.411},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":3.1859999999999999,"15min":4.1669999999999998},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":1.99,"5min":116.78100000000001,"15min":118.075},"last":0.60899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":8.7449999999999992,"5min":6.4420000000000002,"15min":6.327},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":85.994,"5min":155.51300000000001,"15min":210.50299999999999},"last":108.32899999999999},{"interface":"front","average":{"1min":9.2530000000000001,"5min":7.3230000000000004,"15min":6.4809999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":85.971999999999994,"5min":155.77099999999999,"15min":210.59},"last":108.104}]},{"osd":3,"last update":"Sat Sep 23 21:18:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":2.9500000000000002,"15min":4.0309999999999997},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.3900000000000001,"5min":111.22199999999999,"15min":214.64099999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.73899999999999999,"5min":4.0199999999999996,"15min":4.4489999999999998},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":2.0129999999999999,"5min":111.39400000000001,"15min":214.54900000000001},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":2.7829999999999999,"5min":1.7749999999999999,"15min":1.139},"min":{"1min":0.313,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":10.417,"5min":34.497999999999998,"15min":34.497999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.9350000000000001,"5min":2.8399999999999999,"15min":1.49},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.23799999999999999},"max":{"1min":14.675000000000001,"5min":34.883000000000003,"15min":34.883000000000003},"last":0.94099999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:17:54 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":2.6579999999999999,"15min":1.6970000000000001},"min":{"1min":0.35699999999999998,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":41.938000000000002,"5min":41.938000000000002,"15min":41.938000000000002},"last":1.226},{"interface":"front","average":{"1min":5.4660000000000002,"5min":2.5390000000000001,"15min":1.635},"min":{"1min":0.33600000000000002,"5min":0.32300000000000001,"15min":0.27200000000000002},"max":{"1min":40.444000000000003,"5min":40.444000000000003,"15min":40.444000000000003},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 21:17:40 2023","interfaces":[{"interface":"back","average":{"1min":3.0499999999999998,"5min":2.2170000000000001,"15min":1.8740000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.28100000000000003},"max":{"1min":11.859,"5min":18.352,"15min":145.721},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.8600000000000001,"15min":1.893},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"max":{"1min":8.5570000000000004,"5min":10.677,"15min":148.45699999999999},"last":0.89100000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725152,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1701876,"kb_used_data":812972,"kb_used_omap":0,"kb_used_meta":888896,"kb_avail":92043276,"statfs":{"total":95995035648,"available":94252314624,"internally_reserved":0,"allocated":832483328,"data_stored":830921691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910229504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.427,"15min":1.71},"min":{"1min":0.34899999999999998,"5min":0.22600000000000001,"15min":0.217},"max":{"1min":4.1260000000000003,"5min":8.6660000000000004,"15min":82.280000000000001},"last":3.6520000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.798,"15min":1.889},"min":{"1min":0.27600000000000002,"5min":0.23400000000000001,"15min":0.20499999999999999},"max":{"1min":6.8810000000000002,"5min":16.977,"15min":85.236000000000004},"last":0.42599999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3479999999999999,"15min":4.7039999999999997},"min":{"1min":0.255,"5min":0.252,"15min":0.23599999999999999},"max":{"1min":11.374000000000001,"5min":12.032999999999999,"15min":233.77099999999999},"last":2.117},{"interface":"front","average":{"1min":1.3979999999999999,"5min":1.6399999999999999,"15min":4.359},"min":{"1min":0.39600000000000002,"5min":0.29799999999999999,"15min":0.251},"max":{"1min":4.0590000000000002,"5min":12.016,"15min":233.75200000000001},"last":1.2949999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":1.73,"15min":3.2269999999999999},"min":{"1min":0.34999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.1040000000000001,"5min":10.454000000000001,"15min":231.34700000000001},"last":1.6299999999999999},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.9930000000000001,"15min":3.4220000000000002},"min":{"1min":0.374,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9180000000000001,"5min":10.621,"15min":231.381},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":2.403,"15min":3.7440000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":11.42,"5min":11.597,"15min":263.887},"last":2.641},{"interface":"front","average":{"1min":1.7470000000000001,"5min":2.0099999999999998,"15min":3.4380000000000002},"min":{"1min":0.44400000000000001,"5min":0.39700000000000002,"15min":0.30099999999999999},"max":{"1min":4.173,"5min":10.638,"15min":263.70100000000002},"last":1.661}]},{"osd":4,"last update":"Sat Sep 23 21:17:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":1.7709999999999999,"15min":1.0129999999999999},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.25},"max":{"1min":4.0289999999999999,"5min":11.593999999999999,"15min":11.593999999999999},"last":2.048},{"interface":"front","average":{"1min":3.3860000000000001,"5min":1.95,"15min":1.1899999999999999},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.26400000000000001},"max":{"1min":12.49,"5min":12.49,"15min":12.49},"last":3.9590000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:17:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":2.0339999999999998,"15min":1.619},"min":{"1min":0.48499999999999999,"5min":0.28399999999999997,"15min":0.26800000000000002},"max":{"1min":6.984,"5min":8.9130000000000003,"15min":83.451999999999998},"last":3.8479999999999999},{"interface":"front","average":{"1min":3.2999999999999998,"5min":2.73,"15min":1.8859999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":11.696999999999999,"5min":12.725,"15min":83.484999999999999},"last":2.0710000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:17:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1000000000000001,"5min":2.161,"15min":1.419},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.30099999999999999},"max":{"1min":7.4020000000000001,"5min":8.6950000000000003,"15min":9.6329999999999991},"last":3.8159999999999998},{"interface":"front","average":{"1min":2.431,"5min":1.585,"15min":0.995},"min":{"1min":0.45500000000000002,"5min":0.34899999999999998,"15min":0.28899999999999998},"max":{"1min":10.196999999999999,"5min":11.640000000000001,"15min":11.640000000000001},"last":1.5960000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":551157760,"data_stored":551157760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":710672384,"data_stored":710672384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":862650368,"data_stored":862650368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1077739520,"data_stored":1077739520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":434044928,"data_stored":434044928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:18:39.709 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T21:18:39.709 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T21:18:39.752 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:39.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:39.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:39.856 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:39.880 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:39 smithi121 ceph-mon[219709]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.881 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:39 smithi121 ceph-mon[219709]: from='client.37488 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:39.881 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:39 smithi121 ceph-mon[219709]: pgmap v1791: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 130 MiB/s wr, 2.07k op/s 2023-09-23T21:18:39.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:39 smithi121 ceph-mon[211526]: failed to encode map e578 with expected crc 2023-09-23T21:18:39.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:39 smithi121 ceph-mon[211526]: from='client.37488 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:39.881 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:39 smithi121 ceph-mon[211526]: pgmap v1791: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 130 MiB/s wr, 2.07k op/s 2023-09-23T21:18:39.883 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:40.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:39 smithi123 ceph-mon[193903]: failed to encode map e578 with expected crc 2023-09-23T21:18:40.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:39 smithi123 ceph-mon[193903]: from='client.37488 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:18:40.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:39 smithi123 ceph-mon[193903]: pgmap v1791: 89 pgs: 89 active+clean; 1.5 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 130 MiB/s wr, 2.07k op/s 2023-09-23T21:18:40.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 14 16 28771 28755 128.359 99.1875 0.00860402 0.00778568 2023-09-23T21:18:41.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 15 16 30322 30306 126.262 96.9375 0.0072906 0.00791399 2023-09-23T21:18:41.486 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:41.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:41.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:41.594 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:41.645 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:41.879 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:42.058 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:42.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:42.162 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:42.187 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:42.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:42.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:42.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:42.333 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:42.363 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:42.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 16 16 31625 31609 123.46 81.4375 0.00465125 0.00809412 2023-09-23T21:18:42.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:42 smithi123 ceph-mon[193903]: pgmap v1792: 89 pgs: 89 active+clean; 1.6 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T21:18:42.644 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:18:42.644 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5665 12.8 12:15d27e1f:::benchmark_data_smithi121_563784_object5664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.191214+0000", 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 527.45115577399997, 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25962825, 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.645 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5665 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191214+0000", 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.646 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191214+0000", 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191213+0000", 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.647 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191235+0000", 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1696e-05 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191247+0000", 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2706e-05 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.648 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191255+0000", 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9689999999999994e-06 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191315+0000", 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9877999999999998e-05 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191447+0000", 2023-09-23T21:18:42.649 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013184199999999999 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.191579+0000", 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000131815 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.192399+0000", 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00081998399999999995 2023-09-23T21:18:42.650 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.193882+0000", 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001483266 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450745+0000", 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25686271500000002 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.651 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450799+0000", 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4178999999999997e-05 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450842+0000", 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3460000000000001e-05 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.652 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:5678 12.8 12:1b339262:::benchmark_data_smithi121_563784_object5677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e430)", 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:09:55.194613+0000", 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 527.447757153, 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25634367000000002, 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 5678 2023-09-23T21:18:42.653 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194613+0000", 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.654 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194613+0000", 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194611+0000", 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.655 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194633+0000", 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1742e-05 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194644+0000", 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1097999999999999e-05 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194652+0000", 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1380000000000007e-06 2023-09-23T21:18:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194705+0000", 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2296000000000002e-05 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194826+0000", 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121285 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.194992+0000", 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016586699999999999 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.195572+0000", 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00057995500000000005 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.196822+0000", 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0012501229999999999 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450881+0000", 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25405888999999998 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450922+0000", 2023-09-23T21:18:42.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1588000000000002e-05 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:09:55.450956+0000", 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4133e-05 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39654 12.8 12:16533dd6:::benchmark_data_smithi121_563784_object39653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.105135+0000", 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.53723473100001, 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28269233500000002, 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39654 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.661 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105135+0000", 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105135+0000", 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105133+0000", 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105156+0000", 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2793999999999999e-05 2023-09-23T21:18:42.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105168+0000", 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1663e-05 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105177+0000", 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7560000000000005e-06 2023-09-23T21:18:42.664 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105230+0000", 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3582999999999998e-05 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.107274+0000", 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020435539999999999 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.665 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.107403+0000", 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012961799999999999 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112112+0000", 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047086740000000004 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.666 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.117495+0000", 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053830939999999997 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118386+0000", 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00089051600000000005 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387793+0000", 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26940751400000001 2023-09-23T21:18:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387828+0000", 2023-09-23T21:18:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4320999999999999e-05 2023-09-23T21:18:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.669 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39656 12.8 12:13f662b9:::benchmark_data_smithi121_563784_object39655:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.105196+0000", 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.53717417199999, 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28281587899999999, 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.670 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39656 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105196+0000", 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105196+0000", 2023-09-23T21:18:42.671 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105195+0000", 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105216+0000", 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0993999999999999e-05 2023-09-23T21:18:42.672 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105226+0000", 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3770000000000002e-06 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.105232+0000", 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.993e-06 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.110437+0000", 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.005205749 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.110581+0000", 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014404700000000001 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.110712+0000", 2023-09-23T21:18:42.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000130331 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112286+0000", 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0015745100000000001 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.117524+0000", 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052376530000000001 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.675 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387925+0000", 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27040097099999999 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387968+0000", 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3333e-05 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388012+0000", 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3569999999999998e-05 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39666 12.8 12:13f06d34:::benchmark_data_smithi121_563784_object39665:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.112486+0000", 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52988446299997, 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27566012400000001, 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.677 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39666 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112486+0000", 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112486+0000", 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112485+0000", 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112513+0000", 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8552e-05 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112524+0000", 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1253999999999999e-05 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112532+0000", 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8839999999999994e-06 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112589+0000", 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6700999999999998e-05 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112728+0000", 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000139264 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.112919+0000", 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000190896 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.117569+0000", 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046497190000000001 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388030+0000", 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27046082700000001 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388068+0000", 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7994000000000002e-05 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388111+0000", 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3056999999999998e-05 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388146+0000", 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4963999999999999e-05 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:39667 12.d e434/426 12:bc28a1ff:::benchmark_data_smithi121_563784_object39666:head v 434'2460, mlcod=434'2458)", 2023-09-23T21:18:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.113990+0000", 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52837971899999, 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27380560799999998, 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.113990+0000", 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.113990+0000", 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.113989+0000", 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114015+0000", 2023-09-23T21:18:42.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5075e-05 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114027+0000", 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2434e-05 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114035+0000", 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5549999999999997e-06 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114098+0000", 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3459000000000003e-05 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114117+0000", 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9517e-05 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387746+0000", 2023-09-23T21:18:42.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27362806299999998 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387796+0000", 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0308e-05 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:39668 12.1 e434/426 12:80006d8a:::benchmark_data_smithi121_563784_object39667:head v 434'2477, mlcod=434'2475)", 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.114602+0000", 2023-09-23T21:18:42.689 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52776839199998, 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27316755700000001, 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114602+0000", 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114602+0000", 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114601+0000", 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114641+0000", 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0445000000000003e-05 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114653+0000", 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1449e-05 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114660+0000", 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1459999999999999e-06 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114907+0000", 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024668299999999998 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.114926+0000", 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9363000000000001e-05 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118975+0000", 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040494900000000002 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.693 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387769+0000", 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26879371800000001 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:39675 12.8 12:1749fe44:::benchmark_data_smithi121_563784_object39674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.115745+0000", 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52662498799998, 2023-09-23T21:18:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27250877699999998, 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 39675 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115745+0000", 2023-09-23T21:18:42.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115745+0000", 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115743+0000", 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115770+0000", 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6319999999999999e-05 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115783+0000", 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3441000000000001e-05 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115792+0000", 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5739999999999996e-06 2023-09-23T21:18:42.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115843+0000", 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1226e-05 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.115991+0000", 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148052 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.116123+0000", 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013253399999999999 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387870+0000", 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27174642599999999 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388035+0000", 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016523899999999999 2023-09-23T21:18:42.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388186+0000", 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000151216 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388224+0000", 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7710999999999999e-05 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388254+0000", 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9691000000000001e-05 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:39654 12.8 e434/426 ondisk, result = 0)", 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.118308+0000", 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52406234400001, 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26952806499999998, 2023-09-23T21:18:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118308+0000", 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118308+0000", 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118307+0000", 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118313+0000", 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1890000000000004e-06 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118325+0000", 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1575e-05 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118331+0000", 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.63e-06 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118373+0000", 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2197000000000003e-05 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.118381+0000", 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1359999999999999e-06 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387836+0000", 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26945519200000001 2023-09-23T21:18:42.705 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:39675 12.8 e434/426 ondisk, result = 0)", 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.119601+0000", 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52276921200001, 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26827622200000001, 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119601+0000", 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119601+0000", 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119600+0000", 2023-09-23T21:18:42.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.119606+0000", 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9150000000000001e-06 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387737+0000", 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26813114799999999 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387751+0000", 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3926e-05 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387855+0000", 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010456899999999999 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387864+0000", 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3629999999999994e-06 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387877+0000", 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3247999999999999e-05 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.710 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:39656 12.8 e434/426 ondisk, result = 0)", 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52231362700002, 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26797944299999998, 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120056+0000", 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120060+0000", 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.357e-06 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387767+0000", 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.267706369 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387774+0000", 2023-09-23T21:18:42.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9870000000000002e-06 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387910+0000", 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013650700000000001 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.387917+0000", 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3440000000000003e-06 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388036+0000", 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000118429 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:39680 12.d e434/426 12:b632df5a:::benchmark_data_smithi121_563784_object39679:head v 434'2461, mlcod=434'2458)", 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 503.52180070399999, 2023-09-23T21:18:42.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27253574400000002, 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:18:42.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120569+0000", 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.120592+0000", 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3876999999999999e-05 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388029+0000", 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26743616399999998 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388035+0000", 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.144e-06 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388065+0000", 2023-09-23T21:18:42.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0766000000000003e-05 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.388090+0000", 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4352999999999999e-05 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393065+0000", 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049749590000000002 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:19.393105+0000", 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0209000000000001e-05 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:45566 12.2 12:441f91eb:::benchmark_data_smithi121_563784_object45565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:24.926407+0000", 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 497.71596292700002, 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25631767, 2023-09-23T21:18:42.720 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 45566 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926407+0000", 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926407+0000", 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926406+0000", 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926434+0000", 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8530999999999998e-05 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926450+0000", 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6041000000000001e-05 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.926458+0000", 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6760000000000007e-06 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.927760+0000", 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0013016869999999999 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.927887+0000", 2023-09-23T21:18:42.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000126747 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.928045+0000", 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015825400000000001 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.932292+0000", 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042472120000000002 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:24.934877+0000", 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025848360000000001 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.180998+0000", 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24612136300000001 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.181043+0000", 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5117999999999997e-05 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:25.182725+0000", 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016813500000000001 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.727 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64065 12.b 12:dac96c8a:::benchmark_data_smithi121_563784_object64064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.247452+0000", 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 476.394918115, 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33990143499999997, 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64065 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.728 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247452+0000", 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247452+0000", 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247451+0000", 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247472+0000", 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0517999999999999e-05 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247481+0000", 2023-09-23T21:18:42.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8910000000000007e-06 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247488+0000", 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3620000000000002e-06 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247529+0000", 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1146000000000002e-05 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247660+0000", 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000130681 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.247796+0000", 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000136675 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.250174+0000", 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023775049999999998 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.252502+0000", 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002328526 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587275+0000", 2023-09-23T21:18:42.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.33477227999999998 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587320+0000", 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5782999999999999e-05 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.587353+0000", 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2848999999999997e-05 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.734 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:64078 12.8 12:185b8dbf:::benchmark_data_smithi121_563784_object64077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:10:46.270175+0000", 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 476.37219452199997, 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.31567373100000001, 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.735 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 64078 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270175+0000", 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270175+0000", 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270174+0000", 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270196+0000", 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2175e-05 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270208+0000", 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2510000000000001e-05 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270217+0000", 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3990000000000004e-06 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270271+0000", 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4182999999999999e-05 2023-09-23T21:18:42.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270377+0000", 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010607099999999999 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.270477+0000", 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9956000000000003e-05 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.272458+0000", 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001980739 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.272707+0000", 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00024946199999999999 2023-09-23T21:18:42.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.585765+0000", 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.313057431 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.585811+0000", 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6387999999999997e-05 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:10:46.585849+0000", 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8064e-05 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89072 12.8 12:1c946405:::benchmark_data_smithi121_563784_object89071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.116493+0000", 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 441.52587674500001, 2023-09-23T21:18:42.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.29639990999999999, 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89072 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116493+0000", 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116493+0000", 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116492+0000", 2023-09-23T21:18:42.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116516+0000", 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3909000000000001e-05 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116527+0000", 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1314e-05 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116533+0000", 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1700000000000002e-06 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116582+0000", 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9094000000000001e-05 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116715+0000", 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000133008 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.116864+0000", 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000148355 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.128352+0000", 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011488717000000001 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.365879+0000", 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.237526404 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412801+0000", 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.046921870999999997 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412854+0000", 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3232000000000001e-05 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412893+0000", 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9164e-05 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.750 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:89077 12.b 12:d83995cf:::benchmark_data_smithi121_563784_object89076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:11:21.124562+0000", 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 441.51780831100001, 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28833729600000002, 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 89077 2023-09-23T21:18:42.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124562+0000", 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124562+0000", 2023-09-23T21:18:42.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124560+0000", 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124587+0000", 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7463000000000001e-05 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124601+0000", 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3940999999999999e-05 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124610+0000", 2023-09-23T21:18:42.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8389999999999996e-06 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124680+0000", 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0059000000000001e-05 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124804+0000", 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012447900000000001 2023-09-23T21:18:42.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.124899+0000", 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4778000000000006e-05 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.128535+0000", 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0036354719999999998 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.129942+0000", 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0014074910000000001 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412822+0000", 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.28287962100000003 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412864+0000", 2023-09-23T21:18:42.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2592999999999997e-05 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:11:21.412899+0000", 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4516e-05 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116390 12.8 12:18d953d1:::benchmark_data_smithi121_563784_object25987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:18:42.758 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.874159+0000", 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 398.76821075499998, 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25885495400000003, 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116390 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.759 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874159+0000", 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874159+0000", 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874158+0000", 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:42.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874165+0000", 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1600000000000001e-06 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874176+0000", 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1578e-05 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874184+0000", 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2629999999999997e-06 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874244+0000", 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0297000000000002e-05 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874367+0000", 2023-09-23T21:18:42.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012276 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.875397+0000", 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001030148 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.880761+0000", 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0053641790000000002 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.115348+0000", 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.234586402 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.132921+0000", 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017573886 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.132971+0000", 2023-09-23T21:18:42.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9245e-05 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.133014+0000", 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3494000000000001e-05 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:116396 12.8 12:19288291:::benchmark_data_smithi121_563784_object25993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2023-09-23T21:18:42.765 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:12:03.874202+0000", 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 398.768167957, 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25891394899999998, 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 116396 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874202+0000", 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874202+0000", 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874202+0000", 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874204+0000", 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6000000000000001e-06 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874210+0000", 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4659999999999998e-06 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.768 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.874214+0000", 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3889999999999998e-06 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.878233+0000", 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040188159999999997 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.878352+0000", 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011961899999999999 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,2", 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.878422+0000", 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9929000000000001e-05 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:03.880868+0000", 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024459450000000002 2023-09-23T21:18:42.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.115376+0000", 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.234507682 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.133046+0000", 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.017669606000000001 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.133085+0000", 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9826999999999997e-05 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:12:04.133116+0000", 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0514999999999998e-05 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151905 12.2 12:40f63a6a:::benchmark_data_smithi121_563784_object61502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.994394+0000", 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 335.64797595099998, 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26478480100000001, 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151905 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.773 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.994394+0000", 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.994394+0000", 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:18:42.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.994393+0000", 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.994398+0000", 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6999999999999999e-06 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.231854+0000", 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.23745609600000001 2023-09-23T21:18:42.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.231863+0000", 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2499999999999995e-06 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241451+0000", 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0095882420000000003 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241559+0000", 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010814700000000001 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241669+0000", 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010982 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.245457+0000", 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0037880090000000002 2023-09-23T21:18:42.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.246660+0000", 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001203076 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.259089+0000", 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012428349999999999 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.259141+0000", 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2314000000000001e-05 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.259179+0000", 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7824000000000002e-05 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.779 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:18:42.780 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:18:42.780 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:18:42.780 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:42 smithi121 ceph-mon[211526]: pgmap v1792: 89 pgs: 89 active+clean; 1.6 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T21:18:42.780 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:42 smithi121 ceph-mon[219709]: pgmap v1792: 89 pgs: 89 active+clean; 1.6 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-09-23T21:18:42.880 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:42.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:42.984 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:43.056 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.103 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:43.210 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:43.258 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.391 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:43.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:43.406 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:18:43.423 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 17 16 32993 32977 121.227 85.5 0.0112905 0.0082411 2023-09-23T21:18:43.499 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:43.559 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:43.687 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:43.693 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:43.693 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]: dispatch 2023-09-23T21:18:43.693 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]: dispatch 2023-09-23T21:18:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:43.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:43.702 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:43.702 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]: dispatch 2023-09-23T21:18:43.702 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]: dispatch 2023-09-23T21:18:43.703 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:43.703 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]: dispatch 2023-09-23T21:18:43.703 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]: dispatch 2023-09-23T21:18:43.796 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:43.847 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:18:43.848 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:44.114 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:44.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:44.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:44.218 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:44.297 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:44.392 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:44.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:44.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 18 16 34344 34328 119.182 84.4375 0.00235296 0.00837841 2023-09-23T21:18:44.493 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:18:44.503 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:44.534 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:44.602 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.603 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.604 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.604 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.604 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.604 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.604 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:18:44.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:44 smithi123 ceph-mon[193903]: pgmap v1793: 89 pgs: 89 active+clean; 1.8 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T21:18:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:44 smithi123 ceph-mon[193903]: failed to encode map e579 with expected crc 2023-09-23T21:18:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]': finished 2023-09-23T21:18:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:44 smithi123 ceph-mon[193903]: osdmap e579: 8 total, 8 up, 5 in 2023-09-23T21:18:44.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2049642284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[219709]: pgmap v1793: 89 pgs: 89 active+clean; 1.8 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T21:18:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[219709]: failed to encode map e579 with expected crc 2023-09-23T21:18:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]': finished 2023-09-23T21:18:44.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[219709]: osdmap e579: 8 total, 8 up, 5 in 2023-09-23T21:18:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2049642284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:44.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[211526]: pgmap v1793: 89 pgs: 89 active+clean; 1.8 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T21:18:44.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[211526]: failed to encode map e579 with expected crc 2023-09-23T21:18:44.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [6, 2]}]': finished 2023-09-23T21:18:44.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[211526]: osdmap e579: 8 total, 8 up, 5 in 2023-09-23T21:18:44.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2049642284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:45.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:18:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:18:44] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.43.0" 2023-09-23T21:18:45.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 19 16 35476 35460 116.633 70.75 0.00951303 0.00856913 2023-09-23T21:18:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:45 smithi121 ceph-mon[219709]: failed to encode map e580 with expected crc 2023-09-23T21:18:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:45 smithi121 ceph-mon[219709]: failed to encode map e580 with expected crc 2023-09-23T21:18:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:45 smithi121 ceph-mon[219709]: osdmap e580: 8 total, 8 up, 5 in 2023-09-23T21:18:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:45 smithi121 ceph-mon[211526]: failed to encode map e580 with expected crc 2023-09-23T21:18:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:45 smithi121 ceph-mon[211526]: failed to encode map e580 with expected crc 2023-09-23T21:18:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:45 smithi121 ceph-mon[211526]: osdmap e580: 8 total, 8 up, 5 in 2023-09-23T21:18:45.687 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:45.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:45 smithi123 ceph-mon[193903]: failed to encode map e580 with expected crc 2023-09-23T21:18:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:45 smithi123 ceph-mon[193903]: failed to encode map e580 with expected crc 2023-09-23T21:18:45.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:45 smithi123 ceph-mon[193903]: osdmap e580: 8 total, 8 up, 5 in 2023-09-23T21:18:45.792 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:45.859 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:45.974 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:45.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:45.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:46.082 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:46.100 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:46.122 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:46.237 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:46.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:46.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:46.340 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:46.373 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:46.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:18:46.424788+0000 min lat: 0.00109569 max lat: 0.241939 avg lat: 0.00867463 2023-09-23T21:18:46.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:18:46.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 20 16 36884 36868 115.201 88 0.0148588 0.00867463 2023-09-23T21:18:46.436 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:46.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:46.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:46 smithi121 ceph-mon[219709]: pgmap v1796: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-09-23T21:18:46.541 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:46 smithi121 ceph-mon[211526]: pgmap v1796: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-09-23T21:18:46.542 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:46.573 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:46.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:46 smithi123 ceph-mon[193903]: pgmap v1796: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-09-23T21:18:46.780 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:46.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:46.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:46.885 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:46.969 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:47.024 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:47.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:47.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:47.128 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:47.157 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:47.202 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:47.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:47.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:47.305 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:47.381 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:47.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 21 16 38011 37995 113.069 70.4375 0.00724029 0.00883895 2023-09-23T21:18:47.462 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 1 pg_num to 11 2023-09-23T21:18:47.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/421267612' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T21:18:47.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T21:18:47.489 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:47 smithi121 ceph-mon[219709]: pgmap v1797: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T21:18:47.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/421267612' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T21:18:47.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T21:18:47.489 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:47 smithi121 ceph-mon[211526]: pgmap v1797: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T21:18:47.509 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:47.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:47.617 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:47.662 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:47.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/421267612' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T21:18:47.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T21:18:47.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:47 smithi123 ceph-mon[193903]: pgmap v1797: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T21:18:47.843 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:18:48.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 22 16 39206 39190 111.325 74.6875 0.0130364 0.00897751 2023-09-23T21:18:48.488 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:48.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:48.592 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:48.623 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:48.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T21:18:48.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: osdmap e581: 8 total, 8 up, 5 in 2023-09-23T21:18:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:48 smithi123 ceph-mon[193903]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.852 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: osdmap e581: 8 total, 8 up, 5 in 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[219709]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.853 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: osdmap e581: 8 total, 8 up, 5 in 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: failed to encode map e581 with expected crc 2023-09-23T21:18:48.854 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:48 smithi121 ceph-mon[211526]: failed to encode map e581 with expected crc 2023-09-23T21:18:49.050 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:49.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:49.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:49.154 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:49.228 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:49.354 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:49.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:49.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:49.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 23 16 40398 40382 109.723 74.5 0.00835742 0.00910861 2023-09-23T21:18:49.463 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:49.466 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:49.504 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:49.566 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:49.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:49.671 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:49.701 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:49.769 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:49.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:49.873 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:49.903 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.005 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:50.108 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:50.188 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.277 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:18:50.294 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:50.400 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:50.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 24 16 41765 41749 108.711 85.4375 0.0121798 0.00919362 2023-09-23T21:18:50.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:18:50.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:18:50.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: pgmap v1799: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T21:18:50.466 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:50.466 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:50.466 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/277418844' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:50.479 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.576 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:50.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:50.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: pgmap v1799: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/277418844' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: pgmap v1799: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:50.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/277418844' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:18:50.684 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:50.754 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:50.864 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:50.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:50.968 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:50.998 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:51.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:51.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T21:18:51.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/277418844' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:51.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: osdmap e582: 8 total, 8 up, 5 in 2023-09-23T21:18:51.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.252 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[211526]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/277418844' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: osdmap e582: 8 total, 8 up, 5 in 2023-09-23T21:18:51.253 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.254 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:51 smithi121 ceph-mon[219709]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 25 16 43020 43004 107.5 78.4375 0.0115097 0.00929749 2023-09-23T21:18:51.654 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:51.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:51.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:51.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:18:51.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T21:18:51.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/277418844' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:18:51.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: osdmap e582: 8 total, 8 up, 5 in 2023-09-23T21:18:51.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:51 smithi123 ceph-mon[193903]: failed to encode map e582 with expected crc 2023-09-23T21:18:51.765 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:51.798 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:51.901 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:51.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:52.007 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:52.041 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:52.097 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:52.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:52.160 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:18:52.181 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:18:52.202 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:52.202 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:18:52.225 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:18:52.232 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:52.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:18:52.291 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:52.291 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:18:52.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:52.312 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:18:52.336 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:18:52.393 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:52.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 26 16 44336 44320 106.529 82.25 0.0139778 0.00938076 2023-09-23T21:18:52.481 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:52.523 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: osdmap e583: 8 total, 8 up, 5 in 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: pgmap v1802: 99 pgs: 10 unknown, 89 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T21:18:52.524 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3513006431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: osdmap e583: 8 total, 8 up, 5 in 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: pgmap v1802: 99 pgs: 10 unknown, 89 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T21:18:52.525 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3513006431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:52.561 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.561 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: osdmap e583: 8 total, 8 up, 5 in 2023-09-23T21:18:52.561 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.562 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.562 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.562 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.562 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.562 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: failed to encode map e583 with expected crc 2023-09-23T21:18:52.562 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: pgmap v1802: 99 pgs: 10 unknown, 89 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T21:18:52.562 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3513006431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:18:52.638 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:52.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:52.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:52.778 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:52.907 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:52.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:53.013 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:53.102 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:53.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:53.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:53.245 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:18:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:18:53.245 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:45066 14.a 14:50e89666:::benchmark_data_smithi121_684892_object45065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e583)", 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.231822+0000", 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.011621635, 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011655003000000001, 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 45066 2023-09-23T21:18:53.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.231822+0000", 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.231822+0000", 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.247 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.231821+0000", 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.231944+0000", 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000123072 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.231956+0000", 2023-09-23T21:18:53.248 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1341e-05 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.231962+0000", 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5889999999999999e-06 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.231998+0000", 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6022000000000002e-05 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.249 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.232119+0000", 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120914 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.232246+0000", 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000127298 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:18:53.250 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240175+0000", 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079284549999999992 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:45064 14.2 e583/573 14:4efb5a0f:::benchmark_data_smithi121_684892_object45063:head v 583'2827, mlcod=583'2825)", 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.233951+0000", 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0094932130000000003, 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0096132309999999999, 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:18:53.251 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.233951+0000", 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.233951+0000", 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.252 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.233950+0000", 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.233976+0000", 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5558000000000002e-05 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.233986+0000", 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8570000000000007e-06 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.253 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.233992+0000", 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0750000000000003e-06 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234069+0000", 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6940000000000005e-05 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.254 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234086+0000", 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7458000000000001e-05 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:45061 14.d e583/573 14:bf5ebe44:::benchmark_data_smithi121_684892_object45060:head v 583'2790, mlcod=583'2787)", 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.234010+0000", 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0094335559999999992, 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0096131129999999995, 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.255 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234010+0000", 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234010+0000", 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.256 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234010+0000", 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234031+0000", 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1049000000000001e-05 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234037+0000", 2023-09-23T21:18:53.257 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1589999999999998e-06 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234041+0000", 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.489e-06 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234850+0000", 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000809335 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.258 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.234878+0000", 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7813e-05 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:45070 14.9 14:9b1e900f:::benchmark_data_smithi121_684892_object45069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e583)", 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.235815+0000", 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.007629411, 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078712260000000003, 2023-09-23T21:18:53.259 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 45070 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235815+0000", 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.260 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235815+0000", 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235813+0000", 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.261 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235885+0000", 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1552999999999997e-05 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235897+0000", 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1637000000000001e-05 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235903+0000", 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3400000000000003e-06 2023-09-23T21:18:53.262 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.236005+0000", 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010163299999999999 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.236140+0000", 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013564600000000001 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:18:53.263 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.236275+0000", 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013486999999999999 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:45071 14.9 14:946a9143:::benchmark_data_smithi121_684892_object45070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e583)", 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.235916+0000", 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0075279530000000004, 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078452379999999992, 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:18:53.264 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 45071 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235916+0000", 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.265 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235916+0000", 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235916+0000", 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235960+0000", 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4521999999999998e-05 2023-09-23T21:18:53.266 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235967+0000", 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4320000000000004e-06 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.235971+0000", 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1659999999999996e-06 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.267 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.237984+0000", 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002013019 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.238093+0000", 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010965 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.238215+0000", 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121482 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.268 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:45063 14.d e583/573 14:bd698c3e:::benchmark_data_smithi121_684892_object45062:head v 583'2791, mlcod=583'2787)", 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.239899+0000", 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0035451060000000001, 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039318809999999999, 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.269 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239899+0000", 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239899+0000", 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239898+0000", 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.270 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239925+0000", 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6891000000000001e-05 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239935+0000", 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6809999999999993e-06 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.271 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239941+0000", 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6969999999999998e-06 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239977+0000", 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6032000000000003e-05 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239994+0000", 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7461000000000001e-05 2023-09-23T21:18:53.272 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:45068 14.2 e583/573 14:49a8dd39:::benchmark_data_smithi121_684892_object45067:head v 583'2828, mlcod=583'2825)", 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.239961+0000", 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.003482948, 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039271899999999997, 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.273 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239961+0000", 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239961+0000", 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.274 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239961+0000", 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239981+0000", 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0729000000000001e-05 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239988+0000", 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.635e-06 2023-09-23T21:18:53.275 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.239992+0000", 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.386e-06 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240199+0000", 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000206725 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.276 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240218+0000", 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8564e-05 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:45073 14.3 e583/573 14:c793aca9:::benchmark_data_smithi121_684892_object45072:head v 583'2817, mlcod=583'2815)", 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.240133+0000", 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0033112340000000001, 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038149450000000001, 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:18:53.277 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240133+0000", 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240133+0000", 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.278 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240132+0000", 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240152+0000", 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0006000000000001e-05 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240161+0000", 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9649999999999997e-06 2023-09-23T21:18:53.279 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240166+0000", 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7840000000000003e-06 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.241319+0000", 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011526469999999999 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.280 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.241336+0000", 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7469000000000002e-05 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:45072 14.a 14:50774de3:::benchmark_data_smithi121_684892_object45071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e583)", 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.240189+0000", 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0032547219999999998, 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003826772, 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:18:53.281 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 45072 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240189+0000", 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.282 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240189+0000", 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240188+0000", 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240225+0000", 2023-09-23T21:18:53.283 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6909000000000002e-05 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240234+0000", 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7949999999999996e-06 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240239+0000", 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8099999999999997e-06 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.284 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240339+0000", 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100905 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240456+0000", 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000116711 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 5,6", 2023-09-23T21:18:53.285 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.240580+0000", 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012427400000000001 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:45074 14.d e583/573 14:bdf1c7ee:::benchmark_data_smithi121_684892_object45073:head v 583'2792, mlcod=583'2787)", 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.242220+0000", 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0012240930000000001, 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0018707299999999999, 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.286 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.242220+0000", 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.242220+0000", 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.287 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.242219+0000", 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.242241+0000", 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1532999999999999e-05 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.288 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.242250+0000", 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3910000000000004e-06 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.242265+0000", 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4859e-05 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243018+0000", 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00075248199999999996 2023-09-23T21:18:53.289 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243035+0000", 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7867999999999999e-05 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:45075 14.2 e583/573 14:438d0ef0:::benchmark_data_smithi121_684892_object45074:head v 583'2829, mlcod=583'2825)", 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:18:53.243005+0000", 2023-09-23T21:18:53.290 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.00043868800000000001, 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011191009999999999, 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243005+0000", 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243005+0000", 2023-09-23T21:18:53.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243005+0000", 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243025+0000", 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9779000000000002e-05 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243034+0000", 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5999999999999996e-06 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243040+0000", 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4999999999999999e-06 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243115+0000", 2023-09-23T21:18:53.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5476999999999996e-05 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:18:53.243132+0000", 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7110000000000001e-05 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 11 2023-09-23T21:18:53.294 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:18:53.338 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:53.408 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:53.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 27 16 45273 45257 104.752 58.5625 0.0120749 0.00953918 2023-09-23T21:18:53.589 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:53.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:53.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:53.667 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:18:53.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:53.743 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:53.796 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:53.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:53.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:53.902 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:18:53.911 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:18:53.911 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:18:53.911 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:18:53.912 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:18:53.912 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:18:53.912 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:18:53.934 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:53.997 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:18:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:18:54.088 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:54.103 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:54.130 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.171 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:54.276 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:18:54.307 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.368 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:18:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:18:54.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 28 16 46205 46189 103.091 58.25 0.00866471 0.00969664 2023-09-23T21:18:54.433 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:54 smithi123 ceph-mon[193903]: pgmap v1803: 99 pgs: 10 unknown, 89 active+clean; 2.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T21:18:54.473 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:18:54.503 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:54 smithi121 ceph-mon[211526]: pgmap v1803: 99 pgs: 10 unknown, 89 active+clean; 2.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T21:18:54.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:54 smithi121 ceph-mon[219709]: pgmap v1803: 99 pgs: 10 unknown, 89 active+clean; 2.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-09-23T21:18:54.693 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:18:54.706 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:18:54.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:18:54.810 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:54.858 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:54.901 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:18:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:54.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:18:55.007 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:55.090 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:55.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:18:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:18:54] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.43.0" 2023-09-23T21:18:55.201 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:18:55.217 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:55.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:55.315 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:18:55.342 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:55.345 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:55.424 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 29 16 47137 47121 101.545 58.25 0.00936293 0.00984244 2023-09-23T21:18:55.437 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:55.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[211526]: 1.0 scrub starts 2023-09-23T21:18:55.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[211526]: 1.0 scrub ok 2023-09-23T21:18:55.600 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:55.600 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:55.600 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[219709]: 1.0 scrub starts 2023-09-23T21:18:55.600 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[219709]: 1.0 scrub ok 2023-09-23T21:18:55.600 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:55.600 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:55 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:55.707 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:55.719 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:55 smithi123 ceph-mon[193903]: 1.0 scrub starts 2023-09-23T21:18:55.719 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:55 smithi123 ceph-mon[193903]: 1.0 scrub ok 2023-09-23T21:18:55.719 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:55 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:55.719 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:55 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T21:18:55.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:55.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:55.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:55.864 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:56.324 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:56.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:56.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 30 16 48183 48167 100.339 65.375 0.0181126 0.00996163 2023-09-23T21:18:56.428 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:56.452 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:56 smithi123 ceph-mon[193903]: pgmap v1804: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T21:18:56.452 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:56 smithi123 ceph-mon[193903]: failed to encode map e584 with expected crc 2023-09-23T21:18:56.452 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T21:18:56.452 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:56 smithi123 ceph-mon[193903]: osdmap e584: 8 total, 8 up, 5 in 2023-09-23T21:18:56.504 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:56.630 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:56.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:56.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[219709]: pgmap v1804: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T21:18:56.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[219709]: failed to encode map e584 with expected crc 2023-09-23T21:18:56.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T21:18:56.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[219709]: osdmap e584: 8 total, 8 up, 5 in 2023-09-23T21:18:56.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[211526]: pgmap v1804: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T21:18:56.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[211526]: failed to encode map e584 with expected crc 2023-09-23T21:18:56.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T21:18:56.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:56 smithi121 ceph-mon[211526]: osdmap e584: 8 total, 8 up, 5 in 2023-09-23T21:18:56.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:18:56.751 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:18:56.764 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:56.805 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:18:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:56.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:18:56.908 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:18:56.933 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:57.046 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:18:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:57.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:18:57.150 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:57.185 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:57.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 31 16 49033 49017 98.8155 53.125 0.0171901 0.0101148 2023-09-23T21:18:57.453 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:18:57.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: osdmap e585: 8 total, 8 up, 5 in 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/338039355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: pgmap v1807: 99 pgs: 3 peering, 96 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: osdmap e585: 8 total, 8 up, 5 in 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/338039355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: pgmap v1807: 99 pgs: 3 peering, 96 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T21:18:57.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:57.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:57 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:18:57.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: osdmap e585: 8 total, 8 up, 5 in 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: failed to encode map e585 with expected crc 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/338039355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:18:57.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: pgmap v1807: 99 pgs: 3 peering, 96 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T21:18:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:18:57.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:57 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:18:57.923 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:18:57.923 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:18:57.923 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:18:58.194 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:18:58.194 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:18:58.194 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:18:58.194 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:18:58.194 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:18:58.293 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:58.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:58.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:58.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 32 16 50080 50064 97.7723 65.4375 0.0117045 0.0102235 2023-09-23T21:18:58.430 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: osdmap e586: 8 total, 8 up, 5 in 2023-09-23T21:18:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:58.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:58.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:58 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T21:18:58.609 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:58.609 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: osdmap e586: 8 total, 8 up, 5 in 2023-09-23T21:18:58.609 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.609 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.609 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: osdmap e586: 8 total, 8 up, 5 in 2023-09-23T21:18:58.610 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: failed to encode map e586 with expected crc 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:18:58.611 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:58 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T21:18:58.931 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:58.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:59.039 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:18:59.077 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:59.121 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:18:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:59.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:18:59.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:18:59.254 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:59.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 33 16 50724 50708 96.0291 40.25 0.013791 0.0104086 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: osdmap e587: 8 total, 8 up, 5 in 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.608 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.609 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.609 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:18:59 smithi123 ceph-mon[193903]: pgmap v1810: 99 pgs: 3 active+undersized+remapped, 7 peering, 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T21:18:59.637 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:18:59.641 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:18:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:18:59.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:18:59.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: osdmap e587: 8 total, 8 up, 5 in 2023-09-23T21:18:59.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.748 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[211526]: pgmap v1810: 99 pgs: 3 active+undersized+remapped, 7 peering, 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: osdmap e587: 8 total, 8 up, 5 in 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.749 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: failed to encode map e587 with expected crc 2023-09-23T21:18:59.750 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:18:59 smithi121 ceph-mon[219709]: pgmap v1810: 99 pgs: 3 active+undersized+remapped, 7 peering, 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T21:18:59.755 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:18:59.852 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:18:59.963 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:18:59.983 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:00.002 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:00.024 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:00.030 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:00.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:00.047 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:00.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:00.100 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:00.131 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:00.132 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.133 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.133 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.133 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.133 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.133 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.133 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.133 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:00.145 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:00.176 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:00.329 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:00.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:00.415 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:00.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 34 16 51716 51700 95.0282 62 0.0155068 0.0105071 2023-09-23T21:19:00.433 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:00.460 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:00.502 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:00.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:00.606 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:00.635 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:00.874 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: osdmap e588: 8 total, 8 up, 5 in 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.875 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: 1.7 scrub starts 2023-09-23T21:19:00.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: 1.7 scrub ok 2023-09-23T21:19:00.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:00 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/129892051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:00.906 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:00.908 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: osdmap e588: 8 total, 8 up, 5 in 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: 1.7 scrub starts 2023-09-23T21:19:00.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: 1.7 scrub ok 2023-09-23T21:19:00.910 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/129892051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:00.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:00.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: osdmap e588: 8 total, 8 up, 5 in 2023-09-23T21:19:00.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.911 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: failed to encode map e588 with expected crc 2023-09-23T21:19:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: 1.7 scrub starts 2023-09-23T21:19:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: 1.7 scrub ok 2023-09-23T21:19:00.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:00 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/129892051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:00.947 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:19:01.012 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:01.040 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:01.329 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:01.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 35 16 52406 52390 93.545 43.125 0.0154247 0.0106836 2023-09-23T21:19:01.593 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:01.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:01.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:01.700 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:01.728 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:01.798 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:01 smithi121 ceph-mon[219709]: 1.1 scrub starts 2023-09-23T21:19:01.798 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:01 smithi121 ceph-mon[219709]: 1.1 scrub ok 2023-09-23T21:19:01.798 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:01 smithi121 ceph-mon[219709]: pgmap v1812: 99 pgs: 3 active+undersized+remapped, 4 peering, 92 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 59 MiB/s wr, 933 op/s 2023-09-23T21:19:01.798 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:01 smithi121 ceph-mon[211526]: 1.1 scrub starts 2023-09-23T21:19:01.798 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:01 smithi121 ceph-mon[211526]: 1.1 scrub ok 2023-09-23T21:19:01.799 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:01 smithi121 ceph-mon[211526]: pgmap v1812: 99 pgs: 3 active+undersized+remapped, 4 peering, 92 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 59 MiB/s wr, 933 op/s 2023-09-23T21:19:01.822 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:01.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:01.926 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:01.959 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:01 smithi123 ceph-mon[193903]: 1.1 scrub starts 2023-09-23T21:19:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:01 smithi123 ceph-mon[193903]: 1.1 scrub ok 2023-09-23T21:19:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:01 smithi123 ceph-mon[193903]: pgmap v1812: 99 pgs: 3 active+undersized+remapped, 4 peering, 92 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 59 MiB/s wr, 933 op/s 2023-09-23T21:19:02.015 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:02.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:02.119 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:02.144 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:02.185 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:02.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:02.289 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:02.381 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:02.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 36 16 53234 53218 92.384 51.75 0.015004 0.0108202 2023-09-23T21:19:02.524 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:02.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:02.632 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:02.706 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:02.800 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:02.800 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:02.808 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:02.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:02.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[219709]: 1.9 scrub starts 2023-09-23T21:19:02.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[219709]: 1.9 scrub ok 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[219709]: 1.3 scrub starts 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[219709]: 1.3 scrub ok 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[219709]: 1.5 scrub starts 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[219709]: 1.5 scrub ok 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[211526]: 1.9 scrub starts 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[211526]: 1.9 scrub ok 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[211526]: 1.3 scrub starts 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[211526]: 1.3 scrub ok 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[211526]: 1.5 scrub starts 2023-09-23T21:19:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:02 smithi121 ceph-mon[211526]: 1.5 scrub ok 2023-09-23T21:19:02.918 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:02.975 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:02 smithi123 ceph-mon[193903]: 1.9 scrub starts 2023-09-23T21:19:02.975 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:02 smithi123 ceph-mon[193903]: 1.9 scrub ok 2023-09-23T21:19:02.975 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:02 smithi123 ceph-mon[193903]: 1.3 scrub starts 2023-09-23T21:19:02.975 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:02 smithi123 ceph-mon[193903]: 1.3 scrub ok 2023-09-23T21:19:02.975 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:02 smithi123 ceph-mon[193903]: 1.5 scrub starts 2023-09-23T21:19:02.975 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:02 smithi123 ceph-mon[193903]: 1.5 scrub ok 2023-09-23T21:19:02.978 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.037 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:03.134 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:03.243 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:03.276 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.339 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:03.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 37 16 54215 54199 91.5441 61.3125 0.0084551 0.010918 2023-09-23T21:19:03.444 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:03.463 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:03.493 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.670 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:03.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:03.783 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:03.815 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:03.995 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:19:04.066 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:04.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:04.170 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:04.251 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.270 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:19:04.295 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:04.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:04.405 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:04.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:04.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[211526]: pgmap v1813: 99 pgs: 4 peering, 95 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 62 MiB/s wr, 982 op/s 2023-09-23T21:19:04.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/782019465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[219709]: pgmap v1813: 99 pgs: 4 peering, 95 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 62 MiB/s wr, 982 op/s 2023-09-23T21:19:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/782019465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:04.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 38 16 55039 55023 90.4902 51.5 0.015761 0.0110454 2023-09-23T21:19:04.434 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:04.642 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:04.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:04 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:04.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:04 smithi123 ceph-mon[193903]: pgmap v1813: 99 pgs: 4 peering, 95 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 62 MiB/s wr, 982 op/s 2023-09-23T21:19:04.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/782019465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:04.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:04.767 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:19:04.768 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:04.768 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:19:04.768 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:19:04.768 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:19:04.769 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:19:04.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:19:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:19:04] "GET /metrics HTTP/1.1" 200 36665 "" "Prometheus/2.43.0" 2023-09-23T21:19:04.944 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:04.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:05.049 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:05.091 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:05.336 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:05.337 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[211526]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.337 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:05.337 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[211526]: osdmap e589: 8 total, 8 up, 5 in 2023-09-23T21:19:05.337 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[211526]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.337 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[211526]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.337 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[211526]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.339 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:05.339 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[219709]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.339 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:05.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[219709]: osdmap e589: 8 total, 8 up, 5 in 2023-09-23T21:19:05.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[219709]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[219709]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.340 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:05 smithi121 ceph-mon[219709]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.353 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:05.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:05.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 39 16 55999 55983 89.7084 60 0.00815892 0.0111414 2023-09-23T21:19:05.458 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:05.488 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:05.640 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:05.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:05.686 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:05.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:05 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:05 smithi123 ceph-mon[193903]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:05 smithi123 ceph-mon[193903]: osdmap e589: 8 total, 8 up, 5 in 2023-09-23T21:19:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:05 smithi123 ceph-mon[193903]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:05 smithi123 ceph-mon[193903]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:05 smithi123 ceph-mon[193903]: failed to encode map e589 with expected crc 2023-09-23T21:19:05.745 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:05.791 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:05.813 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:05.818 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:05.833 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:05.852 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:05.862 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:05.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:05.966 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:05.993 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:06.035 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:06.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:06.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:06.082 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:06.115 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:06.139 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:06.139 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:06.166 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:06.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.168 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.168 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:06.174 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:06.283 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:06.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:06.387 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:06.413 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:06.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:19:06.426407+0000 min lat: 0.00109569 max lat: 0.255024 avg lat: 0.0112535 2023-09-23T21:19:06.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:19:06.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 40 16 56764 56748 88.6609 47.8125 0.00325239 0.0112535 2023-09-23T21:19:06.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:06 smithi121 ceph-mon[219709]: pgmap v1815: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 57 MiB/s wr, 894 op/s 2023-09-23T21:19:06.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3785787120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:06 smithi121 ceph-mon[211526]: pgmap v1815: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 57 MiB/s wr, 894 op/s 2023-09-23T21:19:06.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3785787120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:06.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:06 smithi123 ceph-mon[193903]: pgmap v1815: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 57 MiB/s wr, 894 op/s 2023-09-23T21:19:06.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3785787120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:07.144 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:57463 14.9 e589/573 14:9732584a:::benchmark_data_smithi121_684892_object57462:head v 589'3531, mlcod=589'3525)", 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.135168+0000", 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0073476339999999996, 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0073790890000000001, 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135168+0000", 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135168+0000", 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135168+0000", 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135186+0000", 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.842e-05 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135192+0000", 2023-09-23T21:19:07.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5690000000000004e-06 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135195+0000", 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5829999999999998e-06 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.139165+0000", 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039700079999999997 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.139184+0000", 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9071000000000001e-05 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:57469 14.7 14:ea4ae573:::benchmark_data_smithi121_684892_object57468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.135209+0000", 2023-09-23T21:19:07.149 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0073065220000000002, 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0074142210000000004, 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57469 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135209+0000", 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135209+0000", 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135208+0000", 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135242+0000", 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3905000000000001e-05 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135251+0000", 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1449999999999997e-06 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135262+0000", 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0959e-05 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.138356+0000", 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0030946010000000002 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.138498+0000", 2023-09-23T21:19:07.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000141576 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.138653+0000", 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015464600000000001 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.154 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:57460 14.3 e589/573 14:c8877abf:::benchmark_data_smithi121_684892_object57459:head v 589'3615, mlcod=589'3611)", 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.137521+0000", 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.004994531, 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051847860000000003, 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137521+0000", 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137521+0000", 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137520+0000", 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137549+0000", 2023-09-23T21:19:07.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8682000000000001e-05 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137561+0000", 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2309e-05 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137568+0000", 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4180000000000002e-06 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137606+0000", 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8075999999999999e-05 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137623+0000", 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7163000000000002e-05 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:57465 14.5 e589/573 14:a3b092dc:::benchmark_data_smithi121_684892_object57464:head v 589'3659, mlcod=589'3657)", 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.140563+0000", 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0019521359999999999, 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022075609999999998, 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140563+0000", 2023-09-23T21:19:07.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140563+0000", 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140562+0000", 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140586+0000", 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3859999999999999e-05 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140602+0000", 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5218000000000001e-05 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140608+0000", 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3729999999999998e-06 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140725+0000", 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011682499999999999 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140742+0000", 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7357999999999999e-05 2023-09-23T21:19:07.162 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:57470 14.9 e589/573 14:9abdedef:::benchmark_data_smithi121_684892_object57469:head v 589'3532, mlcod=589'3525)", 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.140637+0000", 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0018782250000000001, 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022034450000000001, 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140637+0000", 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140637+0000", 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140637+0000", 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:07.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140656+0000", 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8638999999999999e-05 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140663+0000", 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0609999999999998e-06 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140667+0000", 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0729999999999998e-06 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140829+0000", 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016239600000000001 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.140847+0000", 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7776000000000001e-05 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:57474 14.4 e589/573 14:2dbfad43:::benchmark_data_smithi121_684892_object57473:head v 589'3563, mlcod=589'3561)", 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.142323+0000", 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.000192461, 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00059228000000000004, 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.167 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142323+0000", 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142323+0000", 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142322+0000", 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142346+0000", 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4074e-05 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142358+0000", 2023-09-23T21:19:07.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1610000000000001e-05 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142364+0000", 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1009999999999997e-06 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142502+0000", 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013821699999999999 2023-09-23T21:19:07.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.142522+0000", 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9601000000000001e-05 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:57462 14.7 14:ef2ebe50:::benchmark_data_smithi121_684892_object57461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.126976+0000", 2023-09-23T21:19:07.171 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.01553912, 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016012194, 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 57462 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.126976+0000", 2023-09-23T21:19:07.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.126976+0000", 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.126975+0000", 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.127014+0000", 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9542000000000002e-05 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.127028+0000", 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3652000000000001e-05 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.127035+0000", 2023-09-23T21:19:07.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8669999999999996e-06 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.127210+0000", 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000175055 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.127345+0000", 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013465800000000001 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,7", 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.127490+0000", 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014580499999999999 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137177+0000", 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0096865609999999998 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:07.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.138335+0000", 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0011576659999999999 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:57464 14.6 e589/573 14:62c933eb:::benchmark_data_smithi121_684892_object57463:head v 589'3611, mlcod=589'3608)", 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.132012+0000", 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.010503696999999999, 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011068896, 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:07.177 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.132012+0000", 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.132012+0000", 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.132011+0000", 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.132046+0000", 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5660000000000001e-05 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.132066+0000", 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9113000000000001e-05 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.132073+0000", 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.272e-06 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137578+0000", 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005504798 2023-09-23T21:19:07.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137596+0000", 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8547999999999999e-05 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:57457 14.9 e589/573 14:93374309:::benchmark_data_smithi121_684892_object57456:head v 589'3530, mlcod=589'3525)", 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:07.135118+0000", 2023-09-23T21:19:07.181 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0073975400000000002, 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0080282830000000006, 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135118+0000", 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:07.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135118+0000", 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135117+0000", 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135141+0000", 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3507e-05 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135152+0000", 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0574000000000001e-05 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.135156+0000", 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4680000000000003e-06 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137392+0000", 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022356020000000002 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:07.137411+0000", 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9449e-05 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T21:19:07.185 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 9 2023-09-23T21:19:07.186 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:19:07.309 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:07.358 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:19:07.414 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:07.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 41 16 57670 57654 87.8794 56.625 0.00875648 0.0113744 2023-09-23T21:19:07.444 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:07 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:19:07.513 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:07.617 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:07.643 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:07 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:19:07.644 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:07 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:19:07.647 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.753 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:07.857 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:07.886 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:07.936 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:07.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:07.980 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:07.988 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:19:07.988 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:19:07.988 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:19:08.041 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:08.111 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:08.283 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:08 smithi121 ceph-mon[219709]: pgmap v1816: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 650 op/s 2023-09-23T21:19:08.312 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:19:08.325 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:08.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:08.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:08.425 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 42 16 58454 58438 86.9536 49 0.185959 0.0114624 2023-09-23T21:19:08.430 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:08.460 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:08 smithi123 ceph-mon[193903]: pgmap v1816: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 650 op/s 2023-09-23T21:19:08.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:08 smithi121 ceph-mon[211526]: pgmap v1816: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 41 MiB/s wr, 650 op/s 2023-09-23T21:19:08.710 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:08.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:08.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:08.809 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:08.815 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:08.894 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.092 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:09.202 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:09.272 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 43 16 59288 59272 86.1434 52.125 0.00261249 0.011594 2023-09-23T21:19:09.482 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:09.495 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:09.597 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:09.626 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.795 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:09.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:09.899 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:09.929 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:09.933 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:09.975 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:09.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:09.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:10.082 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:10.108 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.158 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:10.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:10.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:10.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:10.293 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 44 16 60018 60002 85.2224 45.625 0.00841074 0.0117294 2023-09-23T21:19:10.450 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:10.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:10.555 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:10.586 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:10.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:10 smithi123 ceph-mon[193903]: pgmap v1817: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 848 op/s 2023-09-23T21:19:10.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:10 smithi121 ceph-mon[211526]: pgmap v1817: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 848 op/s 2023-09-23T21:19:10.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:10 smithi121 ceph-mon[219709]: pgmap v1817: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 848 op/s 2023-09-23T21:19:11.138 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:11.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:11.241 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:11.252 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:19:11.252 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:19:11.252 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:19:11.252 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:19:11.252 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:19:11.335 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 45 16 60945 60929 84.616 57.9375 0.025197 0.0118124 2023-09-23T21:19:11.452 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:19:11.478 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:11.582 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:11.653 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4049405777' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:11.658 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.776 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:11.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:11.782 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:11.801 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:11.820 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:11.838 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:11.856 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:11.881 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:11.906 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:11.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4049405777' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:11.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4049405777' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:11.941 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:11.973 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:12.001 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.003 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:12.017 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:12.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:12.132 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:12.162 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:12.214 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:12.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:12.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:12.318 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:12.347 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:12.349 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:19:12.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 46 16 61645 61629 83.7274 43.75 0.0249196 0.0119364 2023-09-23T21:19:12.595 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:12.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: pgmap v1818: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 53 MiB/s wr, 838 op/s 2023-09-23T21:19:12.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:12.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4049405777' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:12.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: osdmap e590: 8 total, 8 up, 5 in 2023-09-23T21:19:12.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.597 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1013017576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:12.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:12.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:12.598 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: pgmap v1818: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 53 MiB/s wr, 838 op/s 2023-09-23T21:19:12.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:12.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4049405777' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:12.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: osdmap e590: 8 total, 8 up, 5 in 2023-09-23T21:19:12.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.599 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1013017576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:12.699 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:12.728 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: pgmap v1818: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 53 MiB/s wr, 838 op/s 2023-09-23T21:19:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4049405777' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: osdmap e590: 8 total, 8 up, 5 in 2023-09-23T21:19:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: failed to encode map e590 with expected crc 2023-09-23T21:19:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1013017576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:12.901 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:13.321 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:13.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:13.425 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:13.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 47 16 62433 62417 82.9937 49.25 0.0235984 0.0120442 2023-09-23T21:19:13.491 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:13.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:13.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[211526]: pgmap v1820: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 37 MiB/s wr, 582 op/s 2023-09-23T21:19:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:13.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:13 smithi121 ceph-mon[219709]: pgmap v1820: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 37 MiB/s wr, 582 op/s 2023-09-23T21:19:13.687 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:13.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:13.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:13.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:13.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:13 smithi123 ceph-mon[193903]: pgmap v1820: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 37 MiB/s wr, 582 op/s 2023-09-23T21:19:13.796 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:13.894 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:13.918 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:19:13.918 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:13.918 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T21:19:13.918 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.835145 2023-09-23T21:19:13.918 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0.8351445203708832 2023-09-23T21:19:13.941 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:13.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:14.044 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:14.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.119 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:14.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:14.225 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:14.257 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.296 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:14.317 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:14.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:14.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:14.423 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:14.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 48 16 63217 63201 82.2855 49 0.0198701 0.0121479 2023-09-23T21:19:14.451 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.551 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:14.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:14.655 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:14.721 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:14.885 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:19:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:19:14] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.43.0" 2023-09-23T21:19:14.887 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:14.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:15.008 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:15.028 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:19:15.037 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:15.083 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:15.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:15.186 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:15.244 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:15.245 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:15.280 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:15.325 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:15.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 49 16 63982 63966 81.5819 47.8125 0.0102811 0.0122526 2023-09-23T21:19:15.484 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:15.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:15.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:15.588 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:15.664 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:15.734 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:15.785 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:15.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:15.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:15.862 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:15.892 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:15.961 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:16.256 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:19:16.263 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:16.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:16.377 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:16.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 50 16 64906 64890 81.1052 57.75 0.0223076 0.0123216 2023-09-23T21:19:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:16 smithi123 ceph-mon[193903]: pgmap v1821: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T21:19:16.482 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:16.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:16 smithi121 ceph-mon[219709]: pgmap v1821: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T21:19:16.560 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:16 smithi121 ceph-mon[211526]: pgmap v1821: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 52 MiB/s wr, 831 op/s 2023-09-23T21:19:16.610 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:16.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:16.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:16.732 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:16.821 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:16.970 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:16.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:16.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:17.009 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:17.079 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:17.154 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.300 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:17.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:17.408 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:17.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 51 16 65694 65678 80.4805 49.25 0.014517 0.0124201 2023-09-23T21:19:17.439 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/838216968' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.8351445203708832}]: dispatch 2023-09-23T21:19:17.457 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.4 primary-affinity to 0.835145 (8547322) 2023-09-23T21:19:17.497 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.597 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:17.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:17.702 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:17.732 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.781 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/838216968' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.8351445203708832}]: dispatch 2023-09-23T21:19:17.781 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/838216968' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.8351445203708832}]: dispatch 2023-09-23T21:19:17.824 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:17.927 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:17.954 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:17.994 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:17.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:17.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:18.098 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:18.124 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:18.165 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:18.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:18.268 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:18.293 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:18.334 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:18.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:18.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:18.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 52 16 65895 65879 79.1744 12.5625 0.00321219 0.0124249 2023-09-23T21:19:18.437 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:18.472 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:18.636 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[219709]: pgmap v1822: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:19:18.636 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[219709]: failed to encode map e591 with expected crc 2023-09-23T21:19:18.636 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/838216968' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.8351445203708832}]': finished 2023-09-23T21:19:18.636 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[219709]: osdmap e591: 8 total, 8 up, 5 in 2023-09-23T21:19:18.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[219709]: failed to encode map e591 with expected crc 2023-09-23T21:19:18.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[211526]: pgmap v1822: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:19:18.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[211526]: failed to encode map e591 with expected crc 2023-09-23T21:19:18.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/838216968' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.8351445203708832}]': finished 2023-09-23T21:19:18.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[211526]: osdmap e591: 8 total, 8 up, 5 in 2023-09-23T21:19:18.637 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:18 smithi121 ceph-mon[211526]: failed to encode map e591 with expected crc 2023-09-23T21:19:18.721 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:18 smithi123 ceph-mon[193903]: pgmap v1822: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T21:19:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:18 smithi123 ceph-mon[193903]: failed to encode map e591 with expected crc 2023-09-23T21:19:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/838216968' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.8351445203708832}]': finished 2023-09-23T21:19:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:18 smithi123 ceph-mon[193903]: osdmap e591: 8 total, 8 up, 5 in 2023-09-23T21:19:18.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:18 smithi123 ceph-mon[193903]: failed to encode map e591 with expected crc 2023-09-23T21:19:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:18.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:18.856 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:18.896 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.050 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:19.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:19.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:19.135 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:19.155 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:19.159 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:19.185 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:19.190 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.210 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:19.243 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:19.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:19.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:19.247 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:19.280 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:19.311 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:19.337 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:19.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.338 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.339 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.339 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.339 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.339 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.339 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.339 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:19.346 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:19.376 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 53 16 66732 66716 78.6675 52.3125 0.0113426 0.0127058 2023-09-23T21:19:19.470 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[211526]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.470 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[211526]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.475 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:19 smithi123 ceph-mon[193903]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.475 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:19 smithi123 ceph-mon[193903]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.475 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:19 smithi123 ceph-mon[193903]: osdmap e592: 8 total, 8 up, 5 in 2023-09-23T21:19:19.475 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:19 smithi123 ceph-mon[193903]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.475 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3555312255' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:19.475 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1418036728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:19.475 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:19 smithi123 ceph-mon[193903]: pgmap v1825: 99 pgs: 99 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T21:19:19.487 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:19:19.531 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:19.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:19.536 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:66789 14.2 14:4f5fa95c:::benchmark_data_smithi121_684892_object66788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.489979+0000", 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.031909980999999997, 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.031945456999999997, 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:19.537 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 66789 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.489979+0000", 2023-09-23T21:19:19.538 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.489979+0000", 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.489978+0000", 2023-09-23T21:19:19.539 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.490011+0000", 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2608999999999999e-05 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.490024+0000", 2023-09-23T21:19:19.540 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3434999999999999e-05 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.490031+0000", 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9079999999999998e-06 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.490356+0000", 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00032448399999999999 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.541 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op must wait for map", 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.490361+0000", 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8409999999999996e-06 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.498815+0000", 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0084536620000000007 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.542 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.543 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:19.543 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.498932+0000", 2023-09-23T21:19:19.543 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000116783 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.499067+0000", 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013485099999999999 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.501855+0000", 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002788451 2023-09-23T21:19:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518747+0000", 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016892014 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:66801 14.2 14:4a11954a:::benchmark_data_smithi121_684892_object66800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T21:19:19.545 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.500699+0000", 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.021189864999999999, 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021346468, 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 66801 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.546 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.500699+0000", 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.500699+0000", 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.500698+0000", 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.500733+0000", 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5233e-05 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.500746+0000", 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2806000000000001e-05 2023-09-23T21:19:19.548 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.500752+0000", 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3289999999999999e-06 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.505269+0000", 2023-09-23T21:19:19.549 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045166319999999996 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.505404+0000", 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013563200000000001 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:19:19.550 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.505544+0000", 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013999499999999999 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.515465+0000", 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0099208370000000001 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:66807 14.4 14:25b685ad:::benchmark_data_smithi121_684892_object66806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.514493+0000", 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0073958549999999998, 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0076437850000000002, 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:19:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 66807 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.554 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514493+0000", 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514493+0000", 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514492+0000", 2023-09-23T21:19:19.555 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514526+0000", 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3787e-05 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514539+0000", 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3254999999999999e-05 2023-09-23T21:19:19.556 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514545+0000", 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4450000000000001e-06 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514600+0000", 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4814000000000001e-05 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:19.557 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514730+0000", 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012962100000000001 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514869+0000", 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000139207 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.520017+0000", 2023-09-23T21:19:19.558 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051475490000000004 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:66808 14.4 14:2c64bfb0:::benchmark_data_smithi121_684892_object66807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.514923+0000", 2023-09-23T21:19:19.559 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0069663090000000004, 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0073059570000000001, 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 66808 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.560 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514923+0000", 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.561 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514923+0000", 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514922+0000", 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.562 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514967+0000", 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4943e-05 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514977+0000", 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7569999999999999e-06 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.514983+0000", 2023-09-23T21:19:19.563 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8830000000000003e-06 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.517310+0000", 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023275219999999998 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.517423+0000", 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113217 2023-09-23T21:19:19.564 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.517544+0000", 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000121104 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.565 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:66797 14.e e593/573)", 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.518537+0000", 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.003352252, 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037882229999999999, 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518537+0000", 2023-09-23T21:19:19.566 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518537+0000", 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518537+0000", 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:19.567 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518558+0000", 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1574999999999999e-05 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518564+0000", 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.015e-06 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.568 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518568+0000", 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8619999999999996e-06 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.569 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:66798 14.e e593/573)", 2023-09-23T21:19:19.570 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.518579+0000", 2023-09-23T21:19:19.570 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0033100320000000001, 2023-09-23T21:19:19.570 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037951759999999999, 2023-09-23T21:19:19.570 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.570 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T21:19:19.570 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518579+0000", 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518579+0000", 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.571 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518579+0000", 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518600+0000", 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0843e-05 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518606+0000", 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6150000000000002e-06 2023-09-23T21:19:19.572 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518611+0000", 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3420000000000001e-06 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.573 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:66804 14.c e593/573 14:37985cc7:::benchmark_data_smithi121_684892_object66803:head v 593'4251, mlcod=592'4249)", 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.518649+0000", 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0032403520000000002, 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00379218, 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518649+0000", 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.574 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518649+0000", 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518649+0000", 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967296 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.575 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518669+0000", 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0641000000000001e-05 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518675+0000", 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0959999999999997e-06 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.518679+0000", 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0409999999999999e-06 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.520037+0000", 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001357074 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.520052+0000", 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5590000000000002e-05 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.577 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:66810 14.2 14:4a9518ef:::benchmark_data_smithi121_684892_object66809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.521083+0000", 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.00080652600000000001, 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001428831, 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:19.578 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 66810 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521083+0000", 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521083+0000", 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521082+0000", 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521123+0000", 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1664e-05 2023-09-23T21:19:19.580 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521135+0000", 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1657999999999999e-05 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521142+0000", 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4569999999999999e-06 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521248+0000", 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010556 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521407+0000", 2023-09-23T21:19:19.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000158655 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 4,5", 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521551+0000", 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014457900000000001 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_repop(client.47438.0:66806 14.9 e593/573 14:90548ceb:::benchmark_data_smithi121_684892_object66805:head v 593'4066, mlcod=592'4064)", 2023-09-23T21:19:19.583 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.521196+0000", 2023-09-23T21:19:19.584 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.00069369799999999997, 2023-09-23T21:19:19.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00143052, 2023-09-23T21:19:19.584 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.584 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:19:19.584 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.584 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521196+0000", 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521196+0000", 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521195+0000", 2023-09-23T21:19:19.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521220+0000", 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5469999999999998e-05 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521232+0000", 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1618e-05 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521238+0000", 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2360000000000001e-06 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:19:19.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521331+0000", 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3171000000000006e-05 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521348+0000", 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6461e-05 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.588 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:66811 14.2 14.70086ef2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.521434+0000", 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.00045552100000000002, 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0012562490000000001, 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 66811 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521434+0000", 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521434+0000", 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521433+0000", 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521481+0000", 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8547999999999997e-05 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521493+0000", 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1404e-05 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521499+0000", 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5230000000000001e-06 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.592 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:66812 14.d 14.3e4acaad (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:19.521514+0000", 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.00037555599999999998, 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001235407, 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "queued for pg", 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:19:19.593 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 66812 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521514+0000", 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:19:19.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521514+0000", 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521513+0000", 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521579+0000", 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5458999999999998e-05 2023-09-23T21:19:19.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521591+0000", 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2221e-05 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:19.521598+0000", 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.917e-06 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:19:19.596 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.597 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:19:19.597 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T21:19:19.597 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 11 2023-09-23T21:19:19.597 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:19:19.635 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:19.665 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:19.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[211526]: osdmap e592: 8 total, 8 up, 5 in 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[211526]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3555312255' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1418036728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[211526]: pgmap v1825: 99 pgs: 99 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[219709]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[219709]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[219709]: osdmap e592: 8 total, 8 up, 5 in 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[219709]: failed to encode map e592 with expected crc 2023-09-23T21:19:19.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3555312255' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:19.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1418036728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:19.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:19 smithi121 ceph-mon[219709]: pgmap v1825: 99 pgs: 99 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T21:19:20.426 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 54 16 67400 67384 77.9838 41.75 0.0141624 0.0128191 2023-09-23T21:19:20.554 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:19:20.554 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:20.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:20.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:20.658 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1418036728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: osdmap e593: 8 total, 8 up, 5 in 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:20 smithi123 ceph-mon[193903]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.749 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:20.904 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:20.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:20.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1418036728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: osdmap e593: 8 total, 8 up, 5 in 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[211526]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1418036728' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: osdmap e593: 8 total, 8 up, 5 in 2023-09-23T21:19:20.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:20 smithi121 ceph-mon[219709]: failed to encode map e593 with expected crc 2023-09-23T21:19:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:20.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:21.018 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:21.047 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.097 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:21.204 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:21.231 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.280 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:21.384 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:21.391 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:21.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 55 16 68250 68234 77.5317 53.125 0.0211775 0.0128911 2023-09-23T21:19:21.471 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.680 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:21.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:21.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:21 smithi123 ceph-mon[193903]: 1.6 scrub starts 2023-09-23T21:19:21.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:21 smithi123 ceph-mon[193903]: 1.6 scrub ok 2023-09-23T21:19:21.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:21 smithi123 ceph-mon[193903]: pgmap v1827: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 45 MiB/s wr, 713 op/s 2023-09-23T21:19:21.792 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:21.862 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:21.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:21 smithi121 ceph-mon[211526]: 1.6 scrub starts 2023-09-23T21:19:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:21 smithi121 ceph-mon[211526]: 1.6 scrub ok 2023-09-23T21:19:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:21 smithi121 ceph-mon[211526]: pgmap v1827: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 45 MiB/s wr, 713 op/s 2023-09-23T21:19:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:21 smithi121 ceph-mon[219709]: 1.6 scrub starts 2023-09-23T21:19:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:21 smithi121 ceph-mon[219709]: 1.6 scrub ok 2023-09-23T21:19:21.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:21 smithi121 ceph-mon[219709]: pgmap v1827: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 45 MiB/s wr, 713 op/s 2023-09-23T21:19:22.036 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:22.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:22.154 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:22.231 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.363 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:22.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 56 16 69109 69093 77.1058 53.6875 0.0263683 0.0129649 2023-09-23T21:19:22.476 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:22.548 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.611 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:22.619 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:22.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:22.677 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:19:22.722 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:22.747 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:22.788 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:22.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:22.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:22.892 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:22.975 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.043 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:23.062 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:23.202 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:23.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:23.319 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:23.396 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 57 16 69750 69734 76.4559 40.0625 0.0240446 0.0130738 2023-09-23T21:19:23.548 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 6, 2, 7] out_osds: [0, 1, 3] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:19:23.548 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:23.549 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [4, 5, 6, 2, 7] 2023-09-23T21:19:23.549 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T21:19:23.646 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:23.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:23.754 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:23.785 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:23.978 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:19:23.978 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:19:23.978 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:19:23.978 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:19:23.978 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:19:24.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 58 16 70460 70444 75.9027 44.375 0.00847477 0.0131678 2023-09-23T21:19:24.442 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:24.546 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:24.575 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:24.663 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:24.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:24 smithi123 ceph-mon[193903]: pgmap v1828: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 35 MiB/s wr, 552 op/s 2023-09-23T21:19:24.771 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:24.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:24 smithi121 ceph-mon[211526]: pgmap v1828: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 35 MiB/s wr, 552 op/s 2023-09-23T21:19:24.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:24 smithi121 ceph-mon[219709]: pgmap v1828: 99 pgs: 99 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 35 MiB/s wr, 552 op/s 2023-09-23T21:19:24.848 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:24.982 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:24.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:25.024 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:19:25.090 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:25.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:19:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:19:24] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.43.0" 2023-09-23T21:19:25.211 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:25.239 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:25.395 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:25.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:25.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 59 16 71158 71142 75.3555 43.625 0.0129568 0.0132647 2023-09-23T21:19:25.504 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:25.538 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:25 smithi121 ceph-mon[211526]: 1.2 scrub starts 2023-09-23T21:19:25.539 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:25 smithi121 ceph-mon[211526]: 1.2 scrub ok 2023-09-23T21:19:25.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:25 smithi121 ceph-mon[219709]: 1.2 scrub starts 2023-09-23T21:19:25.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:25 smithi121 ceph-mon[219709]: 1.2 scrub ok 2023-09-23T21:19:25.542 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:25.595 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:25.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:25.642 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:25.668 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:25.693 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:25.727 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:25 smithi123 ceph-mon[193903]: 1.2 scrub starts 2023-09-23T21:19:25.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:25 smithi123 ceph-mon[193903]: 1.2 scrub ok 2023-09-23T21:19:25.758 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:25.784 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:25.786 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.786 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.786 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.787 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.787 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.787 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.787 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.787 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:25.813 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:25.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:25.906 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:25.918 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:25.989 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:26.212 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:26.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:26.326 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:26.355 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:26.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:19:26.428222+0000 min lat: 0.00109569 max lat: 1.03451 avg lat: 0.0133403 2023-09-23T21:19:26.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:19:26.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 60 16 71956 71940 74.9308 49.875 0.0116842 0.0133403 2023-09-23T21:19:26.486 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:19:26.549 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:26 smithi123 ceph-mon[193903]: pgmap v1829: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T21:19:26.549 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:26 smithi123 ceph-mon[193903]: 1.4 scrub starts 2023-09-23T21:19:26.549 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:26 smithi123 ceph-mon[193903]: 1.4 scrub ok 2023-09-23T21:19:26.549 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1500609668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:26.549 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3414511867' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:26.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[211526]: pgmap v1829: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T21:19:26.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[211526]: 1.4 scrub starts 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[211526]: 1.4 scrub ok 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1500609668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3414511867' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[219709]: pgmap v1829: 99 pgs: 99 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[219709]: 1.4 scrub starts 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[219709]: 1.4 scrub ok 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1500609668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3414511867' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:27.281 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:27.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:27.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:27.385 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:27.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 61 16 72858 72842 74.6265 56.375 0.015888 0.0133954 2023-09-23T21:19:27.467 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:27.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: 1.8 scrub starts 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: 1.8 scrub ok 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: failed to encode map e594 with expected crc 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: failed to encode map e594 with expected crc 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3414511867' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: osdmap e594: 8 total, 8 up, 5 in 2023-09-23T21:19:27.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[211526]: pgmap v1831: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 35 MiB/s wr, 555 op/s 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: 1.8 scrub starts 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: 1.8 scrub ok 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: failed to encode map e594 with expected crc 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: failed to encode map e594 with expected crc 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3414511867' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: osdmap e594: 8 total, 8 up, 5 in 2023-09-23T21:19:27.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:27 smithi121 ceph-mon[219709]: pgmap v1831: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 35 MiB/s wr, 555 op/s 2023-09-23T21:19:27.710 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:27.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:27.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:27.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: 1.8 scrub starts 2023-09-23T21:19:27.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: 1.8 scrub ok 2023-09-23T21:19:27.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:19:27.723 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:19:27.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: failed to encode map e594 with expected crc 2023-09-23T21:19:27.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: failed to encode map e594 with expected crc 2023-09-23T21:19:27.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3414511867' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:27.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: osdmap e594: 8 total, 8 up, 5 in 2023-09-23T21:19:27.724 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:27 smithi123 ceph-mon[193903]: pgmap v1831: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 35 MiB/s wr, 555 op/s 2023-09-23T21:19:27.823 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:27.855 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.134 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:28.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:28.240 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:28.271 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.322 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:28.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:28.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:28.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 62 16 73585 73569 74.1557 45.4375 0.0229004 0.0134791 2023-09-23T21:19:28.457 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.505 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:28.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:28.530 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:28.530 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:28.609 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:28.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:28 smithi121 conmon[211495]: 2023-09-23T21:19:28.429+0000 7fdc50c66700 -1 mon.a@0(leader).osd e594 definitely_dead 0 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3778674053' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[211526]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3778674053' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:28.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:28.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:28 smithi121 ceph-mon[219709]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:19:28.698 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:28.783 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:28.783 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:28.783 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:28 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3778674053' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:28.783 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:28 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T21:19:28.783 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:28 smithi123 ceph-mon[193903]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T21:19:28.885 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:28.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:28.906 INFO:teuthology.orchestra.run.smithi121.stderr:marked out osd.5. 2023-09-23T21:19:29.000 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:29.027 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.122 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:29.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:29.226 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:29.253 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.295 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:29.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:29.424 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.427 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 63 16 73982 73966 73.3724 24.8125 0.0319255 0.0135179 2023-09-23T21:19:29.465 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:29.569 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:29.595 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.685 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:19:29.727 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:29.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:29.742 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:19:29.831 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:29.857 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:29.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[219709]: failed to encode map e595 with expected crc 2023-09-23T21:19:29.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[219709]: failed to encode map e595 with expected crc 2023-09-23T21:19:29.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:19:29.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[219709]: osdmap e595: 8 total, 8 up, 4 in 2023-09-23T21:19:29.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[219709]: failed to encode map e595 with expected crc 2023-09-23T21:19:29.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[219709]: pgmap v1833: 99 pgs: 99 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 861 op/s 2023-09-23T21:19:29.905 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:29.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:29 smithi123 ceph-mon[193903]: failed to encode map e595 with expected crc 2023-09-23T21:19:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:29 smithi123 ceph-mon[193903]: failed to encode map e595 with expected crc 2023-09-23T21:19:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:29 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:19:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:29 smithi123 ceph-mon[193903]: osdmap e595: 8 total, 8 up, 4 in 2023-09-23T21:19:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:29 smithi123 ceph-mon[193903]: failed to encode map e595 with expected crc 2023-09-23T21:19:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:29 smithi123 ceph-mon[193903]: pgmap v1833: 99 pgs: 99 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 861 op/s 2023-09-23T21:19:30.017 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:30.045 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:30.046 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.106 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:30.131 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:30.140 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:30.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[211526]: failed to encode map e595 with expected crc 2023-09-23T21:19:30.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[211526]: failed to encode map e595 with expected crc 2023-09-23T21:19:30.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T21:19:30.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[211526]: osdmap e595: 8 total, 8 up, 4 in 2023-09-23T21:19:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[211526]: failed to encode map e595 with expected crc 2023-09-23T21:19:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:29 smithi121 ceph-mon[211526]: pgmap v1833: 99 pgs: 99 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 54 MiB/s wr, 861 op/s 2023-09-23T21:19:30.213 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:30.240 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.333 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:30.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 64 16 73982 73966 72.226 0 - 0.0135179 2023-09-23T21:19:30.436 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:30.463 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.557 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:30.661 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:30.689 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.710 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:30.736 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:30.841 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:30.866 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:30.909 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:30.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:31.016 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:31.107 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:31.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[211526]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[211526]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[211526]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[211526]: osdmap e596: 8 total, 8 up, 4 in 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[211526]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[211526]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[211526]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[219709]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[219709]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[219709]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[219709]: osdmap e596: 8 total, 8 up, 4 in 2023-09-23T21:19:31.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[219709]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[219709]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:30 smithi121 ceph-mon[219709]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.241 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:30 smithi123 ceph-mon[193903]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.241 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:30 smithi123 ceph-mon[193903]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.242 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:30 smithi123 ceph-mon[193903]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.242 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:30 smithi123 ceph-mon[193903]: osdmap e596: 8 total, 8 up, 4 in 2023-09-23T21:19:31.242 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:30 smithi123 ceph-mon[193903]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.242 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:30 smithi123 ceph-mon[193903]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.242 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:30 smithi123 ceph-mon[193903]: failed to encode map e596 with expected crc 2023-09-23T21:19:31.371 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:31.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:31.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 65 16 74023 74007 71.1542 1.28125 0.0505492 0.0140469 2023-09-23T21:19:31.481 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:31.531 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:31.656 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:31.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:31.761 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:31.912 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:32.030 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:32.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:32.139 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: osdmap e597: 8 total, 8 up, 4 in 2023-09-23T21:19:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: 14.d starting backfill to osd.2 from (0'0,0'0] MAX to 594'4613 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: 14.7 starting backfill to osd.2 from (0'0,0'0] MAX to 594'4704 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: pgmap v1836: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 94 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 44 MiB/s wr, 699 op/s; 9127/221259 objects degraded (4.125%); 4613/221259 objects misplaced (2.085%) 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[211526]: osdmap e598: 8 total, 8 up, 4 in 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: osdmap e597: 8 total, 8 up, 4 in 2023-09-23T21:19:32.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: 14.d starting backfill to osd.2 from (0'0,0'0] MAX to 594'4613 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: 14.7 starting backfill to osd.2 from (0'0,0'0] MAX to 594'4704 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: pgmap v1836: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 94 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 44 MiB/s wr, 699 op/s; 9127/221259 objects degraded (4.125%); 4613/221259 objects misplaced (2.085%) 2023-09-23T21:19:32.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:31 smithi121 ceph-mon[219709]: osdmap e598: 8 total, 8 up, 4 in 2023-09-23T21:19:32.222 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: osdmap e597: 8 total, 8 up, 4 in 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: failed to encode map e597 with expected crc 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: 14.d starting backfill to osd.2 from (0'0,0'0] MAX to 594'4613 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: 14.7 starting backfill to osd.2 from (0'0,0'0] MAX to 594'4704 2023-09-23T21:19:32.223 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: pgmap v1836: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 94 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 44 MiB/s wr, 699 op/s; 9127/221259 objects degraded (4.125%); 4613/221259 objects misplaced (2.085%) 2023-09-23T21:19:32.224 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:31 smithi123 ceph-mon[193903]: osdmap e598: 8 total, 8 up, 4 in 2023-09-23T21:19:32.226 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:32.226 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:32.230 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:32.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 66 16 74395 74379 70.4283 23.25 0.0106266 0.0141579 2023-09-23T21:19:32.487 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:32.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:32.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:32.592 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:32.677 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:32.857 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:32.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:32.964 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:32.997 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:33.045 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 9127/221259 objects degraded (4.125%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:33.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[211526]: osdmap e599: 8 total, 8 up, 4 in 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 9127/221259 objects degraded (4.125%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:33.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:32 smithi121 ceph-mon[219709]: osdmap e599: 8 total, 8 up, 4 in 2023-09-23T21:19:33.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:33.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:33.154 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:33.192 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:33.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: failed to encode map e598 with expected crc 2023-09-23T21:19:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 9127/221259 objects degraded (4.125%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:33.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:32 smithi123 ceph-mon[193903]: osdmap e599: 8 total, 8 up, 4 in 2023-09-23T21:19:33.353 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:33.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:33.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:33.387 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:33.408 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:33.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 67 16 74837 74821 69.7894 27.625 0.0177339 0.0143089 2023-09-23T21:19:33.429 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:33.450 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:33.458 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:33.480 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:33.490 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:33.518 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:33.543 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:33.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:33.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:33.547 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:33.550 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:19:33.569 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:33.570 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.571 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.571 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.571 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.571 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.571 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.571 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.571 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:33.648 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:33.676 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:33.773 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:33.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:33.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:33.877 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:33.902 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:33.944 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:33.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:33.958 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:19:34.016 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:34.048 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:34.078 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:34.138 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:34.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:34.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/391068587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3218751630' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:34.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:33 smithi123 ceph-mon[193903]: pgmap v1839: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 94 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 6.1 MiB/s wr, 97 op/s; 9127/221259 objects degraded (4.125%); 4613/221259 objects misplaced (2.085%); 0 B/s, 1 objects/s recovering 2023-09-23T21:19:34.242 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:34.334 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/391068587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3218751630' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:34.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[211526]: pgmap v1839: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 94 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 6.1 MiB/s wr, 97 op/s; 9127/221259 objects degraded (4.125%); 4613/221259 objects misplaced (2.085%); 0 B/s, 1 objects/s recovering 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: failed to encode map e599 with expected crc 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/391068587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3218751630' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:34.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:33 smithi121 ceph-mon[219709]: pgmap v1839: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 94 active+clean; 4.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 6.1 MiB/s wr, 97 op/s; 9127/221259 objects degraded (4.125%); 4613/221259 objects misplaced (2.085%); 0 B/s, 1 objects/s recovering 2023-09-23T21:19:34.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 68 16 75221 75205 69.116 24 0.0253662 0.0144606 2023-09-23T21:19:34.458 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:34.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:34.563 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:34.641 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:34.799 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7] out_osds: [0, 1, 3, 5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:19:34.799 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:34.799 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T21:19:34.799 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T21:19:34.799 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T21:19:34.800 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T21:19:34.938 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:34.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:35.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:35.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:35.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[211526]: failed to encode map e600 with expected crc 2023-09-23T21:19:35.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[211526]: failed to encode map e600 with expected crc 2023-09-23T21:19:35.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:35.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[211526]: osdmap e600: 8 total, 8 up, 4 in 2023-09-23T21:19:35.083 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:19:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:19:34] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2023-09-23T21:19:35.083 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:35.083 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[219709]: failed to encode map e600 with expected crc 2023-09-23T21:19:35.083 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[219709]: failed to encode map e600 with expected crc 2023-09-23T21:19:35.083 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:35.083 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:34 smithi121 ceph-mon[219709]: osdmap e600: 8 total, 8 up, 4 in 2023-09-23T21:19:35.179 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:34 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:34 smithi123 ceph-mon[193903]: failed to encode map e600 with expected crc 2023-09-23T21:19:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:34 smithi123 ceph-mon[193903]: failed to encode map e600 with expected crc 2023-09-23T21:19:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:34 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:19:35.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:34 smithi123 ceph-mon[193903]: osdmap e600: 8 total, 8 up, 4 in 2023-09-23T21:19:35.258 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:35.319 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:35.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:35.423 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:35.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 69 16 75716 75700 68.5626 30.9375 0.0176252 0.0145561 2023-09-23T21:19:35.448 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:35.556 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:35.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:35.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:35.666 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:35.696 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:35.904 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:35.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:36.009 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:36.130 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:35 smithi121 ceph-mon[211526]: pgmap v1841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 2 activating+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 19 MiB/s wr, 303 op/s; 41586/224169 objects degraded (18.551%); 13781/224169 objects misplaced (6.148%); 1.2 MiB/s, 20 objects/s recovering 2023-09-23T21:19:36.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:35 smithi121 ceph-mon[219709]: pgmap v1841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 2 activating+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 19 MiB/s wr, 303 op/s; 41586/224169 objects degraded (18.551%); 13781/224169 objects misplaced (6.148%); 1.2 MiB/s, 20 objects/s recovering 2023-09-23T21:19:36.261 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:36.267 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:35 smithi123 ceph-mon[193903]: pgmap v1841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 2 activating+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+undersized+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 19 MiB/s wr, 303 op/s; 41586/224169 objects degraded (18.551%); 13781/224169 objects misplaced (6.148%); 1.2 MiB/s, 20 objects/s recovering 2023-09-23T21:19:36.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:36.369 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:36.401 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 70 16 76251 76235 68.0608 33.4375 0.0410955 0.0146849 2023-09-23T21:19:36.468 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:36.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:36.573 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:36.658 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.748 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:36.768 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:36.859 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:19:36.860 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:19:36.860 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:19:36.860 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:19:36.860 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:19:36.883 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:36.916 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:36.977 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:36.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:37.081 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:37.085 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:19:37.085 INFO:teuthology.orchestra.run.smithi121.stdout: "success": "" 2023-09-23T21:19:37.085 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:19:37.105 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:19:37.181 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:37.213 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:37.296 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:37.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:37.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:37.409 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:37.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 71 16 76803 76787 67.5881 34.5 0.0164908 0.0147886 2023-09-23T21:19:37.450 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:37.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:37 smithi121 ceph-mon[219709]: pgmap v1842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 14 MiB/s wr, 230 op/s; 41586/225417 objects degraded (18.448%); 13924/225417 objects misplaced (6.177%); 885 KiB/s, 14 objects/s recovering 2023-09-23T21:19:38.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:37 smithi121 ceph-mon[211526]: pgmap v1842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 14 MiB/s wr, 230 op/s; 41586/225417 objects degraded (18.448%); 13924/225417 objects misplaced (6.177%); 885 KiB/s, 14 objects/s recovering 2023-09-23T21:19:38.187 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:38.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:37 smithi123 ceph-mon[193903]: pgmap v1842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 14 MiB/s wr, 230 op/s; 41586/225417 objects degraded (18.448%); 13924/225417 objects misplaced (6.177%); 885 KiB/s, 14 objects/s recovering 2023-09-23T21:19:38.295 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:38.349 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:19:38.384 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:38.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 72 16 77299 77283 67.0799 31 0.0445223 0.0149018 2023-09-23T21:19:38.540 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:38.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:38.650 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:38.678 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:38.726 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:38.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:38.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:38.830 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:38.917 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.025 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:19:39.047 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:39.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:39.116 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:39.153 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:39.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:38 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41586/225417 objects degraded (18.448%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:39.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:38 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41586/225417 objects degraded (18.448%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:39.198 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:38 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41586/225417 objects degraded (18.448%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:39.252 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:39.356 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:39.393 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 73 16 77779 77763 66.5719 30 0.0122696 0.0150145 2023-09-23T21:19:39.628 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:39.652 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:39.654 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:39.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:39.675 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:39.696 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:39.732 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:39.755 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:39.759 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:39.785 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:39.825 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:39.827 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.827 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.828 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.828 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.828 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.828 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.828 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.828 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:39.884 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:39.923 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:39 smithi123 ceph-mon[193903]: pgmap v1843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 28 MiB/s wr, 451 op/s; 41586/231408 objects degraded (17.971%); 13943/231408 objects misplaced (6.025%); 1.9 MiB/s, 31 objects/s recovering 2023-09-23T21:19:39.924 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/637268711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:40.023 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:40.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:40.128 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:40.155 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:40.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:39 smithi121 ceph-mon[211526]: pgmap v1843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 28 MiB/s wr, 451 op/s; 41586/231408 objects degraded (17.971%); 13943/231408 objects misplaced (6.025%); 1.9 MiB/s, 31 objects/s recovering 2023-09-23T21:19:40.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/637268711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:39 smithi121 ceph-mon[219709]: pgmap v1843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 28 MiB/s wr, 451 op/s; 41586/231408 objects degraded (17.971%); 13943/231408 objects misplaced (6.025%); 1.9 MiB/s, 31 objects/s recovering 2023-09-23T21:19:40.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/637268711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:40.198 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:40.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:40.318 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:40.351 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:40.428 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 74 16 78229 78213 66.0523 28.125 0.0123551 0.0151258 2023-09-23T21:19:41.043 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:41.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:41.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:41.148 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:41.180 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:41.336 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:41.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:41.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:41.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 75 16 78702 78686 65.5657 29.5625 0.0299839 0.0152454 2023-09-23T21:19:41.439 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:41.473 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:41.528 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:41.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:41.632 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:41.662 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:41.714 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:41.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:41.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:41.881 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:41.881 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:41.930 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.142 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:42.259 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:42.287 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.343 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:42.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:42.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 76 16 79286 79270 65.1833 36.5 0.0290277 0.015336 2023-09-23T21:19:42.447 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:42.462 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:19:42.521 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.580 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:42.637 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:42 smithi121 ceph-mon[219709]: pgmap v1844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 28 MiB/s wr, 452 op/s; 41586/232782 objects degraded (17.865%); 14106/232782 objects misplaced (6.060%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T21:19:42.638 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1265038746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:42.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:42 smithi121 ceph-mon[211526]: pgmap v1844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 28 MiB/s wr, 452 op/s; 41586/232782 objects degraded (17.865%); 14106/232782 objects misplaced (6.060%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T21:19:42.638 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1265038746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:42.683 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:42.713 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:42 smithi123 ceph-mon[193903]: pgmap v1844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 28 MiB/s wr, 452 op/s; 41586/232782 objects degraded (17.865%); 14106/232782 objects misplaced (6.060%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T21:19:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1265038746' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:19:42.762 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:42.865 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:42.891 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:42.937 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:42.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:43.041 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:43.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:19:43.136 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:19:43.215 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7] out_osds: [0, 1, 3, 5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:19:43.215 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:19:43.215 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:19:43.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T21:19:43.325 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:43.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:43.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 77 16 79727 79711 64.6943 27.5625 0.0113496 0.0154516 2023-09-23T21:19:43.440 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1265038746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:43.461 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: osdmap e601: 8 total, 8 up, 4 in 2023-09-23T21:19:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:43.462 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:43 smithi123 ceph-mon[193903]: pgmap v1846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 19 MiB/s wr, 304 op/s; 41586/232782 objects degraded (17.865%); 14106/232782 objects misplaced (6.060%); 1004 KiB/s, 15 objects/s recovering 2023-09-23T21:19:43.493 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.560 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:43.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:43.671 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:43.711 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:43.779 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:43.835 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:43.872 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:43.886 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:43.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:43.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1265038746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: osdmap e601: 8 total, 8 up, 4 in 2023-09-23T21:19:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[211526]: pgmap v1846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 19 MiB/s wr, 304 op/s; 41586/232782 objects degraded (17.865%); 14106/232782 objects misplaced (6.060%); 1004 KiB/s, 15 objects/s recovering 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1265038746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: osdmap e601: 8 total, 8 up, 4 in 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: failed to encode map e601 with expected crc 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:43 smithi121 ceph-mon[219709]: pgmap v1846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 19 MiB/s wr, 304 op/s; 41586/232782 objects degraded (17.865%); 14106/232782 objects misplaced (6.060%); 1004 KiB/s, 15 objects/s recovering 2023-09-23T21:19:44.005 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:44.072 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:44.250 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:44.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:44.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:44.356 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:44.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 78 16 80345 80329 64.36 38.625 0.0153973 0.0155325 2023-09-23T21:19:44.447 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:44.606 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:44.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:44.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:44.719 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:44.745 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:44.788 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:44.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:44.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:44.897 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:44.932 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:45.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:19:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:19:44] "GET /metrics HTTP/1.1" 200 36703 "" "Prometheus/2.43.0" 2023-09-23T21:19:45.242 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:45.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:45.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:45.339 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:19:45.348 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:45.428 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:45.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 79 16 80895 80879 63.9804 34.375 0.0272665 0.0156235 2023-09-23T21:19:45.490 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:45 smithi121 ceph-mon[211526]: pgmap v1847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 31 MiB/s wr, 488 op/s; 41586/238830 objects degraded (17.412%); 14164/238830 objects misplaced (5.931%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T21:19:45.491 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:45 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41586/238830 objects degraded (17.412%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:45.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:45 smithi121 ceph-mon[219709]: pgmap v1847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 31 MiB/s wr, 488 op/s; 41586/238830 objects degraded (17.412%); 14164/238830 objects misplaced (5.931%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T21:19:45.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:45 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41586/238830 objects degraded (17.412%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:45 smithi123 ceph-mon[193903]: pgmap v1847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 31 MiB/s wr, 488 op/s; 41586/238830 objects degraded (17.412%); 14164/238830 objects misplaced (5.931%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T21:19:45.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:45 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41586/238830 objects degraded (17.412%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:46.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:2023-09-23T21:19:46.430562+0000 min lat: 0.00109569 max lat: 2.49902 avg lat: 0.0157148 2023-09-23T21:19:46.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T21:19:46.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 80 16 81444 81428 63.6095 34.3125 0.0286233 0.0157148 2023-09-23T21:19:46.559 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:46.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:46.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:46.664 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:46.775 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:46.790 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:19:46.790 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:19:46.944 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:46.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:47.071 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:47.096 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.149 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:47.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:47.271 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:47.319 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:47.429 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 81 16 81952 81936 63.2161 31.75 0.0362697 0.0158125 2023-09-23T21:19:47.534 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:47.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:47.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:47.672 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.120 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:48.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:48.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:48.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:47 smithi121 ceph-mon[211526]: pgmap v1848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 31 MiB/s wr, 496 op/s; 41586/240309 objects degraded (17.305%); 14334/240309 objects misplaced (5.965%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T21:19:48.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:47 smithi121 ceph-mon[219709]: pgmap v1848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 31 MiB/s wr, 496 op/s; 41586/240309 objects degraded (17.305%); 14334/240309 objects misplaced (5.965%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T21:19:48.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:48.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:47 smithi123 ceph-mon[193903]: pgmap v1848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 4.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 31 MiB/s wr, 496 op/s; 41586/240309 objects degraded (17.305%); 14334/240309 objects misplaced (5.965%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T21:19:48.325 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.334 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:19:48.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 82 16 82431 82415 62.8102 29.9375 0.0144923 0.0159142 2023-09-23T21:19:48.440 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:48.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:48.522 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:48.566 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:19:48.669 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.785 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:19:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:48.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:19:48.890 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:48.902 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:48.908 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:48 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]: dispatch 2023-09-23T21:19:48.908 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:48 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]: dispatch 2023-09-23T21:19:48.908 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:48 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2922612113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:moved 10 / 297 (3.367%) 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:avg 74.25 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 3.26917 -> 5.88961 (expected baseline 7.46241) 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.2 with 69 -> 70 pgs (0.929293 -> 0.942761 * mean) 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.4 with 77 -> 81 pgs (1.03704 -> 1.09091 * mean) 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T21:19:48.922 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:19:48.923 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0416 2023-09-23T21:19:48.923 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0457 2023-09-23T21:19:48.923 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 1.0000 -> 0.8130 2023-09-23T21:19:48.923 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:48.950 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:48.964 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:48.982 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:49.030 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:49.050 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:49.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:49.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:49.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.091 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.092 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.092 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.092 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.092 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.092 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:19:49.116 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:49.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:49.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:49.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:48 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]: dispatch 2023-09-23T21:19:49.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:48 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]: dispatch 2023-09-23T21:19:49.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:48 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2922612113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:48 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]: dispatch 2023-09-23T21:19:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:48 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]: dispatch 2023-09-23T21:19:49.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:48 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2922612113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:49.220 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:49.247 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:49.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 83 16 82785 82769 62.32 22.125 0.00616874 0.0159578 2023-09-23T21:19:49.941 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:49.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:50.046 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:50.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:50.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[211526]: from='client.37605 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:19:50.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]': finished 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[211526]: osdmap e602: 8 total, 8 up, 4 in 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[211526]: failed to encode map e602 with expected crc 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[211526]: failed to encode map e602 with expected crc 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[211526]: pgmap v1850: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 37 MiB/s wr, 593 op/s; 41586/247035 objects degraded (16.834%); 14370/247035 objects misplaced (5.817%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[219709]: from='client.37605 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]': finished 2023-09-23T21:19:50.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[219709]: osdmap e602: 8 total, 8 up, 4 in 2023-09-23T21:19:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[219709]: failed to encode map e602 with expected crc 2023-09-23T21:19:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[219709]: failed to encode map e602 with expected crc 2023-09-23T21:19:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[219709]: pgmap v1850: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 37 MiB/s wr, 593 op/s; 41586/247035 objects degraded (16.834%); 14370/247035 objects misplaced (5.817%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:19:50.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:49 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:19:50.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:49 smithi123 ceph-mon[193903]: from='client.37605 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:19:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:49 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'53281'}"}]': finished 2023-09-23T21:19:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:49 smithi123 ceph-mon[193903]: osdmap e602: 8 total, 8 up, 4 in 2023-09-23T21:19:50.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:49 smithi123 ceph-mon[193903]: failed to encode map e602 with expected crc 2023-09-23T21:19:50.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:49 smithi123 ceph-mon[193903]: failed to encode map e602 with expected crc 2023-09-23T21:19:50.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:49 smithi123 ceph-mon[193903]: pgmap v1850: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 37 MiB/s wr, 593 op/s; 41586/247035 objects degraded (16.834%); 14370/247035 objects misplaced (5.817%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T21:19:50.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:49 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:19:50.379 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:50.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:50.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:50.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 84 16 82818 82802 61.6026 2.0625 0.00500091 0.0160424 2023-09-23T21:19:50.483 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:50.509 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:51.008 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:51.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:51.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:51.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:51.129 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:51.148 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:51.163 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:19:51.164 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T21:19:51.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: osdmap e603: 8 total, 8 up, 4 in 2023-09-23T21:19:51.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.235 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:50 smithi123 ceph-mon[193903]: 14.9 starting backfill to osd.7 from (0'0,0'0] MAX to 601'5063 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: osdmap e603: 8 total, 8 up, 4 in 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.236 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[211526]: 14.9 starting backfill to osd.7 from (0'0,0'0] MAX to 601'5063 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: osdmap e603: 8 total, 8 up, 4 in 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: failed to encode map e603 with expected crc 2023-09-23T21:19:51.237 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:50 smithi121 ceph-mon[219709]: 14.9 starting backfill to osd.7 from (0'0,0'0] MAX to 601'5063 2023-09-23T21:19:51.264 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:51.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:51.368 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:51.413 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:51.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 85 16 82966 82950 60.9867 9.25 0.0481231 0.01639 2023-09-23T21:19:51.591 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:51.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:51.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:51.696 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:51.789 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:51.899 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:19:51.929 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:51.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:52.104 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: osdmap e604: 8 total, 8 up, 4 in 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: 14.b starting backfill to osd.4 from (0'0,0'0] MAX to 601'5293 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: 14.3 starting backfill to osd.2 from (0'0,0'0] MAX to 601'5200 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: 14.d continuing backfill to osd.2 from (577'1217,601'5161] MIN to 601'5161 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: 14.d starting backfill to osd.7 from (0'0,0'0] MAX to 601'5161 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: pgmap v1853: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 30 MiB/s wr, 485 op/s; 37023/247563 objects degraded (14.955%); 18796/247563 objects misplaced (7.592%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1537937650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 41586/247035 objects degraded (16.834%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:52.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[211526]: osdmap e605: 8 total, 8 up, 4 in 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: osdmap e604: 8 total, 8 up, 4 in 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: 14.b starting backfill to osd.4 from (0'0,0'0] MAX to 601'5293 2023-09-23T21:19:52.165 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: 14.3 starting backfill to osd.2 from (0'0,0'0] MAX to 601'5200 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: 14.d continuing backfill to osd.2 from (577'1217,601'5161] MIN to 601'5161 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: 14.d starting backfill to osd.7 from (0'0,0'0] MAX to 601'5161 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: pgmap v1853: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 30 MiB/s wr, 485 op/s; 37023/247563 objects degraded (14.955%); 18796/247563 objects misplaced (7.592%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1537937650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 41586/247035 objects degraded (16.834%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:52.166 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:52.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:52.167 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:51 smithi121 ceph-mon[219709]: osdmap e605: 8 total, 8 up, 4 in 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: osdmap e604: 8 total, 8 up, 4 in 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: 14.b starting backfill to osd.4 from (0'0,0'0] MAX to 601'5293 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.220 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: failed to encode map e604 with expected crc 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: 14.3 starting backfill to osd.2 from (0'0,0'0] MAX to 601'5200 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: 14.d continuing backfill to osd.2 from (577'1217,601'5161] MIN to 601'5161 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: 14.d starting backfill to osd.7 from (0'0,0'0] MAX to 601'5161 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: pgmap v1853: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 30 MiB/s wr, 485 op/s; 37023/247563 objects degraded (14.955%); 18796/247563 objects misplaced (7.592%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1537937650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 41586/247035 objects degraded (16.834%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:19:52.221 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:51 smithi123 ceph-mon[193903]: osdmap e605: 8 total, 8 up, 4 in 2023-09-23T21:19:52.241 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:52.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 86 16 83376 83360 60.5755 25.625 0.0455793 0.0164956 2023-09-23T21:19:52.462 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:19:52.504 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:52.620 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:52.647 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:52.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:52 smithi123 ceph-mon[193903]: failed to encode map e605 with expected crc 2023-09-23T21:19:52.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:52 smithi123 ceph-mon[193903]: failed to encode map e605 with expected crc 2023-09-23T21:19:52.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:52 smithi123 ceph-mon[193903]: failed to encode map e605 with expected crc 2023-09-23T21:19:52.945 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:52 smithi123 ceph-mon[193903]: osdmap e606: 8 total, 8 up, 4 in 2023-09-23T21:19:52.983 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:52.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:53.085 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:53.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[219709]: failed to encode map e605 with expected crc 2023-09-23T21:19:53.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[219709]: failed to encode map e605 with expected crc 2023-09-23T21:19:53.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[219709]: failed to encode map e605 with expected crc 2023-09-23T21:19:53.086 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[219709]: osdmap e606: 8 total, 8 up, 4 in 2023-09-23T21:19:53.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:53.097 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[211526]: failed to encode map e605 with expected crc 2023-09-23T21:19:53.097 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[211526]: failed to encode map e605 with expected crc 2023-09-23T21:19:53.097 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[211526]: failed to encode map e605 with expected crc 2023-09-23T21:19:53.098 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:52 smithi121 ceph-mon[211526]: osdmap e606: 8 total, 8 up, 4 in 2023-09-23T21:19:53.176 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:53.200 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:53.381 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:53.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:53.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:53.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 87 16 83810 83794 60.191 27.125 0.0288149 0.0166087 2023-09-23T21:19:53.485 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:53.527 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:53.665 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:19:53.666 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:19:53.666 INFO:teuthology.orchestra.run.smithi121.stdout: "complaint_time": 30, 2023-09-23T21:19:53.666 INFO:teuthology.orchestra.run.smithi121.stdout: "num_blocked_ops": 0 2023-09-23T21:19:53.666 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:19:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[219709]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[219709]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[219709]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[219709]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[219709]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[219709]: pgmap v1856: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 2.7 MiB/s wr, 43 op/s; 37023/247563 objects degraded (14.955%); 18796/247563 objects misplaced (7.592%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T21:19:54.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[211526]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[211526]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[211526]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[211526]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[211526]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:53 smithi121 ceph-mon[211526]: pgmap v1856: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 2.7 MiB/s wr, 43 op/s; 37023/247563 objects degraded (14.955%); 18796/247563 objects misplaced (7.592%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T21:19:54.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:19:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:19:54.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:53 smithi123 ceph-mon[193903]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:53 smithi123 ceph-mon[193903]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:53 smithi123 ceph-mon[193903]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:53 smithi123 ceph-mon[193903]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:53 smithi123 ceph-mon[193903]: failed to encode map e606 with expected crc 2023-09-23T21:19:54.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:53 smithi123 ceph-mon[193903]: pgmap v1856: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 85 active+clean; 5.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 2.7 MiB/s wr, 43 op/s; 37023/247563 objects degraded (14.955%); 18796/247563 objects misplaced (7.592%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T21:19:54.285 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:54.380 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.431 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 88 16 84271 84255 59.8344 28.8125 0.0300846 0.0167059 2023-09-23T21:19:54.623 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:54.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:54.682 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:19:54.733 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:19:54.771 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:54.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:19:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:19:54] "GET /metrics HTTP/1.1" 200 36703 "" "Prometheus/2.43.0" 2023-09-23T21:19:54.991 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:19:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:54.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:19:55.097 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:55.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:19:55.128 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:55.430 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 89 16 84788 84772 59.5251 32.3125 0.0229973 0.0167917 2023-09-23T21:19:55.833 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:55.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:55.937 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:56.049 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:56.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:56 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:56.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:56 smithi121 ceph-mon[211526]: pgmap v1857: 99 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 15 KiB/s rd, 16 MiB/s wr, 275 op/s; 36429/251346 objects degraded (14.494%); 40432/251346 objects misplaced (16.086%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T21:19:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:56 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:56.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:56 smithi121 ceph-mon[219709]: pgmap v1857: 99 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 15 KiB/s rd, 16 MiB/s wr, 275 op/s; 36429/251346 objects degraded (14.494%); 40432/251346 objects misplaced (16.086%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T21:19:56.183 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:56.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:56.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:56.292 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:56.393 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:56.394 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:56 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:19:56.394 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:56 smithi123 ceph-mon[193903]: pgmap v1857: 99 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 15 KiB/s rd, 16 MiB/s wr, 275 op/s; 36429/251346 objects degraded (14.494%); 40432/251346 objects misplaced (16.086%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T21:19:56.399 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:56.431 INFO:tasks.radosbench.radosbench.0.smithi121.stdout: 90 6 85314 85308 59.2355 33.5 0.0241695 0.0168767 2023-09-23T21:19:56.457 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:56.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:56.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:56.540 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:56.568 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:56.597 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:56.645 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:56.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:56.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:56.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:19:56.752 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:19:56.768 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:19:56.771 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:19:56.779 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:56.793 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:19:56.815 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:19:56.830 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:19:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:56.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:19:56.836 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:19:56.856 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:19:56.875 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:19:56.894 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:19:56.896 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.896 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.896 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.897 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.897 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.897 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.897 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.897 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:19:56.936 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:56.964 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:57.009 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:57.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:57.118 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:19:57.146 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:57.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/849812892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:57.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:57 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 36429/251346 objects degraded (14.494%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/849812892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:57.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:57 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 36429/251346 objects degraded (14.494%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:57.195 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:19:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:57.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:19:57.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/849812892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:19:57.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:57 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 36429/251346 objects degraded (14.494%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:19:57.306 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:19:57.337 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:57.431 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total time run: 90.0173 2023-09-23T21:19:57.431 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Total writes made: 85314 2023-09-23T21:19:57.431 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Write size: 65536 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Object size: 65536 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Bandwidth (MB/sec): 59.2344 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Bandwidth: 37.5091 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max bandwidth (MB/sec): 178.875 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min bandwidth (MB/sec): 0 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average IOPS: 947 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev IOPS: 600.155 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max IOPS: 2862 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min IOPS: 0 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Average Latency(s): 0.0168776 2023-09-23T21:19:57.432 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Stddev Latency(s): 0.0486903 2023-09-23T21:19:57.433 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Max latency(s): 2.49902 2023-09-23T21:19:57.433 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Min latency(s): 0.00109569 2023-09-23T21:19:57.436 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T21:19:57.778 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:19:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:57.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:19:57.883 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:19:57.964 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:58.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[219709]: from='client.37626 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:19:58.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]: dispatch 2023-09-23T21:19:58.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]: dispatch 2023-09-23T21:19:58.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[219709]: pgmap v1858: 99 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 13 KiB/s rd, 17 MiB/s wr, 276 op/s; 36429/252192 objects degraded (14.445%); 40318/252192 objects misplaced (15.987%); 4.7 MiB/s, 73 objects/s recovering 2023-09-23T21:19:58.066 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:58.074 INFO:teuthology.orchestra.run.smithi121.stdout:moved 9 / 297 (3.0303%) 2023-09-23T21:19:58.074 INFO:teuthology.orchestra.run.smithi121.stdout:avg 74.25 2023-09-23T21:19:58.074 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 5.88961 -> 9.83298 (expected baseline 7.46241) 2023-09-23T21:19:58.074 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.6 with 67 -> 58 pgs (0.902357 -> 0.781145 * mean) 2023-09-23T21:19:58.074 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.4 with 81 -> 83 pgs (1.09091 -> 1.11785 * mean) 2023-09-23T21:19:58.074 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:19:58.074 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:19:58.075 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:19:58.075 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:19:58.075 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0434 2023-09-23T21:19:58.075 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0477 2023-09-23T21:19:58.075 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.8130 -> 0.7630 2023-09-23T21:19:58.104 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:19:58.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:58.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:19:58.220 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:19:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:58 smithi123 ceph-mon[193903]: from='client.37626 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:19:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]: dispatch 2023-09-23T21:19:58.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]: dispatch 2023-09-23T21:19:58.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:58 smithi123 ceph-mon[193903]: pgmap v1858: 99 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 13 KiB/s rd, 17 MiB/s wr, 276 op/s; 36429/252192 objects degraded (14.445%); 40318/252192 objects misplaced (15.987%); 4.7 MiB/s, 73 objects/s recovering 2023-09-23T21:19:58.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:58.251 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:19:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[211526]: from='client.37626 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:19:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]: dispatch 2023-09-23T21:19:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]: dispatch 2023-09-23T21:19:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[211526]: pgmap v1858: 99 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 13 KiB/s rd, 17 MiB/s wr, 276 op/s; 36429/252192 objects degraded (14.445%); 40318/252192 objects misplaced (15.987%); 4.7 MiB/s, 73 objects/s recovering 2023-09-23T21:19:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:19:59.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[211526]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]': finished 2023-09-23T21:19:59.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[211526]: osdmap e607: 8 total, 8 up, 4 in 2023-09-23T21:19:59.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[211526]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[211526]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[211526]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.285 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[219709]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.285 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]': finished 2023-09-23T21:19:59.285 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[219709]: osdmap e607: 8 total, 8 up, 4 in 2023-09-23T21:19:59.285 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[219709]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.285 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[219709]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.285 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:19:59 smithi121 ceph-mon[219709]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.455 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T21:19:59.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T21:19:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:59 smithi123 ceph-mon[193903]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:59 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'50005'}"}]': finished 2023-09-23T21:19:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:59 smithi123 ceph-mon[193903]: osdmap e607: 8 total, 8 up, 4 in 2023-09-23T21:19:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:59 smithi123 ceph-mon[193903]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:59 smithi123 ceph-mon[193903]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:19:59 smithi123 ceph-mon[193903]: failed to encode map e607 with expected crc 2023-09-23T21:19:59.720 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:19:59.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:19:59.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:19:59.823 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:19:59.852 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:00.150 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: osdmap e608: 8 total, 8 up, 4 in 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pgmap v1861: 99 pgs: 2 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 13 KiB/s rd, 27 MiB/s wr, 504 op/s; 31800/254838 objects degraded (12.479%); 36304/254838 objects misplaced (14.246%); 5.3 MiB/s, 83 objects/s recovering 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 36429/252192 objects degraded (14.445%), 9 pgs degraded 2023-09-23T21:20:00.152 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: [WRN] PG_DEGRADED: Degraded data redundancy: 36429/252192 objects degraded (14.445%), 9 pgs degraded 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.0 is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.3 is active+recovery_wait+degraded+remapped, acting [4,7,6] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.4 is active+recovery_wait+undersized+degraded+remapped, acting [6,7] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.5 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,7] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.9 is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2023-09-23T21:20:00.153 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[219709]: pg 14.a is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2023-09-23T21:20:00.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: osdmap e608: 8 total, 8 up, 4 in 2023-09-23T21:20:00.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pgmap v1861: 99 pgs: 2 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 13 KiB/s rd, 27 MiB/s wr, 504 op/s; 31800/254838 objects degraded (12.479%); 36304/254838 objects misplaced (14.246%); 5.3 MiB/s, 83 objects/s recovering 2023-09-23T21:20:00.154 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 36429/252192 objects degraded (14.445%), 9 pgs degraded 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: [WRN] PG_DEGRADED: Degraded data redundancy: 36429/252192 objects degraded (14.445%), 9 pgs degraded 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.0 is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.3 is active+recovery_wait+degraded+remapped, acting [4,7,6] 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.4 is active+recovery_wait+undersized+degraded+remapped, acting [6,7] 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.5 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,7] 2023-09-23T21:20:00.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.9 is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2023-09-23T21:20:00.156 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:00 smithi121 ceph-mon[211526]: pg 14.a is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2023-09-23T21:20:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:00.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:00.257 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:00.301 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:00.403 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:00.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:00.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:00.412 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.412 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.412 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: osdmap e608: 8 total, 8 up, 4 in 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: failed to encode map e608 with expected crc 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pgmap v1861: 99 pgs: 2 remapped+peering, 7 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 13 KiB/s rd, 27 MiB/s wr, 504 op/s; 31800/254838 objects degraded (12.479%); 36304/254838 objects misplaced (14.246%); 5.3 MiB/s, 83 objects/s recovering 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 36429/252192 objects degraded (14.445%), 9 pgs degraded 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: [WRN] PG_DEGRADED: Degraded data redundancy: 36429/252192 objects degraded (14.445%), 9 pgs degraded 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.0 is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.3 is active+recovery_wait+degraded+remapped, acting [4,7,6] 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.4 is active+recovery_wait+undersized+degraded+remapped, acting [6,7] 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.5 is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2023-09-23T21:20:00.413 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2023-09-23T21:20:00.414 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,7] 2023-09-23T21:20:00.414 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.9 is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2023-09-23T21:20:00.414 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:00 smithi123 ceph-mon[193903]: pg 14.a is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2023-09-23T21:20:00.417 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:00.417 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:00.513 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:00.549 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:01.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:20:01.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[211526]: osdmap e609: 8 total, 8 up, 4 in 2023-09-23T21:20:01.082 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[211526]: failed to encode map e609 with expected crc 2023-09-23T21:20:01.083 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[211526]: failed to encode map e609 with expected crc 2023-09-23T21:20:01.083 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[211526]: 14.a starting backfill to osd.2 from (0'0,0'0] MAX to 606'5332 2023-09-23T21:20:01.083 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[211526]: 14.f starting backfill to osd.4 from (0'0,0'0] MAX to 606'5362 2023-09-23T21:20:01.304 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:20:01.338 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:20:01.338 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[219709]: osdmap e609: 8 total, 8 up, 4 in 2023-09-23T21:20:01.338 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[219709]: failed to encode map e609 with expected crc 2023-09-23T21:20:01.338 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[219709]: failed to encode map e609 with expected crc 2023-09-23T21:20:01.338 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[219709]: 14.a starting backfill to osd.2 from (0'0,0'0] MAX to 606'5332 2023-09-23T21:20:01.338 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:01 smithi121 ceph-mon[219709]: 14.f starting backfill to osd.4 from (0'0,0'0] MAX to 606'5362 2023-09-23T21:20:01.473 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:01.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:01 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T21:20:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:01 smithi123 ceph-mon[193903]: osdmap e609: 8 total, 8 up, 4 in 2023-09-23T21:20:01.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:01 smithi123 ceph-mon[193903]: failed to encode map e609 with expected crc 2023-09-23T21:20:01.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:01 smithi123 ceph-mon[193903]: failed to encode map e609 with expected crc 2023-09-23T21:20:01.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:01 smithi123 ceph-mon[193903]: 14.a starting backfill to osd.2 from (0'0,0'0] MAX to 606'5332 2023-09-23T21:20:01.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:01 smithi123 ceph-mon[193903]: 14.f starting backfill to osd.4 from (0'0,0'0] MAX to 606'5362 2023-09-23T21:20:01.483 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:20:01.547 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:01.580 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:01.636 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:01.794 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:01.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:01.900 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:01.982 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:02.127 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:02.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:02.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[219709]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[219709]: osdmap e610: 8 total, 8 up, 4 in 2023-09-23T21:20:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[219709]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[219709]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[219709]: pgmap v1864: 99 pgs: 2 remapped+peering, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 255 B/s rd, 20 MiB/s wr, 447 op/s; 35939/254469 objects degraded (14.123%); 31992/254469 objects misplaced (12.572%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T21:20:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 31800/254838 objects degraded (12.479%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[211526]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[211526]: osdmap e610: 8 total, 8 up, 4 in 2023-09-23T21:20:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[211526]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[211526]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[211526]: pgmap v1864: 99 pgs: 2 remapped+peering, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 255 B/s rd, 20 MiB/s wr, 447 op/s; 35939/254469 objects degraded (14.123%); 31992/254469 objects misplaced (12.572%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T21:20:02.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 31800/254838 objects degraded (12.479%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:02.240 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:02.281 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:02.390 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:02.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:02.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:02 smithi123 ceph-mon[193903]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:02 smithi123 ceph-mon[193903]: osdmap e610: 8 total, 8 up, 4 in 2023-09-23T21:20:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:02 smithi123 ceph-mon[193903]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:02 smithi123 ceph-mon[193903]: failed to encode map e610 with expected crc 2023-09-23T21:20:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:02 smithi123 ceph-mon[193903]: pgmap v1864: 99 pgs: 2 remapped+peering, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 255 B/s rd, 20 MiB/s wr, 447 op/s; 35939/254469 objects degraded (14.123%); 31992/254469 objects misplaced (12.572%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T21:20:02.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 31800/254838 objects degraded (12.479%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:02.495 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:02.523 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:02.529 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:02.644 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:02.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:02.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:02.748 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:02.828 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:02.962 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:02.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:03.085 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:03.130 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:20:03.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:03 smithi123 ceph-mon[193903]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:03 smithi123 ceph-mon[193903]: osdmap e611: 8 total, 8 up, 4 in 2023-09-23T21:20:03.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:03 smithi123 ceph-mon[193903]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.156 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:03 smithi123 ceph-mon[193903]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.157 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3583171573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:03.157 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:03.157 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:03 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:03.174 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:03.225 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[211526]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[211526]: osdmap e611: 8 total, 8 up, 4 in 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[211526]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[211526]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3583171573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[219709]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[219709]: osdmap e611: 8 total, 8 up, 4 in 2023-09-23T21:20:03.226 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[219709]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[219709]: failed to encode map e611 with expected crc 2023-09-23T21:20:03.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3583171573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:03.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:03.227 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:03 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:03.279 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:03.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:03.392 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:03.422 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[211526]: failed to encode map e612 with expected crc 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3583171573' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[211526]: osdmap e612: 8 total, 8 up, 4 in 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[211526]: failed to encode map e612 with expected crc 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[211526]: pgmap v1867: 99 pgs: 2 remapped+peering, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 48 KiB/s wr, 32 op/s; 35939/254469 objects degraded (14.123%); 31992/254469 objects misplaced (12.572%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:04.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[219709]: failed to encode map e612 with expected crc 2023-09-23T21:20:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3583171573' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[219709]: osdmap e612: 8 total, 8 up, 4 in 2023-09-23T21:20:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[219709]: failed to encode map e612 with expected crc 2023-09-23T21:20:04.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:04 smithi121 ceph-mon[219709]: pgmap v1867: 99 pgs: 2 remapped+peering, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 48 KiB/s wr, 32 op/s; 35939/254469 objects degraded (14.123%); 31992/254469 objects misplaced (12.572%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:20:04.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:04 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:04.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:04 smithi123 ceph-mon[193903]: failed to encode map e612 with expected crc 2023-09-23T21:20:04.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3583171573' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:04.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:04 smithi123 ceph-mon[193903]: osdmap e612: 8 total, 8 up, 4 in 2023-09-23T21:20:04.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:04 smithi123 ceph-mon[193903]: failed to encode map e612 with expected crc 2023-09-23T21:20:04.481 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:04 smithi123 ceph-mon[193903]: pgmap v1867: 99 pgs: 2 remapped+peering, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 5.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 48 KiB/s wr, 32 op/s; 35939/254469 objects degraded (14.123%); 31992/254469 objects misplaced (12.572%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T21:20:05.048 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:05.076 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:20:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:20:04] "GET /metrics HTTP/1.1" 200 36713 "" "Prometheus/2.43.0" 2023-09-23T21:20:05.347 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:05.349 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:20:05.349 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:20:05.349 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:05 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:05.349 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:05 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:20:05.350 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:05 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:20:05.350 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:05 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:05.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:05.451 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:05 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:20:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:05 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:20:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:05 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:05.526 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:05.667 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:05.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:05.781 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:05.821 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:05.928 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:05.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:05.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:06.034 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 297 (0%) 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout:avg 74.25 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 9.83298 -> 9.83298 (expected baseline 7.46241) 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.6 with 58 -> 58 pgs (0.781145 -> 0.781145 * mean) 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.4 with 83 -> 83 pgs (1.11785 -> 1.11785 * mean) 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:20:06.067 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 3 2023-09-23T21:20:06.068 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:20:06.068 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0438 2023-09-23T21:20:06.068 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.4385 2023-09-23T21:20:06.071 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:06.074 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:20:06.334 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:20:06.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:20:06.361 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:06.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:06.366 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:06 smithi121 ceph-mon[211526]: pgmap v1868: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 36 KiB/s wr, 292 op/s; 39850/250194 objects degraded (15.928%); 42081/250194 objects misplaced (16.819%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:20:06.367 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:06 smithi121 ceph-mon[219709]: pgmap v1868: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 36 KiB/s wr, 292 op/s; 39850/250194 objects degraded (15.928%); 42081/250194 objects misplaced (16.819%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:20:06.387 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:20:06.466 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:06 smithi123 ceph-mon[193903]: pgmap v1868: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 36 KiB/s wr, 292 op/s; 39850/250194 objects degraded (15.928%); 42081/250194 objects misplaced (16.819%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T21:20:06.493 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:06.533 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:06.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:06.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:06.593 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:20:06.593 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:20:06.593 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.594 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:20:06.641 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:06.672 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:06.881 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:06.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:06.985 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:07.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:20:07.081 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:07.213 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:07.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:07.303 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[211526]: from='client.47618 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:07.303 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:20:07.303 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3516042440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:07.303 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39850/250194 objects degraded (15.928%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:07.304 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[219709]: from='client.47618 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:07.304 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:20:07.305 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3516042440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:07.305 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:07 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39850/250194 objects degraded (15.928%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:07.327 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T21:20:07.327 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T21:20:07.332 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:07.402 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:07 smithi123 ceph-mon[193903]: from='client.47618 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:07 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T21:20:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:07 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3516042440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:07 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39850/250194 objects degraded (15.928%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:08.240 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:08.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:08.312 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:08 smithi121 ceph-mon[211526]: pgmap v1869: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 31 KiB/s wr, 299 op/s; 39850/249204 objects degraded (15.991%); 42081/249204 objects misplaced (16.886%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T21:20:08.313 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:08 smithi121 ceph-mon[219709]: pgmap v1869: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 31 KiB/s wr, 299 op/s; 39850/249204 objects degraded (15.991%); 42081/249204 objects misplaced (16.886%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T21:20:08.345 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:08.387 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:08.438 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:08.462 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:08.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:08 smithi123 ceph-mon[193903]: pgmap v1869: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 5.1 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 31 KiB/s wr, 299 op/s; 39850/249204 objects degraded (15.991%); 42081/249204 objects misplaced (16.886%); 5.6 MiB/s, 90 objects/s recovering 2023-09-23T21:20:08.519 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:08.567 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:08.596 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:08.825 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:08.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:08.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:08.929 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:08.934 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:20:08.934 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:20:08.934 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:20:08.934 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:20:08.978 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:09.586 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:09.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:09.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:09.691 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:09.766 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:09.920 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:09.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:10.047 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:20:10.047 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:10.144 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:10.211 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:20:10.288 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:10.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:10.418 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:10.447 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:10.487 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:10.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:10.594 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:10.621 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:10 smithi121 ceph-mon[211526]: pgmap v1870: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 480 op/s; 39571/242943 objects degraded (16.288%); 40741/242943 objects misplaced (16.770%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T21:20:10.622 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:10 smithi121 ceph-mon[219709]: pgmap v1870: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 480 op/s; 39571/242943 objects degraded (16.288%); 40741/242943 objects misplaced (16.770%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T21:20:10.627 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:10.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:10 smithi123 ceph-mon[193903]: pgmap v1870: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 480 op/s; 39571/242943 objects degraded (16.288%); 40741/242943 objects misplaced (16.770%); 6.2 MiB/s, 100 objects/s recovering 2023-09-23T21:20:11.299 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:11.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:11.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:11.366 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:11.407 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:11.534 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:11.639 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:11.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:11.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:11.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:11.801 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:12.069 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:12.128 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:12.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:12.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:12.235 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:12.268 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:12.369 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:12.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:12.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:12.475 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:12.510 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[211526]: pgmap v1871: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 446 op/s; 39571/242049 objects degraded (16.348%); 40741/242049 objects misplaced (16.832%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39571/242049 objects degraded (16.348%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1180388455' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]: dispatch 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]: dispatch 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[219709]: pgmap v1871: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 446 op/s; 39571/242049 objects degraded (16.348%); 40741/242049 objects misplaced (16.832%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:20:12.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39571/242049 objects degraded (16.348%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:12.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1180388455' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:12.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:12.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]: dispatch 2023-09-23T21:20:12.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:12 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]: dispatch 2023-09-23T21:20:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:12 smithi123 ceph-mon[193903]: pgmap v1871: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 446 op/s; 39571/242049 objects degraded (16.348%); 40741/242049 objects misplaced (16.832%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T21:20:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39571/242049 objects degraded (16.348%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1180388455' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]: dispatch 2023-09-23T21:20:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:12 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]: dispatch 2023-09-23T21:20:12.912 INFO:teuthology.orchestra.run.smithi121.stdout:moved 3 / 297 (1.0101%) 2023-09-23T21:20:12.912 INFO:teuthology.orchestra.run.smithi121.stdout:avg 74.25 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 9.83298 -> 11.4755 (expected baseline 7.46241) 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.6 with 58 -> 55 pgs (0.781145 -> 0.740741 * mean) 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.4 with 83 -> 83 pgs (1.11785 -> 1.11785 * mean) 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:oload 110 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 0.05 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0426 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.0468 2023-09-23T21:20:12.913 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 weight 0.7630 -> 0.7130 2023-09-23T21:20:12.917 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:20:13.010 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:13.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:13.114 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:13.142 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:13.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: from='client.37653 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:13.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:13.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:13.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:13.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:20:13.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]': finished 2023-09-23T21:20:13.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: osdmap e613: 8 total, 8 up, 4 in 2023-09-23T21:20:13.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: failed to encode map e613 with expected crc 2023-09-23T21:20:13.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:13 smithi123 ceph-mon[193903]: failed to encode map e613 with expected crc 2023-09-23T21:20:13.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: from='client.37653 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]': finished 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: osdmap e613: 8 total, 8 up, 4 in 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: failed to encode map e613 with expected crc 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[211526]: failed to encode map e613 with expected crc 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: from='client.37653 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:13.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:20:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'6':'46729'}"}]': finished 2023-09-23T21:20:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: osdmap e613: 8 total, 8 up, 4 in 2023-09-23T21:20:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: failed to encode map e613 with expected crc 2023-09-23T21:20:13.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:13 smithi121 ceph-mon[219709]: failed to encode map e613 with expected crc 2023-09-23T21:20:14.620 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[219709]: failed to encode map e613 with expected crc 2023-09-23T21:20:14.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[219709]: pgmap v1873: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 413 op/s; 39571/242049 objects degraded (16.348%); 40741/242049 objects misplaced (16.832%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:20:14.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[219709]: osdmap e614: 8 total, 8 up, 4 in 2023-09-23T21:20:14.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[219709]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[219709]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[219709]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.621 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[219709]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.621 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[211526]: failed to encode map e613 with expected crc 2023-09-23T21:20:14.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[211526]: pgmap v1873: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 413 op/s; 39571/242049 objects degraded (16.348%); 40741/242049 objects misplaced (16.832%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:20:14.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[211526]: osdmap e614: 8 total, 8 up, 4 in 2023-09-23T21:20:14.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[211526]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[211526]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[211526]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.622 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:14 smithi121 ceph-mon[211526]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.622 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:14.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:14.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:14.645 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T21:20:14.646 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T21:20:14.725 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:14 smithi123 ceph-mon[193903]: failed to encode map e613 with expected crc 2023-09-23T21:20:14.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:14 smithi123 ceph-mon[193903]: pgmap v1873: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 4.9 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 413 op/s; 39571/242049 objects degraded (16.348%); 40741/242049 objects misplaced (16.832%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T21:20:14.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:14 smithi123 ceph-mon[193903]: osdmap e614: 8 total, 8 up, 4 in 2023-09-23T21:20:14.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:14 smithi123 ceph-mon[193903]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:14 smithi123 ceph-mon[193903]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:14 smithi123 ceph-mon[193903]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:14 smithi123 ceph-mon[193903]: failed to encode map e614 with expected crc 2023-09-23T21:20:14.763 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:14.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:20:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:20:14] "GET /metrics HTTP/1.1" 200 36711 "" "Prometheus/2.43.0" 2023-09-23T21:20:15.518 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:15.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:15.623 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:15.656 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:15.714 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:15.716 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:15 smithi121 ceph-mon[211526]: failed to encode map e614 with expected crc 2023-09-23T21:20:15.716 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:15 smithi121 ceph-mon[211526]: pgmap v1875: 99 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 567 op/s; 39202/235569 objects degraded (16.641%); 39459/235569 objects misplaced (16.751%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T21:20:15.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:15.717 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:15 smithi121 ceph-mon[219709]: failed to encode map e614 with expected crc 2023-09-23T21:20:15.717 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:15 smithi121 ceph-mon[219709]: pgmap v1875: 99 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 567 op/s; 39202/235569 objects degraded (16.641%); 39459/235569 objects misplaced (16.751%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T21:20:15.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:15.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:15 smithi123 ceph-mon[193903]: failed to encode map e614 with expected crc 2023-09-23T21:20:15.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:15 smithi123 ceph-mon[193903]: pgmap v1875: 99 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 567 op/s; 39202/235569 objects degraded (16.641%); 39459/235569 objects misplaced (16.751%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T21:20:15.802 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:20:15.803 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:15.819 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:15.825 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:20:15.853 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:20:15.857 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:15.881 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:20:15.918 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:20:15.919 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:15.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:15.960 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:20:15.991 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:20:16.025 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:16.029 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:20:16.031 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.031 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.031 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.031 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.031 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.032 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.032 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.032 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:20:16.107 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:16.209 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:16.210 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:16.268 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:16.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:16.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:16.379 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:20:16.389 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:16.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:16 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2046690586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:16.481 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:16.526 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:16.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:16.634 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:16.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:16 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2046690586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:16.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:16 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2046690586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:16.708 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:16.800 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:16.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:16.912 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:16.943 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:17.696 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:17.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:17.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:17.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:17 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 39202/235569 objects degraded (16.641%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:17.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:17 smithi123 ceph-mon[193903]: pgmap v1876: 99 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 342 op/s; 39202/234711 objects degraded (16.702%); 39459/234711 objects misplaced (16.812%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:20:17.800 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:17.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:17 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 39202/235569 objects degraded (16.641%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:17.803 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:17 smithi121 ceph-mon[211526]: pgmap v1876: 99 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 342 op/s; 39202/234711 objects degraded (16.702%); 39459/234711 objects misplaced (16.812%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:20:17.804 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:17 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 39202/235569 objects degraded (16.641%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:17.804 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:17 smithi121 ceph-mon[219709]: pgmap v1876: 99 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 342 op/s; 39202/234711 objects degraded (16.702%); 39459/234711 objects misplaced (16.812%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T21:20:17.825 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:20:17.881 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:17.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:17.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:18.094 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:18.149 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:18.285 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:18.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:18.394 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:18.492 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:18.530 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:18.572 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:18.630 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:18.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:18.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:18.757 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:18.799 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:18.916 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:18.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:18.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:19.020 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:19.111 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:19.249 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:19.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:19.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:19.371 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:19.404 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:19.500 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:20:19.563 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:19.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:19.575 INFO:teuthology.orchestra.run.smithi121.stdout:moved 0 / 297 (0%) 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:avg 74.25 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:stddev 11.4755 -> 11.4755 (expected baseline 7.46241) 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:min osd.6 with 55 -> 55 pgs (0.740741 -> 0.740741 * mean) 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:max osd.4 with 83 -> 83 pgs (1.11785 -> 1.11785 * mean) 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:oload 1000 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:max_change 1 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:max_change_osds 4 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:average_utilization 0.0410 2023-09-23T21:20:19.576 INFO:teuthology.orchestra.run.smithi121.stdout:overload_utilization 0.4102 2023-09-23T21:20:19.579 INFO:teuthology.orchestra.run.smithi121.stderr:no change 2023-09-23T21:20:19.669 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:19.695 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:19.735 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:19.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:19.841 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:19.871 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:20.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:20 smithi121 ceph-mon[211526]: pgmap v1877: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.7 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 553 op/s; 38778/228762 objects degraded (16.951%); 38336/228762 objects misplaced (16.758%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T21:20:20.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:20 smithi121 ceph-mon[219709]: pgmap v1877: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.7 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 553 op/s; 38778/228762 objects degraded (16.951%); 38336/228762 objects misplaced (16.758%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T21:20:20.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:20 smithi123 ceph-mon[193903]: pgmap v1877: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.7 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 553 op/s; 38778/228762 objects degraded (16.951%); 38336/228762 objects misplaced (16.758%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T21:20:20.920 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:20.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:21.025 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:21.061 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:21.334 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:21.465 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:21.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:21.572 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:21.609 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:21.694 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:21.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:21.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:21 smithi123 ceph-mon[193903]: from='client.47648 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:21.785 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:21.786 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:21.802 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:21.811 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:21 smithi121 ceph-mon[211526]: from='client.47648 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:21.811 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:21 smithi121 ceph-mon[219709]: from='client.47648 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:20:21.834 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:21.987 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:21.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:22.092 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:22.123 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:22.500 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:20:22.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:22 smithi121 ceph-mon[219709]: pgmap v1878: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 556 op/s; 38778/227889 objects degraded (17.016%); 38336/227889 objects misplaced (16.822%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:20:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:22 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2705104995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 38778/227889 objects degraded (17.016%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:22.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:22 smithi121 ceph-mon[211526]: pgmap v1878: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 556 op/s; 38778/227889 objects degraded (17.016%); 38336/227889 objects misplaced (16.822%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:20:22.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:22 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2705104995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:22.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 38778/227889 objects degraded (17.016%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:22.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:22 smithi123 ceph-mon[193903]: pgmap v1878: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 556 op/s; 38778/227889 objects degraded (17.016%); 38336/227889 objects misplaced (16.822%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T21:20:22.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:22 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2705104995' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:22.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 38778/227889 objects degraded (17.016%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:23.183 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:20:23.276 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:23.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:23.384 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:23.488 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:23.621 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:23.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:23.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:23.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:23.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2705104995' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: osdmap e615: 8 total, 8 up, 4 in 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[211526]: pgmap v1880: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 270 op/s; 38778/227889 objects degraded (17.016%); 38336/227889 objects misplaced (16.822%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2705104995' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:23.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: osdmap e615: 8 total, 8 up, 4 in 2023-09-23T21:20:23.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:23 smithi121 ceph-mon[219709]: pgmap v1880: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 270 op/s; 38778/227889 objects degraded (17.016%); 38336/227889 objects misplaced (16.822%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T21:20:23.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:23.813 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:23.813 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.813 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2705104995' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:23.813 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: osdmap e615: 8 total, 8 up, 4 in 2023-09-23T21:20:23.813 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.813 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.814 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: failed to encode map e615 with expected crc 2023-09-23T21:20:23.814 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:23 smithi123 ceph-mon[193903]: pgmap v1880: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 270 op/s; 38778/227889 objects degraded (17.016%); 38336/227889 objects misplaced (16.822%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T21:20:23.829 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:24.006 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:24.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:24.122 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:24.214 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:24.220 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:24.240 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:20:24.266 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:20:24.291 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:20:24.314 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:20:24.340 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:20:24.341 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:24.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:24.372 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:20:24.396 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:20:24.419 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:20:24.422 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.422 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.422 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.422 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.423 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.423 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.423 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.423 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:20:24.458 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:24.492 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:24.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3136137605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:24.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3136137605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:24.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3136137605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:25.056 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:25.058 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:20:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:20:24] "GET /metrics HTTP/1.1" 200 36711 "" "Prometheus/2.43.0" 2023-09-23T21:20:25.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:25.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:25.160 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:25.191 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:25.447 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:25.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:25.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:25.551 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:25.582 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:25.631 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:25.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:25.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:25.735 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:25.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:25 smithi121 ceph-mon[219709]: pgmap v1881: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 461 op/s; 38312/221709 objects degraded (17.280%); 37179/221709 objects misplaced (16.769%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:20:25.779 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:25 smithi121 ceph-mon[211526]: pgmap v1881: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 461 op/s; 38312/221709 objects degraded (17.280%); 37179/221709 objects misplaced (16.769%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:20:25.812 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:25 smithi123 ceph-mon[193903]: pgmap v1881: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 461 op/s; 38312/221709 objects degraded (17.280%); 37179/221709 objects misplaced (16.769%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:20:25.816 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:25.931 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7] out_osds: [0, 1, 3, 5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:20:25.931 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:20:25.932 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T21:20:25.932 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T21:20:25.946 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:25.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:26.055 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:26.079 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:20:26.084 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:26.306 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:26.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:26.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:26.414 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:26.460 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:26.490 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:26.537 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:26.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:26.558 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:26.644 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:26.709 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:26.833 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:26.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:26.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:26.953 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:26.960 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:26 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 38312/221709 objects degraded (17.280%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:26.990 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:27.020 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:20:27.020 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:20:27.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.020 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47438.0:98596 14.b 14:d64db483:::benchmark_data_smithi121_684892_object13279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:20:27.020 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:27.008032+0000", 2023-09-23T21:20:27.020 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.011773907, 2023-09-23T21:20:27.020 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011798094, 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47438", 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 98596 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.008032+0000", 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.021 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.008032+0000", 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.008031+0000", 2023-09-23T21:20:27.022 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.008037+0000", 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6640000000000003e-06 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013226+0000", 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051899800000000003 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.023 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013236+0000", 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0389999999999993e-06 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013288+0000", 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2951000000000002e-05 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.024 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013433+0000", 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014407499999999999 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013570+0000", 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013695299999999999 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:20:27.025 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015972+0000", 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024028420000000001 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:98597 14.5 e615/596 14:a3b429a4:::benchmark_data_smithi121_684892_object13280:head v 615'6346, mlcod=0'0)", 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:27.013339+0000", 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0064668490000000002, 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0065445349999999998, 2023-09-23T21:20:27.026 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013339+0000", 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013339+0000", 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013339+0000", 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013347+0000", 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6170000000000001e-06 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:20:27.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013352+0000", 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.604e-06 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.013357+0000", 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3479999999999997e-06 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015200+0000", 2023-09-23T21:20:27.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0018430670000000001 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015223+0000", 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3062999999999999e-05 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:98600 14.5 e615/596 14:a9825079:::benchmark_data_smithi121_684892_object13283:head v 615'6347, mlcod=0'0)", 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:27.015678+0000", 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.004127772, 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0042405990000000003, 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015678+0000", 2023-09-23T21:20:27.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015678+0000", 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015678+0000", 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.032 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015686+0000", 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5229999999999995e-06 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015695+0000", 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.0450000000000006e-06 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:20:27.033 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.015702+0000", 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.9589999999999998e-06 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.019718+0000", 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0040154650000000002 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.019740+0000", 2023-09-23T21:20:27.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2977999999999999e-05 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47438.0:98601 14.6 14:6682d00d:::benchmark_data_smithi121_684892_object13284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:27.017280+0000", 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0025256940000000002, 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026786079999999999, 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:20:27.035 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "waiting for sub ops", 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47438", 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 98601 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017280+0000", 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017280+0000", 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017279+0000", 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017287+0000", 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9039999999999992e-06 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017301+0000", 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4129e-05 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:20:27.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017310+0000", 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1300000000000007e-06 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017353+0000", 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3195999999999998e-05 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017482+0000", 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000129102 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,6", 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.017590+0000", 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107664 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:27.019498+0000", 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001908033 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.040 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:20:27.041 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.041 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:27.041 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:20:27.041 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 4 2023-09-23T21:20:27.041 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:20:27.047 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:26 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 38312/221709 objects degraded (17.280%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:27.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:26 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 38312/221709 objects degraded (17.280%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:27.543 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:20:28.052 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:28.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:28.158 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:28.187 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:28.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:27 smithi123 ceph-mon[193903]: pgmap v1882: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 463 op/s; 38312/220812 objects degraded (17.351%); 37179/220812 objects misplaced (16.837%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:20:28.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:28.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:28.387 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:27 smithi121 ceph-mon[211526]: pgmap v1882: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 463 op/s; 38312/220812 objects degraded (17.351%); 37179/220812 objects misplaced (16.837%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:20:28.387 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:28.387 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:28.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:27 smithi121 ceph-mon[219709]: pgmap v1882: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.5 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 463 op/s; 38312/220812 objects degraded (17.351%); 37179/220812 objects misplaced (16.837%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T21:20:28.387 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:28.388 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:29.230 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:20:29.327 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:29.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:29.389 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:29.440 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:29.485 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:29.907 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:29.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:29.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:30.010 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:30.105 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:30.252 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:30.253 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:30.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:30.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:30.312 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:30 smithi121 conmon[211495]: 2023-09-23T21:20:30.308+0000 7fdc50c66700 -1 mon.a@0(leader).osd e615 definitely_dead 0 2023-09-23T21:20:30.373 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:30.407 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:30.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:30 smithi123 ceph-mon[193903]: pgmap v1883: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.4 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 482 op/s; 37866/214272 objects degraded (17.672%); 35988/214272 objects misplaced (16.795%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T21:20:30.463 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1239228979' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:20:30.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/155782654' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:20:30.464 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:30 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:20:30.485 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:20:30.496 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.3. 2023-09-23T21:20:30.535 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:30.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:30.595 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[211526]: pgmap v1883: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.4 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 482 op/s; 37866/214272 objects degraded (17.672%); 35988/214272 objects misplaced (16.795%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T21:20:30.595 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1239228979' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:20:30.595 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/155782654' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:20:30.595 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:20:30.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[219709]: pgmap v1883: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 4.4 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 482 op/s; 37866/214272 objects degraded (17.672%); 35988/214272 objects misplaced (16.795%); 7.6 MiB/s, 122 objects/s recovering 2023-09-23T21:20:30.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1239228979' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:20:30.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/155782654' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:20:30.596 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:30 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T21:20:30.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:30.669 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1239228979' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T21:20:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: osdmap e616: 8 total, 8 up, 5 in 2023-09-23T21:20:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[211526]: pgmap v1885: 99 pgs: 2 remapped+peering, 26 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 29163/200157 objects degraded (14.570%); 35988/200157 objects misplaced (17.980%); 8.5 MiB/s, 137 objects/s recovering 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1239228979' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: osdmap e616: 8 total, 8 up, 5 in 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:31 smithi121 ceph-mon[219709]: pgmap v1885: 99 pgs: 2 remapped+peering, 26 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 29163/200157 objects degraded (14.570%); 35988/200157 objects misplaced (17.980%); 8.5 MiB/s, 137 objects/s recovering 2023-09-23T21:20:31.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:31.734 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1239228979' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: osdmap e616: 8 total, 8 up, 5 in 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.735 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: failed to encode map e616 with expected crc 2023-09-23T21:20:31.736 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:31 smithi123 ceph-mon[193903]: pgmap v1885: 99 pgs: 2 remapped+peering, 26 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 29163/200157 objects degraded (14.570%); 35988/200157 objects misplaced (17.980%); 8.5 MiB/s, 137 objects/s recovering 2023-09-23T21:20:32.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: osdmap e617: 8 total, 8 up, 5 in 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.a starting backfill to osd.3 from (0'0,0'0] MAX to 615'6224 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.f starting backfill to osd.3 from (0'0,0'0] MAX to 615'6309 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 29163/200157 objects degraded (14.570%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: osdmap e618: 8 total, 8 up, 5 in 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.9 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6163 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.3 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6322 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.4 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6180 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.5 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6458 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.2 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6181 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.1 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6211 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.8 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6267 2023-09-23T21:20:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[211526]: 14.6 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6297 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: osdmap e617: 8 total, 8 up, 5 in 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.a starting backfill to osd.3 from (0'0,0'0] MAX to 615'6224 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.f starting backfill to osd.3 from (0'0,0'0] MAX to 615'6309 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 29163/200157 objects degraded (14.570%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: osdmap e618: 8 total, 8 up, 5 in 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.9 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6163 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.3 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6322 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.4 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6180 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.5 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6458 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.2 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6181 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.1 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6211 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.8 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6267 2023-09-23T21:20:32.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:32 smithi121 ceph-mon[219709]: 14.6 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6297 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: osdmap e617: 8 total, 8 up, 5 in 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e617 with expected crc 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.a starting backfill to osd.3 from (0'0,0'0] MAX to 615'6224 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.f starting backfill to osd.3 from (0'0,0'0] MAX to 615'6309 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 29163/200157 objects degraded (14.570%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: osdmap e618: 8 total, 8 up, 5 in 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: failed to encode map e618 with expected crc 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.9 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6163 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.3 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6322 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.4 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6180 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.5 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6458 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.2 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6181 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.1 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6211 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.8 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6267 2023-09-23T21:20:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:32 smithi123 ceph-mon[193903]: 14.6 starting backfill to osd.3 from (0'0,0'0] MAX to 615'6297 2023-09-23T21:20:33.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:33.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:33.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:33.286 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:33.367 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:33.485 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:33.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:33.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:33.595 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:33.672 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e618 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: osdmap e619: 8 total, 8 up, 5 in 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:33 smithi123 ceph-mon[193903]: pgmap v1889: 99 pgs: 2 remapped+peering, 26 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 29163/200157 objects degraded (14.570%); 35988/200157 objects misplaced (17.980%) 2023-09-23T21:20:33.677 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:33.803 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:33.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:33.844 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e618 with expected crc 2023-09-23T21:20:33.844 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: osdmap e619: 8 total, 8 up, 5 in 2023-09-23T21:20:33.844 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.845 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.846 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.846 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[211526]: pgmap v1889: 99 pgs: 2 remapped+peering, 26 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 29163/200157 objects degraded (14.570%); 35988/200157 objects misplaced (17.980%) 2023-09-23T21:20:33.846 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e618 with expected crc 2023-09-23T21:20:33.846 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: osdmap e619: 8 total, 8 up, 5 in 2023-09-23T21:20:33.846 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:33.848 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:33 smithi121 ceph-mon[219709]: pgmap v1889: 99 pgs: 2 remapped+peering, 26 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 29163/200157 objects degraded (14.570%); 35988/200157 objects misplaced (17.980%) 2023-09-23T21:20:33.868 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T21:20:33.915 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:33.994 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:34.123 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:34.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:34.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:34.234 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:34.262 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:34.347 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:34.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:34.471 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:34.501 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:34.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: failed to encode map e619 with expected crc 2023-09-23T21:20:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: osdmap e620: 8 total, 8 up, 5 in 2023-09-23T21:20:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:34 smithi123 ceph-mon[193903]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.771 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: failed to encode map e619 with expected crc 2023-09-23T21:20:34.771 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: osdmap e620: 8 total, 8 up, 5 in 2023-09-23T21:20:34.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.772 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[219709]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: failed to encode map e619 with expected crc 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: osdmap e620: 8 total, 8 up, 5 in 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:34 smithi121 ceph-mon[211526]: failed to encode map e620 with expected crc 2023-09-23T21:20:34.815 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:34.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:34.922 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:34.994 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:35.041 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:20:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:20:34] "GET /metrics HTTP/1.1" 200 36710 "" "Prometheus/2.43.0" 2023-09-23T21:20:35.140 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:35.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:35.256 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:35.304 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:35.379 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:20:35.384 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:35.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:35.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:35.406 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:20:35.432 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:20:35.461 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:20:35.494 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:35.498 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:20:35.528 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:20:35.556 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:20:35.564 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:35.580 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.582 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:20:35.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1836652059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:35.613 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:35 smithi121 ceph-mon[211526]: pgmap v1891: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 27 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 57 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 83 MiB/s wr, 1.87k op/s; 28393/209469 objects degraded (13.555%); 39232/209469 objects misplaced (18.729%); 22 MiB/s, 360 objects/s recovering 2023-09-23T21:20:35.613 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1836652059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:35.614 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:35 smithi121 ceph-mon[219709]: pgmap v1891: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 27 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 57 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 83 MiB/s wr, 1.87k op/s; 28393/209469 objects degraded (13.555%); 39232/209469 objects misplaced (18.729%); 22 MiB/s, 360 objects/s recovering 2023-09-23T21:20:35.623 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:35.623 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:35.728 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:35.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:35.841 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:35.916 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1836652059' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:35.917 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:35 smithi123 ceph-mon[193903]: pgmap v1891: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 27 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 57 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 83 MiB/s wr, 1.87k op/s; 28393/209469 objects degraded (13.555%); 39232/209469 objects misplaced (18.729%); 22 MiB/s, 360 objects/s recovering 2023-09-23T21:20:35.921 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:36.039 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:36.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:36.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:36.156 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:36.195 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:36.801 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:20:37.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:36 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 28393/209469 objects degraded (13.555%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:37.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:36 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 28393/209469 objects degraded (13.555%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:37.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:36 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 28393/209469 objects degraded (13.555%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:37.764 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:20:37.857 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:37.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:37.909 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:37.969 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:38.021 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:38.143 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:38.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:38.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:37 smithi121 ceph-mon[211526]: pgmap v1892: 99 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 56 MiB/s wr, 1.30k op/s; 36594/208782 objects degraded (17.527%); 39398/208782 objects misplaced (18.870%); 18 MiB/s, 283 objects/s recovering 2023-09-23T21:20:38.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:37 smithi121 ceph-mon[219709]: pgmap v1892: 99 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 56 MiB/s wr, 1.30k op/s; 36594/208782 objects degraded (17.527%); 39398/208782 objects misplaced (18.870%); 18 MiB/s, 283 objects/s recovering 2023-09-23T21:20:38.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:37 smithi123 ceph-mon[193903]: pgmap v1892: 99 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 81 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 56 MiB/s wr, 1.30k op/s; 36594/208782 objects degraded (17.527%); 39398/208782 objects misplaced (18.870%); 18 MiB/s, 283 objects/s recovering 2023-09-23T21:20:38.261 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:38.345 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:38.466 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:38.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:38.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:38.579 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:38.620 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:38.680 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:38.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:38.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:38.784 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:38.818 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:38.868 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7, 3] out_osds: [0, 1, 5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:20:38.868 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:20:38.869 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T21:20:38.869 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T21:20:38.871 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:38.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:38.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:38.978 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:39.013 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:39.063 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:39.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:39.167 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:39.200 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:39.529 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:39.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:39.583 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:39.636 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:39.694 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:39.771 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:39.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:39.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:39.779 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:39.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:20:39.878 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:39.977 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:40.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:40.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:40.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:40.240 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:40.458 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:40 smithi121 ceph-mon[211526]: pgmap v1893: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 44 MiB/s wr, 1.28k op/s; 35770/203190 objects degraded (17.604%); 37729/203190 objects misplaced (18.568%); 26 MiB/s, 420 objects/s recovering 2023-09-23T21:20:40.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:40 smithi123 ceph-mon[193903]: pgmap v1893: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 44 MiB/s wr, 1.28k op/s; 35770/203190 objects degraded (17.604%); 37729/203190 objects misplaced (18.568%); 26 MiB/s, 420 objects/s recovering 2023-09-23T21:20:40.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:40 smithi121 ceph-mon[219709]: pgmap v1893: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 44 MiB/s wr, 1.28k op/s; 35770/203190 objects degraded (17.604%); 37729/203190 objects misplaced (18.568%); 26 MiB/s, 420 objects/s recovering 2023-09-23T21:20:41.037 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:41.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:41.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:41.141 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:41.169 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:41.468 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:20:41.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4117067364' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:41.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:41 smithi123 ceph-mon[193903]: pgmap v1894: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 39 MiB/s wr, 1.16k op/s; 35497/202371 objects degraded (17.541%); 37729/202371 objects misplaced (18.643%); 25 MiB/s, 403 objects/s recovering 2023-09-23T21:20:41.911 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4117067364' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:41 smithi121 ceph-mon[211526]: pgmap v1894: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 39 MiB/s wr, 1.16k op/s; 35497/202371 objects degraded (17.541%); 37729/202371 objects misplaced (18.643%); 25 MiB/s, 403 objects/s recovering 2023-09-23T21:20:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4117067364' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:20:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:41 smithi121 ceph-mon[219709]: pgmap v1894: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 39 MiB/s wr, 1.16k op/s; 35497/202371 objects degraded (17.541%); 37729/202371 objects misplaced (18.643%); 25 MiB/s, 403 objects/s recovering 2023-09-23T21:20:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:41.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:42.015 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:42.079 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:42.125 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:42.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:42.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:42.232 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:42.269 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:42 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4117067364' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:42.465 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:42 smithi123 ceph-mon[193903]: osdmap e621: 8 total, 8 up, 5 in 2023-09-23T21:20:42.466 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:42 smithi123 ceph-mon[193903]: failed to encode map e621 with expected crc 2023-09-23T21:20:42.466 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:42 smithi123 ceph-mon[193903]: failed to encode map e621 with expected crc 2023-09-23T21:20:42.466 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 35497/202371 objects degraded (17.541%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:42.488 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:42.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:42.534 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:42.535 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4117067364' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:42.535 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[211526]: osdmap e621: 8 total, 8 up, 5 in 2023-09-23T21:20:42.535 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[211526]: failed to encode map e621 with expected crc 2023-09-23T21:20:42.535 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[211526]: failed to encode map e621 with expected crc 2023-09-23T21:20:42.535 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 35497/202371 objects degraded (17.541%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:42.537 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:20:42.537 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4117067364' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:20:42.537 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[219709]: osdmap e621: 8 total, 8 up, 5 in 2023-09-23T21:20:42.537 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[219709]: failed to encode map e621 with expected crc 2023-09-23T21:20:42.537 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[219709]: failed to encode map e621 with expected crc 2023-09-23T21:20:42.537 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 35497/202371 objects degraded (17.541%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:42.597 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:42.641 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:42.711 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:42.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:42.818 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:42.908 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:42.983 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:42.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:43.091 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:43.125 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:43.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:43.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:43 smithi121 ceph-mon[211526]: pgmap v1896: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 35497/202371 objects degraded (17.541%); 37729/202371 objects misplaced (18.643%); 13 MiB/s, 208 objects/s recovering 2023-09-23T21:20:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:43 smithi121 ceph-mon[219709]: pgmap v1896: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 35497/202371 objects degraded (17.541%); 37729/202371 objects misplaced (18.643%); 13 MiB/s, 208 objects/s recovering 2023-09-23T21:20:43.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:43.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:43.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:43 smithi123 ceph-mon[193903]: pgmap v1896: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 35497/202371 objects degraded (17.541%); 37729/202371 objects misplaced (18.643%); 13 MiB/s, 208 objects/s recovering 2023-09-23T21:20:44.123 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:44.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:44.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:44.235 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:44.317 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:44.324 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:44.454 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:44.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:44.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:44.486 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:20:44.486 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:20:44.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:44.611 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:44.777 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:20:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:20:44] "GET /metrics HTTP/1.1" 200 36725 "" "Prometheus/2.43.0" 2023-09-23T21:20:44.877 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:44.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:44.984 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:45.018 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:45.107 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:20:45.487 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:45.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:45.592 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:45.631 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:45 smithi121 conmon[211495]: 2023-09-23T21:20:45.620+0000 7fdc50c66700 -1 mon.a@0(leader).osd e621 definitely_dead 0 2023-09-23T21:20:45.784 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:45.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:45.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:45.899 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:45.934 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:46.477 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.5. 2023-09-23T21:20:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:46 smithi121 ceph-mon[219709]: pgmap v1897: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 34748/197253 objects degraded (17.616%); 36143/197253 objects misplaced (18.323%); 21 MiB/s, 341 objects/s recovering 2023-09-23T21:20:46.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2036497230' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:20:46.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:46 smithi121 ceph-mon[211526]: pgmap v1897: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 34748/197253 objects degraded (17.616%); 36143/197253 objects misplaced (18.323%); 21 MiB/s, 341 objects/s recovering 2023-09-23T21:20:46.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2036497230' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:20:46.904 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:46.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:46.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:46.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:46 smithi123 ceph-mon[193903]: pgmap v1897: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 34748/197253 objects degraded (17.616%); 36143/197253 objects misplaced (18.323%); 21 MiB/s, 341 objects/s recovering 2023-09-23T21:20:46.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2036497230' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T21:20:47.008 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:47.069 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:47.176 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:47.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:47.288 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:47.300 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:20:47.316 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:47.578 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2036497230' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 34748/197253 objects degraded (17.616%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.580 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: 14.3 continuing backfill to osd.3 from (583'2698,621'6660] MIN to 621'6660 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[219709]: pgmap v1900: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 34473/196431 objects degraded (17.550%); 36144/196431 objects misplaced (18.400%); 18 MiB/s, 286 objects/s recovering 2023-09-23T21:20:47.581 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2036497230' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 34748/197253 objects degraded (17.616%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.582 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.583 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: 14.3 continuing backfill to osd.3 from (583'2698,621'6660] MIN to 621'6660 2023-09-23T21:20:47.583 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:47 smithi121 ceph-mon[211526]: pgmap v1900: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 34473/196431 objects degraded (17.550%); 36144/196431 objects misplaced (18.400%); 18 MiB/s, 286 objects/s recovering 2023-09-23T21:20:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:47.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:47.684 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:47.712 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:47.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2036497230' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T21:20:47.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T21:20:47.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e622 with expected crc 2023-09-23T21:20:47.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 34748/197253 objects degraded (17.616%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T21:20:47.986 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: 14.3 continuing backfill to osd.3 from (583'2698,621'6660] MIN to 621'6660 2023-09-23T21:20:47.987 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:47 smithi123 ceph-mon[193903]: pgmap v1900: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 34473/196431 objects degraded (17.550%); 36144/196431 objects misplaced (18.400%); 18 MiB/s, 286 objects/s recovering 2023-09-23T21:20:48.204 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:48.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:48.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:48.305 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T21:20:48.308 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:20:48.344 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:48.370 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:48.410 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:48.539 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: 14.8 continuing backfill to osd.3 from (580'2246,621'6629] MIN to 621'6629 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.540 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: 14.f continuing backfill to osd.3 from (582'2686,621'6719] MIN to 621'6719 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: 14.a continuing backfill to osd.3 from (581'2518,621'6607] MIN to 621'6607 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: 14.d continuing backfill to osd.2 from (581'2548,621'6607] 14:b0078553:::benchmark_data_smithi121_684892_object78377:head to 621'6607 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: 14.2 continuing backfill to osd.3 from (580'2267,621'6601] MIN to 621'6601 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[219709]: 14.9 starting backfill to osd.3 from (581'2544,621'6525] MIN to 621'6527 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.541 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: 14.8 continuing backfill to osd.3 from (580'2246,621'6629] MIN to 621'6629 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: 14.f continuing backfill to osd.3 from (582'2686,621'6719] MIN to 621'6719 2023-09-23T21:20:48.542 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: 14.a continuing backfill to osd.3 from (581'2518,621'6607] MIN to 621'6607 2023-09-23T21:20:48.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: 14.d continuing backfill to osd.2 from (581'2548,621'6607] 14:b0078553:::benchmark_data_smithi121_684892_object78377:head to 621'6607 2023-09-23T21:20:48.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: 14.2 continuing backfill to osd.3 from (580'2267,621'6601] MIN to 621'6601 2023-09-23T21:20:48.543 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:48 smithi121 ceph-mon[211526]: 14.9 starting backfill to osd.3 from (581'2544,621'6525] MIN to 621'6527 2023-09-23T21:20:48.564 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:20:48.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:48.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:20:48.672 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:48.709 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:48.871 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:20:48.898 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:20:48.924 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:20:48.963 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:20:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e623 with expected crc 2023-09-23T21:20:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T21:20:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: 14.8 continuing backfill to osd.3 from (580'2246,621'6629] MIN to 621'6629 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: 14.f continuing backfill to osd.3 from (582'2686,621'6719] MIN to 621'6719 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: 14.a continuing backfill to osd.3 from (581'2518,621'6607] MIN to 621'6607 2023-09-23T21:20:48.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: 14.d continuing backfill to osd.2 from (581'2548,621'6607] 14:b0078553:::benchmark_data_smithi121_684892_object78377:head to 621'6607 2023-09-23T21:20:48.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: 14.2 continuing backfill to osd.3 from (580'2267,621'6601] MIN to 621'6601 2023-09-23T21:20:48.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:48 smithi123 ceph-mon[193903]: 14.9 starting backfill to osd.3 from (581'2544,621'6525] MIN to 621'6527 2023-09-23T21:20:48.983 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:48.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:48.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:49.002 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:20:49.048 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:20:49.080 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:20:49.093 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:49.110 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:20:49.119 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.119 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.120 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.120 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.120 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.120 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.120 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.120 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:20:49.197 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:49.320 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:49.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:49.427 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:49.492 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:49.589 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:49.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:49.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:49.728 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:49.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e624 with expected crc 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1900190123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:49 smithi123 ceph-mon[193903]: pgmap v1903: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 404 op/s; 28820/192393 objects degraded (14.980%); 27586/192393 objects misplaced (14.338%); 23 MiB/s, 374 objects/s recovering 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e624 with expected crc 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1900190123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[219709]: pgmap v1903: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 404 op/s; 28820/192393 objects degraded (14.980%); 27586/192393 objects misplaced (14.338%); 23 MiB/s, 374 objects/s recovering 2023-09-23T21:20:49.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:49.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e624 with expected crc 2023-09-23T21:20:49.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1900190123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:49.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T21:20:49.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: failed to encode map e625 with expected crc 2023-09-23T21:20:49.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:49 smithi121 ceph-mon[211526]: pgmap v1903: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 404 op/s; 28820/192393 objects degraded (14.980%); 27586/192393 objects misplaced (14.338%); 23 MiB/s, 374 objects/s recovering 2023-09-23T21:20:51.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T21:20:51.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.231 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:50 smithi123 ceph-mon[193903]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[219709]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T21:20:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:50 smithi121 ceph-mon[211526]: failed to encode map e626 with expected crc 2023-09-23T21:20:51.696 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:51.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:51.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:51.801 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:51.889 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:52.061 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:52.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:52.167 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:52.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:51 smithi123 ceph-mon[193903]: pgmap v1905: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 344 op/s; 27032/191784 objects degraded (14.095%); 31210/191784 objects misplaced (16.274%); 18 MiB/s, 297 objects/s recovering 2023-09-23T21:20:52.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:51 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 28820/192393 objects degraded (14.980%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:52.241 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:52.369 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:52.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:52.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:52.394 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:20:52.394 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:20:52.394 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.394 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:108284 14.1 e626/617 14:8065a87f:::benchmark_data_smithi121_684892_object22967:head v 626'6657, mlcod=626'6654)", 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:52.393995+0000", 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.00046697699999999998, 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00049808600000000001, 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.393995+0000", 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:52.395 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.393995+0000", 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.393994+0000", 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:20:52.396 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.394005+0000", 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1216000000000001e-05 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.394026+0000", 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0928e-05 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.394038+0000", 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1751e-05 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:52.397 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.394092+0000", 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4054000000000001e-05 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:52.394123+0000", 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1597999999999998e-05 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:52.398 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:20:52.399 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:20:52.399 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 1 2023-09-23T21:20:52.399 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:20:52.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:51 smithi121 ceph-mon[211526]: pgmap v1905: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 344 op/s; 27032/191784 objects degraded (14.095%); 31210/191784 objects misplaced (16.274%); 18 MiB/s, 297 objects/s recovering 2023-09-23T21:20:52.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:51 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 28820/192393 objects degraded (14.980%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:52.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:51 smithi121 ceph-mon[219709]: pgmap v1905: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 344 op/s; 27032/191784 objects degraded (14.095%); 31210/191784 objects misplaced (16.274%); 18 MiB/s, 297 objects/s recovering 2023-09-23T21:20:52.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:51 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 28820/192393 objects degraded (14.980%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:52.482 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:52.575 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:52.692 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:52.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:52.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:52.809 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:52.846 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:52.901 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:52.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:52.966 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:20:53.006 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:20:53.094 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:53.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2093509085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:53.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:52 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2093509085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:53.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:52 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:53.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2093509085' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:53.234 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:52 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:20:53.275 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:20:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:53.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:20:53.306 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7, 3, 5] out_osds: [0, 1] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:20:53.307 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:20:53.307 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T21:20:53.307 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T21:20:53.389 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:53.535 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:53.707 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:53.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:53.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:53.815 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:20:53.827 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:20:53.919 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: failed to encode map e627 with expected crc 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: failed to encode map e627 with expected crc 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: failed to encode map e627 with expected crc 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: failed to encode map e627 with expected crc 2023-09-23T21:20:53.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:53 smithi123 ceph-mon[193903]: pgmap v1907: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 27032/191784 objects degraded (14.095%); 31210/191784 objects misplaced (16.274%); 15 MiB/s, 244 objects/s recovering 2023-09-23T21:20:54.043 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:20:54.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:54.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:20:54.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[219709]: pgmap v1907: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 27032/191784 objects degraded (14.095%); 31210/191784 objects misplaced (16.274%); 15 MiB/s, 244 objects/s recovering 2023-09-23T21:20:54.146 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:20:54.147 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.147 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:20:54.147 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T21:20:54.147 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.147 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.147 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: failed to encode map e627 with expected crc 2023-09-23T21:20:54.147 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:53 smithi121 ceph-mon[211526]: pgmap v1907: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 79 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 282 op/s; 27032/191784 objects degraded (14.095%); 31210/191784 objects misplaced (16.274%); 15 MiB/s, 244 objects/s recovering 2023-09-23T21:20:54.164 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:54.200 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:54.295 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:20:54.368 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:54.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:54.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:54.449 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:54.452 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:54.480 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:20:54.552 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:54.636 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:20:54.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:54.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:20:54.740 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:54.819 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:54.915 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:20:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:20:54] "GET /metrics HTTP/1.1" 200 36725 "" "Prometheus/2.43.0" 2023-09-23T21:20:54.926 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:54.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:54.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:55.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:55.109 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:55.252 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:55.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:55.385 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:55.419 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:56.172 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:20:56.422 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:56.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:56.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:56.470 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:56 smithi121 ceph-mon[211526]: pgmap v1908: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 353 op/s; 26594/185529 objects degraded (14.334%); 29883/185529 objects misplaced (16.107%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T21:20:56.530 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:20:56.562 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:56.739 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:56 smithi123 ceph-mon[193903]: pgmap v1908: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 353 op/s; 26594/185529 objects degraded (14.334%); 29883/185529 objects misplaced (16.107%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T21:20:56.841 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:56 smithi121 ceph-mon[219709]: pgmap v1908: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 353 op/s; 26594/185529 objects degraded (14.334%); 29883/185529 objects misplaced (16.107%); 9.6 MiB/s, 154 objects/s recovering 2023-09-23T21:20:57.056 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:20:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:57.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:20:57.153 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:57.161 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:57.267 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:57.429 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:57.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:57.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:57.549 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:57.590 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:57.650 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:20:57.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:57 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 26594/185529 objects degraded (14.334%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:57.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:57 smithi123 ceph-mon[193903]: pgmap v1909: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 26441/184584 objects degraded (14.325%); 29721/184584 objects misplaced (16.102%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:20:57.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:57 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 26594/185529 objects degraded (14.334%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:57.731 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:57 smithi121 ceph-mon[219709]: pgmap v1909: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 26441/184584 objects degraded (14.325%); 29721/184584 objects misplaced (16.102%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:20:57.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:57 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 26594/185529 objects degraded (14.334%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T21:20:57.731 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:57 smithi121 ceph-mon[211526]: pgmap v1909: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 325 op/s; 26441/184584 objects degraded (14.325%); 29721/184584 objects misplaced (16.102%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T21:20:57.915 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:20:57.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:57.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:20:57.964 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:20:57.989 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:20:58.018 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:20:58.020 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:58.049 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:20:58.077 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:20:58.093 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:58.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:20:58.142 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:20:58.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:57 smithi121 conmon[211495]: 2023-09-23T21:20:57.907+0000 7fdc50c66700 -1 mon.a@0(leader).osd e627 definitely_dead 0 2023-09-23T21:20:58.165 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.167 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:20:58.241 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:58.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:58.361 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:20:58.445 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:58.516 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.1. 2023-09-23T21:20:58.572 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:20:58.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:58.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:20:58.681 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:20:58.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:58.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:58.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/695580214' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:58.706 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2109513373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:58.709 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:58.754 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:20:58.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:20:58.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:20:58.861 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:20:58.890 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:58.890 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:58.890 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/695580214' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:58.890 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2109513373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:58.891 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:20:58.891 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:20:58.891 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/695580214' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T21:20:58.891 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2109513373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:20:58.894 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:20:59.767 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/695580214' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T21:20:59.767 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: osdmap e628: 8 total, 8 up, 7 in 2023-09-23T21:20:59.767 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.767 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.768 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[211526]: pgmap v1911: 99 pgs: 1 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 21 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 3.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 11093/178200 objects degraded (6.225%); 10497/178200 objects misplaced (5.891%); 15 MiB/s, 247 objects/s recovering 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/695580214' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: osdmap e628: 8 total, 8 up, 7 in 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.769 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.770 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.770 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.770 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.770 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:20:59 smithi121 ceph-mon[219709]: pgmap v1911: 99 pgs: 1 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 21 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 3.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 11093/178200 objects degraded (6.225%); 10497/178200 objects misplaced (5.891%); 15 MiB/s, 247 objects/s recovering 2023-09-23T21:20:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/695580214' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T21:20:59.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: osdmap e628: 8 total, 8 up, 7 in 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: failed to encode map e628 with expected crc 2023-09-23T21:20:59.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:20:59 smithi123 ceph-mon[193903]: pgmap v1911: 99 pgs: 1 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 21 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 3.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 565 op/s; 11093/178200 objects degraded (6.225%); 10497/178200 objects misplaced (5.891%); 15 MiB/s, 247 objects/s recovering 2023-09-23T21:21:00.101 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:00.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:00.206 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:00.234 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: osdmap e629: 8 total, 8 up, 7 in 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: 14.1 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6879 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: 14.b starting backfill to osd.1 from (0'0,0'0] MAX to 627'7079 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[219709]: 14.1 starting backfill to osd.3 from (580'2225,627'6878] MIN to 627'6879 2023-09-23T21:21:00.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: osdmap e629: 8 total, 8 up, 7 in 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: 14.1 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6879 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: 14.b starting backfill to osd.1 from (0'0,0'0] MAX to 627'7079 2023-09-23T21:21:00.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.917 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.918 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:00 smithi121 ceph-mon[211526]: 14.1 starting backfill to osd.3 from (580'2225,627'6878] MIN to 627'6879 2023-09-23T21:21:00.961 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:00.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: osdmap e629: 8 total, 8 up, 7 in 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: 14.1 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6879 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: 14.b starting backfill to osd.1 from (0'0,0'0] MAX to 627'7079 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: failed to encode map e629 with expected crc 2023-09-23T21:21:00.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:00 smithi123 ceph-mon[193903]: 14.1 starting backfill to osd.3 from (580'2225,627'6878] MIN to 627'6879 2023-09-23T21:21:01.065 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:01.091 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:01.138 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:01.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:01.247 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:01.302 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:01.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.8 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6948 2023-09-23T21:21:01.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.6 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6981 2023-09-23T21:21:01.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: osdmap e630: 8 total, 8 up, 7 in 2023-09-23T21:21:01.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.625 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.a starting backfill to osd.1 from (0'0,0'0] MAX to 627'6919 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.3 starting backfill to osd.1 from (0'0,0'0] MAX to 627'7000 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.2 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6930 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.6 starting backfill to osd.3 from (580'2298,627'6980] MIN to 627'6981 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.8 continuing backfill to osd.3 from (580'2246,627'6948] MIN to 627'6948 2023-09-23T21:21:01.626 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.f starting backfill to osd.1 from (0'0,0'0] MAX to 627'7034 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.3 continuing backfill to osd.3 from (583'2698,627'7000] MIN to 627'7000 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.a continuing backfill to osd.3 from (581'2518,627'6919] MIN to 627'6919 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.2 continuing backfill to osd.3 from (580'2267,627'6930] MIN to 627'6930 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: 14.f continuing backfill to osd.3 from (582'2686,627'7034] MIN to 627'7034 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[219709]: pgmap v1914: 99 pgs: 1 activating+remapped, 2 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 22 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 431 op/s; 7419/177756 objects degraded (4.174%); 14719/177756 objects misplaced (8.280%); 13 MiB/s, 216 objects/s recovering 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.8 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6948 2023-09-23T21:21:01.627 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.6 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6981 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: osdmap e630: 8 total, 8 up, 7 in 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.a starting backfill to osd.1 from (0'0,0'0] MAX to 627'6919 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.3 starting backfill to osd.1 from (0'0,0'0] MAX to 627'7000 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.2 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6930 2023-09-23T21:21:01.628 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.6 starting backfill to osd.3 from (580'2298,627'6980] MIN to 627'6981 2023-09-23T21:21:01.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.8 continuing backfill to osd.3 from (580'2246,627'6948] MIN to 627'6948 2023-09-23T21:21:01.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.f starting backfill to osd.1 from (0'0,0'0] MAX to 627'7034 2023-09-23T21:21:01.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.3 continuing backfill to osd.3 from (583'2698,627'7000] MIN to 627'7000 2023-09-23T21:21:01.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.a continuing backfill to osd.3 from (581'2518,627'6919] MIN to 627'6919 2023-09-23T21:21:01.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.2 continuing backfill to osd.3 from (580'2267,627'6930] MIN to 627'6930 2023-09-23T21:21:01.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: 14.f continuing backfill to osd.3 from (582'2686,627'7034] MIN to 627'7034 2023-09-23T21:21:01.629 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:01 smithi121 ceph-mon[211526]: pgmap v1914: 99 pgs: 1 activating+remapped, 2 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 22 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 431 op/s; 7419/177756 objects degraded (4.174%); 14719/177756 objects misplaced (8.280%); 13 MiB/s, 216 objects/s recovering 2023-09-23T21:21:01.678 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:01.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:01.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:01.784 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:01.862 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T21:21:01.865 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.865 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.8 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6948 2023-09-23T21:21:01.865 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.6 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6981 2023-09-23T21:21:01.865 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: osdmap e630: 8 total, 8 up, 7 in 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.a starting backfill to osd.1 from (0'0,0'0] MAX to 627'6919 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.3 starting backfill to osd.1 from (0'0,0'0] MAX to 627'7000 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: failed to encode map e630 with expected crc 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.2 starting backfill to osd.1 from (0'0,0'0] MAX to 627'6930 2023-09-23T21:21:01.866 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.6 starting backfill to osd.3 from (580'2298,627'6980] MIN to 627'6981 2023-09-23T21:21:01.867 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.8 continuing backfill to osd.3 from (580'2246,627'6948] MIN to 627'6948 2023-09-23T21:21:01.867 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.f starting backfill to osd.1 from (0'0,0'0] MAX to 627'7034 2023-09-23T21:21:01.867 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.3 continuing backfill to osd.3 from (583'2698,627'7000] MIN to 627'7000 2023-09-23T21:21:01.867 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.a continuing backfill to osd.3 from (581'2518,627'6919] MIN to 627'6919 2023-09-23T21:21:01.867 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.2 continuing backfill to osd.3 from (580'2267,627'6930] MIN to 627'6930 2023-09-23T21:21:01.867 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: 14.f continuing backfill to osd.3 from (582'2686,627'7034] MIN to 627'7034 2023-09-23T21:21:01.867 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:01 smithi123 ceph-mon[193903]: pgmap v1914: 99 pgs: 1 activating+remapped, 2 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 22 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 431 op/s; 7419/177756 objects degraded (4.174%); 14719/177756 objects misplaced (8.280%); 13 MiB/s, 216 objects/s recovering 2023-09-23T21:21:01.874 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:01.912 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:21:01.913 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:21:02.000 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:02.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:02.126 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:02.230 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:02.325 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:02.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:02.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:02.433 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:02.535 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:02.582 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:02.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:02.687 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:02.724 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:02.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: osdmap e631: 8 total, 8 up, 7 in 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7419/177756 objects degraded (4.174%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:02.899 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[219709]: osdmap e632: 8 total, 8 up, 7 in 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: osdmap e631: 8 total, 8 up, 7 in 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7419/177756 objects degraded (4.174%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:02.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:02 smithi121 ceph-mon[211526]: osdmap e632: 8 total, 8 up, 7 in 2023-09-23T21:21:02.916 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:02.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:02.929 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: osdmap e631: 8 total, 8 up, 7 in 2023-09-23T21:21:02.929 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.929 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: failed to encode map e631 with expected crc 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7419/177756 objects degraded (4.174%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:02.930 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:02 smithi123 ceph-mon[193903]: osdmap e632: 8 total, 8 up, 7 in 2023-09-23T21:21:03.028 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:03.072 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:03.342 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:21:03.507 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:21:03.510 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:03.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:03.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:03.618 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[211526]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[211526]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[211526]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[211526]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[211526]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[211526]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[211526]: pgmap v1917: 99 pgs: 2 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 23 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 7419/177756 objects degraded (4.174%); 14719/177756 objects misplaced (8.280%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T21:21:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[219709]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[219709]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[219709]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[219709]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[219709]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[219709]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:03 smithi121 ceph-mon[219709]: pgmap v1917: 99 pgs: 2 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 23 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 7419/177756 objects degraded (4.174%); 14719/177756 objects misplaced (8.280%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T21:21:03.678 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:03.889 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:03.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:03.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:03 smithi123 ceph-mon[193903]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:03 smithi123 ceph-mon[193903]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:03 smithi123 ceph-mon[193903]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:03 smithi123 ceph-mon[193903]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:03 smithi123 ceph-mon[193903]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:03 smithi123 ceph-mon[193903]: failed to encode map e632 with expected crc 2023-09-23T21:21:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:03 smithi123 ceph-mon[193903]: pgmap v1917: 99 pgs: 2 active+recovering+remapped, 7 remapped+peering, 1 active+recovering+undersized+remapped, 23 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 7419/177756 objects degraded (4.174%); 14719/177756 objects misplaced (8.280%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T21:21:03.995 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:04.082 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:04.150 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:04.175 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:04.224 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:04.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:04.341 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:04.405 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:04.524 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:04.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:04.569 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:21:04.642 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:04.693 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/748133547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:04.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:04 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:04.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:21:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:21:04] "GET /metrics HTTP/1.1" 200 36709 "" "Prometheus/2.43.0" 2023-09-23T21:21:04.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/748133547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:04.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:04 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:04.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/748133547' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:04.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:04 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[211526]: failed to encode map e633 with expected crc 2023-09-23T21:21:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[211526]: osdmap e633: 8 total, 8 up, 7 in 2023-09-23T21:21:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[211526]: failed to encode map e633 with expected crc 2023-09-23T21:21:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:21:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[211526]: pgmap v1919: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 83 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 24902/174489 objects degraded (14.271%); 47253/174489 objects misplaced (27.081%); 6.8 MiB/s, 215 objects/s recovering 2023-09-23T21:21:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[219709]: failed to encode map e633 with expected crc 2023-09-23T21:21:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[219709]: osdmap e633: 8 total, 8 up, 7 in 2023-09-23T21:21:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[219709]: failed to encode map e633 with expected crc 2023-09-23T21:21:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:21:05.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:05 smithi121 ceph-mon[219709]: pgmap v1919: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 83 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 24902/174489 objects degraded (14.271%); 47253/174489 objects misplaced (27.081%); 6.8 MiB/s, 215 objects/s recovering 2023-09-23T21:21:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:05 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:05 smithi123 ceph-mon[193903]: failed to encode map e633 with expected crc 2023-09-23T21:21:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:05 smithi123 ceph-mon[193903]: osdmap e633: 8 total, 8 up, 7 in 2023-09-23T21:21:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:05 smithi123 ceph-mon[193903]: failed to encode map e633 with expected crc 2023-09-23T21:21:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:05 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:21:05.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:05 smithi123 ceph-mon[193903]: pgmap v1919: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 83 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 24902/174489 objects degraded (14.271%); 47253/174489 objects misplaced (27.081%); 6.8 MiB/s, 215 objects/s recovering 2023-09-23T21:21:06.821 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:06.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:06.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:06.863 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7, 3, 5, 1] out_osds: [0] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:21:06.863 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:21:06.925 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:07.006 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:07.122 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:07.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:07.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 24902/174489 objects degraded (14.271%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:07.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 24902/174489 objects degraded (14.271%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:07.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 24902/174489 objects degraded (14.271%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:07.230 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:07.378 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:07.924 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:07.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:07.956 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:08.015 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:21:08.048 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:08.058 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:21:08.058 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:21:08.148 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:08.232 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:07 smithi123 ceph-mon[193903]: pgmap v1920: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 24902/173646 objects degraded (14.341%); 46841/173646 objects misplaced (26.975%); 5.1 MiB/s, 169 objects/s recovering 2023-09-23T21:21:08.296 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:07 smithi121 ceph-mon[219709]: pgmap v1920: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 24902/173646 objects degraded (14.341%); 46841/173646 objects misplaced (26.975%); 5.1 MiB/s, 169 objects/s recovering 2023-09-23T21:21:08.296 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:07 smithi121 ceph-mon[211526]: pgmap v1920: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 24902/173646 objects degraded (14.341%); 46841/173646 objects misplaced (26.975%); 5.1 MiB/s, 169 objects/s recovering 2023-09-23T21:21:08.327 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:08.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:08.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:08.446 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:08.583 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:08.698 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:08.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:08.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:08.810 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:08.844 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:09.145 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/288798594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:09.145 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/288798594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:09.242 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:21:09.243 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:21:09.243 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:21:09.243 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:21:09.243 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:21:09.243 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:21:09.243 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:21:09.259 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:21:09.261 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.261 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.261 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.262 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:21:09.424 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:21:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/288798594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:09.493 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:09.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:09.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:09.599 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:09.636 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:09.892 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:09.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:09.996 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:10.023 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:10.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:10 smithi121 ceph-mon[211526]: pgmap v1921: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 24418/167373 objects degraded (14.589%); 44927/167373 objects misplaced (26.842%); 7.7 MiB/s, 227 objects/s recovering 2023-09-23T21:21:10.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:10 smithi121 ceph-mon[219709]: pgmap v1921: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 24418/167373 objects degraded (14.589%); 44927/167373 objects misplaced (26.842%); 7.7 MiB/s, 227 objects/s recovering 2023-09-23T21:21:10.267 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:10 smithi123 ceph-mon[193903]: pgmap v1921: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 24418/167373 objects degraded (14.589%); 44927/167373 objects misplaced (26.842%); 7.7 MiB/s, 227 objects/s recovering 2023-09-23T21:21:10.591 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:10.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:10.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:10.696 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:10.798 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:10.888 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:10.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:10.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:11.000 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:11.035 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:11.220 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:11.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:11.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:11.324 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:11.429 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:11.532 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:12.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:12 smithi123 ceph-mon[193903]: pgmap v1922: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 24418/167373 objects degraded (14.589%); 44927/167373 objects misplaced (26.842%); 6.8 MiB/s, 201 objects/s recovering 2023-09-23T21:21:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 24418/167373 objects degraded (14.589%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4085646587' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:21:12.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:12 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:21:12.891 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[219709]: pgmap v1922: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 24418/167373 objects degraded (14.589%); 44927/167373 objects misplaced (26.842%); 6.8 MiB/s, 201 objects/s recovering 2023-09-23T21:21:12.892 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 24418/167373 objects degraded (14.589%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:12.892 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4085646587' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:21:12.892 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:21:12.893 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[211526]: pgmap v1922: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 24418/167373 objects degraded (14.589%); 44927/167373 objects misplaced (26.842%); 6.8 MiB/s, 201 objects/s recovering 2023-09-23T21:21:12.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 24418/167373 objects degraded (14.589%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:12.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4085646587' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:21:12.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:12 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:21:12.951 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:21:13.272 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:21:13.466 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:13.467 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:21:13.467 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[211526]: osdmap e634: 8 total, 8 up, 7 in 2023-09-23T21:21:13.467 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[211526]: failed to encode map e634 with expected crc 2023-09-23T21:21:13.467 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[211526]: failed to encode map e634 with expected crc 2023-09-23T21:21:13.467 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[211526]: pgmap v1924: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 24418/166539 objects degraded (14.662%); 44473/166539 objects misplaced (26.704%); 5.8 MiB/s, 125 objects/s recovering 2023-09-23T21:21:13.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:13.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:21:13.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[219709]: osdmap e634: 8 total, 8 up, 7 in 2023-09-23T21:21:13.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[219709]: failed to encode map e634 with expected crc 2023-09-23T21:21:13.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[219709]: failed to encode map e634 with expected crc 2023-09-23T21:21:13.810 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:13 smithi121 ceph-mon[219709]: pgmap v1924: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 24418/166539 objects degraded (14.662%); 44473/166539 objects misplaced (26.704%); 5.8 MiB/s, 125 objects/s recovering 2023-09-23T21:21:13.831 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:21:13.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:13.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:13 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:21:13.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:13 smithi123 ceph-mon[193903]: osdmap e634: 8 total, 8 up, 7 in 2023-09-23T21:21:13.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:13 smithi123 ceph-mon[193903]: failed to encode map e634 with expected crc 2023-09-23T21:21:13.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:13 smithi123 ceph-mon[193903]: failed to encode map e634 with expected crc 2023-09-23T21:21:13.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:13 smithi123 ceph-mon[193903]: pgmap v1924: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 24418/166539 objects degraded (14.662%); 44473/166539 objects misplaced (26.704%); 5.8 MiB/s, 125 objects/s recovering 2023-09-23T21:21:14.510 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:14.601 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:14.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:14 smithi123 ceph-mon[193903]: failed to encode map e634 with expected crc 2023-09-23T21:21:14.779 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:14 smithi121 ceph-mon[211526]: failed to encode map e634 with expected crc 2023-09-23T21:21:14.779 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:21:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:21:14] "GET /metrics HTTP/1.1" 200 36710 "" "Prometheus/2.43.0" 2023-09-23T21:21:14.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:14 smithi121 ceph-mon[219709]: failed to encode map e634 with expected crc 2023-09-23T21:21:15.505 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:21:15.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3905989539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:15.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:15.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:15 smithi121 ceph-mon[211526]: pgmap v1925: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 23921/160764 objects degraded (14.880%); 42822/160764 objects misplaced (26.637%); 8.2 MiB/s, 190 objects/s recovering 2023-09-23T21:21:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3905989539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:15.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:15.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:15 smithi121 ceph-mon[219709]: pgmap v1925: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 23921/160764 objects degraded (14.880%); 42822/160764 objects misplaced (26.637%); 8.2 MiB/s, 190 objects/s recovering 2023-09-23T21:21:15.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3905989539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:15.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:15.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:15 smithi123 ceph-mon[193903]: pgmap v1925: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 23921/160764 objects degraded (14.880%); 42822/160764 objects misplaced (26.637%); 8.2 MiB/s, 190 objects/s recovering 2023-09-23T21:21:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:16.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:21:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: osdmap e635: 8 total, 8 up, 7 in 2023-09-23T21:21:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[211526]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:16.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:21:16.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: osdmap e635: 8 total, 8 up, 7 in 2023-09-23T21:21:16.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.917 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:16 smithi121 ceph-mon[219709]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: osdmap e635: 8 total, 8 up, 7 in 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: failed to encode map e635 with expected crc 2023-09-23T21:21:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:16 smithi123 ceph-mon[193903]: failed to encode map e635 with expected crc 2023-09-23T21:21:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T21:21:17.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 23921/160764 objects degraded (14.880%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[211526]: pgmap v1928: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 23921/159960 objects degraded (14.954%); 42364/159960 objects misplaced (26.484%); 9.1 MiB/s, 196 objects/s recovering 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 23921/160764 objects degraded (14.880%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:17.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:17 smithi121 ceph-mon[219709]: pgmap v1928: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 23921/159960 objects degraded (14.954%); 42364/159960 objects misplaced (26.484%); 9.1 MiB/s, 196 objects/s recovering 2023-09-23T21:21:17.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T21:21:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: failed to encode map e636 with expected crc 2023-09-23T21:21:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 23921/160764 objects degraded (14.880%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:17.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:17 smithi123 ceph-mon[193903]: pgmap v1928: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 85 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 23921/159960 objects degraded (14.954%); 42364/159960 objects misplaced (26.484%); 9.1 MiB/s, 196 objects/s recovering 2023-09-23T21:21:18.151 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:21:18.177 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:21:18.198 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:21:18.224 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:21:18.251 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:21:18.278 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:21:18.315 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:21:18.353 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:21:18.356 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.357 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.357 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.357 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.357 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.357 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.358 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.358 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3094503839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:18.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: failed to encode map e637 with expected crc 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3094503839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:18.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:18 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:18.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: failed to encode map e637 with expected crc 2023-09-23T21:21:19.041 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T21:21:19.041 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: failed to encode map e637 with expected crc 2023-09-23T21:21:19.041 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: failed to encode map e637 with expected crc 2023-09-23T21:21:19.042 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: failed to encode map e637 with expected crc 2023-09-23T21:21:19.042 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3094503839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:19.042 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:19.042 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:18 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:19.076 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:21:19.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:19 smithi121 ceph-mon[211526]: pgmap v1930: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 86 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 23353/154146 objects degraded (15.150%); 40673/154146 objects misplaced (26.386%); 13 MiB/s, 289 objects/s recovering 2023-09-23T21:21:19.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:19 smithi121 ceph-mon[219709]: pgmap v1930: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 86 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 23353/154146 objects degraded (15.150%); 40673/154146 objects misplaced (26.386%); 13 MiB/s, 289 objects/s recovering 2023-09-23T21:21:19.707 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:19.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:19 smithi123 ceph-mon[193903]: pgmap v1930: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 86 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 23353/154146 objects degraded (15.150%); 40673/154146 objects misplaced (26.386%); 13 MiB/s, 289 objects/s recovering 2023-09-23T21:21:20.014 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:20.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:20.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:20.132 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:20.234 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:20.347 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:20.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:20.460 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:20.494 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:20.538 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:21:20.542 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:20.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:20.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:20.646 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:20.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/967792042' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:21:20.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/967792042' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:21:20.672 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/967792042' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:21:20.675 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:20.913 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:20.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:21.026 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:21.058 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:21.111 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:21.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:21.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:21.227 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:21:21.227 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:21:21.242 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:21.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:21:21.594 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:21.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:21.676 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:21:21.698 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:21.725 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:21.774 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:21.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:21.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:21.837 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: failed to encode map e638 with expected crc 2023-09-23T21:21:21.837 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/967792042' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:21:21.837 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T21:21:21.837 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: failed to encode map e638 with expected crc 2023-09-23T21:21:21.837 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.837 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.837 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: pgmap v1932: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 86 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 23353/154146 objects degraded (15.150%); 40673/154146 objects misplaced (26.386%); 8.3 MiB/s, 158 objects/s recovering 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: failed to encode map e638 with expected crc 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/967792042' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: failed to encode map e638 with expected crc 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.839 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: pgmap v1932: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 86 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 23353/154146 objects degraded (15.150%); 40673/154146 objects misplaced (26.386%); 8.3 MiB/s, 158 objects/s recovering 2023-09-23T21:21:21.839 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:21:21.839 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:21:21.839 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:21 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.877 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:21.909 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: failed to encode map e638 with expected crc 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/967792042' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: failed to encode map e638 with expected crc 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: pgmap v1932: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 86 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 23353/154146 objects degraded (15.150%); 40673/154146 objects misplaced (26.386%); 8.3 MiB/s, 158 objects/s recovering 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:21:21.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:21 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:22.005 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:22.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:22.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:22.065 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:22.109 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:22.136 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:22.180 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:22.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:22.221 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:22.283 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:22.319 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:22.403 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:22.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:22.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:22.507 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:22.534 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:22.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.556 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.556 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.556 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.560 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:21:22.619 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:22.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:22.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:22.734 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:22.779 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:22.874 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:22.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T21:21:22.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.876 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 23353/154146 objects degraded (15.150%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1743938973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:22 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:22.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:22.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 23353/154146 objects degraded (15.150%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1743938973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: failed to encode map e639 with expected crc 2023-09-23T21:21:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 23353/154146 objects degraded (15.150%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1743938973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:22.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:22 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:22.978 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:21:22.979 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:23.051 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:23.198 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:23.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:23.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:23.322 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:23.361 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:23.498 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:23.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:23.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:23.659 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:23.824 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:23.827 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:21:23.827 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.827 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.827 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:23.827 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T21:21:23.827 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: 14.b continuing backfill to osd.1 from (582'2656,637'7613] MIN to 637'7613 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[211526]: pgmap v1935: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 84 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 23353/153429 objects degraded (15.221%); 40141/153429 objects misplaced (26.163%); 9.2 MiB/s, 172 objects/s recovering 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T21:21:23.828 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: 14.b continuing backfill to osd.1 from (582'2656,637'7613] MIN to 637'7613 2023-09-23T21:21:23.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.829 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:23 smithi121 ceph-mon[219709]: pgmap v1935: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 84 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 23353/153429 objects degraded (15.221%); 40141/153429 objects misplaced (26.163%); 9.2 MiB/s, 172 objects/s recovering 2023-09-23T21:21:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:23.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:23.857 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:23.931 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:23.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: 14.b continuing backfill to osd.1 from (582'2656,637'7613] MIN to 637'7613 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: failed to encode map e640 with expected crc 2023-09-23T21:21:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:23 smithi123 ceph-mon[193903]: pgmap v1935: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 84 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 368 op/s; 23353/153429 objects degraded (15.221%); 40141/153429 objects misplaced (26.163%); 9.2 MiB/s, 172 objects/s recovering 2023-09-23T21:21:24.050 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:24.247 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:24.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:24.367 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:24.406 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:24.522 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:24.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:24.625 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:24.673 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:24.936 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:21:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:21:24] "GET /metrics HTTP/1.1" 200 36710 "" "Prometheus/2.43.0" 2023-09-23T21:21:25.000 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:25.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:25.112 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:25.157 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:25.172 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:21:25.203 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:25 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3683555252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:25.203 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:25 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3683555252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:25.204 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:21:25.236 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:25.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:25.242 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:21:25.269 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:21:25.304 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:21:25.339 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:25.343 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:21:25.381 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:25.392 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:21:25.431 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:21:25.439 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.439 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.439 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.439 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.440 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.440 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.440 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.440 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:21:25.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:25 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3683555252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:25.716 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:25.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:25.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:25.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:25.852 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:25.897 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:25.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:25.913 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:21:26.000 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:26.045 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:26.171 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:26.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:26.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:26.211 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:21:26.292 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:26.333 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:26.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:26 smithi123 ceph-mon[193903]: pgmap v1936: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 22953/149868 objects degraded (15.315%); 39171/149868 objects misplaced (26.137%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:21:26.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2259626080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:21:26.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:26 smithi121 ceph-mon[211526]: pgmap v1936: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 22953/149868 objects degraded (15.315%); 39171/149868 objects misplaced (26.137%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:21:26.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2259626080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:21:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:26 smithi121 ceph-mon[219709]: pgmap v1936: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 22953/149868 objects degraded (15.315%); 39171/149868 objects misplaced (26.137%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T21:21:26.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2259626080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:21:26.873 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:21:26.873 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:21:26.873 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:21:26.873 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:21:26.873 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:21:26.961 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:26.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:27.068 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:27.117 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:21:27.173 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:27.189 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2259626080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.233 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.234 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 22953/149868 objects degraded (15.315%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:27.300 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:27.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:27.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:27.417 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2259626080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:27 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 22953/149868 objects degraded (15.315%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:27.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.485 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2259626080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: failed to encode map e641 with expected crc 2023-09-23T21:21:27.486 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:27 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 22953/149868 objects degraded (15.315%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:27.518 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:27.637 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:27.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:27.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:27.764 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:27.797 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:27.857 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:27.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:27.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:27.963 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:27.993 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:28.037 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:28.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:28.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:28.140 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:28.178 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:28.250 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:21:28.451 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:28.457 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:28.463 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:28 smithi121 ceph-mon[211526]: pgmap v1938: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 3.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 302 op/s; 22953/148695 objects degraded (15.436%); 35371/148695 objects misplaced (23.788%); 11 MiB/s, 170 objects/s recovering 2023-09-23T21:21:28.463 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:28.463 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:28.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:28.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:28 smithi121 ceph-mon[219709]: pgmap v1938: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 3.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 302 op/s; 22953/148695 objects degraded (15.436%); 35371/148695 objects misplaced (23.788%); 11 MiB/s, 170 objects/s recovering 2023-09-23T21:21:28.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:28.464 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:28.565 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:28.600 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:28.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:28 smithi123 ceph-mon[193903]: pgmap v1938: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 3.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 302 op/s; 22953/148695 objects degraded (15.436%); 35371/148695 objects misplaced (23.788%); 11 MiB/s, 170 objects/s recovering 2023-09-23T21:21:28.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:28.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:28.761 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:28.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:28.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:28.867 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:28.907 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:28.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:21:29.091 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:29.288 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:29.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:29.394 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:29.445 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:29 smithi121 ceph-mon[219709]: pgmap v1939: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 22430/143265 objects degraded (15.656%); 33861/143265 objects misplaced (23.635%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:21:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3908288371' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:21:29.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:29 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:21:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:29 smithi121 ceph-mon[211526]: pgmap v1939: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 22430/143265 objects degraded (15.656%); 33861/143265 objects misplaced (23.635%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:21:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3908288371' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:21:29.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:29 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:21:29.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:29 smithi123 ceph-mon[193903]: pgmap v1939: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 22430/143265 objects degraded (15.656%); 33861/143265 objects misplaced (23.635%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:21:29.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3908288371' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:21:29.732 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:29 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:21:29.891 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:29.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:29.997 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:30.080 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:30.114 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:30.225 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:30.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:30.343 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:30.423 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:30.482 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:21:30.506 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:30.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:30.610 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:30.640 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:30.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:30 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:30 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:21:30.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:30 smithi121 ceph-mon[219709]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T21:21:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:30 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:30 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:21:30.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:30 smithi121 ceph-mon[211526]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T21:21:30.685 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:30.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:30 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:30.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:30 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:21:30.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:30 smithi123 ceph-mon[193903]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T21:21:30.789 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:30.815 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:30.857 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:30.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:30.960 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:31.005 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:31.171 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:31.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:31.275 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:31.360 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:31.550 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:21:31.657 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:31.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[211526]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[211526]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[211526]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[211526]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[211526]: pgmap v1941: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 22430/143265 objects degraded (15.656%); 33861/143265 objects misplaced (23.635%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1113989976' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[219709]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[219709]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[219709]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[219709]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[219709]: pgmap v1941: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 22430/143265 objects degraded (15.656%); 33861/143265 objects misplaced (23.635%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:21:31.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1113989976' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:21:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:31.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:31.777 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:31.821 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:31.935 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:31.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:31.983 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:31 smithi123 ceph-mon[193903]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:31 smithi123 ceph-mon[193903]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:31 smithi123 ceph-mon[193903]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:31 smithi123 ceph-mon[193903]: failed to encode map e642 with expected crc 2023-09-23T21:21:31.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:31 smithi123 ceph-mon[193903]: pgmap v1941: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 22430/143265 objects degraded (15.656%); 33861/143265 objects misplaced (23.635%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T21:21:31.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1113989976' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:21:32.039 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:32.048 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:21:32.048 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:21:32.048 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:21:32.048 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:21:32.048 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.048 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29157 13.d 13:ba073a5b:::benchmark_data_smithi121_621349_object29156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.335153+0000", 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.70825867500002, 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.090469696, 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29157 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.049 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335153+0000", 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335153+0000", 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.050 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335151+0000", 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335199+0000", 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7856999999999999e-05 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335212+0000", 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2886e-05 2023-09-23T21:21:32.051 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335218+0000", 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.003e-06 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335933+0000", 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00071500999999999997 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.052 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335942+0000", 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9309999999999997e-06 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.410242+0000", 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0743003360000001 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.410356+0000", 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113804 2023-09-23T21:21:32.053 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.410462+0000", 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000105679 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415027+0000", 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045649250000000001 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.054 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425478+0000", 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010451577 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425533+0000", 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5121000000000003e-05 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425583+0000", 2023-09-23T21:21:32.055 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9027999999999999e-05 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425623+0000", 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0564e-05 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.056 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29163 13.d 13:bd0f7f4a:::benchmark_data_smithi121_621349_object29162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.704927806, 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.087340899, 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29163 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.057 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:21:32.058 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.059 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.060 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.060 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.060 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338517+0000", 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3837999999999997e-05 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338524+0000", 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.764e-06 2023-09-23T21:21:32.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338529+0000", 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.34e-06 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.340130+0000", 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016010110000000001 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.062 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.063 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.340139+0000", 2023-09-23T21:21:32.063 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4180000000000003e-06 2023-09-23T21:21:32.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.063 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.063 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.411491+0000", 2023-09-23T21:21:32.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0713520459999999 2023-09-23T21:21:32.065 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.065 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.065 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.065 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.411569+0000", 2023-09-23T21:21:32.065 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7754000000000001e-05 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.411661+0000", 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1989000000000003e-05 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415037+0000", 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033760830000000002 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.066 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425672+0000", 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01063513 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425768+0000", 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5523e-05 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.067 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425798+0000", 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0213e-05 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425825+0000", 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7376e-05 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.068 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29167 13.d 13:b8b3085c:::benchmark_data_smithi121_621349_object29166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.344226+0000", 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.69918633200001, 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0817864850000001, 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.069 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29167 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344226+0000", 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344226+0000", 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344224+0000", 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344292+0000", 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7875999999999999e-05 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.071 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344313+0000", 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1047e-05 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344323+0000", 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8160000000000005e-06 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.072 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344393+0000", 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0419999999999993e-05 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344405+0000", 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1279e-05 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.412407+0000", 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0680027459999999 2023-09-23T21:21:32.073 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.412493+0000", 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5641000000000002e-05 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.412577+0000", 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3573999999999996e-05 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.074 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416806+0000", 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042291220000000001 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425728+0000", 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0089224749999999992 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.075 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425945+0000", 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021669500000000001 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425990+0000", 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4672999999999997e-05 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426012+0000", 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2650000000000002e-05 2023-09-23T21:21:32.076 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29184 13.d 13:b1899bbe:::benchmark_data_smithi121_621349_object29183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.352357+0000", 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.69105524700001, 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.073765817, 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.077 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29184 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352357+0000", 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352357+0000", 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352355+0000", 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352398+0000", 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2236e-05 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352421+0000", 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3785999999999999e-05 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352429+0000", 2023-09-23T21:21:32.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1829999999999997e-06 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352523+0000", 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4435e-05 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352563+0000", 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0054e-05 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.413400+0000", 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0608368429999999 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.413493+0000", 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3127000000000005e-05 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.413606+0000", 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011261500000000001 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.417613+0000", 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040076959999999998 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426059+0000", 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0084452539999999993 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426080+0000", 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1024000000000001e-05 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426104+0000", 2023-09-23T21:21:32.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4549000000000002e-05 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426123+0000", 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8369999999999999e-05 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.085 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29197 13.d 13:b78b711b:::benchmark_data_smithi121_621349_object29196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.362494+0000", 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.68091845200001, 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0637530639999999, 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29197 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362494+0000", 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362494+0000", 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362491+0000", 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999971 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362525+0000", 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4663000000000003e-05 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362542+0000", 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6629999999999998e-05 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362549+0000", 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3860000000000001e-06 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362612+0000", 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3017999999999999e-05 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362624+0000", 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1355999999999999e-05 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.414206+0000", 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0515827600000001 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.414326+0000", 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119527 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.414446+0000", 2023-09-23T21:21:32.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119704 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.419652+0000", 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0052065820000000004 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426167+0000", 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065143290000000001 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426189+0000", 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2019999999999999e-05 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426213+0000", 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4711000000000001e-05 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.093 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426247+0000", 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3303000000000003e-05 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29198 13.d 13:b1f63f48:::benchmark_data_smithi121_621349_object29197:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.364267+0000", 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.67914481000003, 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.062194168, 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.094 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29198 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364267+0000", 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364267+0000", 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364266+0000", 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364319+0000", 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3294000000000002e-05 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364336+0000", 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6586e-05 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364342+0000", 2023-09-23T21:21:32.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3879999999999997e-06 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364395+0000", 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2553999999999999e-05 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364403+0000", 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6710000000000005e-06 2023-09-23T21:21:32.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415080+0000", 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0506764390000001 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415203+0000", 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122834 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415360+0000", 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015729000000000001 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.419658+0000", 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042984269999999996 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426326+0000", 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0066676870000000003 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426380+0000", 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3671999999999997e-05 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.101 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426424+0000", 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4722999999999999e-05 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426461+0000", 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6999000000000003e-05 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.102 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29204 13.d 13:bda7deaf:::benchmark_data_smithi121_621349_object29203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.366715+0000", 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.67669700699997, 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0599175940000001, 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.103 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29204 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366715+0000", 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366715+0000", 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366714+0000", 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366785+0000", 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0659000000000002e-05 2023-09-23T21:21:32.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366797+0000", 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2165e-05 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366803+0000", 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3450000000000002e-06 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368437+0000", 2023-09-23T21:21:32.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016336370000000001 2023-09-23T21:21:32.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368443+0000", 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4799999999999998e-06 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415992+0000", 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.047549072 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416070+0000", 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8032000000000001e-05 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416163+0000", 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2848000000000003e-05 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.419663+0000", 2023-09-23T21:21:32.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035002639999999999 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426506+0000", 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0068424100000000002 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426545+0000", 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9292000000000003e-05 2023-09-23T21:21:32.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426596+0000", 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0487e-05 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426633+0000", 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6959000000000003e-05 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.111 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29207 13.d 13:ba8daafe:::benchmark_data_smithi121_621349_object29206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.367965+0000", 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.67544750100001, 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.058811653, 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:32.112 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29207 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.367965+0000", 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.367965+0000", 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.367964+0000", 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368049+0000", 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5740000000000002e-05 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368061+0000", 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1823e-05 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368067+0000", 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1789999999999996e-06 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368459+0000", 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039199200000000002 2023-09-23T21:21:32.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368466+0000", 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5799999999999997e-06 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416834+0000", 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.048368228 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416945+0000", 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110939 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.417030+0000", 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4515e-05 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.420777+0000", 2023-09-23T21:21:32.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037476340000000001 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426676+0000", 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058991240000000004 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426712+0000", 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5243999999999997e-05 2023-09-23T21:21:32.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426748+0000", 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6112999999999998e-05 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426776+0000", 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8425999999999999e-05 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.119 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82776 14.9 14:9f17e60f:::benchmark_data_smithi121_684892_object82775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.046431+0000", 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 102.996980751, 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.162395522, 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.120 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82776 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046431+0000", 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046431+0000", 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046430+0000", 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046479+0000", 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9322e-05 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046491+0000", 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2165e-05 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046498+0000", 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.63e-06 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046566+0000", 2023-09-23T21:21:32.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7835000000000004e-05 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046580+0000", 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4644000000000001e-05 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.182033+0000", 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1354527400000001 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.182235+0000", 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00020247700000000001 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.185181+0000", 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029460900000000002 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.125 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.197562+0000", 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012380406 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.197634+0000", 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1888e-05 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.201505+0000", 2023-09-23T21:21:32.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038713459999999999 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.205146+0000", 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003640505 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.205201+0000", 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5662999999999999e-05 2023-09-23T21:21:32.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.208827+0000", 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036255250000000001 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82782 14.3 14:ca144c45:::benchmark_data_smithi121_684892_object82781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.075782+0000", 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 102.967629824, 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.204296958, 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82782 2023-09-23T21:21:32.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075782+0000", 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075782+0000", 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075781+0000", 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075838+0000", 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6950000000000002e-05 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075848+0000", 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9799999999999993e-06 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075857+0000", 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.388e-06 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075910+0000", 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3829000000000002e-05 2023-09-23T21:21:32.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075917+0000", 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0659999999999998e-06 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939495+0000", 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86357769500000003 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939502+0000", 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.567e-06 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.227405+0000", 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.287903112 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.227578+0000", 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000173096 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.227776+0000", 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00019853500000000001 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.238175+0000", 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010398154999999999 2023-09-23T21:21:32.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.250085+0000", 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011910732 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.259611+0000", 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095251820000000001 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.276319+0000", 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016708074999999999 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.276371+0000", 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2423999999999998e-05 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.280079+0000", 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003708154 2023-09-23T21:21:32.137 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82750 14.d 14:bde3e4ca:::benchmark_data_smithi121_684892_object82749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929044+0000", 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 102.114367915, 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1129718479999999, 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.138 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82750 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929044+0000", 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929044+0000", 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929043+0000", 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929154+0000", 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011115700000000001 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929167+0000", 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3256e-05 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929176+0000", 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5490000000000006e-06 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929348+0000", 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017200599999999999 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op must wait for map", 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929357+0000", 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2399999999999996e-06 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939839+0000", 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010482152999999999 2023-09-23T21:21:32.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939848+0000", 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1969999999999996e-06 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.022667+0000", 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0828196860000001 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.024357+0000", 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001689664 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.024620+0000", 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00026303900000000001 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040525+0000", 2023-09-23T21:21:32.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015905097999999999 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040582+0000", 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6847e-05 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040690+0000", 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107917 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041614+0000", 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00092387299999999997 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041825+0000", 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000211048 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041891+0000", 2023-09-23T21:21:32.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6196000000000001e-05 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.042016+0000", 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012504399999999999 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82756 14.d 14:bbd11e56:::benchmark_data_smithi121_684892_object82755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:21:32.147 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929186+0000", 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 102.114226179, 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1161783830000001, 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82756 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.148 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929186+0000", 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929186+0000", 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929185+0000", 2023-09-23T21:21:32.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929224+0000", 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8248999999999999e-05 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929233+0000", 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3559999999999999e-06 2023-09-23T21:21:32.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929238+0000", 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1250000000000001e-06 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929414+0000", 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017563499999999999 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:21:32.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929420+0000", 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5549999999999999e-06 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939865+0000", 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010444759 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939871+0000", 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8429999999999998e-06 2023-09-23T21:21:32.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.027891+0000", 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0880204819999999 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.028035+0000", 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143812 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:21:32.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.028207+0000", 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017190700000000001 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040556+0000", 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012348675 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040608+0000", 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2113000000000001e-05 2023-09-23T21:21:32.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040716+0000", 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010831399999999999 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041620+0000", 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00090397299999999997 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045178+0000", 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035583540000000001 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045234+0000", 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5095e-05 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045364+0000", 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013066799999999999 2023-09-23T21:21:32.156 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82763 14.d 14:b667732c:::benchmark_data_smithi121_684892_object82762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929244+0000", 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 102.114168507, 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1163038169999999, 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.157 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82763 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929244+0000", 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929244+0000", 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929243+0000", 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929306+0000", 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3299999999999994e-05 2023-09-23T21:21:32.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929314+0000", 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3209999999999999e-06 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929319+0000", 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8319999999999996e-06 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929444+0000", 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012564 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929451+0000", 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4729999999999997e-06 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939889+0000", 2023-09-23T21:21:32.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010438215000000001 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939895+0000", 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7130000000000002e-06 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.030421+0000", 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0905261260000001 2023-09-23T21:21:32.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.030565+0000", 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000144411 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.030732+0000", 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016633699999999999 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040635+0000", 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0099034659999999997 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040743+0000", 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107513 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041626+0000", 2023-09-23T21:21:32.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00088339600000000005 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.043700+0000", 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002074186 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045399+0000", 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016993799999999999 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045441+0000", 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1952000000000001e-05 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045547+0000", 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010592899999999999 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82785 14.d 14:b7c2ba0e:::benchmark_data_smithi121_684892_object82784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 102.114077831, 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1163883370000001, 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82785 2023-09-23T21:21:32.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929369+0000", 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5110000000000001e-05 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929376+0000", 2023-09-23T21:21:32.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8369999999999999e-06 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929380+0000", 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3649999999999997e-06 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929475+0000", 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5144999999999998e-05 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929482+0000", 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0609999999999998e-06 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939914+0000", 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010431874000000001 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939920+0000", 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5099999999999998e-06 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.032921+0000", 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0930011749999999 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.033068+0000", 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014687200000000001 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.033247+0000", 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017891000000000001 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040662+0000", 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0074158979999999998 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040793+0000", 2023-09-23T21:21:32.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013099699999999999 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041632+0000", 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00083883899999999997 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.043734+0000", 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021020280000000001 2023-09-23T21:21:32.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045579+0000", 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0018450109999999999 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045619+0000", 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9159000000000002e-05 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045723+0000", 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000104026 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100246 14.3 14:cc38e1d4:::benchmark_data_smithi121_684892_object14929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.435396+0000", 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.608016202000002, 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7781351010000002, 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:32.176 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100246 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435396+0000", 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435396+0000", 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435394+0000", 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435402+0000", 2023-09-23T21:21:32.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.498e-06 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435415+0000", 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.306e-05 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435421+0000", 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3099999999999997e-06 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.460157+0000", 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024736042 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.461010+0000", 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00085269499999999999 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.483883+0000", 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022873072000000001 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.483970+0000", 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6707999999999997e-05 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:21:32.181 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.484100+0000", 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013049700000000001 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.497294+0000", 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013194188000000001 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504390+0000", 2023-09-23T21:21:32.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070955189999999998 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504398+0000", 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4169999999999992e-06 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488059+0000", 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98366098099999999 2023-09-23T21:21:32.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488067+0000", 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3329999999999999e-06 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118250+0000", 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63018284499999999 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118407+0000", 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015762799999999999 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.213461+0000", 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.095054270999999996 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.213531+0000", 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9522000000000003e-05 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100261 14.a 14:53619746:::benchmark_data_smithi121_684892_object14944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.456188+0000", 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.587224487, 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2422673900000001, 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.186 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100261 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456188+0000", 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456188+0000", 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456187+0000", 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456189+0000", 2023-09-23T21:21:32.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.069e-06 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456194+0000", 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7820000000000001e-06 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456197+0000", 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2509999999999999e-06 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456237+0000", 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9221999999999998e-05 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456979+0000", 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00074260199999999998 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.481936+0000", 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024957032000000001 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.482046+0000", 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000109895 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.482209+0000", 2023-09-23T21:21:32.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000162895 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.486894+0000", 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0046854690000000003 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.526405+0000", 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.039510085 2023-09-23T21:21:32.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.526414+0000", 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6690000000000001e-06 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.698361+0000", 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.171946878 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.698455+0000", 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3826000000000006e-05 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100262 14.3 14:cdb0c55f:::benchmark_data_smithi121_684892_object14945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:21:32.194 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.587202808000001, 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.761741352, 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100262 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:21:32.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.197 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.197 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.197 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456211+0000", 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1469999999999999e-06 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456216+0000", 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9259999999999999e-06 2023-09-23T21:21:32.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456219+0000", 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2210000000000002e-06 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.461047+0000", 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048276259999999998 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.461643+0000", 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00059657600000000003 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.492320+0000", 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.030677013999999999 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.492430+0000", 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000109405 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.492588+0000", 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015804099999999999 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504428+0000", 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011839861 2023-09-23T21:21:32.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504434+0000", 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8289999999999996e-06 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488092+0000", 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98365893699999996 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488098+0000", 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2140000000000004e-06 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118481+0000", 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63038356299999998 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118622+0000", 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140362 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.217866+0000", 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.099244350999999995 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.217951+0000", 2023-09-23T21:21:32.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4611999999999996e-05 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100269 14.a 14:5ecb1e34:::benchmark_data_smithi121_684892_object14952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.473910+0000", 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.569502053999997, 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.350025773, 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.205 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100269 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473910+0000", 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.206 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473910+0000", 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473909+0000", 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473916+0000", 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9379999999999992e-06 2023-09-23T21:21:32.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473933+0000", 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6674e-05 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473941+0000", 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.537e-06 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473995+0000", 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4669000000000001e-05 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.474764+0000", 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00076869500000000001 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.495964+0000", 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021199908 2023-09-23T21:21:32.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.496032+0000", 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8529999999999996e-05 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.496123+0000", 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0557999999999996e-05 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.499899+0000", 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037764249999999999 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.533301+0000", 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.033401695000000002 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.533308+0000", 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3289999999999996e-06 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.787436+0000", 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.254127725 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.787602+0000", 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165642 2023-09-23T21:21:32.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.823870+0000", 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.036268056 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.823936+0000", 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5850000000000001e-05 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.213 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100280 14.9 14:92d6aeda:::benchmark_data_smithi121_684892_object14963:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.543047+0000", 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.500365269, 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.815507285, 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.214 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100280 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543047+0000", 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543047+0000", 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543046+0000", 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543051+0000", 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4709999999999998e-06 2023-09-23T21:21:32.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543060+0000", 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6009999999999995e-06 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543065+0000", 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9529999999999998e-06 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543115+0000", 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9855000000000003e-05 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543126+0000", 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1042e-05 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.485495+0000", 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94236904700000002 2023-09-23T21:21:32.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.485504+0000", 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0690000000000008e-06 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.077364+0000", 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.59186067399999998 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:21:32.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.078112+0000", 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00074786899999999996 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.170404+0000", 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.092291976999999997 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.170501+0000", 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6801999999999999e-05 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3,6", 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.170629+0000", 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012787700000000001 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.175959+0000", 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053303320000000001 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.221 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.175992+0000", 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2880999999999999e-05 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.184319+0000", 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083270949999999996 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.355737+0000", 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17141735899999999 2023-09-23T21:21:32.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.355801+0000", 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4055999999999998e-05 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.358554+0000", 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002753326 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.223 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100281 14.3 14:c07e95ed:::benchmark_data_smithi121_684892_object14964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.543561+0000", 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 61.499851114000002, 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.67612417, 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:21:32.224 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100281 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543561+0000", 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543561+0000", 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:32.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543560+0000", 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543565+0000", 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5770000000000004e-06 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:32.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543576+0000", 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1257000000000001e-05 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543581+0000", 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2229999999999998e-06 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543629+0000", 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8146999999999998e-05 2023-09-23T21:21:32.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543639+0000", 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5459999999999997e-06 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488369+0000", 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94473046900000002 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:32.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488377+0000", 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9200000000000004e-06 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118649+0000", 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63027127500000002 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118806+0000", 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015786700000000001 2023-09-23T21:21:32.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118980+0000", 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017401299999999999 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.160178+0000", 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.041197960999999998 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.205774+0000", 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.045596043000000003 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.208482+0000", 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027076510000000002 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.218017+0000", 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095350959999999998 2023-09-23T21:21:32.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.218079+0000", 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2071999999999998e-05 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.219685+0000", 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016058649999999999 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.232 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.233 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:32.233 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:32.233 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1113989976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.903 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 22430/143265 objects degraded (15.656%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:32.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1113989976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:21:32.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T21:21:32.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.905 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 22430/143265 objects degraded (15.656%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:32.918 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:21:32.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1113989976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:21:32.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: failed to encode map e643 with expected crc 2023-09-23T21:21:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 22430/143265 objects degraded (15.656%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:33.110 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:33.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:33.213 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:33.303 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:21:33.311 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:33.505 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:33.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:33.554 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:33 smithi123 ceph-mon[193903]: pgmap v1943: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 22430/142062 objects degraded (15.789%); 33376/142062 objects misplaced (23.494%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T21:21:33.615 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:21:33.616 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:33.619 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:33.645 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:21:33.673 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:21:33.699 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:21:33.740 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:33.747 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:21:33.782 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:21:33.821 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:21:33.851 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:33 smithi121 ceph-mon[219709]: pgmap v1943: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 22430/142062 objects degraded (15.789%); 33376/142062 objects misplaced (23.494%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T21:21:33.852 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:33 smithi121 ceph-mon[211526]: pgmap v1943: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 360 op/s; 22430/142062 objects degraded (15.789%); 33376/142062 objects misplaced (23.494%); 9.1 MiB/s, 145 objects/s recovering 2023-09-23T21:21:33.879 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.880 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:21:33.889 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:33.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:34.006 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:34.049 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:34.090 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:34.210 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:34.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:34.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:34.315 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:34.474 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:34.519 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:34.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:34.627 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:34.709 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:34 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2894981338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:34.711 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:34.889 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:34 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2894981338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:34.889 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:34 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2894981338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:34.890 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:21:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:21:34] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T21:21:34.920 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:34.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:34.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:35.028 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:35.105 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:35.228 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:35.236 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:21:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:35.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:35.342 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:35.372 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:35.428 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:35.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:35.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:35.546 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:35.562 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:35 smithi121 ceph-mon[211526]: pgmap v1944: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 21870/136260 objects degraded (16.050%); 31681/136260 objects misplaced (23.250%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:21:35.569 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:35 smithi121 ceph-mon[219709]: pgmap v1944: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 21870/136260 objects degraded (16.050%); 31681/136260 objects misplaced (23.250%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:21:35.588 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:35.854 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:35.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:35.915 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:35 smithi123 ceph-mon[193903]: pgmap v1944: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 21870/136260 objects degraded (16.050%); 31681/136260 objects misplaced (23.250%); 10 MiB/s, 159 objects/s recovering 2023-09-23T21:21:35.925 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:35.961 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:36.004 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:36.097 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:36.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:36.200 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:36.306 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:36.472 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:36.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:36.566 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3848765910' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:21:36.578 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:36.596 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:21:36.627 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:36.677 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:36.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:36.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:36.720 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:21:36.720 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:21:36.781 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:36.805 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3848765910' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:21:36.805 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3848765910' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:21:36.807 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:36.849 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:36.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:36.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:36.952 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:36.994 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:37.444 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:21:37.466 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:37.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:37.571 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:37.636 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3848765910' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 21870/136260 objects degraded (16.050%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[211526]: pgmap v1946: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 21870/134967 objects degraded (16.204%); 31201/134967 objects misplaced (23.118%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3848765910' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 21870/136260 objects degraded (16.050%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:37.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:37 smithi121 ceph-mon[219709]: pgmap v1946: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 21870/134967 objects degraded (16.204%); 31201/134967 objects misplaced (23.118%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T21:21:37.780 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:37.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:37.820 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T21:21:37.902 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:37.960 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3848765910' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: failed to encode map e644 with expected crc 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 21870/136260 objects degraded (16.050%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:37.961 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:37 smithi123 ceph-mon[193903]: pgmap v1946: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 21870/134967 objects degraded (16.204%); 31201/134967 objects misplaced (23.118%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T21:21:37.989 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:38.026 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:21:38.148 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:38.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:38.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:38.277 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:38.349 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:38.397 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:38.435 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:38.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:38.547 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:38.596 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:38.703 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:38.763 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:38.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:38.868 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:38.912 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:38.914 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:38.970 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:21:39.152 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:39.155 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3590473673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:39.155 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3590473673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:39.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:39.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3590473673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:39.257 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:39.291 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3590473673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T21:21:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:39 smithi123 ceph-mon[193903]: pgmap v1948: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21263/129078 objects degraded (16.473%); 29565/129078 objects misplaced (22.905%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:21:40.312 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:40.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:40.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3590473673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:40.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T21:21:40.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[211526]: pgmap v1948: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21263/129078 objects degraded (16.473%); 29565/129078 objects misplaced (22.905%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:21:40.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:40.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3590473673' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:40.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T21:21:40.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: failed to encode map e645 with expected crc 2023-09-23T21:21:40.324 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:39 smithi121 ceph-mon[219709]: pgmap v1948: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21263/129078 objects degraded (16.473%); 29565/129078 objects misplaced (22.905%); 11 MiB/s, 171 objects/s recovering 2023-09-23T21:21:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:40.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:40.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:40.481 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:40.606 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:40.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:40.710 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:40.802 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:40.932 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:40.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:40.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:41.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:41.133 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:41.193 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:21:41.227 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:21:41.298 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:41.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:41.410 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:41.433 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:21:41.433 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:21:41.434 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:21:41.434 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:21:41.434 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:21:41.436 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3918519466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:41.436 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3918519466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:41.439 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:41.446 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:21:41.446 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:21:41.446 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:21:41.446 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:21:41.446 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.447 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:21:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3918519466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:41.505 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:41.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:41.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:41.612 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:41.655 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:42 smithi123 ceph-mon[193903]: pgmap v1949: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 21263/129078 objects degraded (16.473%); 29565/129078 objects misplaced (22.905%); 11 MiB/s, 168 objects/s recovering 2023-09-23T21:21:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 21263/129078 objects degraded (16.473%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:42.481 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:42.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:42.487 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:42 smithi121 ceph-mon[211526]: pgmap v1949: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 21263/129078 objects degraded (16.473%); 29565/129078 objects misplaced (22.905%); 11 MiB/s, 168 objects/s recovering 2023-09-23T21:21:42.487 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 21263/129078 objects degraded (16.473%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:42 smithi121 ceph-mon[219709]: pgmap v1949: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 21263/129078 objects degraded (16.473%); 29565/129078 objects misplaced (22.905%); 11 MiB/s, 168 objects/s recovering 2023-09-23T21:21:42.487 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 21263/129078 objects degraded (16.473%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:42.587 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:42.623 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:21:42.627 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:42.673 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T21:21:42.695 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:42.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:42.800 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:42.921 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:43.058 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:43.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:43.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:43.177 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:43.199 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:43.310 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:43.425 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:43.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:43.466 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:43.500 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:21:43.542 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:43.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:43.663 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:43.663 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:43 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3245646625' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:21:43.663 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:43 smithi123 ceph-mon[193903]: pgmap v1950: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 351 op/s; 21263/127827 objects degraded (16.634%); 29050/127827 objects misplaced (22.726%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:21:43.671 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:43.847 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:43.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3245646625' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:21:43.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[211526]: pgmap v1950: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 351 op/s; 21263/127827 objects degraded (16.634%); 29050/127827 objects misplaced (22.726%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:21:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3245646625' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:21:43.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:43 smithi121 ceph-mon[219709]: pgmap v1950: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 351 op/s; 21263/127827 objects degraded (16.634%); 29050/127827 objects misplaced (22.726%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T21:21:43.964 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:44.033 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:44.350 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:44.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:44.468 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:44.492 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:21:44.550 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:44.658 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:44.667 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3245646625' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:21:44.667 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:44 smithi123 ceph-mon[193903]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T21:21:44.667 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:44 smithi123 ceph-mon[193903]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.668 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:44 smithi123 ceph-mon[193903]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.668 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:44 smithi123 ceph-mon[193903]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.668 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:44 smithi123 ceph-mon[193903]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.668 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:44 smithi123 ceph-mon[193903]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:44.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:44.769 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3245646625' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:21:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[211526]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T21:21:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[211526]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[211526]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[211526]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[211526]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[211526]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3245646625' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:21:44.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[219709]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T21:21:44.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[219709]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[219709]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[219709]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[219709]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.779 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:44 smithi121 ceph-mon[219709]: failed to encode map e646 with expected crc 2023-09-23T21:21:44.857 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:44.940 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:44.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:45.043 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:45.072 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:21:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:21:44] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T21:21:45.102 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:21:45.116 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:45.517 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:45.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:45.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:45.604 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:45.624 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:45.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:45 smithi121 ceph-mon[211526]: pgmap v1952: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 20667/122214 objects degraded (16.911%); 27469/122214 objects misplaced (22.476%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:21:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:45 smithi121 ceph-mon[219709]: pgmap v1952: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 20667/122214 objects degraded (16.911%); 27469/122214 objects misplaced (22.476%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:21:45.682 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:45.755 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:45.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:45.861 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:45.912 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:45.935 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:45.956 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:45 smithi123 ceph-mon[193903]: pgmap v1952: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 20667/122214 objects degraded (16.911%); 27469/122214 objects misplaced (22.476%); 11 MiB/s, 179 objects/s recovering 2023-09-23T21:21:46.065 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:46.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:46.169 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:46.258 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:46.316 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:43997 13.c 13:3ce0f102:::benchmark_data_smithi121_621349_object43996:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e500)", 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.347985+0000", 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.96447151299998, 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.039016079, 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.317 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 43997 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.347985+0000", 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.347985+0000", 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.347984+0000", 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.348013+0000", 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.919e-05 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.348023+0000", 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0148e-05 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.348030+0000", 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0110000000000003e-06 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.348144+0000", 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114193 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.348282+0000", 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013795600000000001 2023-09-23T21:21:46.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.348445+0000", 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016265499999999999 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.350478+0000", 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002033244 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.322 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361866+0000", 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011387598000000001 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375071+0000", 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013205000999999999 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375083+0000", 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2418e-05 2023-09-23T21:21:46.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.386931+0000", 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.011848007 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.387001+0000", 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9708000000000004e-05 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.324 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44005 13.c 13:36791aaa:::benchmark_data_smithi121_621349_object44004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e500)", 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.355429+0000", 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.957027031, 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0318314319999999, 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.325 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44005 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355429+0000", 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355429+0000", 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355428+0000", 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355470+0000", 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1900000000000002e-05 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355481+0000", 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0927000000000001e-05 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355486+0000", 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0560000000000004e-06 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355520+0000", 2023-09-23T21:21:46.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3964000000000002e-05 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355627+0000", 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000106767 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.355731+0000", 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010416099999999999 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.329 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358691+0000", 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029604430000000001 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.368847+0000", 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010155733 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375540+0000", 2023-09-23T21:21:46.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0066932149999999998 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375547+0000", 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5289999999999997e-06 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.387211+0000", 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.011663829 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.387261+0000", 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0189999999999999e-05 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44011 13.c 13:3e3490c6:::benchmark_data_smithi121_621349_object44010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e500)", 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.358859+0000", 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.953597823, 2023-09-23T21:21:46.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0338803320000001, 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44011 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358859+0000", 2023-09-23T21:21:46.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358859+0000", 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358857+0000", 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358892+0000", 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4598000000000003e-05 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358904+0000", 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2339e-05 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358911+0000", 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4309999999999999e-06 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.358967+0000", 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6088000000000002e-05 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.359120+0000", 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015343799999999999 2023-09-23T21:21:46.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.359296+0000", 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017560699999999999 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.367199+0000", 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079033799999999998 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.337 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375594+0000", 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083953020000000003 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375602+0000", 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7570000000000004e-06 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.387303+0000", 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0117007039999999 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.387839+0000", 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00053653900000000005 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.392670+0000", 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048309770000000002 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.392739+0000", 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8485999999999994e-05 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44012 13.c 13:3f873033:::benchmark_data_smithi121_621349_object44011:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e500)", 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.361617+0000", 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.95083894700002, 2023-09-23T21:21:46.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0364750730000001, 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44012 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361617+0000", 2023-09-23T21:21:46.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361617+0000", 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361616+0000", 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361663+0000", 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6548000000000002e-05 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361673+0000", 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0659000000000001e-05 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361680+0000", 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2009999999999996e-06 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.361889+0000", 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00020936500000000001 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.362021+0000", 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013195399999999999 2023-09-23T21:21:46.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.362171+0000", 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000149612 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.367207+0000", 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050365189999999997 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375632+0000", 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0084246939999999999 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.375638+0000", 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6420000000000001e-06 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.387883+0000", 2023-09-23T21:21:46.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.012244616 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.392335+0000", 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044518309999999998 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.397999+0000", 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0056641069999999998 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.398093+0000", 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3522000000000003e-05 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44022 13.c 13:3d397a9b:::benchmark_data_smithi121_621349_object44021:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.473839+0000", 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.83861703999997, 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93147686500000004, 2023-09-23T21:21:46.348 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44022 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.473839+0000", 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.473839+0000", 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.473838+0000", 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.473919+0000", 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0939999999999994e-05 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.473935+0000", 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5548000000000001e-05 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.473940+0000", 2023-09-23T21:21:46.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6520000000000003e-06 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.473993+0000", 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2791999999999999e-05 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.474005+0000", 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1874e-05 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.392767+0000", 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.91876213699999998 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.392895+0000", 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012730700000000001 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.393005+0000", 2023-09-23T21:21:46.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011014900000000001 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.399971+0000", 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0069667740000000002 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.401868+0000", 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00189672 2023-09-23T21:21:46.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405153+0000", 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0032852290000000002 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405214+0000", 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.071e-05 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405316+0000", 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010208399999999999 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44047 13.c 13:35354d71:::benchmark_data_smithi121_621349_object44046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.498299+0000", 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.81415690199998, 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90717514600000004, 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.356 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44047 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.498299+0000", 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.498299+0000", 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.498298+0000", 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.498329+0000", 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1194e-05 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.498345+0000", 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6541e-05 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.498352+0000", 2023-09-23T21:21:46.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3269999999999997e-06 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.499237+0000", 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00088500500000000001 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.499245+0000", 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7390000000000005e-06 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.360 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.394316+0000", 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89507084000000003 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.394436+0000", 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120296 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.394586+0000", 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014913900000000001 2023-09-23T21:21:46.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.399979+0000", 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053937300000000002 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.401919+0000", 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001939648 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405353+0000", 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034341200000000001 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405391+0000", 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8154000000000001e-05 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405475+0000", 2023-09-23T21:21:46.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3368999999999994e-05 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44067 13.c 13:3d1f593f:::benchmark_data_smithi121_621349_object44066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.517699+0000", 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.79475719700002, 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88800127799999995, 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.364 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44067 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517699+0000", 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517699+0000", 2023-09-23T21:21:46.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517698+0000", 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517738+0000", 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9922000000000002e-05 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517754+0000", 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6767000000000001e-05 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517762+0000", 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3189999999999997e-06 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517826+0000", 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4300000000000004e-05 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.517837+0000", 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1408e-05 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.395847+0000", 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.87800937700000004 2023-09-23T21:21:46.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.395968+0000", 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012139 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.396118+0000", 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014944399999999999 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.369 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.399986+0000", 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038686469999999998 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405526+0000", 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0055397010000000002 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405572+0000", 2023-09-23T21:21:46.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5748999999999999e-05 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405617+0000", 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5429999999999997e-05 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405700+0000", 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3376999999999998e-05 2023-09-23T21:21:46.371 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44086 13.c 13:387654b8:::benchmark_data_smithi121_621349_object44085:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.558836+0000", 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.75362031999998, 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85175557999999996, 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.372 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44086 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558836+0000", 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558836+0000", 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558835+0000", 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558864+0000", 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8895999999999999e-05 2023-09-23T21:21:46.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558876+0000", 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2346e-05 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558883+0000", 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.601e-06 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558941+0000", 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8050999999999997e-05 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.558950+0000", 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0690000000000008e-06 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.398120+0000", 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83917014999999995 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.398242+0000", 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012208499999999999 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.398427+0000", 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00018484099999999999 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.401927+0000", 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035002470000000002 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405728+0000", 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038013880000000002 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410472+0000", 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0047432309999999997 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410503+0000", 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1526000000000003e-05 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410592+0000", 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8424999999999994e-05 2023-09-23T21:21:46.379 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44110 13.c 13:3a3dfd5b:::benchmark_data_smithi121_621349_object44109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.579756+0000", 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.73270080200001, 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83096674999999998, 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.380 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44110 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579756+0000", 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579756+0000", 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579755+0000", 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579788+0000", 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2941999999999997e-05 2023-09-23T21:21:46.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579805+0000", 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7748999999999999e-05 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579812+0000", 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8009999999999998e-06 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579873+0000", 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1126999999999999e-05 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.579882+0000", 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8380000000000008e-06 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.400028+0000", 2023-09-23T21:21:46.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82014556199999999 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.400142+0000", 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113952 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.400291+0000", 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014975200000000001 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405537+0000", 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0052452150000000001 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405759+0000", 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000222446 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410632+0000", 2023-09-23T21:21:46.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048727370000000002 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410673+0000", 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0856999999999999e-05 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410722+0000", 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9665e-05 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.387 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44117 13.c 13:30536b5b:::benchmark_data_smithi121_621349_object44116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.584681+0000", 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.72777490800001, 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82614510500000005, 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44117 2023-09-23T21:21:46.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584681+0000", 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584681+0000", 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584680+0000", 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584735+0000", 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4679000000000003e-05 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584750+0000", 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4752e-05 2023-09-23T21:21:46.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584756+0000", 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9920000000000001e-06 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584814+0000", 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7326e-05 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.584824+0000", 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0002e-05 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.401951+0000", 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.81712745399999998 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.402077+0000", 2023-09-23T21:21:46.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000125246 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.402212+0000", 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013586699999999999 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405707+0000", 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034947630000000001 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.409006+0000", 2023-09-23T21:21:46.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0032984910000000002 2023-09-23T21:21:46.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410757+0000", 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00175117 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410781+0000", 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4552000000000001e-05 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410827+0000", 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5215e-05 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44127 13.c 13:3553d955:::benchmark_data_smithi121_621349_object44126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.638225+0000", 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.67423132699997, 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77268782599999997, 2023-09-23T21:21:46.396 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44127 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638225+0000", 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638225+0000", 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638224+0000", 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638302+0000", 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7791000000000001e-05 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638320+0000", 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8451e-05 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638326+0000", 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3099999999999997e-06 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638384+0000", 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7936000000000002e-05 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638391+0000", 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7259999999999996e-06 2023-09-23T21:21:46.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.403535+0000", 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76514390300000001 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.403662+0000", 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000127489 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.403785+0000", 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122244 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.407487+0000", 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037020439999999998 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.409032+0000", 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0015455600000000001 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410844+0000", 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0018121509999999999 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410868+0000", 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3762000000000001e-05 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410913+0000", 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4662e-05 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44129 13.c 13:3c979a86:::benchmark_data_smithi121_621349_object44128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.638618+0000", 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.67383832199999, 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.77693406700000001, 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44129 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.405 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638618+0000", 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638618+0000", 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638617+0000", 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638660+0000", 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2654999999999997e-05 2023-09-23T21:21:46.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638674+0000", 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4293e-05 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638680+0000", 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5210000000000002e-06 2023-09-23T21:21:46.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638726+0000", 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5853000000000003e-05 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.638736+0000", 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0862e-05 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405781+0000", 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76704440500000004 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.405913+0000", 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013258599999999999 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.406041+0000", 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012712300000000001 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410615+0000", 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004574285 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.413921+0000", 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033062489999999998 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.415412+0000", 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014906469999999999 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.415452+0000", 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0602000000000002e-05 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.415552+0000", 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9835999999999998e-05 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.413 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44149 13.c 13:328d061b:::benchmark_data_smithi121_621349_object44148:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.656463+0000", 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.655993857, 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75931500200000002, 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44149 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656463+0000", 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656463+0000", 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656461+0000", 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656513+0000", 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1913999999999998e-05 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656529+0000", 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5758000000000001e-05 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656535+0000", 2023-09-23T21:21:46.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.443e-06 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656597+0000", 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1909000000000006e-05 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.656609+0000", 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1904e-05 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.407532+0000", 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75092327199999998 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.407653+0000", 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012016899999999999 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.407790+0000", 2023-09-23T21:21:46.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000137844 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410741+0000", 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029507769999999999 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.413933+0000", 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031917920000000002 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.415630+0000", 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016967709999999999 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.415677+0000", 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7018000000000003e-05 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.415778+0000", 2023-09-23T21:21:46.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100702 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44150 13.c 13:30c3947d:::benchmark_data_smithi121_621349_object44149:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.657484+0000", 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.654972499, 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.76311543999999998, 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.421 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44150 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657484+0000", 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657484+0000", 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657483+0000", 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657545+0000", 2023-09-23T21:21:46.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1540999999999999e-05 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657556+0000", 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1646e-05 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657562+0000", 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1789999999999996e-06 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657610+0000", 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7741999999999998e-05 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.657618+0000", 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2190000000000004e-06 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.409049+0000", 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.75143014100000005 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.409168+0000", 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011995299999999999 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.409320+0000", 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000151316 2023-09-23T21:21:46.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.413944+0000", 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0046243559999999996 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.418430+0000", 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044862319999999997 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.420456+0000", 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020255189999999999 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.420505+0000", 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9032000000000001e-05 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.420599+0000", 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4388000000000005e-05 2023-09-23T21:21:46.428 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:44194 13.c 13:3417833b:::benchmark_data_smithi121_621349_object44193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:50.723221+0000", 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 415.58923567099998, 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.69757941700000003, 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.429 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 44194 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723221+0000", 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723221+0000", 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723220+0000", 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723275+0000", 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5897999999999999e-05 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723288+0000", 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2452e-05 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.432 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723295+0000", 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7390000000000002e-06 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723349+0000", 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3968000000000002e-05 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:50.723357+0000", 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7399999999999993e-06 2023-09-23T21:21:46.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410926+0000", 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.68756892800000002 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.410998+0000", 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1664999999999999e-05 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,7", 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.411080+0000", 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1965999999999994e-05 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.414059+0000", 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029790210000000001 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.420279+0000", 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0062199330000000004 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.420656+0000", 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00037730800000000003 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.420706+0000", 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9801e-05 2023-09-23T21:21:46.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:51.420800+0000", 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4152000000000003e-05 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:123842 13.16 13:6d7b9520:::benchmark_data_smithi121_621349_object31348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:16:35.472958+0000", 2023-09-23T21:21:46.437 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 310.83949884600003, 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.037113918, 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 123842 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.472958+0000", 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.472958+0000", 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.472956+0000", 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.472963+0000", 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1189999999999999e-06 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.472976+0000", 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2765000000000001e-05 2023-09-23T21:21:46.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.472983+0000", 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5649999999999998e-06 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.473039+0000", 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6348000000000003e-05 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.473184+0000", 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014522999999999999 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.473628+0000", 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000443931 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.481901+0000", 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0082732629999999995 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.491437+0000", 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095354629999999992 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.491451+0000", 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4394e-05 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.509995+0000", 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0185440219999999 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.510071+0000", 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6178000000000001e-05 2023-09-23T21:21:46.444 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:123876 13.16 13:69c2de88:::benchmark_data_smithi121_621349_object31382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:16:35.528056+0000", 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 310.78440084800002, 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99117372000000004, 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.445 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 123876 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528056+0000", 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528056+0000", 2023-09-23T21:21:46.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528054+0000", 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528061+0000", 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6080000000000001e-06 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528073+0000", 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.273e-05 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528080+0000", 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2600000000000002e-06 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528150+0000", 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0443000000000003e-05 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.528161+0000", 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.101e-05 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.510425+0000", 2023-09-23T21:21:46.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98226343400000005 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.510543+0000", 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011842500000000001 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.510953+0000", 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00040964199999999999 2023-09-23T21:21:46.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.517076+0000", 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061229479999999996 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.519114+0000", 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002038832 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.519152+0000", 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7138999999999998e-05 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.519190+0000", 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8560999999999999e-05 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.519229+0000", 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9076000000000003e-05 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:123908 13.16 13:6d9a570d:::benchmark_data_smithi121_621349_object31414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:16:35.570166+0000", 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 310.74229067599998, 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95424609800000004, 2023-09-23T21:21:46.453 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 123908 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.570166+0000", 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.570166+0000", 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.570165+0000", 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.570172+0000", 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9410000000000001e-06 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.570181+0000", 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4979999999999995e-06 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.570186+0000", 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0030000000000002e-06 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.572848+0000", 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0026620820000000001 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.572858+0000", 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0266e-05 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.512904+0000", 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94004559700000001 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.513007+0000", 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010275799999999999 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:21:46.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.513351+0000", 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00034460600000000001 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.519080+0000", 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0057283569999999999 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.524266+0000", 2023-09-23T21:21:46.459 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0051862820000000004 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.524331+0000", 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4641999999999995e-05 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.524371+0000", 2023-09-23T21:21:46.460 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0049000000000002e-05 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.524412+0000", 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1068000000000001e-05 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:123921 13.16 13:6f858597:::benchmark_data_smithi121_621349_object31427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-09-23T21:21:46.461 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:16:35.589482+0000", 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 310.72297435299998, 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94316944700000005, 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 123921 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.462 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589482+0000", 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589482+0000", 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589481+0000", 2023-09-23T21:21:46.463 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589490+0000", 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8119999999999997e-06 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589499+0000", 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3640000000000005e-06 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589504+0000", 2023-09-23T21:21:46.464 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0459999999999996e-06 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589592+0000", 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7865e-05 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589608+0000", 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6101999999999999e-05 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.465 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.515039+0000", 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92543056099999998 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.515128+0000", 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9234000000000007e-05 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.515429+0000", 2023-09-23T21:21:46.466 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00030076799999999998 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.519120+0000", 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036915049999999999 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.525957+0000", 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0068369829999999996 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.467 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.532554+0000", 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065967359999999997 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.532610+0000", 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6039999999999999e-05 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.532651+0000", 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1520999999999998e-05 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.468 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:123923 13.16 13:69513cca:::benchmark_data_smithi121_621349_object31429:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:16:35.589661+0000", 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 310.72279542000001, 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94313882199999999, 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:21:46.469 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 123923 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589661+0000", 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589661+0000", 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:21:46.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589660+0000", 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589664+0000", 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.782e-06 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:21:46.471 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589674+0000", 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4909999999999994e-06 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589679+0000", 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3449999999999996e-06 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589714+0000", 2023-09-23T21:21:46.472 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4683e-05 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:35.589719+0000", 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0180000000000001e-06 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.517104+0000", 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92738587900000002 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.473 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.517171+0000", 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6767999999999995e-05 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,4", 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.517395+0000", 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022371500000000001 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:21:46.474 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.522421+0000", 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050258710000000003 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.532692+0000", 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010271583000000001 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.532723+0000", 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0244e-05 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.475 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.532764+0000", 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1458999999999997e-05 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:16:36.532800+0000", 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5672e-05 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.476 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.477 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:21:46.477 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:21:46.477 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:21:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:46.478 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:46.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:46.521 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:46 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3792414219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:46.532 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:21:46.579 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:46.691 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:46.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:46.822 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:46.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:46.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:46 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3792414219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:46 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3792414219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:21:46.923 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:46.964 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:47.572 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:21:47.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[211526]: failed to encode map e647 with expected crc 2023-09-23T21:21:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[211526]: failed to encode map e647 with expected crc 2023-09-23T21:21:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3792414219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:21:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[211526]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T21:21:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 20667/122214 objects degraded (16.911%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:47.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[211526]: pgmap v1954: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 20667/120864 objects degraded (17.099%); 26991/120864 objects misplaced (22.332%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:21:47.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[219709]: failed to encode map e647 with expected crc 2023-09-23T21:21:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[219709]: failed to encode map e647 with expected crc 2023-09-23T21:21:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3792414219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:21:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[219709]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T21:21:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 20667/122214 objects degraded (16.911%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:47.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:47 smithi121 ceph-mon[219709]: pgmap v1954: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 20667/120864 objects degraded (17.099%); 26991/120864 objects misplaced (22.332%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:21:47.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:47 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:21:47.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:47 smithi123 ceph-mon[193903]: failed to encode map e647 with expected crc 2023-09-23T21:21:47.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:47 smithi123 ceph-mon[193903]: failed to encode map e647 with expected crc 2023-09-23T21:21:47.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3792414219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:21:47.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:47 smithi123 ceph-mon[193903]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T21:21:47.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:47 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 20667/122214 objects degraded (16.911%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:21:47.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:47 smithi123 ceph-mon[193903]: pgmap v1954: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 342 op/s; 20667/120864 objects degraded (17.099%); 26991/120864 objects misplaced (22.332%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T21:21:48.217 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:48.633 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:48.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:48.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:48.744 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:48.876 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:49.149 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:49.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:49.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:49.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:49.318 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:49.430 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:49.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:49.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:49.437 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:21:49.475 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:21:49.498 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:21:49.517 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:21:49.536 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:49.567 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:49.618 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:21:49.634 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:49.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:49.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:49.653 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:21:49.676 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:21:49.704 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:21:49.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.705 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.706 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:21:49.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:49 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/370775787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:49.733 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:49 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/370775787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:49.734 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:49 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/370775787' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:49.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:49.848 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:50.145 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:50.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:50.251 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:50.282 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:50.341 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:50.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:50.446 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:50.499 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:21:50.562 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:50.655 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:21:50.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:50 smithi121 ceph-mon[211526]: pgmap v1955: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 20049/114954 objects degraded (17.441%); 25343/114954 objects misplaced (22.046%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:21:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/718309633' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:50.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:50 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:50 smithi121 ceph-mon[219709]: pgmap v1955: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 20049/114954 objects degraded (17.441%); 25343/114954 objects misplaced (22.046%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:21:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/718309633' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:50.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:50 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:50.726 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:21:50.726 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:21:50.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:50 smithi123 ceph-mon[193903]: pgmap v1955: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 20049/114954 objects degraded (17.441%); 25343/114954 objects misplaced (22.046%); 13 MiB/s, 204 objects/s recovering 2023-09-23T21:21:50.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/718309633' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:50.733 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:50 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:21:50.742 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:50.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:50.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:50.864 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:50.947 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:51.043 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:51.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:51.147 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:51.251 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:51.357 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:51.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:51.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:51.442 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:21:51.443 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:21:51.475 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:51.556 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:51.634 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:51.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:51.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:51.673 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:51.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:21:51.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T21:21:51.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:51 smithi123 ceph-mon[193903]: pgmap v1957: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 20049/114954 objects degraded (17.441%); 25343/114954 objects misplaced (22.046%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:21:51.737 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:51.847 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:51.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:21:51.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[211526]: pgmap v1957: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 20049/114954 objects degraded (17.441%); 25343/114954 objects misplaced (22.046%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:21:51.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T21:21:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: failed to encode map e648 with expected crc 2023-09-23T21:21:51.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:51 smithi121 ceph-mon[219709]: pgmap v1957: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 20049/114954 objects degraded (17.441%); 25343/114954 objects misplaced (22.046%); 11 MiB/s, 176 objects/s recovering 2023-09-23T21:21:51.960 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:21:51.985 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:52.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:52.106 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:52.209 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:52.360 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:52.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:52.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:52.487 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:52.551 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:52.602 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:52.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:52.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:52.625 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:52.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 20049/114954 objects degraded (17.441%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:21:52.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 20049/114954 objects degraded (17.441%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:21:52.709 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:52.736 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:52.831 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:21:52.869 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 20049/114954 objects degraded (17.441%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:21:52.922 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:52.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:52.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:53.036 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:53.083 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:53.126 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:53.166 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:53.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:53.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:53.272 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:53.308 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:53.468 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:53.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:53.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:53.573 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:53.651 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:53.745 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:53.784 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:53.793 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:53 smithi123 ceph-mon[193903]: pgmap v1958: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 20049/113673 objects degraded (17.637%); 24810/113673 objects misplaced (21.826%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T21:21:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:53.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:53.894 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:53.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:53 smithi121 ceph-mon[211526]: pgmap v1958: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 20049/113673 objects degraded (17.637%); 24810/113673 objects misplaced (21.826%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T21:21:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:53 smithi121 ceph-mon[219709]: pgmap v1958: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 20049/113673 objects degraded (17.637%); 24810/113673 objects misplaced (21.826%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T21:21:53.924 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:54.021 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:54.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:54.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:54.125 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:54.227 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:54.564 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:21:54.568 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:54.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:54.692 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:21:54.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/470771042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:54.778 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:54 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:54.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/470771042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:54.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:54 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:54.789 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:54.976 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/470771042' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:54.976 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:54 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:21:54.977 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:21:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:54.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:21:55.033 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:21:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:21:54] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T21:21:55.093 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:21:55.143 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:55.322 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:21:55.322 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:21:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:21:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T21:21:55.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:55 smithi123 ceph-mon[193903]: pgmap v1960: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 19501/107847 objects degraded (18.082%); 23138/107847 objects misplaced (21.454%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:21:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:21:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T21:21:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[211526]: pgmap v1960: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 19501/107847 objects degraded (18.082%); 23138/107847 objects misplaced (21.454%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: failed to encode map e649 with expected crc 2023-09-23T21:21:55.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:55 smithi121 ceph-mon[219709]: pgmap v1960: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 19501/107847 objects degraded (18.082%); 23138/107847 objects misplaced (21.454%); 11 MiB/s, 178 objects/s recovering 2023-09-23T21:21:56.513 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:21:56.549 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:21:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:56.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:21:56.655 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:56.758 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:56.942 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:56.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:57.063 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:57.111 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:57.188 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:57.191 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:21:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:57.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:57.195 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:21:57.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:56 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 19501/107847 objects degraded (18.082%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:21:57.295 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:57.316 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:21:57.343 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:57.344 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:56 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 19501/107847 objects degraded (18.082%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:21:57.344 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:56 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 19501/107847 objects degraded (18.082%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:21:57.348 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:21:57.372 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:21:57.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:21:57.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:21:57.445 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:21:57.478 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:21:57.484 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:21:57.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:57.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:21:57.531 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:21:57.590 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:57.617 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:57.667 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:57.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:57.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:57.771 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:21:57.771 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.771 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.771 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.771 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.771 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.771 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.772 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.772 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:21:57.772 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:21:57.842 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:58.057 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:21:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:58.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:21:58.175 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:58.208 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:58.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:58 smithi123 ceph-mon[193903]: from='client.47813 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:21:58.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3367315179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:58.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:58 smithi123 ceph-mon[193903]: pgmap v1961: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 19501/106476 objects degraded (18.315%); 22581/106476 objects misplaced (21.208%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:21:58.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:58 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:58.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:21:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:58.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[219709]: from='client.47813 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:21:58.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3367315179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:58.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[219709]: pgmap v1961: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 19501/106476 objects degraded (18.315%); 22581/106476 objects misplaced (21.208%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:21:58.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:58.268 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:58.275 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[211526]: from='client.47813 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:21:58.275 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3367315179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:21:58.275 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[211526]: pgmap v1961: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 353 op/s; 19501/106476 objects degraded (18.315%); 22581/106476 objects misplaced (21.208%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T21:21:58.276 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:21:58.276 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:21:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:21:58.291 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1960,"stamp":"2023-09-23T21:21:55.422081+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2355018499,"num_objects":35949,"num_object_clones":0,"num_object_copies":107847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19501,"num_objects_misplaced":23138,"num_objects_unfound":0,"num_objects_dirty":35949,"num_whiteouts":0,"num_read":92,"num_read_kb":169,"num_write":134809,"num_write_kb":5460565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22460,"num_bytes_recovered":1374172517,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85145,"ondisk_log_size":85145,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15074260,"kb_used_data":8599452,"kb_used_omap":16,"kb_used_meta":6474479,"kb_avail":641141804,"statfs":{"total":671965249536,"available":656529207296,"internally_reserved":0,"allocated":8805838848,"data_stored":8788875036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6629867369},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-284360704,"num_objects":-4339,"num_object_clones":0,"num_object_copies":-13017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1166,"num_objects_misplaced":-3853,"num_objects_unfound":0,"num_objects_dirty":-4339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4339,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1430,"num_bytes_recovered":93716480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003118"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":223,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553816+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:21:46.553816+0000","last_peered":"2023-09-23T21:21:46.553816+0000","last_clean":"2023-09-23T21:21:46.553816+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:21:46.553816+0000","last_undegraded":"2023-09-23T21:21:46.553816+0000","last_fullsized":"2023-09-23T21:21:46.553816+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":60,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540748+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:21:46.540748+0000","last_peered":"2023-09-23T21:21:46.540748+0000","last_clean":"2023-09-23T21:21:46.540748+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:21:46.540748+0000","last_undegraded":"2023-09-23T21:21:46.540748+0000","last_fullsized":"2023-09-23T21:21:46.540748+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553949+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:21:46.553949+0000","last_peered":"2023-09-23T21:21:46.553949+0000","last_clean":"2023-09-23T21:21:46.553949+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:21:46.553949+0000","last_undegraded":"2023-09-23T21:21:46.553949+0000","last_fullsized":"2023-09-23T21:21:46.553949+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":335,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545881+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:21:46.545881+0000","last_peered":"2023-09-23T21:21:46.545881+0000","last_clean":"2023-09-23T21:21:46.545881+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:21:46.545881+0000","last_undegraded":"2023-09-23T21:21:46.545881+0000","last_fullsized":"2023-09-23T21:21:46.545881+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":51,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540417+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:21:46.540417+0000","last_peered":"2023-09-23T21:21:46.540417+0000","last_clean":"2023-09-23T21:21:46.540417+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:21:46.540417+0000","last_undegraded":"2023-09-23T21:21:46.540417+0000","last_fullsized":"2023-09-23T21:21:46.540417+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":153,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.533680+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:21:46.533680+0000","last_peered":"2023-09-23T21:21:46.533680+0000","last_clean":"2023-09-23T21:21:46.533680+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:21:46.533680+0000","last_undegraded":"2023-09-23T21:21:46.533680+0000","last_fullsized":"2023-09-23T21:21:46.533680+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":55,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.548022+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:21:46.548022+0000","last_peered":"2023-09-23T21:21:46.548022+0000","last_clean":"2023-09-23T21:21:46.548022+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:21:46.548022+0000","last_undegraded":"2023-09-23T21:21:46.548022+0000","last_fullsized":"2023-09-23T21:21:46.548022+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":69,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540502+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:21:46.540502+0000","last_peered":"2023-09-23T21:21:46.540502+0000","last_clean":"2023-09-23T21:21:46.540502+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:21:46.540502+0000","last_undegraded":"2023-09-23T21:21:46.540502+0000","last_fullsized":"2023-09-23T21:21:46.540502+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":45,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.543728+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:21:46.543728+0000","last_peered":"2023-09-23T21:21:46.543728+0000","last_clean":"2023-09-23T21:21:46.543728+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:21:46.543728+0000","last_undegraded":"2023-09-23T21:21:46.543728+0000","last_fullsized":"2023-09-23T21:21:46.543728+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":42,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540670+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:21:46.540670+0000","last_peered":"2023-09-23T21:21:46.540670+0000","last_clean":"2023-09-23T21:21:46.540670+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:21:46.540670+0000","last_undegraded":"2023-09-23T21:21:46.540670+0000","last_fullsized":"2023-09-23T21:21:46.540670+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":283,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.546566+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:21:46.546566+0000","last_peered":"2023-09-23T21:21:46.546566+0000","last_clean":"2023-09-23T21:21:46.546566+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:21:46.546566+0000","last_undegraded":"2023-09-23T21:21:46.546566+0000","last_fullsized":"2023-09-23T21:21:46.546566+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":39,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553630+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:21:46.553630+0000","last_peered":"2023-09-23T21:21:46.553630+0000","last_clean":"2023-09-23T21:21:46.553630+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:21:46.553630+0000","last_undegraded":"2023-09-23T21:21:46.553630+0000","last_fullsized":"2023-09-23T21:21:46.553630+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":379,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553463+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:21:46.553463+0000","last_peered":"2023-09-23T21:21:46.553463+0000","last_clean":"2023-09-23T21:21:46.553463+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:21:46.553463+0000","last_undegraded":"2023-09-23T21:21:46.553463+0000","last_fullsized":"2023-09-23T21:21:46.553463+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":606,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.550588+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:21:46.550588+0000","last_peered":"2023-09-23T21:21:46.550588+0000","last_clean":"2023-09-23T21:21:46.550588+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:21:46.550588+0000","last_undegraded":"2023-09-23T21:21:46.550588+0000","last_fullsized":"2023-09-23T21:21:46.550588+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":231,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.546222+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:21:46.546222+0000","last_peered":"2023-09-23T21:21:46.546222+0000","last_clean":"2023-09-23T21:21:46.546222+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:21:46.546222+0000","last_undegraded":"2023-09-23T21:21:46.546222+0000","last_fullsized":"2023-09-23T21:21:46.546222+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":51,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540930+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:21:46.540930+0000","last_peered":"2023-09-23T21:21:46.540930+0000","last_clean":"2023-09-23T21:21:46.540930+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:21:46.540930+0000","last_undegraded":"2023-09-23T21:21:46.540930+0000","last_fullsized":"2023-09-23T21:21:46.540930+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":257,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545837+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:21:46.545837+0000","last_peered":"2023-09-23T21:21:46.545837+0000","last_clean":"2023-09-23T21:21:46.545837+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:21:46.545837+0000","last_undegraded":"2023-09-23T21:21:46.545837+0000","last_fullsized":"2023-09-23T21:21:46.545837+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":265,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.546699+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:21:46.546699+0000","last_peered":"2023-09-23T21:21:46.546699+0000","last_clean":"2023-09-23T21:21:46.546699+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:21:46.546699+0000","last_undegraded":"2023-09-23T21:21:46.546699+0000","last_fullsized":"2023-09-23T21:21:46.546699+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545717+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:21:46.545717+0000","last_peered":"2023-09-23T21:21:46.545717+0000","last_clean":"2023-09-23T21:21:46.545717+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:21:46.545717+0000","last_undegraded":"2023-09-23T21:21:46.545717+0000","last_fullsized":"2023-09-23T21:21:46.545717+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":42,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.547128+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:21:46.547128+0000","last_peered":"2023-09-23T21:21:46.547128+0000","last_clean":"2023-09-23T21:21:46.547128+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:21:46.547128+0000","last_undegraded":"2023-09-23T21:21:46.547128+0000","last_fullsized":"2023-09-23T21:21:46.547128+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":744,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.533124+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:21:46.533124+0000","last_peered":"2023-09-23T21:21:46.533124+0000","last_clean":"2023-09-23T21:21:46.533124+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:21:46.533124+0000","last_undegraded":"2023-09-23T21:21:46.533124+0000","last_fullsized":"2023-09-23T21:21:46.533124+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":51,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.550419+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:21:46.550419+0000","last_peered":"2023-09-23T21:21:46.550419+0000","last_clean":"2023-09-23T21:21:46.550419+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:21:46.550419+0000","last_undegraded":"2023-09-23T21:21:46.550419+0000","last_fullsized":"2023-09-23T21:21:46.550419+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"648'8267","reported_seq":10088,"reported_epoch":648,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:51.407206+0000","last_change":"2023-09-23T21:21:01.215269+0000","last_active":"2023-09-23T21:21:51.407206+0000","last_peered":"2023-09-23T21:21:51.407206+0000","last_clean":"2023-09-23T21:19:28.877182+0000","last_became_active":"2023-09-23T21:21:01.190149+0000","last_became_peered":"2023-09-23T21:21:01.190149+0000","last_unstale":"2023-09-23T21:21:51.407206+0000","last_undegraded":"2023-09-23T21:21:00.607176+0000","last_fullsized":"2023-09-23T21:21:51.407206+0000","mapping_epoch":629,"log_start":"583'2767","ondisk_log_start":"583'2767","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:05.989327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145424384,"num_objects":2219,"num_object_clones":0,"num_object_copies":6657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2038,"num_objects_misplaced":2400,"num_objects_unfound":0,"num_objects_dirty":2219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8267,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":844,"num_bytes_recovered":55312384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,4,5],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,5,6","objects":1539},{"shards":"4,6,7","objects":680}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":285,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.547432+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:21:46.547432+0000","last_peered":"2023-09-23T21:21:46.547432+0000","last_clean":"2023-09-23T21:21:46.547432+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:21:46.547432+0000","last_undegraded":"2023-09-23T21:21:46.547432+0000","last_fullsized":"2023-09-23T21:21:46.547432+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540336+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:21:46.540336+0000","last_peered":"2023-09-23T21:21:46.540336+0000","last_clean":"2023-09-23T21:21:46.540336+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:21:46.540336+0000","last_undegraded":"2023-09-23T21:21:46.540336+0000","last_fullsized":"2023-09-23T21:21:46.540336+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"648'8509","reported_seq":10531,"reported_epoch":648,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:54.080614+0000","last_change":"2023-09-23T21:21:01.210342+0000","last_active":"2023-09-23T21:21:54.080614+0000","last_peered":"2023-09-23T21:21:54.080614+0000","last_clean":"2023-09-23T21:19:28.923531+0000","last_became_active":"2023-09-23T21:21:01.189847+0000","last_became_peered":"2023-09-23T21:21:01.189847+0000","last_unstale":"2023-09-23T21:21:54.080614+0000","last_undegraded":"2023-09-23T21:21:00.610088+0000","last_fullsized":"2023-09-23T21:21:54.080614+0000","mapping_epoch":629,"log_start":"589'3498","ondisk_log_start":"589'3498","created":573,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5011,"log_dups_size":0,"ondisk_log_size":5011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:58.686882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148307968,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2000,"num_objects_misplaced":2526,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8509,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":938,"num_bytes_recovered":61341696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[4,6,5],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":1537},{"shards":"4,6,7","objects":726}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":299,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545963+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:21:46.545963+0000","last_peered":"2023-09-23T21:21:46.545963+0000","last_clean":"2023-09-23T21:21:46.545963+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:21:46.545963+0000","last_undegraded":"2023-09-23T21:21:46.545963+0000","last_fullsized":"2023-09-23T21:21:46.545963+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553874+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:21:46.553874+0000","last_peered":"2023-09-23T21:21:46.553874+0000","last_clean":"2023-09-23T21:21:46.553874+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:21:46.553874+0000","last_undegraded":"2023-09-23T21:21:46.553874+0000","last_fullsized":"2023-09-23T21:21:46.553874+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"648'8311","reported_seq":13542,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:53.578164+0000","last_change":"2023-09-23T21:21:17.539683+0000","last_active":"2023-09-23T21:21:53.578164+0000","last_peered":"2023-09-23T21:21:53.578164+0000","last_clean":"2023-09-23T21:21:53.578164+0000","last_became_active":"2023-09-23T21:21:17.521633+0000","last_became_peered":"2023-09-23T21:21:17.521633+0000","last_unstale":"2023-09-23T21:21:53.578164+0000","last_undegraded":"2023-09-23T21:21:53.578164+0000","last_fullsized":"2023-09-23T21:21:53.578164+0000","mapping_epoch":636,"log_start":"588'3365","ondisk_log_start":"588'3365","created":573,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:05.844201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8311,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2555,"num_bytes_recovered":80478208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":271,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.546743+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:21:46.546743+0000","last_peered":"2023-09-23T21:21:46.546743+0000","last_clean":"2023-09-23T21:21:46.546743+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:21:46.546743+0000","last_undegraded":"2023-09-23T21:21:46.546743+0000","last_fullsized":"2023-09-23T21:21:46.546743+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.536652+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:21:46.536652+0000","last_peered":"2023-09-23T21:21:46.536652+0000","last_clean":"2023-09-23T21:21:46.536652+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:21:46.536652+0000","last_undegraded":"2023-09-23T21:21:46.536652+0000","last_fullsized":"2023-09-23T21:21:46.536652+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"648'8366","reported_seq":8465,"reported_epoch":648,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:21:53.686609+0000","last_change":"2023-09-23T21:20:59.644218+0000","last_active":"2023-09-23T21:21:53.686609+0000","last_peered":"2023-09-23T21:21:53.686609+0000","last_clean":"2023-09-23T21:20:30.430127+0000","last_became_active":"2023-09-23T21:20:59.632855+0000","last_became_peered":"2023-09-23T21:20:59.632855+0000","last_unstale":"2023-09-23T21:21:53.686609+0000","last_undegraded":"2023-09-23T21:21:53.686609+0000","last_fullsized":"2023-09-23T21:21:53.686609+0000","mapping_epoch":628,"log_start":"589'3425","ondisk_log_start":"589'3425","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4941,"log_dups_size":0,"ondisk_log_size":4941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:33.014367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146276352,"num_objects":2232,"num_object_clones":0,"num_object_copies":6696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4464,"num_objects_unfound":0,"num_objects_dirty":2232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8366,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2232}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":51,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540973+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:21:46.540973+0000","last_peered":"2023-09-23T21:21:46.540973+0000","last_clean":"2023-09-23T21:21:46.540973+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:21:46.540973+0000","last_undegraded":"2023-09-23T21:21:46.540973+0000","last_fullsized":"2023-09-23T21:21:46.540973+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.554187+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:21:46.554187+0000","last_peered":"2023-09-23T21:21:46.554187+0000","last_clean":"2023-09-23T21:21:46.554187+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:21:46.554187+0000","last_undegraded":"2023-09-23T21:21:46.554187+0000","last_fullsized":"2023-09-23T21:21:46.554187+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1074,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545531+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:21:46.545531+0000","last_peered":"2023-09-23T21:21:46.545531+0000","last_clean":"2023-09-23T21:21:46.545531+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:21:46.545531+0000","last_undegraded":"2023-09-23T21:21:46.545531+0000","last_fullsized":"2023-09-23T21:21:46.545531+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.6","version":"648'8467","reported_seq":12923,"reported_epoch":648,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:53.725109+0000","last_change":"2023-09-23T21:21:01.239919+0000","last_active":"2023-09-23T21:21:53.725109+0000","last_peered":"2023-09-23T21:21:53.725109+0000","last_clean":"2023-09-23T21:19:28.802255+0000","last_became_active":"2023-09-23T21:21:01.197858+0000","last_became_peered":"2023-09-23T21:21:01.197858+0000","last_unstale":"2023-09-23T21:21:53.725109+0000","last_undegraded":"2023-09-23T21:21:00.673065+0000","last_fullsized":"2023-09-23T21:21:53.725109+0000","mapping_epoch":629,"log_start":"589'3498","ondisk_log_start":"589'3498","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4969,"log_dups_size":0,"ondisk_log_size":4969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:39.315049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2259,"num_objects_misplaced":2259,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8467,"num_write_kb":343232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2171,"num_bytes_recovered":142213120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,4,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,4,6","objects":680},{"shards":"2,5,6","objects":1579}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":59,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540690+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:21:46.540690+0000","last_peered":"2023-09-23T21:21:46.540690+0000","last_clean":"2023-09-23T21:21:46.540690+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:21:46.540690+0000","last_undegraded":"2023-09-23T21:21:46.540690+0000","last_fullsized":"2023-09-23T21:21:46.540690+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":406,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553530+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:21:46.553530+0000","last_peered":"2023-09-23T21:21:46.553530+0000","last_clean":"2023-09-23T21:21:46.553530+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:21:46.553530+0000","last_undegraded":"2023-09-23T21:21:46.553530+0000","last_fullsized":"2023-09-23T21:21:46.553530+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":942,"reported_epoch":647,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:21:46.542912+0000","last_change":"2023-09-23T21:20:48.157415+0000","last_active":"2023-09-23T21:21:46.542912+0000","last_peered":"2023-09-23T21:21:46.542912+0000","last_clean":"2023-09-23T21:20:41.603469+0000","last_became_active":"2023-09-23T21:20:48.156886+0000","last_became_peered":"2023-09-23T21:20:48.156886+0000","last_unstale":"2023-09-23T21:21:46.542912+0000","last_undegraded":"2023-09-23T21:20:47.922781+0000","last_fullsized":"2023-09-23T21:20:47.921128+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:30.898986+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4,6","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.7","version":"648'8599","reported_seq":9801,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:53.838456+0000","last_change":"2023-09-23T21:20:59.594443+0000","last_active":"2023-09-23T21:21:53.838456+0000","last_peered":"2023-09-23T21:21:53.838456+0000","last_clean":"2023-09-23T21:21:53.838456+0000","last_became_active":"2023-09-23T21:20:48.149995+0000","last_became_peered":"2023-09-23T21:20:48.149995+0000","last_unstale":"2023-09-23T21:21:53.838456+0000","last_undegraded":"2023-09-23T21:21:53.838456+0000","last_fullsized":"2023-09-23T21:21:53.838456+0000","mapping_epoch":623,"log_start":"582'2665","ondisk_log_start":"582'2665","created":573,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5934,"log_dups_size":0,"ondisk_log_size":5934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.278714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150274048,"num_objects":2293,"num_object_clones":0,"num_object_copies":6879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8599,"num_write_kb":348544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":143,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.538379+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:21:46.538379+0000","last_peered":"2023-09-23T21:21:46.538379+0000","last_clean":"2023-09-23T21:21:46.538379+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:21:46.538379+0000","last_undegraded":"2023-09-23T21:21:46.538379+0000","last_fullsized":"2023-09-23T21:21:46.538379+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":656,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.558658+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:21:46.558658+0000","last_peered":"2023-09-23T21:21:46.558658+0000","last_clean":"2023-09-23T21:21:46.558658+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:21:46.558658+0000","last_undegraded":"2023-09-23T21:21:46.558658+0000","last_fullsized":"2023-09-23T21:21:46.558658+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"648'8203","reported_seq":14467,"reported_epoch":648,"state":"active+recovering+remapped","last_fresh":"2023-09-23T21:21:51.405836+0000","last_change":"2023-09-23T21:20:59.612076+0000","last_active":"2023-09-23T21:21:51.405836+0000","last_peered":"2023-09-23T21:21:51.405836+0000","last_clean":"2023-09-23T21:19:28.911487+0000","last_became_active":"2023-09-23T21:20:32.169326+0000","last_became_peered":"2023-09-23T21:20:32.169326+0000","last_unstale":"2023-09-23T21:21:51.405836+0000","last_undegraded":"2023-09-23T21:21:51.405836+0000","last_fullsized":"2023-09-23T21:21:51.405836+0000","mapping_epoch":617,"log_start":"583'2668","ondisk_log_start":"583'2668","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5535,"log_dups_size":0,"ondisk_log_size":5535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:09.080183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148701184,"num_objects":2269,"num_object_clones":0,"num_object_copies":6807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":542,"num_objects_unfound":0,"num_objects_dirty":2269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8203,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3078,"num_bytes_recovered":201719808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,4],"avail_no_missing":["6","5","7"],"object_location_counts":[{"shards":"4,6,7","objects":1727},{"shards":"5,6,7","objects":542}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":285,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.557238+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:21:46.557238+0000","last_peered":"2023-09-23T21:21:46.557238+0000","last_clean":"2023-09-23T21:21:46.557238+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:21:46.557238+0000","last_undegraded":"2023-09-23T21:21:46.557238+0000","last_fullsized":"2023-09-23T21:21:46.557238+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":693,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.563822+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:21:46.563822+0000","last_peered":"2023-09-23T21:21:46.563822+0000","last_clean":"2023-09-23T21:21:46.563822+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:21:46.563822+0000","last_undegraded":"2023-09-23T21:21:46.563822+0000","last_fullsized":"2023-09-23T21:21:46.563822+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":911,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.543323+0000","last_change":"2023-09-23T21:20:49.968107+0000","last_active":"2023-09-23T21:21:46.543323+0000","last_peered":"2023-09-23T21:21:46.543323+0000","last_clean":"2023-09-23T21:21:46.543323+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:21:46.543323+0000","last_undegraded":"2023-09-23T21:21:46.543323+0000","last_fullsized":"2023-09-23T21:21:46.543323+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":429358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.5","version":"648'8633","reported_seq":14973,"reported_epoch":648,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:54.085999+0000","last_change":"2023-09-23T21:20:59.554347+0000","last_active":"2023-09-23T21:21:54.085999+0000","last_peered":"2023-09-23T21:21:54.085999+0000","last_clean":"2023-09-23T21:19:28.864495+0000","last_became_active":"2023-09-23T21:20:32.158375+0000","last_became_peered":"2023-09-23T21:20:32.158375+0000","last_unstale":"2023-09-23T21:21:54.085999+0000","last_undegraded":"2023-09-23T21:20:59.551954+0000","last_fullsized":"2023-09-23T21:21:54.085999+0000","mapping_epoch":617,"log_start":"589'3592","ondisk_log_start":"589'3592","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:46.553959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152502272,"num_objects":2327,"num_object_clones":0,"num_object_copies":6981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":2327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8633,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3116,"num_bytes_recovered":204210176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"2,4,7","objects":750},{"shards":"4,5,7","objects":1577}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":51,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540324+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:21:46.540324+0000","last_peered":"2023-09-23T21:21:46.540324+0000","last_clean":"2023-09-23T21:21:46.540324+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:21:46.540324+0000","last_undegraded":"2023-09-23T21:21:46.540324+0000","last_fullsized":"2023-09-23T21:21:46.540324+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.554145+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:21:46.554145+0000","last_peered":"2023-09-23T21:21:46.554145+0000","last_clean":"2023-09-23T21:21:46.554145+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:21:46.554145+0000","last_undegraded":"2023-09-23T21:21:46.554145+0000","last_fullsized":"2023-09-23T21:21:46.554145+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":730,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.533555+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:21:46.533555+0000","last_peered":"2023-09-23T21:21:46.533555+0000","last_clean":"2023-09-23T21:21:46.533555+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:21:46.533555+0000","last_undegraded":"2023-09-23T21:21:46.533555+0000","last_fullsized":"2023-09-23T21:21:46.533555+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"648'8390","reported_seq":9240,"reported_epoch":648,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:53.696329+0000","last_change":"2023-09-23T21:21:01.233673+0000","last_active":"2023-09-23T21:21:53.696329+0000","last_peered":"2023-09-23T21:21:53.696329+0000","last_clean":"2023-09-23T21:19:28.898608+0000","last_became_active":"2023-09-23T21:21:01.195228+0000","last_became_peered":"2023-09-23T21:21:01.195228+0000","last_unstale":"2023-09-23T21:21:53.696329+0000","last_undegraded":"2023-09-23T21:21:00.570455+0000","last_fullsized":"2023-09-23T21:21:53.696329+0000","mapping_epoch":629,"log_start":"589'3446","ondisk_log_start":"589'3446","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4944,"log_dups_size":0,"ondisk_log_size":4944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:03.164901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146538496,"num_objects":2236,"num_object_clones":0,"num_object_copies":6708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2032,"num_objects_misplaced":2440,"num_objects_unfound":0,"num_objects_dirty":2236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8390,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[2,7,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1519},{"shards":"2,6,7","objects":717}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553747+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:21:46.553747+0000","last_peered":"2023-09-23T21:21:46.553747+0000","last_clean":"2023-09-23T21:21:46.553747+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:21:46.553747+0000","last_undegraded":"2023-09-23T21:21:46.553747+0000","last_fullsized":"2023-09-23T21:21:46.553747+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":35,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.543998+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:21:46.543998+0000","last_peered":"2023-09-23T21:21:46.543998+0000","last_clean":"2023-09-23T21:21:46.543998+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:21:46.543998+0000","last_undegraded":"2023-09-23T21:21:46.543998+0000","last_fullsized":"2023-09-23T21:21:46.543998+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553700+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:21:46.553700+0000","last_peered":"2023-09-23T21:21:46.553700+0000","last_clean":"2023-09-23T21:21:46.553700+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:21:46.553700+0000","last_undegraded":"2023-09-23T21:21:46.553700+0000","last_fullsized":"2023-09-23T21:21:46.553700+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":923,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.543108+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:21:46.543108+0000","last_peered":"2023-09-23T21:21:46.543108+0000","last_clean":"2023-09-23T21:21:46.543108+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:21:46.543108+0000","last_undegraded":"2023-09-23T21:21:46.543108+0000","last_fullsized":"2023-09-23T21:21:46.543108+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"648'8358","reported_seq":8441,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.071758+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:21:54.071758+0000","last_peered":"2023-09-23T21:21:54.071758+0000","last_clean":"2023-09-23T21:21:54.071758+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:21:54.071758+0000","last_undegraded":"2023-09-23T21:21:54.071758+0000","last_fullsized":"2023-09-23T21:21:54.071758+0000","mapping_epoch":573,"log_start":"588'3287","ondisk_log_start":"588'3287","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143130624,"num_objects":2184,"num_object_clones":0,"num_object_copies":6552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8358,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553874+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:21:46.553874+0000","last_peered":"2023-09-23T21:21:46.553874+0000","last_clean":"2023-09-23T21:21:46.553874+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:21:46.553874+0000","last_undegraded":"2023-09-23T21:21:46.553874+0000","last_fullsized":"2023-09-23T21:21:46.553874+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":561,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.550457+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:21:46.550457+0000","last_peered":"2023-09-23T21:21:46.550457+0000","last_clean":"2023-09-23T21:21:46.550457+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:21:46.550457+0000","last_undegraded":"2023-09-23T21:21:46.550457+0000","last_fullsized":"2023-09-23T21:21:46.550457+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553692+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:21:46.553692+0000","last_peered":"2023-09-23T21:21:46.553692+0000","last_clean":"2023-09-23T21:21:46.553692+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:21:46.553692+0000","last_undegraded":"2023-09-23T21:21:46.553692+0000","last_fullsized":"2023-09-23T21:21:46.553692+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1034,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545824+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:21:46.545824+0000","last_peered":"2023-09-23T21:21:46.545824+0000","last_clean":"2023-09-23T21:21:46.545824+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:21:46.545824+0000","last_undegraded":"2023-09-23T21:21:46.545824+0000","last_fullsized":"2023-09-23T21:21:46.545824+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.d","version":"648'8343","reported_seq":8501,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:51.378296+0000","last_change":"2023-09-23T21:21:22.585504+0000","last_active":"2023-09-23T21:21:51.378296+0000","last_peered":"2023-09-23T21:21:51.378296+0000","last_clean":"2023-09-23T21:21:51.378296+0000","last_became_active":"2023-09-23T21:21:22.568285+0000","last_became_peered":"2023-09-23T21:21:22.568285+0000","last_unstale":"2023-09-23T21:21:51.378296+0000","last_undegraded":"2023-09-23T21:21:51.378296+0000","last_fullsized":"2023-09-23T21:21:51.378296+0000","mapping_epoch":639,"log_start":"583'2848","ondisk_log_start":"583'2848","created":573,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5495,"log_dups_size":0,"ondisk_log_size":5495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:01.251062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152436760,"num_objects":2327,"num_object_clones":0,"num_object_copies":6981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2327,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8343,"num_write_kb":341377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":677,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:48.734598+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:21:48.734598+0000","last_peered":"2023-09-23T21:21:48.734598+0000","last_clean":"2023-09-23T21:21:48.734598+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:21:48.734598+0000","last_undegraded":"2023-09-23T21:21:48.734598+0000","last_fullsized":"2023-09-23T21:21:48.734598+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553155+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:21:46.553155+0000","last_peered":"2023-09-23T21:21:46.553155+0000","last_clean":"2023-09-23T21:21:46.553155+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:21:46.553155+0000","last_undegraded":"2023-09-23T21:21:46.553155+0000","last_fullsized":"2023-09-23T21:21:46.553155+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.554098+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:21:46.554098+0000","last_peered":"2023-09-23T21:21:46.554098+0000","last_clean":"2023-09-23T21:21:46.554098+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:21:46.554098+0000","last_undegraded":"2023-09-23T21:21:46.554098+0000","last_fullsized":"2023-09-23T21:21:46.554098+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"648'8479","reported_seq":11378,"reported_epoch":648,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:53.569772+0000","last_change":"2023-09-23T21:21:01.214995+0000","last_active":"2023-09-23T21:21:53.569772+0000","last_peered":"2023-09-23T21:21:53.569772+0000","last_clean":"2023-09-23T21:19:28.902983+0000","last_became_active":"2023-09-23T21:21:01.189928+0000","last_became_peered":"2023-09-23T21:21:01.189928+0000","last_unstale":"2023-09-23T21:21:53.569772+0000","last_undegraded":"2023-09-23T21:21:00.625306+0000","last_fullsized":"2023-09-23T21:21:53.569772+0000","mapping_epoch":629,"log_start":"582'2686","ondisk_log_start":"582'2686","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5793,"log_dups_size":0,"ondisk_log_size":5793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:45.309159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144244736,"num_objects":2201,"num_object_clones":0,"num_object_copies":6603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2109,"num_objects_misplaced":2293,"num_objects_unfound":0,"num_objects_dirty":2201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8479,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1379,"num_bytes_recovered":90374144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,2,5],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,5,7","objects":1471},{"shards":"2,6,7","objects":730}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":505,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.543998+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:21:46.543998+0000","last_peered":"2023-09-23T21:21:46.543998+0000","last_clean":"2023-09-23T21:21:46.543998+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:21:46.543998+0000","last_undegraded":"2023-09-23T21:21:46.543998+0000","last_fullsized":"2023-09-23T21:21:46.543998+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1035,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545493+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:21:46.545493+0000","last_peered":"2023-09-23T21:21:46.545493+0000","last_clean":"2023-09-23T21:21:46.545493+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:21:46.545493+0000","last_undegraded":"2023-09-23T21:21:46.545493+0000","last_fullsized":"2023-09-23T21:21:46.545493+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":35,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.541118+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:21:46.541118+0000","last_peered":"2023-09-23T21:21:46.541118+0000","last_clean":"2023-09-23T21:21:46.541118+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:21:46.541118+0000","last_undegraded":"2023-09-23T21:21:46.541118+0000","last_fullsized":"2023-09-23T21:21:46.541118+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.533836+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:21:46.533836+0000","last_peered":"2023-09-23T21:21:46.533836+0000","last_clean":"2023-09-23T21:21:46.533836+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:21:46.533836+0000","last_undegraded":"2023-09-23T21:21:46.533836+0000","last_fullsized":"2023-09-23T21:21:46.533836+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":682,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.554273+0000","last_change":"2023-09-23T21:21:21.571375+0000","last_active":"2023-09-23T21:21:46.554273+0000","last_peered":"2023-09-23T21:21:46.554273+0000","last_clean":"2023-09-23T21:21:46.554273+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:21:46.554273+0000","last_undegraded":"2023-09-23T21:21:46.554273+0000","last_fullsized":"2023-09-23T21:21:46.554273+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.551943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.551943+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.c","version":"648'8526","reported_seq":8609,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.093892+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:21:54.093892+0000","last_peered":"2023-09-23T21:21:54.093892+0000","last_clean":"2023-09-23T21:21:54.093892+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:21:54.093892+0000","last_undegraded":"2023-09-23T21:21:54.093892+0000","last_fullsized":"2023-09-23T21:21:54.093892+0000","mapping_epoch":573,"log_start":"589'3523","ondisk_log_start":"589'3523","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149159936,"num_objects":2276,"num_object_clones":0,"num_object_copies":6828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8526,"num_write_kb":345664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":315,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.545564+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:21:46.545564+0000","last_peered":"2023-09-23T21:21:46.545564+0000","last_clean":"2023-09-23T21:21:46.545564+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:21:46.545564+0000","last_undegraded":"2023-09-23T21:21:46.545564+0000","last_fullsized":"2023-09-23T21:21:46.545564+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.557121+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:21:46.557121+0000","last_peered":"2023-09-23T21:21:46.557121+0000","last_clean":"2023-09-23T21:21:46.557121+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:21:46.557121+0000","last_undegraded":"2023-09-23T21:21:46.557121+0000","last_fullsized":"2023-09-23T21:21:46.557121+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.554529+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:21:46.554529+0000","last_peered":"2023-09-23T21:21:46.554529+0000","last_clean":"2023-09-23T21:21:46.554529+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:21:46.554529+0000","last_undegraded":"2023-09-23T21:21:46.554529+0000","last_fullsized":"2023-09-23T21:21:46.554529+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":710,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.538963+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:21:46.538963+0000","last_peered":"2023-09-23T21:21:46.538963+0000","last_clean":"2023-09-23T21:21:46.538963+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:21:46.538963+0000","last_undegraded":"2023-09-23T21:21:46.538963+0000","last_fullsized":"2023-09-23T21:21:46.538963+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"648'8301","reported_seq":13662,"reported_epoch":648,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:21:54.089556+0000","last_change":"2023-09-23T21:20:48.304799+0000","last_active":"2023-09-23T21:21:54.089556+0000","last_peered":"2023-09-23T21:21:54.089556+0000","last_clean":"2023-09-23T21:19:28.920085+0000","last_became_active":"2023-09-23T21:20:48.224556+0000","last_became_peered":"2023-09-23T21:20:48.224556+0000","last_unstale":"2023-09-23T21:21:54.089556+0000","last_undegraded":"2023-09-23T21:20:48.204030+0000","last_fullsized":"2023-09-23T21:20:47.933463+0000","mapping_epoch":623,"log_start":"588'3244","ondisk_log_start":"588'3244","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:02.563744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140836864,"num_objects":2149,"num_object_clones":0,"num_object_copies":6447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3911,"num_objects_misplaced":272,"num_objects_unfound":0,"num_objects_dirty":2149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8301,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2617,"num_bytes_recovered":171376640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[4,2],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"2,4","objects":1},{"shards":"2,4,6","objects":734},{"shards":"4,5,6","objects":1414}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":255,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.546659+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:21:46.546659+0000","last_peered":"2023-09-23T21:21:46.546659+0000","last_clean":"2023-09-23T21:21:46.546659+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:21:46.546659+0000","last_undegraded":"2023-09-23T21:21:46.546659+0000","last_fullsized":"2023-09-23T21:21:46.546659+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553672+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:21:46.553672+0000","last_peered":"2023-09-23T21:21:46.553672+0000","last_clean":"2023-09-23T21:21:46.553672+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:21:46.553672+0000","last_undegraded":"2023-09-23T21:21:46.553672+0000","last_fullsized":"2023-09-23T21:21:46.553672+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":45,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.543699+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:21:46.543699+0000","last_peered":"2023-09-23T21:21:46.543699+0000","last_clean":"2023-09-23T21:21:46.543699+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:21:46.543699+0000","last_undegraded":"2023-09-23T21:21:46.543699+0000","last_fullsized":"2023-09-23T21:21:46.543699+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":949,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.547758+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:21:46.547758+0000","last_peered":"2023-09-23T21:21:46.547758+0000","last_clean":"2023-09-23T21:21:46.547758+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:21:46.547758+0000","last_undegraded":"2023-09-23T21:21:46.547758+0000","last_fullsized":"2023-09-23T21:21:46.547758+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.a","version":"648'8401","reported_seq":14675,"reported_epoch":648,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:54.067875+0000","last_change":"2023-09-23T21:21:01.218382+0000","last_active":"2023-09-23T21:21:54.067875+0000","last_peered":"2023-09-23T21:21:54.067875+0000","last_clean":"2023-09-23T21:19:28.896530+0000","last_became_active":"2023-09-23T21:21:01.190094+0000","last_became_peered":"2023-09-23T21:21:01.190094+0000","last_unstale":"2023-09-23T21:21:54.067875+0000","last_undegraded":"2023-09-23T21:21:00.719860+0000","last_fullsized":"2023-09-23T21:21:54.067875+0000","mapping_epoch":629,"log_start":"588'3318","ondisk_log_start":"588'3318","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:06.621301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146079744,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3537,"num_objects_misplaced":2925,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8401,"num_write_kb":340160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3068,"num_bytes_recovered":200933376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[4,5,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":1517},{"shards":"4,6,7","objects":712}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":345,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.547522+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:21:46.547522+0000","last_peered":"2023-09-23T21:21:46.547522+0000","last_clean":"2023-09-23T21:21:46.547522+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:21:46.547522+0000","last_undegraded":"2023-09-23T21:21:46.547522+0000","last_fullsized":"2023-09-23T21:21:46.547522+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":30,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.554643+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:21:46.554643+0000","last_peered":"2023-09-23T21:21:46.554643+0000","last_clean":"2023-09-23T21:21:46.554643+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:21:46.554643+0000","last_undegraded":"2023-09-23T21:21:46.554643+0000","last_fullsized":"2023-09-23T21:21:46.554643+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":60,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540328+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:21:46.540328+0000","last_peered":"2023-09-23T21:21:46.540328+0000","last_clean":"2023-09-23T21:21:46.540328+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:21:46.540328+0000","last_undegraded":"2023-09-23T21:21:46.540328+0000","last_fullsized":"2023-09-23T21:21:46.540328+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1042,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540877+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:21:46.540877+0000","last_peered":"2023-09-23T21:21:46.540877+0000","last_clean":"2023-09-23T21:21:46.540877+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:21:46.540877+0000","last_undegraded":"2023-09-23T21:21:46.540877+0000","last_fullsized":"2023-09-23T21:21:46.540877+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"648'8552","reported_seq":11859,"reported_epoch":648,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:21:53.701810+0000","last_change":"2023-09-23T21:21:22.582611+0000","last_active":"2023-09-23T21:21:53.701810+0000","last_peered":"2023-09-23T21:21:53.701810+0000","last_clean":"2023-09-23T21:19:48.922364+0000","last_became_active":"2023-09-23T21:21:22.566477+0000","last_became_peered":"2023-09-23T21:21:22.566477+0000","last_unstale":"2023-09-23T21:21:53.701810+0000","last_undegraded":"2023-09-23T21:21:53.701810+0000","last_fullsized":"2023-09-23T21:21:53.701810+0000","mapping_epoch":639,"log_start":"589'3556","ondisk_log_start":"589'3556","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4996,"log_dups_size":0,"ondisk_log_size":4996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:46.195361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150994944,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2304,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8552,"num_write_kb":347392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104660992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":2304}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":554,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.533748+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:21:46.533748+0000","last_peered":"2023-09-23T21:21:46.533748+0000","last_clean":"2023-09-23T21:21:46.533748+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:21:46.533748+0000","last_undegraded":"2023-09-23T21:21:46.533748+0000","last_fullsized":"2023-09-23T21:21:46.533748+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":34,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.554454+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:21:46.554454+0000","last_peered":"2023-09-23T21:21:46.554454+0000","last_clean":"2023-09-23T21:21:46.554454+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:21:46.554454+0000","last_undegraded":"2023-09-23T21:21:46.554454+0000","last_fullsized":"2023-09-23T21:21:46.554454+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":35,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.541453+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:21:46.541453+0000","last_peered":"2023-09-23T21:21:46.541453+0000","last_clean":"2023-09-23T21:21:46.541453+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:21:46.541453+0000","last_undegraded":"2023-09-23T21:21:46.541453+0000","last_fullsized":"2023-09-23T21:21:46.541453+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":99,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.550525+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:21:46.550525+0000","last_peered":"2023-09-23T21:21:46.550525+0000","last_clean":"2023-09-23T21:21:46.550525+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:21:46.550525+0000","last_undegraded":"2023-09-23T21:21:46.550525+0000","last_fullsized":"2023-09-23T21:21:46.550525+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":740,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.483366+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:21:50.483366+0000","last_peered":"2023-09-23T21:21:50.483366+0000","last_clean":"2023-09-23T21:21:50.483366+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:21:50.483366+0000","last_undegraded":"2023-09-23T21:21:50.483366+0000","last_fullsized":"2023-09-23T21:21:50.483366+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":408,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.553325+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:21:46.553325+0000","last_peered":"2023-09-23T21:21:46.553325+0000","last_clean":"2023-09-23T21:21:46.553325+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:21:46.553325+0000","last_undegraded":"2023-09-23T21:21:46.553325+0000","last_fullsized":"2023-09-23T21:21:46.553325+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":251,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.550282+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:21:46.550282+0000","last_peered":"2023-09-23T21:21:46.550282+0000","last_clean":"2023-09-23T21:21:46.550282+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:21:46.550282+0000","last_undegraded":"2023-09-23T21:21:46.550282+0000","last_fullsized":"2023-09-23T21:21:46.550282+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":159,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.534042+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:21:46.534042+0000","last_peered":"2023-09-23T21:21:46.534042+0000","last_clean":"2023-09-23T21:21:46.534042+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:21:46.534042+0000","last_undegraded":"2023-09-23T21:21:46.534042+0000","last_fullsized":"2023-09-23T21:21:46.534042+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540707+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:21:46.540707+0000","last_peered":"2023-09-23T21:21:46.540707+0000","last_clean":"2023-09-23T21:21:46.540707+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:21:46.540707+0000","last_undegraded":"2023-09-23T21:21:46.540707+0000","last_fullsized":"2023-09-23T21:21:46.540707+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":60,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.547198+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:21:46.547198+0000","last_peered":"2023-09-23T21:21:46.547198+0000","last_clean":"2023-09-23T21:21:46.547198+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:21:46.547198+0000","last_undegraded":"2023-09-23T21:21:46.547198+0000","last_fullsized":"2023-09-23T21:21:46.547198+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":51,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.540588+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:21:46.540588+0000","last_peered":"2023-09-23T21:21:46.540588+0000","last_clean":"2023-09-23T21:21:46.540588+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:21:46.540588+0000","last_undegraded":"2023-09-23T21:21:46.540588+0000","last_fullsized":"2023-09-23T21:21:46.540588+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":636,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.551726+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:21:46.551726+0000","last_peered":"2023-09-23T21:21:46.551726+0000","last_clean":"2023-09-23T21:21:46.551726+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:21:46.551726+0000","last_undegraded":"2023-09-23T21:21:46.551726+0000","last_fullsized":"2023-09-23T21:21:46.551726+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":355,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T21:21:46.548018+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:21:46.548018+0000","last_peered":"2023-09-23T21:21:46.548018+0000","last_clean":"2023-09-23T21:21:46.548018+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:21:46.548018+0000","last_undegraded":"2023-09-23T21:21:46.548018+0000","last_fullsized":"2023-09-23T21:21:46.548018+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":2354315288,"num_objects":35925,"num_object_clones":0,"num_object_copies":107775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19500,"num_objects_misplaced":23138,"num_objects_unfound":0,"num_objects_dirty":35925,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134705,"num_write_kb":5460097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22281,"num_bytes_recovered":1372585984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8686088192,"data_stored":8686075976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83319,"ondisk_log_size":83319,"up":48,"acting":47,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697707,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1550763,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3297280,"data_stored":3279520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201918,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1909524,"kb_used_data":1015564,"kb_used_omap":0,"kb_used_meta":893952,"kb_avail":91835628,"statfs":{"total":95995035648,"available":94039683072,"internally_reserved":0,"allocated":1039937536,"data_stored":1038156320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915406848},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.605,"15min":2.3159999999999998},"min":{"1min":0.27300000000000002,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":6.6390000000000002,"5min":16.468,"15min":141.33199999999999},"last":2.7309999999999999},{"interface":"front","average":{"1min":2.4790000000000001,"5min":1.992,"15min":2.4780000000000002},"min":{"1min":0.29799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":12.962,"5min":12.962,"15min":141.31700000000001},"last":2.8929999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":2.0840000000000001,"15min":3.4689999999999999},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.24099999999999999},"max":{"1min":13.054,"5min":13.054,"15min":114.16500000000001},"last":2.9510000000000001},{"interface":"front","average":{"1min":3.0009999999999999,"5min":1.9350000000000001,"15min":3.8170000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":10.881,"5min":16.289000000000001,"15min":117.637},"last":2.8199999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":8.1029999999999998,"5min":6.1260000000000003,"15min":5.5369999999999999},"min":{"1min":0.252,"5min":0.246,"15min":0.224},"max":{"1min":68.793999999999997,"5min":158.726,"15min":183.44200000000001},"last":2.9249999999999998},{"interface":"front","average":{"1min":7.2279999999999998,"5min":5.7489999999999997,"15min":5.7119999999999997},"min":{"1min":0.32800000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":63.264000000000003,"5min":158.70400000000001,"15min":183.46199999999999},"last":0.27200000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":4.3310000000000004,"5min":2.2050000000000001,"15min":4.1589999999999998},"min":{"1min":0.41099999999999998,"5min":0.33700000000000002,"15min":0.26700000000000002},"max":{"1min":30.667999999999999,"5min":30.667999999999999,"15min":135.66200000000001},"last":2.7629999999999999},{"interface":"front","average":{"1min":3.8620000000000001,"5min":2.1019999999999999,"15min":4.383},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":30.798999999999999,"5min":30.798999999999999,"15min":135.47},"last":2.6970000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.1849999999999996,"5min":3.653,"15min":1.9690000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":13.821999999999999,"5min":31.695,"15min":31.695},"last":8.5909999999999993},{"interface":"front","average":{"1min":6.8120000000000003,"5min":3.7869999999999999,"15min":2.0179999999999998},"min":{"1min":0.48499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":27.821999999999999,"5min":34.994,"15min":34.994},"last":2.669}]},{"osd":5,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":4.7350000000000003,"15min":2.46},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":10.712,"5min":156.30099999999999,"15min":156.30099999999999},"last":2.7970000000000002},{"interface":"front","average":{"1min":4.5389999999999997,"5min":4.8739999999999997,"15min":2.548},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":12.151,"5min":156.22,"15min":156.22},"last":3.1920000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":3.75,"15min":2.8140000000000001},"min":{"1min":0.45700000000000002,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":14.066000000000001,"5min":23.574999999999999,"15min":143.18700000000001},"last":8.4429999999999996},{"interface":"front","average":{"1min":6.4370000000000003,"5min":3.242,"15min":2.528},"min":{"1min":0.504,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":21.081,"5min":21.081,"15min":142.70400000000001},"last":2.621}]}]},{"osd":6,"up_from":47,"seq":201863463538,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2987984,"kb_used_data":2098184,"kb_used_omap":0,"kb_used_meta":889728,"kb_avail":90757168,"statfs":{"total":95995035648,"available":92935340032,"internally_reserved":0,"allocated":2148540416,"data_stored":2146764065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911081472},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.6100000000000001,"15min":2.8340000000000001},"min":{"1min":0.246,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":10.004,"5min":12.659000000000001,"15min":149.61600000000001},"last":1.6599999999999999},{"interface":"front","average":{"1min":2.1960000000000002,"5min":2.077,"15min":3.0529999999999999},"min":{"1min":0.27100000000000002,"5min":0.223,"15min":0.223},"max":{"1min":7.0090000000000003,"5min":16.021999999999998,"15min":149.303},"last":3.8769999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.4020000000000001,"15min":2.9990000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":12.013,"5min":16.126999999999999,"15min":129.37899999999999},"last":3.9199999999999999},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.8540000000000001,"15min":2.742},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":14.366,"5min":14.366,"15min":129.40100000000001},"last":4.0960000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":6.4450000000000003,"15min":5.9790000000000001},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.9540000000000006,"5min":181.846,"15min":199.23400000000001},"last":1.6319999999999999},{"interface":"front","average":{"1min":2.2120000000000002,"5min":7.1200000000000001,"15min":6.1239999999999997},"min":{"1min":0.35899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.1059999999999999,"5min":182.14500000000001,"15min":199.08000000000001},"last":3.948}]},{"osd":3,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":1.9330000000000001,"15min":3.4630000000000001},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.218},"max":{"1min":14.411,"5min":14.411,"15min":137.875},"last":4.1299999999999999},{"interface":"front","average":{"1min":2.089,"5min":1.8080000000000001,"15min":3.3839999999999999},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.29799999999999999},"max":{"1min":10.106,"5min":12.676,"15min":137.60900000000001},"last":1.607}]},{"osd":4,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":4.0439999999999996,"5min":4.5910000000000002,"15min":2.2559999999999998},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":15.361000000000001,"5min":55.494,"15min":55.494},"last":8.4429999999999996},{"interface":"front","average":{"1min":3.823,"5min":4.2039999999999997,"15min":2.0510000000000002},"min":{"1min":0.33700000000000002,"5min":0.33400000000000002,"15min":0.23999999999999999},"max":{"1min":14.622,"5min":55.305,"15min":55.305},"last":7.8920000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":5.5620000000000003,"15min":2.6419999999999999},"min":{"1min":0.41699999999999998,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":7.1630000000000003,"5min":222.90899999999999,"15min":222.90899999999999},"last":3.9729999999999999},{"interface":"front","average":{"1min":2.6819999999999999,"5min":5.4749999999999996,"15min":2.6230000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.28499999999999998},"max":{"1min":17.927,"5min":224.655,"15min":224.655},"last":5.6269999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":4.8970000000000002,"15min":2.2599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36399999999999999,"15min":0.28000000000000003},"max":{"1min":12.114000000000001,"5min":165.96799999999999,"15min":165.96799999999999},"last":1.5680000000000001},{"interface":"front","average":{"1min":3.5030000000000001,"5min":5.4870000000000001,"15min":2.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.251},"max":{"1min":14.457000000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":7.9240000000000004}]}]},{"osd":1,"up_from":276,"seq":1185410973985,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":625368,"kb_used_data":8144,"kb_used_omap":5,"kb_used_meta":617210,"kb_avail":93119784,"statfs":{"total":95995035648,"available":95354658816,"internally_reserved":0,"allocated":8339456,"data_stored":5264536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":632023779},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.38200000000000001,"15min":2.0009999999999999},"min":{"1min":0.22600000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":0.505,"5min":1.9450000000000001,"15min":108.47},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.40100000000000002,"15min":1.952},"min":{"1min":0.26200000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":109.532},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":7.6120000000000001,"5min":8.4109999999999996,"15min":7.8899999999999997},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.20200000000000001},"max":{"1min":72.966999999999999,"5min":149.35300000000001,"15min":171.35400000000001},"last":0.26700000000000002},{"interface":"front","average":{"1min":7.6459999999999999,"5min":8.5180000000000007,"15min":7.5659999999999998},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":74.450000000000003,"5min":147.50399999999999,"15min":170.69900000000001},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":0.55200000000000005,"15min":3.593},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":17.247,"5min":17.247,"15min":99.120999999999995},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.66000000000000003,"15min":4.1600000000000001},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.21299999999999999},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":150.38},"last":0.30099999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4770000000000001,"5min":3.2519999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.5800000000000001,"5min":174.84399999999999,"15min":174.84399999999999},"last":2.468},{"interface":"front","average":{"1min":0.91000000000000003,"5min":3.0230000000000001,"15min":2.1850000000000001},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.28199999999999997},"max":{"1min":2.931,"5min":174.82400000000001,"15min":174.82400000000001},"last":2.4620000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":4.2670000000000003,"15min":2.7269999999999999},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.31900000000000001},"max":{"1min":3.8839999999999999,"5min":110.955,"15min":110.955},"last":1.4019999999999999},{"interface":"front","average":{"1min":1.53,"5min":4.5679999999999996,"15min":2.774},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.29199999999999998},"max":{"1min":9.2300000000000004,"5min":110.89,"15min":110.89},"last":2.7010000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1989999999999998,"5min":3.1160000000000001,"15min":2.4329999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":7.7859999999999996,"5min":119.999,"15min":119.999},"last":12.42},{"interface":"front","average":{"1min":1.788,"5min":3.1269999999999998,"15min":2.3370000000000002},"min":{"1min":0.40500000000000003,"5min":0.38300000000000001,"15min":0.31900000000000001},"max":{"1min":8.9550000000000001,"5min":122.328,"15min":122.328},"last":2.2029999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":4.8159999999999998,"15min":2.9239999999999999},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":10.648999999999999,"5min":285.47500000000002,"15min":285.47500000000002},"last":1.7070000000000001},{"interface":"front","average":{"1min":1.875,"5min":4.5430000000000001,"15min":2.7330000000000001},"min":{"1min":0.38600000000000001,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":10.664999999999999,"5min":285.48599999999999,"15min":285.48599999999999},"last":1.716}]}]},{"osd":0,"up_from":0,"seq":395136991725,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781616,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2597804,"kb_used_data":1292068,"kb_used_omap":7,"kb_used_meta":1305720,"kb_avail":91147348,"statfs":{"total":95995035648,"available":93334884352,"internally_reserved":0,"allocated":1323077632,"data_stored":1319916060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1337057361},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.67400000000000004,"15min":1.929},"min":{"1min":0.19900000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":4.2880000000000003,"5min":8.0079999999999991,"15min":103.021},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.621,"15min":2.5099999999999998},"min":{"1min":0.253,"5min":0.218,"15min":0.218},"max":{"1min":4.6840000000000002,"5min":4.9820000000000002,"15min":103.154},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.66300000000000003,"15min":2.7160000000000002},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.19700000000000001},"max":{"1min":4.7350000000000003,"5min":5.2670000000000003,"15min":143.482},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.71999999999999997,"15min":2.7490000000000001},"min":{"1min":0.23899999999999999,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":1.6659999999999999,"5min":8.048,"15min":143.61000000000001},"last":0.26700000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.70399999999999996,"15min":3.5369999999999999},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.7679999999999998,"5min":5.2939999999999996,"15min":143.50800000000001},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.77300000000000002,"15min":3.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.2670000000000003,"5min":8.2189999999999994,"15min":143.27799999999999},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":4.3929999999999998,"15min":3.3849999999999998},"min":{"1min":0.318,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":6.8659999999999997,"5min":220.82900000000001,"15min":225.80799999999999},"last":1.196},{"interface":"front","average":{"1min":2.6160000000000001,"5min":4.3259999999999996,"15min":3.9769999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":6.8029999999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":1.1599999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":4.3010000000000002,"15min":3.48},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":9.0579999999999998,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.5049999999999999,"5min":4.3789999999999996,"15min":3.5089999999999999},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.8129999999999997,"5min":220.31999999999999,"15min":220.31999999999999},"last":3.2440000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":5.0289999999999999,"15min":3.3460000000000001},"min":{"1min":0.41099999999999998,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":5.1980000000000004,"5min":210.74700000000001,"15min":210.74700000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":1.865,"5min":4.782,"15min":2.6890000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.8170000000000002,"5min":210.72900000000001,"15min":210.72900000000001},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.0169999999999999,"15min":1.954},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.5250000000000004,"5min":14.019,"15min":25.260000000000002},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.758,"15min":2.3279999999999998},"min":{"1min":0.41099999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.6200000000000001,"5min":7.5289999999999999,"15min":16.875},"last":0.47599999999999998}]}]},{"osd":3,"up_from":148,"seq":635655160209,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":851720,"kb_used_data":9280,"kb_used_omap":2,"kb_used_meta":842429,"kb_avail":92893432,"statfs":{"total":95995035648,"available":95122874368,"internally_reserved":0,"allocated":9502720,"data_stored":5894461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":862647349},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.42199999999999999,"15min":1.7749999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":4.7430000000000003,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.253},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.42499999999999999,"15min":1.5369999999999999},"min":{"1min":0.252,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":2.6480000000000001,"5min":2.6480000000000001,"15min":18.277999999999999},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:21:33 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.504,"15min":6.5910000000000002},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8980000000000001,"5min":2.8980000000000001,"15min":237.072},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.54200000000000004,"15min":6.5460000000000003},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":3.8359999999999999,"5min":3.8359999999999999,"15min":234.21899999999999},"last":0.26900000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":5.04,"15min":4.6239999999999997},"min":{"1min":0.32500000000000001,"5min":0.219,"15min":0.215},"max":{"1min":6.665,"5min":102.404,"15min":102.404},"last":130.55799999999999},{"interface":"front","average":{"1min":0.73299999999999998,"5min":5.024,"15min":4.1959999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":5.8010000000000002,"5min":104.529,"15min":104.529},"last":133.19999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":2.9169999999999998,"15min":2.5270000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":13.640000000000001,"5min":117.717,"15min":117.717},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.1499999999999999,"5min":3.415,"15min":2.5059999999999998},"min":{"1min":0.42299999999999999,"5min":0.32100000000000001,"15min":0.27300000000000002},"max":{"1min":11.968999999999999,"5min":117.771,"15min":117.771},"last":0.308}]},{"osd":5,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":3.96,"15min":2.8290000000000002},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":6.5449999999999999,"5min":123.458,"15min":123.458},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.706,"5min":3.8410000000000002,"15min":2.8250000000000002},"min":{"1min":0.38,"5min":0.34899999999999998,"15min":0.315},"max":{"1min":4.7619999999999996,"5min":123.292,"15min":123.292},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":4.9569999999999999,"15min":4.7210000000000001},"min":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.316},"max":{"1min":4.0140000000000002,"5min":130.613,"15min":304.22399999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.8089999999999999,"5min":5.3140000000000001,"15min":4.7770000000000001},"min":{"1min":0.41699999999999998,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":4.5179999999999998,"5min":130.50999999999999,"15min":304.15800000000002},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":2.532,"15min":2.1080000000000001},"min":{"1min":0.39100000000000001,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":19.298999999999999,"5min":44.253,"15min":44.253},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.9199999999999999,"5min":2.6669999999999998,"15min":2.5},"min":{"1min":0.434,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":8.6300000000000008,"5min":45.706000000000003,"15min":45.706000000000003},"last":0.872}]}]},{"osd":4,"up_from":31,"seq":133143986828,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2707356,"kb_used_data":1752788,"kb_used_omap":0,"kb_used_meta":954368,"kb_avail":91037796,"statfs":{"total":95995035648,"available":93222703104,"internally_reserved":0,"allocated":1794854912,"data_stored":1793066325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":977272832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":2.0099999999999998,"15min":2.8170000000000002},"min":{"1min":0.32600000000000001,"5min":0.246,"15min":0.20699999999999999},"max":{"1min":6.8390000000000004,"5min":10.436,"15min":226.751},"last":3.5619999999999998},{"interface":"front","average":{"1min":1.492,"5min":1.5469999999999999,"15min":2.5990000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":4.6900000000000004,"5min":12.754,"15min":226.43299999999999},"last":0.79600000000000004}]},{"osd":1,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.583,"15min":3.6789999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.0709999999999997,"5min":12.805,"15min":127.411},"last":0.73999999999999999},{"interface":"front","average":{"1min":2.4689999999999999,"5min":2.1619999999999999,"15min":3.194},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.625,"5min":10.625,"15min":116.78100000000001},"last":3.6059999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:21:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2390000000000008,"5min":8.0739999999999998,"15min":6.0629999999999997},"min":{"1min":0.33600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":70.393000000000001,"5min":108.32899999999999,"15min":210.50299999999999},"last":4.0670000000000002},{"interface":"front","average":{"1min":8.7129999999999992,"5min":8.0129999999999999,"15min":6.1669999999999998},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":70.340000000000003,"5min":108.104,"15min":210.59},"last":3.6339999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.2999999999999998,"15min":4.4119999999999999},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.122999999999999,"5min":14.323,"15min":214.64099999999999},"last":4.0990000000000002},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.391,"15min":4.8159999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":10.587,"5min":10.587,"15min":214.54900000000001},"last":3.657}]},{"osd":5,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.3410000000000002,"15min":1.8280000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":12.997999999999999,"5min":17.879999999999999,"15min":34.497999999999998},"last":13.315},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.3050000000000002,"15min":2.0760000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":19.425000000000001,"5min":19.425000000000001,"15min":34.883000000000003},"last":8.7249999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:21:02 2023","interfaces":[{"interface":"back","average":{"1min":7.1280000000000001,"5min":3.952,"15min":2.1869999999999998},"min":{"1min":0.34899999999999998,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":67.697999999999993,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.77200000000000002},{"interface":"front","average":{"1min":8.2810000000000006,"5min":4.7009999999999996,"15min":2.4020000000000001},"min":{"1min":0.38600000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":67.596999999999994,"5min":67.596999999999994,"15min":67.596999999999994},"last":4.4219999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":3.3159999999999998,"15min":1.9710000000000001},"min":{"1min":0.38700000000000001,"5min":0.317,"15min":0.28100000000000003},"max":{"1min":15.050000000000001,"5min":17.817,"15min":18.352},"last":16.998999999999999},{"interface":"front","average":{"1min":3.323,"5min":3.177,"15min":1.8819999999999999},"min":{"1min":0.441,"5min":0.307,"15min":0.307},"max":{"1min":16.332999999999998,"5min":30.117000000000001,"15min":30.117000000000001},"last":1.734}]}]},{"osd":5,"up_from":39,"seq":167503725192,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3394504,"kb_used_data":2423424,"kb_used_omap":0,"kb_used_meta":971072,"kb_avail":90350648,"statfs":{"total":95995035648,"available":92519063552,"internally_reserved":0,"allocated":2481586176,"data_stored":2479813269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":994377728},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":1.762,"15min":1.9730000000000001},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.217},"max":{"1min":12.558999999999999,"5min":12.558999999999999,"15min":82.280000000000001},"last":2.9329999999999998},{"interface":"front","average":{"1min":2.617,"5min":1.8089999999999999,"15min":2.0840000000000001},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":9.407,"5min":9.407,"15min":85.236000000000004},"last":3.0459999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":2.2440000000000002,"15min":3.9900000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":18.859999999999999,"5min":18.859999999999999,"15min":233.77099999999999},"last":3.2549999999999999},{"interface":"front","average":{"1min":1.198,"5min":1.544,"15min":3.5899999999999999},"min":{"1min":0.27400000000000002,"5min":0.27000000000000002,"15min":0.251},"max":{"1min":7.5220000000000002,"5min":12.654999999999999,"15min":233.75200000000001},"last":2.8780000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":6.7320000000000002,"5min":4.5449999999999999,"15min":4.3639999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":82.072000000000003,"5min":160.749,"15min":231.34700000000001},"last":2.8450000000000002},{"interface":"front","average":{"1min":6.2869999999999999,"5min":4.5359999999999996,"15min":4.4699999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":82.099999999999994,"5min":161.18000000000001,"15min":231.381},"last":3.089}]},{"osd":3,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":2.1869999999999998,"15min":3.9049999999999998},"min":{"1min":0.36099999999999999,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":18.826000000000001,"5min":18.826000000000001,"15min":263.887},"last":3.2879999999999998},{"interface":"front","average":{"1min":3.0680000000000001,"5min":1.895,"15min":3.6440000000000001},"min":{"1min":0.45300000000000001,"5min":0.35799999999999998,"15min":0.30099999999999999},"max":{"1min":12.683999999999999,"5min":12.683999999999999,"15min":263.70100000000002},"last":2.9039999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":3.0649999999999999,"15min":1.7649999999999999},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":7.6299999999999999,"5min":77.804000000000002,"15min":77.804000000000002},"last":2.9849999999999999},{"interface":"front","average":{"1min":2.7719999999999998,"5min":4.2880000000000003,"15min":2.1619999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":8.3810000000000002,"5min":77.825999999999993,"15min":77.825999999999993},"last":3.1709999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":3.9420000000000002,"5min":2.786,"15min":1.913},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":13.965999999999999,"5min":13.965999999999999,"15min":83.451999999999998},"last":3.1190000000000002},{"interface":"front","average":{"1min":4.0800000000000001,"5min":3.0960000000000001,"15min":2.1920000000000002},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":19.029,"5min":19.029,"15min":83.484999999999999},"last":3.5169999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:21:42 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":2.6309999999999998,"15min":1.792},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":10.654999999999999,"5min":10.654999999999999,"15min":10.654999999999999},"last":3.145},{"interface":"front","average":{"1min":2.7160000000000002,"5min":2.2629999999999999,"15min":1.411},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":18.257999999999999,"5min":18.257999999999999,"15min":18.257999999999999},"last":2.9590000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1300692992,"data_stored":1300692992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1773146112,"data_stored":1773142040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2459308032,"data_stored":2459303960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2124156928,"data_stored":2124152856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1028784128,"data_stored":1028784128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:21:58.299 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:21:58.337 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:58.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:58.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:21:58.444 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:58.486 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:58.574 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:58.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:58.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:58.678 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:21:58.704 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:58.745 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:21:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:58.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:21:58.849 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:21:58.917 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:59.021 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:21:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:59.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:21:59.091 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:59.127 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:21:59.150 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:21:59.227 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:59.375 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:21:59.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:59.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:21:59.488 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:21:59.525 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:21:59.661 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:21:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:21:59.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:21:59.766 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:21:59.809 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:00.076 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:147730 12.1 12:806729b5:::benchmark_data_smithi121_563784_object57327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:58.648450+0000", 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 541.42507992200001, 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0452919920000001, 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:22:00.077 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 147730 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648450+0000", 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.078 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648450+0000", 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648449+0000", 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.079 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648452+0000", 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4420000000000001e-06 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648457+0000", 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5079999999999996e-06 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648462+0000", 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4680000000000003e-06 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.080 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648498+0000", 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6494e-05 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648608+0000", 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010921699999999999 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T21:22:00.081 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648701+0000", 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3184999999999996e-05 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.659355+0000", 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010654083 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.661711+0000", 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023556670000000001 2023-09-23T21:22:00.082 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.668680+0000", 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0069692670000000003 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676424+0000", 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0077446140000000004 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.083 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676434+0000", 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3880000000000006e-06 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.693662+0000", 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0172280970000001 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.693742+0000", 2023-09-23T21:22:00.084 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9950999999999994e-05 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:147732 12.1 12:883b207e:::benchmark_data_smithi121_563784_object57329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e464)", 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:58.648473+0000", 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 541.42505721400005, 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0457270919999999, 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:22:00.085 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 147732 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648473+0000", 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.086 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648473+0000", 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648472+0000", 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648474+0000", 2023-09-23T21:22:00.087 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0200000000000001e-06 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648480+0000", 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7660000000000004e-06 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.648483+0000", 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.428e-06 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.088 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.652857+0000", 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043731270000000001 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.652967+0000", 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011064 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.653071+0000", 2023-09-23T21:22:00.089 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010343700000000001 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.663197+0000", 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010125881999999999 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.663414+0000", 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000217906 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.090 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.674405+0000", 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010990541 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676477+0000", 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020718540000000001 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.091 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.676484+0000", 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4919999999999996e-06 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694141+0000", 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0176565399999999 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694200+0000", 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8783000000000003e-05 2023-09-23T21:22:00.092 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:147747 12.1 12:8d405dd7:::benchmark_data_smithi121_563784_object57344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:12:58.843469+0000", 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 541.23006102099998, 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88383067199999998, 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.093 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 147747 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843469+0000", 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.094 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843469+0000", 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843467+0000", 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999976 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843480+0000", 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3183000000000001e-05 2023-09-23T21:22:00.095 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843513+0000", 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3658999999999997e-05 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843526+0000", 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2847000000000001e-05 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.096 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843632+0000", 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010543 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:58.843650+0000", 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8427000000000002e-05 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694222+0000", 2023-09-23T21:22:00.097 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85057223400000004 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694338+0000", 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011524800000000001 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.694455+0000", 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117815 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.098 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.705071+0000", 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010615591000000001 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.705272+0000", 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00020080400000000001 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.099 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.727207+0000", 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021935316 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.727258+0000", 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0495999999999999e-05 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:12:59.727299+0000", 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1851999999999998e-05 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.100 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29430 13.4 13:2f637a96:::benchmark_data_smithi121_621349_object29429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.625873727, 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90678336900000001, 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.101 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29430 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:22:00.102 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447655+0000", 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447720+0000", 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4391000000000001e-05 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.103 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447728+0000", 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4300000000000006e-06 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447738+0000", 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0118999999999999e-05 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447838+0000", 2023-09-23T21:22:00.104 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9207000000000001e-05 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447846+0000", 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2439999999999993e-06 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338557+0000", 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.890711217 2023-09-23T21:22:00.105 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338700+0000", 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143361 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338864+0000", 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016315700000000001 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.106 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342396+0000", 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035323749999999999 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354238+0000", 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011842363 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.107 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354340+0000", 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101962 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354396+0000", 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6174000000000001e-05 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354439+0000", 2023-09-23T21:22:00.108 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2994999999999997e-05 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29438 13.4 13:2ddaff29:::benchmark_data_smithi121_621349_object29437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.62192737599997, 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90300633600000002, 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.109 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29438 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.110 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451601+0000", 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451653+0000", 2023-09-23T21:22:00.111 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2207999999999999e-05 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451664+0000", 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0692e-05 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451669+0000", 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2660000000000001e-06 2023-09-23T21:22:00.112 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.452098+0000", 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00042897200000000001 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.452110+0000", 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.202e-05 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.113 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.339881+0000", 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88777067300000001 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.339995+0000", 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114695 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.114 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340148+0000", 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015272199999999999 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343455+0000", 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003307454 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354283+0000", 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010827448999999999 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.115 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354522+0000", 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00023920500000000001 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354571+0000", 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8539999999999999e-05 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354609+0000", 2023-09-23T21:22:00.116 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8046999999999997e-05 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29448 13.4 13:22d32e47:::benchmark_data_smithi121_621349_object29447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.61718415899998, 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89842967699999998, 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.117 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29448 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.118 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456345+0000", 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456397+0000", 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2176999999999999e-05 2023-09-23T21:22:00.119 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456411+0000", 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4219000000000001e-05 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456417+0000", 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6210000000000001e-06 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.120 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456475+0000", 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8056000000000001e-05 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456485+0000", 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0071000000000001e-05 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340736+0000", 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88425111199999995 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.121 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340816+0000", 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0597000000000001e-05 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340916+0000", 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9607999999999999e-05 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343463+0000", 2023-09-23T21:22:00.122 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002547278 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354482+0000", 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011018857 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354722+0000", 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024004300000000001 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.123 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354754+0000", 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1299e-05 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354775+0000", 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1764e-05 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.124 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29452 13.4 13:23edd8ec:::benchmark_data_smithi121_621349_object29451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.615610774, 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89701267699999998, 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29452 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.125 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:22:00.126 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457953+0000", 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4428000000000003e-05 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457961+0000", 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6809999999999998e-06 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.127 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457965+0000", 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6999999999999999e-06 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.458018+0000", 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2782999999999999e-05 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.458027+0000", 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3030000000000005e-06 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.128 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341620+0000", 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88359242400000004 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341748+0000", 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012769800000000001 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341830+0000", 2023-09-23T21:22:00.129 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2670999999999995e-05 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345316+0000", 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034856290000000001 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354632+0000", 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0093166240000000008 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.130 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354861+0000", 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022811500000000001 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354897+0000", 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6327e-05 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.131 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354932+0000", 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4768999999999999e-05 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29475 13.4 13:21b7c357:::benchmark_data_smithi121_621349_object29474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.60109840600001, 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88263625400000001, 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.132 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29475 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.133 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472430+0000", 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472465+0000", 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4814000000000002e-05 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.134 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472485+0000", 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0435e-05 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472493+0000", 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5240000000000003e-06 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472551+0000", 2023-09-23T21:22:00.135 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8548999999999998e-05 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472562+0000", 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0855e-05 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342433+0000", 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86987049900000002 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.136 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342547+0000", 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114554 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342679+0000", 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000131984 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347274+0000", 2023-09-23T21:22:00.137 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045946049999999999 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354817+0000", 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0075430949999999997 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354987+0000", 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017053899999999999 2023-09-23T21:22:00.138 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355035+0000", 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7305999999999997e-05 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355068+0000", 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2978000000000002e-05 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.139 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29478 13.4 13:2842156f:::benchmark_data_smithi121_621349_object29477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.60032089100002, 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88206223699999997, 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.140 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29478 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:22:00.141 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473208+0000", 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473278+0000", 2023-09-23T21:22:00.142 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0062999999999996e-05 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473296+0000", 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7852000000000001e-05 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473303+0000", 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3479999999999998e-06 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473375+0000", 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1407000000000003e-05 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473396+0000", 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1634999999999999e-05 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.144 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343498+0000", 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.87010185600000001 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343568+0000", 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0224999999999999e-05 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343651+0000", 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2723000000000001e-05 2023-09-23T21:22:00.145 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347284+0000", 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003633168 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355131+0000", 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078466939999999995 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.146 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355175+0000", 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4586999999999998e-05 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355226+0000", 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0695000000000002e-05 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355271+0000", 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4932000000000003e-05 2023-09-23T21:22:00.147 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29505 13.4 13:2e31c810:::benchmark_data_smithi121_621349_object29504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.58338245599998, 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.865266541, 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.148 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29505 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:22:00.149 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490146+0000", 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490797+0000", 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00065088600000000004 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.150 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490813+0000", 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6189999999999999e-05 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490820+0000", 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6730000000000003e-06 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490881+0000", 2023-09-23T21:22:00.151 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0927999999999997e-05 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490890+0000", 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4599999999999992e-06 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344328+0000", 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85343784700000003 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.152 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344457+0000", 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128563 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344619+0000", 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016246899999999999 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349443+0000", 2023-09-23T21:22:00.153 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048236540000000001 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355307+0000", 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058641479999999996 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355337+0000", 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9658999999999998e-05 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.154 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.155 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355380+0000", 2023-09-23T21:22:00.155 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3547999999999997e-05 2023-09-23T21:22:00.155 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.155 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.155 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.156 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355414+0000", 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3769999999999997e-05 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29527 13.4 13:29cd74c5:::benchmark_data_smithi121_621349_object29526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.56851634200001, 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85053144899999999, 2023-09-23T21:22:00.157 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29527 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.158 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505012+0000", 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.159 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505051+0000", 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8639000000000001e-05 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505066+0000", 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5812999999999999e-05 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.160 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505073+0000", 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4250000000000003e-06 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505137+0000", 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4097999999999995e-05 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505144+0000", 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2559999999999996e-06 2023-09-23T21:22:00.161 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345347+0000", 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84020272100000004 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345451+0000", 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103905 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.162 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345554+0000", 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103387 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349453+0000", 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003898641 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355452+0000", 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059990449999999997 2023-09-23T21:22:00.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355477+0000", 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5531999999999999e-05 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355516+0000", 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8699000000000003e-05 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.164 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355545+0000", 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8785999999999998e-05 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29536 13.4 13:2bfcf8e1:::benchmark_data_smithi121_621349_object29535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.56330009499999, 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84544850299999996, 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.165 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29536 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.166 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510216+0000", 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999866 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.167 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510277+0000", 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0891999999999999e-05 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510295+0000", 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7482999999999998e-05 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510301+0000", 2023-09-23T21:22:00.168 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4749999999999998e-06 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510359+0000", 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7927000000000003e-05 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510368+0000", 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9919999999999996e-06 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.169 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346282+0000", 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83591370700000001 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346367+0000", 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4648999999999995e-05 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.170 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346467+0000", 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100895 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351395+0000", 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0049271159999999996 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355574+0000", 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041792699999999997 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.171 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355598+0000", 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4198000000000001e-05 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355645+0000", 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6449999999999997e-05 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.172 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355678+0000", 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3657e-05 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29540 13.4 13:221f9937:::benchmark_data_smithi121_621349_object29539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.562044797, 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84431284100000004, 2023-09-23T21:22:00.173 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29540 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.174 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511484+0000", 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.175 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511579+0000", 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5329999999999997e-05 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511591+0000", 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.145e-05 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511606+0000", 2023-09-23T21:22:00.176 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5325e-05 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511668+0000", 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1458999999999996e-05 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511677+0000", 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0380000000000005e-06 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.177 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347329+0000", 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.835651904 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347442+0000", 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011388899999999999 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.178 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347532+0000", 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9306000000000002e-05 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351403+0000", 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038715400000000001 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355708+0000", 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004304446 2023-09-23T21:22:00.179 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355732+0000", 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4731000000000001e-05 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355770+0000", 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7104000000000003e-05 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.180 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355798+0000", 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8242000000000002e-05 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29542 13.4 13:25d6cd22:::benchmark_data_smithi121_621349_object29541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.55993480799998, 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.842315747, 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.181 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29542 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.182 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513594+0000", 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.183 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513646+0000", 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1647999999999997e-05 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513656+0000", 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0375e-05 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513663+0000", 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6549999999999998e-06 2023-09-23T21:22:00.184 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513727+0000", 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3992999999999995e-05 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513736+0000", 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2660000000000007e-06 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.185 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348465+0000", 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83472863200000003 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348553+0000", 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8218000000000001e-05 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.186 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348636+0000", 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3065000000000005e-05 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352933+0000", 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042976840000000004 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355826+0000", 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028929429999999998 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.187 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355851+0000", 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4511e-05 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355883+0000", 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2462000000000002e-05 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.188 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355911+0000", 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7293000000000001e-05 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29561 13.4 13:2d5ffecf:::benchmark_data_smithi121_621349_object29560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.54545553999998, 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82794979099999999, 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.189 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29561 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.190 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528073+0000", 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.191 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528107+0000", 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4207e-05 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528127+0000", 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9539999999999999e-05 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528133+0000", 2023-09-23T21:22:00.192 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4559999999999997e-06 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528211+0000", 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8057000000000002e-05 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528224+0000", 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2646000000000001e-05 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.193 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349494+0000", 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82127059199999997 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349617+0000", 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012300200000000001 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.194 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349779+0000", 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016171800000000001 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352944+0000", 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031647020000000001 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355939+0000", 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029952239999999999 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.195 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355964+0000", 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4788999999999999e-05 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355997+0000", 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3025999999999999e-05 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.196 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356024+0000", 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7231999999999999e-05 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29582 13.4 13:29c0d8e7:::benchmark_data_smithi121_621349_object29581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.525348941, 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80795742199999998, 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.197 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29582 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.198 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548179+0000", 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.199 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548222+0000", 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2599999999999999e-05 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548233+0000", 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1537e-05 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548238+0000", 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7700000000000001e-06 2023-09-23T21:22:00.200 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548305+0000", 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6441999999999998e-05 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548314+0000", 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0410000000000003e-06 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.201 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350381+0000", 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80206683400000001 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350462+0000", 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1507000000000004e-05 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350589+0000", 2023-09-23T21:22:00.202 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012726299999999999 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353762+0000", 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031722870000000002 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356053+0000", 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022911889999999999 2023-09-23T21:22:00.203 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356078+0000", 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4754999999999999e-05 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356110+0000", 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2641999999999997e-05 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.204 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356138+0000", 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8017e-05 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29585 13.4 13:234913ee:::benchmark_data_smithi121_621349_object29584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.52455411199998, 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80729236000000004, 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.205 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29585 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.206 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548975+0000", 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.207 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549041+0000", 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6148000000000004e-05 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549057+0000", 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5491000000000001e-05 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549064+0000", 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8620000000000003e-06 2023-09-23T21:22:00.208 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549105+0000", 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.104e-05 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549112+0000", 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2879999999999998e-06 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.209 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351443+0000", 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80233120000000002 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351547+0000", 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103696 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351687+0000", 2023-09-23T21:22:00.210 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140529 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353772+0000", 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020847539999999999 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356167+0000", 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023954800000000002 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.211 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356196+0000", 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8331999999999999e-05 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356229+0000", 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3577000000000001e-05 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356268+0000", 2023-09-23T21:22:00.212 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8692000000000002e-05 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29594 13.4 13:299d3c93:::benchmark_data_smithi121_621349_object29593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 442.51325641699998, 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.79612242, 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.213 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29594 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.214 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560272+0000", 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560317+0000", 2023-09-23T21:22:00.215 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4302999999999999e-05 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560329+0000", 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2690999999999999e-05 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560334+0000", 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0429999999999998e-06 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.216 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560389+0000", 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4363000000000001e-05 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560395+0000", 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6510000000000004e-06 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352241+0000", 2023-09-23T21:22:00.217 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.79184583500000005 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352323+0000", 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1592000000000001e-05 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352403+0000", 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0591000000000001e-05 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.218 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353852+0000", 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014487339999999999 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356303+0000", 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024512969999999999 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.219 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356331+0000", 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7708e-05 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356366+0000", 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5049999999999998e-05 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356396+0000", 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9581999999999999e-05 2023-09-23T21:22:00.220 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82785 14.d 14.705d43ed (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.093456+0000", 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 130.98007420900001, 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84462227700000003, 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.221 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82785 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093456+0000", 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093456+0000", 2023-09-23T21:22:00.222 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093452+0000", 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999967 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093496+0000", 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3893000000000001e-05 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.223 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093515+0000", 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9273e-05 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093524+0000", 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9099999999999994e-06 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093575+0000", 2023-09-23T21:22:00.224 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0584999999999998e-05 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093589+0000", 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3782e-05 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.938071+0000", 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84448238600000003 2023-09-23T21:22:00.225 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.938078+0000", 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8669999999999996e-06 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100286 14.4 14:26363854:::benchmark_data_smithi121_684892_object14969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.555543+0000", 2023-09-23T21:22:00.226 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 89.517987120000001, 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.651170026, 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100286 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:00.227 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555543+0000", 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555543+0000", 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555541+0000", 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.228 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555548+0000", 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7730000000000002e-06 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555563+0000", 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5026e-05 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555571+0000", 2023-09-23T21:22:00.229 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1629999999999996e-06 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555633+0000", 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2039999999999996e-05 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555644+0000", 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0987e-05 2023-09-23T21:22:00.230 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.486375+0000", 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.930731062 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.486385+0000", 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8640000000000008e-06 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.231 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.183175+0000", 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.69678952400000005 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.183809+0000", 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00063464999999999995 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:00.232 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.191729+0000", 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079193400000000004 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.191848+0000", 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118924 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.192006+0000", 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000158794 2023-09-23T21:22:00.233 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.197596+0000", 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005589152 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.198336+0000", 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00074035799999999999 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.234 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.201513+0000", 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031775050000000002 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.203954+0000", 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024403469999999998 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:00.235 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.204009+0000", 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5402999999999998e-05 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.206713+0000", 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027034440000000002 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:00.236 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:22:00.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:00 smithi121 ceph-mon[219709]: pgmap v1962: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 18883/100848 objects degraded (18.724%); 21088/100848 objects misplaced (20.911%); 12 MiB/s, 187 objects/s recovering 2023-09-23T21:22:00.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:00 smithi121 ceph-mon[211526]: pgmap v1962: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 18883/100848 objects degraded (18.724%); 21088/100848 objects misplaced (20.911%); 12 MiB/s, 187 objects/s recovering 2023-09-23T21:22:00.967 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:22:00.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:00 smithi123 ceph-mon[193903]: pgmap v1962: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 18883/100848 objects degraded (18.724%); 21088/100848 objects misplaced (20.911%); 12 MiB/s, 187 objects/s recovering 2023-09-23T21:22:01.224 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:22:01.250 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:01.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:01.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:01.355 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:01.399 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:01.467 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:01.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:01.516 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:01 smithi121 ceph-mon[219709]: pgmap v1963: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 18883/100848 objects degraded (18.724%); 21088/100848 objects misplaced (20.911%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:22:01.574 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:01.693 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:01.772 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:01 smithi121 ceph-mon[211526]: pgmap v1963: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 18883/100848 objects degraded (18.724%); 21088/100848 objects misplaced (20.911%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:22:01.857 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:01.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:01.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:01 smithi123 ceph-mon[193903]: pgmap v1963: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 469 op/s; 18883/100848 objects degraded (18.724%); 21088/100848 objects misplaced (20.911%); 10 MiB/s, 167 objects/s recovering 2023-09-23T21:22:01.890 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:01.974 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:01.993 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:22:02.034 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:02.089 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:22:02.112 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:22:02.223 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:02.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:02.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:02.330 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:02.420 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:02.571 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 18883/100848 objects degraded (18.724%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:02.571 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:02 smithi123 ceph-mon[193903]: from='client.37842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:02.572 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/602355750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:22:02.638 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:02.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:02.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:02.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 18883/100848 objects degraded (18.724%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:02.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:02 smithi121 ceph-mon[219709]: from='client.37842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:02.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/602355750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:22:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 18883/100848 objects degraded (18.724%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:02 smithi121 ceph-mon[211526]: from='client.37842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:02.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/602355750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:22:02.749 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:02.818 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:02.919 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:02.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:02.946 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:22:03.025 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:03.096 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:03.220 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:03.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:03.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:03.336 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:03.398 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:03.439 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:03.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:03.543 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:03.570 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:03.629 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:03.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:03.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:03.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[211526]: failed to encode map e650 with expected crc 2023-09-23T21:22:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/602355750' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:22:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[211526]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T21:22:03.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[211526]: pgmap v1965: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 18806/99579 objects degraded (18.886%); 20664/99579 objects misplaced (20.751%); 7.8 MiB/s, 138 objects/s recovering 2023-09-23T21:22:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[219709]: failed to encode map e650 with expected crc 2023-09-23T21:22:03.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/602355750' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:22:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[219709]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T21:22:03.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:03 smithi121 ceph-mon[219709]: pgmap v1965: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 18806/99579 objects degraded (18.886%); 20664/99579 objects misplaced (20.751%); 7.8 MiB/s, 138 objects/s recovering 2023-09-23T21:22:03.751 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:03.798 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:03.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:03 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:03.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:03 smithi123 ceph-mon[193903]: failed to encode map e650 with expected crc 2023-09-23T21:22:03.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/602355750' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:22:03.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:03 smithi123 ceph-mon[193903]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T21:22:03.982 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:03 smithi123 ceph-mon[193903]: pgmap v1965: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 18806/99579 objects degraded (18.886%); 20664/99579 objects misplaced (20.751%); 7.8 MiB/s, 138 objects/s recovering 2023-09-23T21:22:04.028 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:04.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:04.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:04.132 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:04.162 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:04.345 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:04.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:04.388 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1963,"stamp":"2023-09-23T21:22:01.429074+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2202123011,"num_objects":33616,"num_object_clones":0,"num_object_copies":100848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18883,"num_objects_misplaced":21088,"num_objects_unfound":0,"num_objects_dirty":33616,"num_whiteouts":0,"num_read":92,"num_read_kb":169,"num_write":137142,"num_write_kb":5460565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23327,"num_bytes_recovered":1430992229,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85578,"ondisk_log_size":85578,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14663044,"kb_used_data":8182860,"kb_used_omap":16,"kb_used_meta":6479855,"kb_avail":641553020,"statfs":{"total":671965249536,"available":656950292480,"internally_reserved":0,"allocated":8379248640,"data_stored":8362238696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6635372393},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-308150272,"num_objects":-4702,"num_object_clones":0,"num_object_copies":-14106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1166,"num_objects_misplaced":-4255,"num_objects_unfound":0,"num_objects_dirty":-4702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4702,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1680,"num_bytes_recovered":110100480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008136"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":224,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.508080+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:21:50.508080+0000","last_peered":"2023-09-23T21:21:50.508080+0000","last_clean":"2023-09-23T21:21:50.508080+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:21:50.508080+0000","last_undegraded":"2023-09-23T21:21:50.508080+0000","last_fullsized":"2023-09-23T21:21:50.508080+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.488668+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:21:50.488668+0000","last_peered":"2023-09-23T21:21:50.488668+0000","last_clean":"2023-09-23T21:21:50.488668+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:21:50.488668+0000","last_undegraded":"2023-09-23T21:21:50.488668+0000","last_fullsized":"2023-09-23T21:21:50.488668+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.498136+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:21:50.498136+0000","last_peered":"2023-09-23T21:21:50.498136+0000","last_clean":"2023-09-23T21:21:50.498136+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:21:50.498136+0000","last_undegraded":"2023-09-23T21:21:50.498136+0000","last_fullsized":"2023-09-23T21:21:50.498136+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":336,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.492930+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:21:50.492930+0000","last_peered":"2023-09-23T21:21:50.492930+0000","last_clean":"2023-09-23T21:21:50.492930+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:21:50.492930+0000","last_undegraded":"2023-09-23T21:21:50.492930+0000","last_fullsized":"2023-09-23T21:21:50.492930+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":52,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.486731+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:21:50.486731+0000","last_peered":"2023-09-23T21:21:50.486731+0000","last_clean":"2023-09-23T21:21:50.486731+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:21:50.486731+0000","last_undegraded":"2023-09-23T21:21:50.486731+0000","last_fullsized":"2023-09-23T21:21:50.486731+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":154,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.517217+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:21:50.517217+0000","last_peered":"2023-09-23T21:21:50.517217+0000","last_clean":"2023-09-23T21:21:50.517217+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:21:50.517217+0000","last_undegraded":"2023-09-23T21:21:50.517217+0000","last_fullsized":"2023-09-23T21:21:50.517217+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":56,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.503157+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:21:50.503157+0000","last_peered":"2023-09-23T21:21:50.503157+0000","last_clean":"2023-09-23T21:21:50.503157+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:21:50.503157+0000","last_undegraded":"2023-09-23T21:21:50.503157+0000","last_fullsized":"2023-09-23T21:21:50.503157+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":70,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.486798+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:21:50.486798+0000","last_peered":"2023-09-23T21:21:50.486798+0000","last_clean":"2023-09-23T21:21:50.486798+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:21:50.486798+0000","last_undegraded":"2023-09-23T21:21:50.486798+0000","last_fullsized":"2023-09-23T21:21:50.486798+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":46,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.502052+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:21:50.502052+0000","last_peered":"2023-09-23T21:21:50.502052+0000","last_clean":"2023-09-23T21:21:50.502052+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:21:50.502052+0000","last_undegraded":"2023-09-23T21:21:50.502052+0000","last_fullsized":"2023-09-23T21:21:50.502052+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":43,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.489221+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:21:50.489221+0000","last_peered":"2023-09-23T21:21:50.489221+0000","last_clean":"2023-09-23T21:21:50.489221+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:21:50.489221+0000","last_undegraded":"2023-09-23T21:21:50.489221+0000","last_fullsized":"2023-09-23T21:21:50.489221+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":284,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.493408+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:21:50.493408+0000","last_peered":"2023-09-23T21:21:50.493408+0000","last_clean":"2023-09-23T21:21:50.493408+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:21:50.493408+0000","last_undegraded":"2023-09-23T21:21:50.493408+0000","last_fullsized":"2023-09-23T21:21:50.493408+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.498412+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:21:50.498412+0000","last_peered":"2023-09-23T21:21:50.498412+0000","last_clean":"2023-09-23T21:21:50.498412+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:21:50.498412+0000","last_undegraded":"2023-09-23T21:21:50.498412+0000","last_fullsized":"2023-09-23T21:21:50.498412+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":380,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.503507+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:21:50.503507+0000","last_peered":"2023-09-23T21:21:50.503507+0000","last_clean":"2023-09-23T21:21:50.503507+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:21:50.503507+0000","last_undegraded":"2023-09-23T21:21:50.503507+0000","last_fullsized":"2023-09-23T21:21:50.503507+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":607,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.504507+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:21:50.504507+0000","last_peered":"2023-09-23T21:21:50.504507+0000","last_clean":"2023-09-23T21:21:50.504507+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:21:50.504507+0000","last_undegraded":"2023-09-23T21:21:50.504507+0000","last_fullsized":"2023-09-23T21:21:50.504507+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":232,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.496684+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:21:50.496684+0000","last_peered":"2023-09-23T21:21:50.496684+0000","last_clean":"2023-09-23T21:21:50.496684+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:21:50.496684+0000","last_undegraded":"2023-09-23T21:21:50.496684+0000","last_fullsized":"2023-09-23T21:21:50.496684+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":52,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.486972+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:21:50.486972+0000","last_peered":"2023-09-23T21:21:50.486972+0000","last_clean":"2023-09-23T21:21:50.486972+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:21:50.486972+0000","last_undegraded":"2023-09-23T21:21:50.486972+0000","last_fullsized":"2023-09-23T21:21:50.486972+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":258,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.493131+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:21:50.493131+0000","last_peered":"2023-09-23T21:21:50.493131+0000","last_clean":"2023-09-23T21:21:50.493131+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:21:50.493131+0000","last_undegraded":"2023-09-23T21:21:50.493131+0000","last_fullsized":"2023-09-23T21:21:50.493131+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.493617+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:21:50.493617+0000","last_peered":"2023-09-23T21:21:50.493617+0000","last_clean":"2023-09-23T21:21:50.493617+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:21:50.493617+0000","last_undegraded":"2023-09-23T21:21:50.493617+0000","last_fullsized":"2023-09-23T21:21:50.493617+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":226,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.492571+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:21:50.492571+0000","last_peered":"2023-09-23T21:21:50.492571+0000","last_clean":"2023-09-23T21:21:50.492571+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:21:50.492571+0000","last_undegraded":"2023-09-23T21:21:50.492571+0000","last_fullsized":"2023-09-23T21:21:50.492571+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":43,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.488234+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:21:50.488234+0000","last_peered":"2023-09-23T21:21:50.488234+0000","last_clean":"2023-09-23T21:21:50.488234+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:21:50.488234+0000","last_undegraded":"2023-09-23T21:21:50.488234+0000","last_fullsized":"2023-09-23T21:21:50.488234+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":745,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.509152+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:21:50.509152+0000","last_peered":"2023-09-23T21:21:50.509152+0000","last_clean":"2023-09-23T21:21:50.509152+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:21:50.509152+0000","last_undegraded":"2023-09-23T21:21:50.509152+0000","last_fullsized":"2023-09-23T21:21:50.509152+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":52,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.487284+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:21:50.487284+0000","last_peered":"2023-09-23T21:21:50.487284+0000","last_clean":"2023-09-23T21:21:50.487284+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:21:50.487284+0000","last_undegraded":"2023-09-23T21:21:50.487284+0000","last_fullsized":"2023-09-23T21:21:50.487284+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"649'8395","reported_seq":10216,"reported_epoch":649,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:56.408135+0000","last_change":"2023-09-23T21:21:01.215269+0000","last_active":"2023-09-23T21:21:56.408135+0000","last_peered":"2023-09-23T21:21:56.408135+0000","last_clean":"2023-09-23T21:19:28.877182+0000","last_became_active":"2023-09-23T21:21:01.190149+0000","last_became_peered":"2023-09-23T21:21:01.190149+0000","last_unstale":"2023-09-23T21:21:56.408135+0000","last_undegraded":"2023-09-23T21:21:00.607176+0000","last_fullsized":"2023-09-23T21:21:56.408135+0000","mapping_epoch":629,"log_start":"583'2867","ondisk_log_start":"583'2867","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5528,"log_dups_size":0,"ondisk_log_size":5528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:05.989327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137035776,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2038,"num_objects_misplaced":2144,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8395,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":844,"num_bytes_recovered":55312384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,4,5],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,5,6","objects":1411},{"shards":"4,6,7","objects":680}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":286,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.495225+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:21:50.495225+0000","last_peered":"2023-09-23T21:21:50.495225+0000","last_clean":"2023-09-23T21:21:50.495225+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:21:50.495225+0000","last_undegraded":"2023-09-23T21:21:50.495225+0000","last_fullsized":"2023-09-23T21:21:50.495225+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":46,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.501370+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:21:50.501370+0000","last_peered":"2023-09-23T21:21:50.501370+0000","last_clean":"2023-09-23T21:21:50.501370+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:21:50.501370+0000","last_undegraded":"2023-09-23T21:21:50.501370+0000","last_fullsized":"2023-09-23T21:21:50.501370+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"649'8650","reported_seq":10672,"reported_epoch":649,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:59.036108+0000","last_change":"2023-09-23T21:21:01.210342+0000","last_active":"2023-09-23T21:21:59.036108+0000","last_peered":"2023-09-23T21:21:59.036108+0000","last_clean":"2023-09-23T21:19:28.923531+0000","last_became_active":"2023-09-23T21:21:01.189847+0000","last_became_peered":"2023-09-23T21:21:01.189847+0000","last_unstale":"2023-09-23T21:21:59.036108+0000","last_undegraded":"2023-09-23T21:21:00.610088+0000","last_fullsized":"2023-09-23T21:21:59.036108+0000","mapping_epoch":629,"log_start":"589'3598","ondisk_log_start":"589'3598","created":573,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5052,"log_dups_size":0,"ondisk_log_size":5052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:58.686882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139067392,"num_objects":2122,"num_object_clones":0,"num_object_copies":6366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2000,"num_objects_misplaced":2244,"num_objects_unfound":0,"num_objects_dirty":2122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8650,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":938,"num_bytes_recovered":61341696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[4,6,5],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":1396},{"shards":"4,6,7","objects":726}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":300,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.493015+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:21:50.493015+0000","last_peered":"2023-09-23T21:21:50.493015+0000","last_clean":"2023-09-23T21:21:50.493015+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:21:50.493015+0000","last_undegraded":"2023-09-23T21:21:50.493015+0000","last_fullsized":"2023-09-23T21:21:50.493015+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.498122+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:21:50.498122+0000","last_peered":"2023-09-23T21:21:50.498122+0000","last_clean":"2023-09-23T21:21:50.498122+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:21:50.498122+0000","last_undegraded":"2023-09-23T21:21:50.498122+0000","last_fullsized":"2023-09-23T21:21:50.498122+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"649'8460","reported_seq":13692,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:58.481283+0000","last_change":"2023-09-23T21:21:17.539683+0000","last_active":"2023-09-23T21:21:58.481283+0000","last_peered":"2023-09-23T21:21:58.481283+0000","last_clean":"2023-09-23T21:21:58.481283+0000","last_became_active":"2023-09-23T21:21:17.521633+0000","last_became_peered":"2023-09-23T21:21:17.521633+0000","last_unstale":"2023-09-23T21:21:58.481283+0000","last_undegraded":"2023-09-23T21:21:58.481283+0000","last_fullsized":"2023-09-23T21:21:58.481283+0000","mapping_epoch":636,"log_start":"589'3465","ondisk_log_start":"589'3465","created":573,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4995,"log_dups_size":0,"ondisk_log_size":4995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:05.844201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131596288,"num_objects":2008,"num_object_clones":0,"num_object_copies":6024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8460,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2555,"num_bytes_recovered":80478208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":272,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.493733+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:21:50.493733+0000","last_peered":"2023-09-23T21:21:50.493733+0000","last_clean":"2023-09-23T21:21:50.493733+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:21:50.493733+0000","last_undegraded":"2023-09-23T21:21:50.493733+0000","last_fullsized":"2023-09-23T21:21:50.493733+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":20,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.509463+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:21:50.509463+0000","last_peered":"2023-09-23T21:21:50.509463+0000","last_clean":"2023-09-23T21:21:50.509463+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:21:50.509463+0000","last_undegraded":"2023-09-23T21:21:50.509463+0000","last_fullsized":"2023-09-23T21:21:50.509463+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"649'8483","reported_seq":8584,"reported_epoch":649,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:21:58.527100+0000","last_change":"2023-09-23T21:20:59.644218+0000","last_active":"2023-09-23T21:21:58.527100+0000","last_peered":"2023-09-23T21:21:58.527100+0000","last_clean":"2023-09-23T21:20:30.430127+0000","last_became_active":"2023-09-23T21:20:59.632855+0000","last_became_peered":"2023-09-23T21:20:59.632855+0000","last_unstale":"2023-09-23T21:21:58.527100+0000","last_undegraded":"2023-09-23T21:21:58.527100+0000","last_fullsized":"2023-09-23T21:21:58.527100+0000","mapping_epoch":628,"log_start":"589'3525","ondisk_log_start":"589'3525","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4958,"log_dups_size":0,"ondisk_log_size":4958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:33.014367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138608640,"num_objects":2115,"num_object_clones":0,"num_object_copies":6345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4230,"num_objects_unfound":0,"num_objects_dirty":2115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8483,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2115}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":52,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.487046+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:21:50.487046+0000","last_peered":"2023-09-23T21:21:50.487046+0000","last_clean":"2023-09-23T21:21:50.487046+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:21:50.487046+0000","last_undegraded":"2023-09-23T21:21:50.487046+0000","last_fullsized":"2023-09-23T21:21:50.487046+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499095+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:21:50.499095+0000","last_peered":"2023-09-23T21:21:50.499095+0000","last_clean":"2023-09-23T21:21:50.499095+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:21:50.499095+0000","last_undegraded":"2023-09-23T21:21:50.499095+0000","last_fullsized":"2023-09-23T21:21:50.499095+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1075,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.492434+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:21:50.492434+0000","last_peered":"2023-09-23T21:21:50.492434+0000","last_clean":"2023-09-23T21:21:50.492434+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:21:50.492434+0000","last_undegraded":"2023-09-23T21:21:50.492434+0000","last_fullsized":"2023-09-23T21:21:50.492434+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.6","version":"649'8592","reported_seq":13299,"reported_epoch":649,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:58.571461+0000","last_change":"2023-09-23T21:21:01.239919+0000","last_active":"2023-09-23T21:21:58.571461+0000","last_peered":"2023-09-23T21:21:58.571461+0000","last_clean":"2023-09-23T21:19:28.802255+0000","last_became_active":"2023-09-23T21:21:01.197858+0000","last_became_peered":"2023-09-23T21:21:01.197858+0000","last_unstale":"2023-09-23T21:21:58.571461+0000","last_undegraded":"2023-09-23T21:21:00.673065+0000","last_fullsized":"2023-09-23T21:21:58.571461+0000","mapping_epoch":629,"log_start":"589'3598","ondisk_log_start":"589'3598","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4994,"log_dups_size":0,"ondisk_log_size":4994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:39.315049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139853824,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2134,"num_objects_misplaced":2134,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8592,"num_write_kb":343232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2296,"num_bytes_recovered":150405120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,4,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,4,6","objects":680},{"shards":"2,5,6","objects":1454}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":60,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.487471+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:21:50.487471+0000","last_peered":"2023-09-23T21:21:50.487471+0000","last_clean":"2023-09-23T21:21:50.487471+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:21:50.487471+0000","last_undegraded":"2023-09-23T21:21:50.487471+0000","last_fullsized":"2023-09-23T21:21:50.487471+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":407,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.504129+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:21:50.504129+0000","last_peered":"2023-09-23T21:21:50.504129+0000","last_clean":"2023-09-23T21:21:50.504129+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:21:50.504129+0000","last_undegraded":"2023-09-23T21:21:50.504129+0000","last_fullsized":"2023-09-23T21:21:50.504129+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":943,"reported_epoch":648,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:21:50.482232+0000","last_change":"2023-09-23T21:20:48.157415+0000","last_active":"2023-09-23T21:21:50.482232+0000","last_peered":"2023-09-23T21:21:50.482232+0000","last_clean":"2023-09-23T21:20:41.603469+0000","last_became_active":"2023-09-23T21:20:48.156886+0000","last_became_peered":"2023-09-23T21:20:48.156886+0000","last_unstale":"2023-09-23T21:21:50.482232+0000","last_undegraded":"2023-09-23T21:20:47.922781+0000","last_fullsized":"2023-09-23T21:20:47.921128+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:30.898986+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4,6","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.7","version":"649'8748","reported_seq":9951,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:58.828906+0000","last_change":"2023-09-23T21:20:59.594443+0000","last_active":"2023-09-23T21:21:58.828906+0000","last_peered":"2023-09-23T21:21:58.828906+0000","last_clean":"2023-09-23T21:21:58.828906+0000","last_became_active":"2023-09-23T21:20:48.149995+0000","last_became_peered":"2023-09-23T21:20:48.149995+0000","last_unstale":"2023-09-23T21:21:58.828906+0000","last_undegraded":"2023-09-23T21:21:58.828906+0000","last_fullsized":"2023-09-23T21:21:58.828906+0000","mapping_epoch":623,"log_start":"582'2665","ondisk_log_start":"582'2665","created":573,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":6083,"log_dups_size":0,"ondisk_log_size":6083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.278714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140509184,"num_objects":2144,"num_object_clones":0,"num_object_copies":6432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8748,"num_write_kb":348544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":144,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.512230+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:21:50.512230+0000","last_peered":"2023-09-23T21:21:50.512230+0000","last_clean":"2023-09-23T21:21:50.512230+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:21:50.512230+0000","last_undegraded":"2023-09-23T21:21:50.512230+0000","last_fullsized":"2023-09-23T21:21:50.512230+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":657,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.518326+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:21:50.518326+0000","last_peered":"2023-09-23T21:21:50.518326+0000","last_clean":"2023-09-23T21:21:50.518326+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:21:50.518326+0000","last_undegraded":"2023-09-23T21:21:50.518326+0000","last_fullsized":"2023-09-23T21:21:50.518326+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"649'8381","reported_seq":15248,"reported_epoch":649,"state":"active+recovering+remapped","last_fresh":"2023-09-23T21:21:56.436438+0000","last_change":"2023-09-23T21:20:59.612076+0000","last_active":"2023-09-23T21:21:56.436438+0000","last_peered":"2023-09-23T21:21:56.436438+0000","last_clean":"2023-09-23T21:19:28.911487+0000","last_became_active":"2023-09-23T21:20:32.169326+0000","last_became_peered":"2023-09-23T21:20:32.169326+0000","last_unstale":"2023-09-23T21:21:56.436438+0000","last_undegraded":"2023-09-23T21:21:56.436438+0000","last_fullsized":"2023-09-23T21:21:56.436438+0000","mapping_epoch":617,"log_start":"583'2868","ondisk_log_start":"583'2868","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:09.080183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137035776,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":241,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8381,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3379,"num_bytes_recovered":221446144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,4],"avail_no_missing":["6","5","7"],"object_location_counts":[{"shards":"4,6,7","objects":1850},{"shards":"5,6,7","objects":241}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":286,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.507683+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:21:50.507683+0000","last_peered":"2023-09-23T21:21:50.507683+0000","last_clean":"2023-09-23T21:21:50.507683+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:21:50.507683+0000","last_undegraded":"2023-09-23T21:21:50.507683+0000","last_fullsized":"2023-09-23T21:21:50.507683+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":694,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.506693+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:21:50.506693+0000","last_peered":"2023-09-23T21:21:50.506693+0000","last_clean":"2023-09-23T21:21:50.506693+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:21:50.506693+0000","last_undegraded":"2023-09-23T21:21:50.506693+0000","last_fullsized":"2023-09-23T21:21:50.506693+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":912,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.482758+0000","last_change":"2023-09-23T21:20:49.968107+0000","last_active":"2023-09-23T21:21:50.482758+0000","last_peered":"2023-09-23T21:21:50.482758+0000","last_clean":"2023-09-23T21:21:50.482758+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:21:50.482758+0000","last_undegraded":"2023-09-23T21:21:50.482758+0000","last_fullsized":"2023-09-23T21:21:50.482758+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":429358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.5","version":"649'8800","reported_seq":15476,"reported_epoch":649,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:59.100933+0000","last_change":"2023-09-23T21:20:59.554347+0000","last_active":"2023-09-23T21:21:59.100933+0000","last_peered":"2023-09-23T21:21:59.100933+0000","last_clean":"2023-09-23T21:19:28.864495+0000","last_became_active":"2023-09-23T21:20:32.158375+0000","last_became_peered":"2023-09-23T21:20:32.158375+0000","last_unstale":"2023-09-23T21:21:59.100933+0000","last_undegraded":"2023-09-23T21:20:59.551954+0000","last_fullsized":"2023-09-23T21:21:59.100933+0000","mapping_epoch":617,"log_start":"589'3792","ondisk_log_start":"589'3792","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5008,"log_dups_size":0,"ondisk_log_size":5008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:46.553959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141557760,"num_objects":2160,"num_object_clones":0,"num_object_copies":6480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1447,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":2160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8800,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3283,"num_bytes_recovered":215154688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"2,4,7","objects":750},{"shards":"4,5,7","objects":1410}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":52,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.486580+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:21:50.486580+0000","last_peered":"2023-09-23T21:21:50.486580+0000","last_clean":"2023-09-23T21:21:50.486580+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:21:50.486580+0000","last_undegraded":"2023-09-23T21:21:50.486580+0000","last_fullsized":"2023-09-23T21:21:50.486580+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499460+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:21:50.499460+0000","last_peered":"2023-09-23T21:21:50.499460+0000","last_clean":"2023-09-23T21:21:50.499460+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:21:50.499460+0000","last_undegraded":"2023-09-23T21:21:50.499460+0000","last_fullsized":"2023-09-23T21:21:50.499460+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":731,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.517079+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:21:50.517079+0000","last_peered":"2023-09-23T21:21:50.517079+0000","last_clean":"2023-09-23T21:21:50.517079+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:21:50.517079+0000","last_undegraded":"2023-09-23T21:21:50.517079+0000","last_fullsized":"2023-09-23T21:21:50.517079+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"649'8552","reported_seq":9403,"reported_epoch":649,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:58.564489+0000","last_change":"2023-09-23T21:21:01.233673+0000","last_active":"2023-09-23T21:21:58.564489+0000","last_peered":"2023-09-23T21:21:58.564489+0000","last_clean":"2023-09-23T21:19:28.898608+0000","last_became_active":"2023-09-23T21:21:01.195228+0000","last_became_peered":"2023-09-23T21:21:01.195228+0000","last_unstale":"2023-09-23T21:21:58.564489+0000","last_undegraded":"2023-09-23T21:21:00.570455+0000","last_fullsized":"2023-09-23T21:21:58.564489+0000","mapping_epoch":629,"log_start":"589'3546","ondisk_log_start":"589'3546","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:03.164901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135921664,"num_objects":2074,"num_object_clones":0,"num_object_copies":6222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2032,"num_objects_misplaced":2116,"num_objects_unfound":0,"num_objects_dirty":2074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8552,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[2,7,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1357},{"shards":"2,6,7","objects":717}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499233+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:21:50.499233+0000","last_peered":"2023-09-23T21:21:50.499233+0000","last_clean":"2023-09-23T21:21:50.499233+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:21:50.499233+0000","last_undegraded":"2023-09-23T21:21:50.499233+0000","last_fullsized":"2023-09-23T21:21:50.499233+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":36,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.502348+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:21:50.502348+0000","last_peered":"2023-09-23T21:21:50.502348+0000","last_clean":"2023-09-23T21:21:50.502348+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:21:50.502348+0000","last_undegraded":"2023-09-23T21:21:50.502348+0000","last_fullsized":"2023-09-23T21:21:50.502348+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.498482+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:21:50.498482+0000","last_peered":"2023-09-23T21:21:50.498482+0000","last_clean":"2023-09-23T21:21:50.498482+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:21:50.498482+0000","last_undegraded":"2023-09-23T21:21:50.498482+0000","last_fullsized":"2023-09-23T21:21:50.498482+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":924,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.482364+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:21:50.482364+0000","last_peered":"2023-09-23T21:21:50.482364+0000","last_clean":"2023-09-23T21:21:50.482364+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:21:50.482364+0000","last_undegraded":"2023-09-23T21:21:50.482364+0000","last_fullsized":"2023-09-23T21:21:50.482364+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"649'8500","reported_seq":8583,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:59.046962+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:21:59.046962+0000","last_peered":"2023-09-23T21:21:59.046962+0000","last_clean":"2023-09-23T21:21:59.046962+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:21:59.046962+0000","last_undegraded":"2023-09-23T21:21:59.046962+0000","last_fullsized":"2023-09-23T21:21:59.046962+0000","mapping_epoch":573,"log_start":"589'3487","ondisk_log_start":"589'3487","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5013,"log_dups_size":0,"ondisk_log_size":5013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133824512,"num_objects":2042,"num_object_clones":0,"num_object_copies":6126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8500,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.498653+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:21:50.498653+0000","last_peered":"2023-09-23T21:21:50.498653+0000","last_clean":"2023-09-23T21:21:50.498653+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:21:50.498653+0000","last_undegraded":"2023-09-23T21:21:50.498653+0000","last_fullsized":"2023-09-23T21:21:50.498653+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":562,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.504367+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:21:50.504367+0000","last_peered":"2023-09-23T21:21:50.504367+0000","last_clean":"2023-09-23T21:21:50.504367+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:21:50.504367+0000","last_undegraded":"2023-09-23T21:21:50.504367+0000","last_fullsized":"2023-09-23T21:21:50.504367+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499165+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:21:50.499165+0000","last_peered":"2023-09-23T21:21:50.499165+0000","last_clean":"2023-09-23T21:21:50.499165+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:21:50.499165+0000","last_undegraded":"2023-09-23T21:21:50.499165+0000","last_fullsized":"2023-09-23T21:21:50.499165+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1035,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.493059+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:21:50.493059+0000","last_peered":"2023-09-23T21:21:50.493059+0000","last_clean":"2023-09-23T21:21:50.493059+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:21:50.493059+0000","last_undegraded":"2023-09-23T21:21:50.493059+0000","last_fullsized":"2023-09-23T21:21:50.493059+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.d","version":"649'8494","reported_seq":8653,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:56.379751+0000","last_change":"2023-09-23T21:21:22.585504+0000","last_active":"2023-09-23T21:21:56.379751+0000","last_peered":"2023-09-23T21:21:56.379751+0000","last_clean":"2023-09-23T21:21:56.379751+0000","last_became_active":"2023-09-23T21:21:22.568285+0000","last_became_peered":"2023-09-23T21:21:22.568285+0000","last_unstale":"2023-09-23T21:21:56.379751+0000","last_undegraded":"2023-09-23T21:21:56.379751+0000","last_fullsized":"2023-09-23T21:21:56.379751+0000","mapping_epoch":639,"log_start":"584'2948","ondisk_log_start":"584'2948","created":573,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5546,"log_dups_size":0,"ondisk_log_size":5546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:01.251062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142540824,"num_objects":2176,"num_object_clones":0,"num_object_copies":6528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2176,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8494,"num_write_kb":341377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":678,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.482511+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:21:50.482511+0000","last_peered":"2023-09-23T21:21:50.482511+0000","last_clean":"2023-09-23T21:21:50.482511+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:21:50.482511+0000","last_undegraded":"2023-09-23T21:21:50.482511+0000","last_fullsized":"2023-09-23T21:21:50.482511+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499134+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:21:50.499134+0000","last_peered":"2023-09-23T21:21:50.499134+0000","last_clean":"2023-09-23T21:21:50.499134+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:21:50.499134+0000","last_undegraded":"2023-09-23T21:21:50.499134+0000","last_fullsized":"2023-09-23T21:21:50.499134+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499387+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:21:50.499387+0000","last_peered":"2023-09-23T21:21:50.499387+0000","last_clean":"2023-09-23T21:21:50.499387+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:21:50.499387+0000","last_undegraded":"2023-09-23T21:21:50.499387+0000","last_fullsized":"2023-09-23T21:21:50.499387+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"649'8617","reported_seq":11517,"reported_epoch":649,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:58.524341+0000","last_change":"2023-09-23T21:21:01.214995+0000","last_active":"2023-09-23T21:21:58.524341+0000","last_peered":"2023-09-23T21:21:58.524341+0000","last_clean":"2023-09-23T21:19:28.902983+0000","last_became_active":"2023-09-23T21:21:01.189928+0000","last_became_peered":"2023-09-23T21:21:01.189928+0000","last_unstale":"2023-09-23T21:21:58.524341+0000","last_undegraded":"2023-09-23T21:21:00.625306+0000","last_fullsized":"2023-09-23T21:21:58.524341+0000","mapping_epoch":629,"log_start":"582'2686","ondisk_log_start":"582'2686","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:45.309159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135200768,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2063,"num_objects_misplaced":2063,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8617,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1379,"num_bytes_recovered":90374144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,2,5],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,5,7","objects":1333},{"shards":"2,6,7","objects":730}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":507,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.560013+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:21:54.560013+0000","last_peered":"2023-09-23T21:21:54.560013+0000","last_clean":"2023-09-23T21:21:54.560013+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:21:54.560013+0000","last_undegraded":"2023-09-23T21:21:54.560013+0000","last_fullsized":"2023-09-23T21:21:54.560013+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1036,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.492343+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:21:50.492343+0000","last_peered":"2023-09-23T21:21:50.492343+0000","last_clean":"2023-09-23T21:21:50.492343+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:21:50.492343+0000","last_undegraded":"2023-09-23T21:21:50.492343+0000","last_fullsized":"2023-09-23T21:21:50.492343+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.501426+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:21:50.501426+0000","last_peered":"2023-09-23T21:21:50.501426+0000","last_clean":"2023-09-23T21:21:50.501426+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:21:50.501426+0000","last_undegraded":"2023-09-23T21:21:50.501426+0000","last_fullsized":"2023-09-23T21:21:50.501426+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":20,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.516864+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:21:50.516864+0000","last_peered":"2023-09-23T21:21:50.516864+0000","last_clean":"2023-09-23T21:21:50.516864+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:21:50.516864+0000","last_undegraded":"2023-09-23T21:21:50.516864+0000","last_fullsized":"2023-09-23T21:21:50.516864+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":683,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499177+0000","last_change":"2023-09-23T21:21:21.571375+0000","last_active":"2023-09-23T21:21:50.499177+0000","last_peered":"2023-09-23T21:21:50.499177+0000","last_clean":"2023-09-23T21:21:50.499177+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:21:50.499177+0000","last_undegraded":"2023-09-23T21:21:50.499177+0000","last_fullsized":"2023-09-23T21:21:50.499177+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.551943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.551943+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.c","version":"649'8687","reported_seq":8771,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:59.093321+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:21:59.093321+0000","last_peered":"2023-09-23T21:21:59.093321+0000","last_clean":"2023-09-23T21:21:59.093321+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:21:59.093321+0000","last_undegraded":"2023-09-23T21:21:59.093321+0000","last_fullsized":"2023-09-23T21:21:59.093321+0000","mapping_epoch":573,"log_start":"589'3623","ondisk_log_start":"589'3623","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138608640,"num_objects":2115,"num_object_clones":0,"num_object_copies":6345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8687,"num_write_kb":345664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":316,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.492413+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:21:50.492413+0000","last_peered":"2023-09-23T21:21:50.492413+0000","last_clean":"2023-09-23T21:21:50.492413+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:21:50.492413+0000","last_undegraded":"2023-09-23T21:21:50.492413+0000","last_fullsized":"2023-09-23T21:21:50.492413+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":121,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.507605+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:21:50.507605+0000","last_peered":"2023-09-23T21:21:50.507605+0000","last_clean":"2023-09-23T21:21:50.507605+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:21:50.507605+0000","last_undegraded":"2023-09-23T21:21:50.507605+0000","last_fullsized":"2023-09-23T21:21:50.507605+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499313+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:21:50.499313+0000","last_peered":"2023-09-23T21:21:50.499313+0000","last_clean":"2023-09-23T21:21:50.499313+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:21:50.499313+0000","last_undegraded":"2023-09-23T21:21:50.499313+0000","last_fullsized":"2023-09-23T21:21:50.499313+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":711,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.518182+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:21:50.518182+0000","last_peered":"2023-09-23T21:21:50.518182+0000","last_clean":"2023-09-23T21:21:50.518182+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:21:50.518182+0000","last_undegraded":"2023-09-23T21:21:50.518182+0000","last_fullsized":"2023-09-23T21:21:50.518182+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"649'8422","reported_seq":14026,"reported_epoch":649,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:21:59.100500+0000","last_change":"2023-09-23T21:20:48.304799+0000","last_active":"2023-09-23T21:21:59.100500+0000","last_peered":"2023-09-23T21:21:59.100500+0000","last_clean":"2023-09-23T21:19:28.920085+0000","last_became_active":"2023-09-23T21:20:48.224556+0000","last_became_peered":"2023-09-23T21:20:48.224556+0000","last_unstale":"2023-09-23T21:21:59.100500+0000","last_undegraded":"2023-09-23T21:20:48.204030+0000","last_fullsized":"2023-09-23T21:20:47.933463+0000","mapping_epoch":623,"log_start":"588'3344","ondisk_log_start":"588'3344","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5078,"log_dups_size":0,"ondisk_log_size":5078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:02.563744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132907008,"num_objects":2028,"num_object_clones":0,"num_object_copies":6084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3784,"num_objects_misplaced":278,"num_objects_unfound":0,"num_objects_dirty":2028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8422,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2738,"num_bytes_recovered":179306496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[4,2],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"2,4","objects":1},{"shards":"2,4,6","objects":734},{"shards":"4,5,6","objects":1293}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":256,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.493547+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:21:50.493547+0000","last_peered":"2023-09-23T21:21:50.493547+0000","last_clean":"2023-09-23T21:21:50.493547+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:21:50.493547+0000","last_undegraded":"2023-09-23T21:21:50.493547+0000","last_fullsized":"2023-09-23T21:21:50.493547+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499088+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:21:50.499088+0000","last_peered":"2023-09-23T21:21:50.499088+0000","last_clean":"2023-09-23T21:21:50.499088+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:21:50.499088+0000","last_undegraded":"2023-09-23T21:21:50.499088+0000","last_fullsized":"2023-09-23T21:21:50.499088+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":46,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.502641+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:21:50.502641+0000","last_peered":"2023-09-23T21:21:50.502641+0000","last_clean":"2023-09-23T21:21:50.502641+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:21:50.502641+0000","last_undegraded":"2023-09-23T21:21:50.502641+0000","last_fullsized":"2023-09-23T21:21:50.502641+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":950,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.517013+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:21:50.517013+0000","last_peered":"2023-09-23T21:21:50.517013+0000","last_clean":"2023-09-23T21:21:50.517013+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:21:50.517013+0000","last_undegraded":"2023-09-23T21:21:50.517013+0000","last_fullsized":"2023-09-23T21:21:50.517013+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.a","version":"649'8554","reported_seq":15135,"reported_epoch":649,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:21:59.093842+0000","last_change":"2023-09-23T21:21:01.218382+0000","last_active":"2023-09-23T21:21:59.093842+0000","last_peered":"2023-09-23T21:21:59.093842+0000","last_clean":"2023-09-23T21:19:28.896530+0000","last_became_active":"2023-09-23T21:21:01.190094+0000","last_became_peered":"2023-09-23T21:21:01.190094+0000","last_unstale":"2023-09-23T21:21:59.093842+0000","last_undegraded":"2023-09-23T21:21:00.719860+0000","last_fullsized":"2023-09-23T21:21:59.093842+0000","mapping_epoch":629,"log_start":"589'3518","ondisk_log_start":"589'3518","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:06.621301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136052736,"num_objects":2076,"num_object_clones":0,"num_object_copies":6228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3384,"num_objects_misplaced":2772,"num_objects_unfound":0,"num_objects_dirty":2076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8554,"num_write_kb":340160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3221,"num_bytes_recovered":210960384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[4,5,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":1364},{"shards":"4,6,7","objects":712}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":346,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.495315+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:21:50.495315+0000","last_peered":"2023-09-23T21:21:50.495315+0000","last_clean":"2023-09-23T21:21:50.495315+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:21:50.495315+0000","last_undegraded":"2023-09-23T21:21:50.495315+0000","last_fullsized":"2023-09-23T21:21:50.495315+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":31,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499441+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:21:50.499441+0000","last_peered":"2023-09-23T21:21:50.499441+0000","last_clean":"2023-09-23T21:21:50.499441+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:21:50.499441+0000","last_undegraded":"2023-09-23T21:21:50.499441+0000","last_fullsized":"2023-09-23T21:21:50.499441+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":61,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.486655+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:21:50.486655+0000","last_peered":"2023-09-23T21:21:50.486655+0000","last_clean":"2023-09-23T21:21:50.486655+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:21:50.486655+0000","last_undegraded":"2023-09-23T21:21:50.486655+0000","last_fullsized":"2023-09-23T21:21:50.486655+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1043,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.486899+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:21:50.486899+0000","last_peered":"2023-09-23T21:21:50.486899+0000","last_clean":"2023-09-23T21:21:50.486899+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:21:50.486899+0000","last_undegraded":"2023-09-23T21:21:50.486899+0000","last_fullsized":"2023-09-23T21:21:50.486899+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"649'8703","reported_seq":12011,"reported_epoch":649,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:21:58.552372+0000","last_change":"2023-09-23T21:21:22.582611+0000","last_active":"2023-09-23T21:21:58.552372+0000","last_peered":"2023-09-23T21:21:58.552372+0000","last_clean":"2023-09-23T21:19:48.922364+0000","last_became_active":"2023-09-23T21:21:22.566477+0000","last_became_peered":"2023-09-23T21:21:22.566477+0000","last_unstale":"2023-09-23T21:21:58.552372+0000","last_undegraded":"2023-09-23T21:21:58.552372+0000","last_fullsized":"2023-09-23T21:21:58.552372+0000","mapping_epoch":639,"log_start":"589'3756","ondisk_log_start":"589'3756","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4947,"log_dups_size":0,"ondisk_log_size":4947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:46.195361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141099008,"num_objects":2153,"num_object_clones":0,"num_object_copies":6459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2153,"num_objects_unfound":0,"num_objects_dirty":2153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8703,"num_write_kb":347392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104660992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":2153}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":555,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.517308+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:21:50.517308+0000","last_peered":"2023-09-23T21:21:50.517308+0000","last_clean":"2023-09-23T21:21:50.517308+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:21:50.517308+0000","last_undegraded":"2023-09-23T21:21:50.517308+0000","last_fullsized":"2023-09-23T21:21:50.517308+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":35,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.499229+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:21:50.499229+0000","last_peered":"2023-09-23T21:21:50.499229+0000","last_clean":"2023-09-23T21:21:50.499229+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:21:50.499229+0000","last_undegraded":"2023-09-23T21:21:50.499229+0000","last_fullsized":"2023-09-23T21:21:50.499229+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":36,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.501084+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:21:50.501084+0000","last_peered":"2023-09-23T21:21:50.501084+0000","last_clean":"2023-09-23T21:21:50.501084+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:21:50.501084+0000","last_undegraded":"2023-09-23T21:21:50.501084+0000","last_fullsized":"2023-09-23T21:21:50.501084+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":100,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.504441+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:21:50.504441+0000","last_peered":"2023-09-23T21:21:50.504441+0000","last_clean":"2023-09-23T21:21:50.504441+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:21:50.504441+0000","last_undegraded":"2023-09-23T21:21:50.504441+0000","last_fullsized":"2023-09-23T21:21:50.504441+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":741,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:53.735087+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:21:53.735087+0000","last_peered":"2023-09-23T21:21:53.735087+0000","last_clean":"2023-09-23T21:21:53.735087+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:21:53.735087+0000","last_undegraded":"2023-09-23T21:21:53.735087+0000","last_fullsized":"2023-09-23T21:21:53.735087+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":409,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.503970+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:21:50.503970+0000","last_peered":"2023-09-23T21:21:50.503970+0000","last_clean":"2023-09-23T21:21:50.503970+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:21:50.503970+0000","last_undegraded":"2023-09-23T21:21:50.503970+0000","last_fullsized":"2023-09-23T21:21:50.503970+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":252,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.504129+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:21:50.504129+0000","last_peered":"2023-09-23T21:21:50.504129+0000","last_clean":"2023-09-23T21:21:50.504129+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:21:50.504129+0000","last_undegraded":"2023-09-23T21:21:50.504129+0000","last_fullsized":"2023-09-23T21:21:50.504129+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":160,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.517108+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:21:50.517108+0000","last_peered":"2023-09-23T21:21:50.517108+0000","last_clean":"2023-09-23T21:21:50.517108+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:21:50.517108+0000","last_undegraded":"2023-09-23T21:21:50.517108+0000","last_fullsized":"2023-09-23T21:21:50.517108+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":43,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.487649+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:21:50.487649+0000","last_peered":"2023-09-23T21:21:50.487649+0000","last_clean":"2023-09-23T21:21:50.487649+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:21:50.487649+0000","last_undegraded":"2023-09-23T21:21:50.487649+0000","last_fullsized":"2023-09-23T21:21:50.487649+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":61,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.488328+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:21:50.488328+0000","last_peered":"2023-09-23T21:21:50.488328+0000","last_clean":"2023-09-23T21:21:50.488328+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:21:50.488328+0000","last_undegraded":"2023-09-23T21:21:50.488328+0000","last_fullsized":"2023-09-23T21:21:50.488328+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":52,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.489157+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:21:50.489157+0000","last_peered":"2023-09-23T21:21:50.489157+0000","last_clean":"2023-09-23T21:21:50.489157+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:21:50.489157+0000","last_undegraded":"2023-09-23T21:21:50.489157+0000","last_fullsized":"2023-09-23T21:21:50.489157+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":637,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.483717+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:21:50.483717+0000","last_peered":"2023-09-23T21:21:50.483717+0000","last_clean":"2023-09-23T21:21:50.483717+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:21:50.483717+0000","last_undegraded":"2023-09-23T21:21:50.483717+0000","last_fullsized":"2023-09-23T21:21:50.483717+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":356,"reported_epoch":648,"state":"active+clean","last_fresh":"2023-09-23T21:21:50.502608+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:21:50.502608+0000","last_peered":"2023-09-23T21:21:50.502608+0000","last_clean":"2023-09-23T21:21:50.502608+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:21:50.502608+0000","last_undegraded":"2023-09-23T21:21:50.502608+0000","last_fullsized":"2023-09-23T21:21:50.502608+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":2201419800,"num_objects":33592,"num_object_clones":0,"num_object_copies":100776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18882,"num_objects_misplaced":21088,"num_objects_unfound":0,"num_objects_dirty":33592,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137038,"num_write_kb":5460097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23148,"num_bytes_recovered":1429405696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8287498240,"data_stored":8287486024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83752,"ondisk_log_size":83752,"up":48,"acting":47,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697707,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1550763,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3297280,"data_stored":3279520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201919,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1853536,"kb_used_data":953752,"kb_used_omap":0,"kb_used_meta":899712,"kb_avail":91891616,"statfs":{"total":95995035648,"available":94097014784,"internally_reserved":0,"allocated":976642048,"data_stored":974856976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921305088},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.605,"15min":2.3159999999999998},"min":{"1min":0.27300000000000002,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":6.6390000000000002,"5min":16.468,"15min":141.33199999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.4790000000000001,"5min":1.992,"15min":2.4780000000000002},"min":{"1min":0.29799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":12.962,"5min":12.962,"15min":141.31700000000001},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":2.0840000000000001,"15min":3.4689999999999999},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.24099999999999999},"max":{"1min":13.054,"5min":13.054,"15min":114.16500000000001},"last":4.2060000000000004},{"interface":"front","average":{"1min":3.0009999999999999,"5min":1.9350000000000001,"15min":3.8170000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":10.881,"5min":16.289000000000001,"15min":117.637},"last":3.1720000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":8.1029999999999998,"5min":6.1260000000000003,"15min":5.5369999999999999},"min":{"1min":0.252,"5min":0.246,"15min":0.224},"max":{"1min":68.793999999999997,"5min":158.726,"15min":183.44200000000001},"last":3.198},{"interface":"front","average":{"1min":7.2279999999999998,"5min":5.7489999999999997,"15min":5.7119999999999997},"min":{"1min":0.32800000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":63.264000000000003,"5min":158.70400000000001,"15min":183.46199999999999},"last":3.6669999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":4.3310000000000004,"5min":2.2050000000000001,"15min":4.1589999999999998},"min":{"1min":0.41099999999999998,"5min":0.33700000000000002,"15min":0.26700000000000002},"max":{"1min":30.667999999999999,"5min":30.667999999999999,"15min":135.66200000000001},"last":6.1630000000000003},{"interface":"front","average":{"1min":3.8620000000000001,"5min":2.1019999999999999,"15min":4.383},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":30.798999999999999,"5min":30.798999999999999,"15min":135.47},"last":3.3140000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.1849999999999996,"5min":3.653,"15min":1.9690000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":13.821999999999999,"5min":31.695,"15min":31.695},"last":0.39400000000000002},{"interface":"front","average":{"1min":6.8120000000000003,"5min":3.7869999999999999,"15min":2.0179999999999998},"min":{"1min":0.48499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":27.821999999999999,"5min":34.994,"15min":34.994},"last":0.73399999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":4.7350000000000003,"15min":2.46},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":10.712,"5min":156.30099999999999,"15min":156.30099999999999},"last":6.2149999999999999},{"interface":"front","average":{"1min":4.5389999999999997,"5min":4.8739999999999997,"15min":2.548},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":12.151,"5min":156.22,"15min":156.22},"last":0.33500000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":3.75,"15min":2.8140000000000001},"min":{"1min":0.45700000000000002,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":14.066000000000001,"5min":23.574999999999999,"15min":143.18700000000001},"last":3.7250000000000001},{"interface":"front","average":{"1min":6.4370000000000003,"5min":3.242,"15min":2.528},"min":{"1min":0.504,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":21.081,"5min":21.081,"15min":142.70400000000001},"last":0.75900000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463539,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2896796,"kb_used_data":2001236,"kb_used_omap":0,"kb_used_meta":895360,"kb_avail":90848356,"statfs":{"total":95995035648,"available":93028716544,"internally_reserved":0,"allocated":2049265664,"data_stored":2047485457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916848640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5940000000000001,"5min":1.6100000000000001,"15min":2.8340000000000001},"min":{"1min":0.246,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":10.004,"5min":12.659000000000001,"15min":149.61600000000001},"last":2.867},{"interface":"front","average":{"1min":2.1960000000000002,"5min":2.077,"15min":3.0529999999999999},"min":{"1min":0.27100000000000002,"5min":0.223,"15min":0.223},"max":{"1min":7.0090000000000003,"5min":16.021999999999998,"15min":149.303},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.4020000000000001,"15min":2.9990000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":12.013,"5min":16.126999999999999,"15min":129.37899999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.8540000000000001,"15min":2.742},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":14.366,"5min":14.366,"15min":129.40100000000001},"last":7.6399999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":6.4450000000000003,"15min":5.9790000000000001},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.9540000000000006,"5min":181.846,"15min":199.23400000000001},"last":2.919},{"interface":"front","average":{"1min":2.2120000000000002,"5min":7.1200000000000001,"15min":6.1239999999999997},"min":{"1min":0.35899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.1059999999999999,"5min":182.14500000000001,"15min":199.08000000000001},"last":0.35699999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":1.9330000000000001,"15min":3.4630000000000001},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.218},"max":{"1min":14.411,"5min":14.411,"15min":137.875},"last":6.516},{"interface":"front","average":{"1min":2.089,"5min":1.8080000000000001,"15min":3.3839999999999999},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.29799999999999999},"max":{"1min":10.106,"5min":12.676,"15min":137.60900000000001},"last":2.9510000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":4.0439999999999996,"5min":4.5910000000000002,"15min":2.2559999999999998},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":15.361000000000001,"5min":55.494,"15min":55.494},"last":3.5790000000000002},{"interface":"front","average":{"1min":3.823,"5min":4.2039999999999997,"15min":2.0510000000000002},"min":{"1min":0.33700000000000002,"5min":0.33400000000000002,"15min":0.23999999999999999},"max":{"1min":14.622,"5min":55.305,"15min":55.305},"last":10.577}]},{"osd":5,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":5.5620000000000003,"15min":2.6419999999999999},"min":{"1min":0.41699999999999998,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":7.1630000000000003,"5min":222.90899999999999,"15min":222.90899999999999},"last":0.443},{"interface":"front","average":{"1min":2.6819999999999999,"5min":5.4749999999999996,"15min":2.6230000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.28499999999999998},"max":{"1min":17.927,"5min":224.655,"15min":224.655},"last":7.2430000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":4.8970000000000002,"15min":2.2599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36399999999999999,"15min":0.28000000000000003},"max":{"1min":12.114000000000001,"5min":165.96799999999999,"15min":165.96799999999999},"last":3.1779999999999999},{"interface":"front","average":{"1min":3.5030000000000001,"5min":5.4870000000000001,"15min":2.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.251},"max":{"1min":14.457000000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":6.5720000000000001}]}]},{"osd":1,"up_from":276,"seq":1185410973986,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":629480,"kb_used_data":8160,"kb_used_omap":5,"kb_used_meta":621306,"kb_avail":93115672,"statfs":{"total":95995035648,"available":95350448128,"internally_reserved":0,"allocated":8355840,"data_stored":5273241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":636218083},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.38200000000000001,"15min":2.0009999999999999},"min":{"1min":0.22600000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":0.505,"5min":1.9450000000000001,"15min":108.47},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.40100000000000002,"15min":1.952},"min":{"1min":0.26200000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":109.532},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":7.6120000000000001,"5min":8.4109999999999996,"15min":7.8899999999999997},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.20200000000000001},"max":{"1min":72.966999999999999,"5min":149.35300000000001,"15min":171.35400000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":7.6459999999999999,"5min":8.5180000000000007,"15min":7.5659999999999998},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":74.450000000000003,"5min":147.50399999999999,"15min":170.69900000000001},"last":0.20899999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":0.55200000000000005,"15min":3.593},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":17.247,"5min":17.247,"15min":99.120999999999995},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.66000000000000003,"15min":4.1600000000000001},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.21299999999999999},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":150.38},"last":0.22700000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4770000000000001,"5min":3.2519999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.5800000000000001,"5min":174.84399999999999,"15min":174.84399999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.91000000000000003,"5min":3.0230000000000001,"15min":2.1850000000000001},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.28199999999999997},"max":{"1min":2.931,"5min":174.82400000000001,"15min":174.82400000000001},"last":0.65200000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":4.2670000000000003,"15min":2.7269999999999999},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.31900000000000001},"max":{"1min":3.8839999999999999,"5min":110.955,"15min":110.955},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.53,"5min":4.5679999999999996,"15min":2.774},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.29199999999999998},"max":{"1min":9.2300000000000004,"5min":110.89,"15min":110.89},"last":3.3700000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1989999999999998,"5min":3.1160000000000001,"15min":2.4329999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":7.7859999999999996,"5min":119.999,"15min":119.999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.788,"5min":3.1269999999999998,"15min":2.3370000000000002},"min":{"1min":0.40500000000000003,"5min":0.38300000000000001,"15min":0.31900000000000001},"max":{"1min":8.9550000000000001,"5min":122.328,"15min":122.328},"last":1.7110000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":4.8159999999999998,"15min":2.9239999999999999},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":10.648999999999999,"5min":285.47500000000002,"15min":285.47500000000002},"last":0.501},{"interface":"front","average":{"1min":1.875,"5min":4.5430000000000001,"15min":2.7330000000000001},"min":{"1min":0.38600000000000001,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":10.664999999999999,"5min":285.48599999999999,"15min":285.48599999999999},"last":0.49199999999999999}]}]},{"osd":0,"up_from":0,"seq":395136991726,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781617,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2512124,"kb_used_data":1214580,"kb_used_omap":7,"kb_used_meta":1297528,"kb_avail":91233028,"statfs":{"total":95995035648,"available":93422620672,"internally_reserved":0,"allocated":1243729920,"data_stored":1240560669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1328668753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.67400000000000004,"15min":1.929},"min":{"1min":0.19900000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":4.2880000000000003,"5min":8.0079999999999991,"15min":103.021},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.621,"15min":2.5099999999999998},"min":{"1min":0.253,"5min":0.218,"15min":0.218},"max":{"1min":4.6840000000000002,"5min":4.9820000000000002,"15min":103.154},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.66300000000000003,"15min":2.7160000000000002},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.19700000000000001},"max":{"1min":4.7350000000000003,"5min":5.2670000000000003,"15min":143.482},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.71999999999999997,"15min":2.7490000000000001},"min":{"1min":0.23899999999999999,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":1.6659999999999999,"5min":8.048,"15min":143.61000000000001},"last":0.52200000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.70399999999999996,"15min":3.5369999999999999},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.7679999999999998,"5min":5.2939999999999996,"15min":143.50800000000001},"last":1.5429999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.77300000000000002,"15min":3.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.2670000000000003,"5min":8.2189999999999994,"15min":143.27799999999999},"last":0.55700000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":4.3929999999999998,"15min":3.3849999999999998},"min":{"1min":0.318,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":6.8659999999999997,"5min":220.82900000000001,"15min":225.80799999999999},"last":1.5580000000000001},{"interface":"front","average":{"1min":2.6160000000000001,"5min":4.3259999999999996,"15min":3.9769999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":6.8029999999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":1.496}]},{"osd":5,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":4.3010000000000002,"15min":3.48},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":9.0579999999999998,"5min":220.63900000000001,"15min":220.63900000000001},"last":7.048},{"interface":"front","average":{"1min":1.5049999999999999,"5min":4.3789999999999996,"15min":3.5089999999999999},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.8129999999999997,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.68899999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":5.0289999999999999,"15min":3.3460000000000001},"min":{"1min":0.41099999999999998,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":5.1980000000000004,"5min":210.74700000000001,"15min":210.74700000000001},"last":2.2050000000000001},{"interface":"front","average":{"1min":1.865,"5min":4.782,"15min":2.6890000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.8170000000000002,"5min":210.72900000000001,"15min":210.72900000000001},"last":2.3559999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.0169999999999999,"15min":1.954},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.5250000000000004,"5min":14.019,"15min":25.260000000000002},"last":5.3360000000000003},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.758,"15min":2.3279999999999998},"min":{"1min":0.41099999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.6200000000000001,"5min":7.5289999999999999,"15min":16.875},"last":2.0939999999999999}]}]},{"osd":3,"up_from":148,"seq":635655160211,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":865704,"kb_used_data":9312,"kb_used_omap":2,"kb_used_meta":856381,"kb_avail":92879448,"statfs":{"total":95995035648,"available":95108554752,"internally_reserved":0,"allocated":9535488,"data_stored":5911879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":876934197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.42199999999999999,"15min":1.7749999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":4.7430000000000003,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.42499999999999999,"15min":1.5369999999999999},"min":{"1min":0.252,"5min":0.219,"15min":0.19500000000000001},"max":{"1min":2.6480000000000001,"5min":2.6480000000000001,"15min":18.277999999999999},"last":1.3460000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:21:33 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.504,"15min":6.5910000000000002},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8980000000000001,"5min":2.8980000000000001,"15min":237.072},"last":1.278},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.54200000000000004,"15min":6.5460000000000003},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":3.8359999999999999,"5min":3.8359999999999999,"15min":234.21899999999999},"last":0.24299999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":5.04,"15min":4.6239999999999997},"min":{"1min":0.32500000000000001,"5min":0.219,"15min":0.215},"max":{"1min":6.665,"5min":102.404,"15min":102.404},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.73299999999999998,"5min":5.024,"15min":4.1959999999999997},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":5.8010000000000002,"5min":104.529,"15min":104.529},"last":0.318}]},{"osd":4,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":3.339,"5min":2.9169999999999998,"15min":2.5270000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":13.640000000000001,"5min":117.717,"15min":117.717},"last":6.1219999999999999},{"interface":"front","average":{"1min":3.1499999999999999,"5min":3.415,"15min":2.5059999999999998},"min":{"1min":0.42299999999999999,"5min":0.32100000000000001,"15min":0.27300000000000002},"max":{"1min":11.968999999999999,"5min":117.771,"15min":117.771},"last":6.7910000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":3.96,"15min":2.8290000000000002},"min":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":6.5449999999999999,"5min":123.458,"15min":123.458},"last":4.0540000000000003},{"interface":"front","average":{"1min":1.706,"5min":3.8410000000000002,"15min":2.8250000000000002},"min":{"1min":0.38,"5min":0.34899999999999998,"15min":0.315},"max":{"1min":4.7619999999999996,"5min":123.292,"15min":123.292},"last":3.956}]},{"osd":6,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":4.9569999999999999,"15min":4.7210000000000001},"min":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.316},"max":{"1min":4.0140000000000002,"5min":130.613,"15min":304.22399999999999},"last":1.4139999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":5.3140000000000001,"15min":4.7770000000000001},"min":{"1min":0.41699999999999998,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":4.5179999999999998,"5min":130.50999999999999,"15min":304.15800000000002},"last":1.601}]},{"osd":7,"last update":"Sat Sep 23 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":2.532,"15min":2.1080000000000001},"min":{"1min":0.39100000000000001,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":19.298999999999999,"5min":44.253,"15min":44.253},"last":5.4379999999999997},{"interface":"front","average":{"1min":2.9199999999999999,"5min":2.6669999999999998,"15min":2.5},"min":{"1min":0.434,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":8.6300000000000008,"5min":45.706000000000003,"15min":45.706000000000003},"last":1.3819999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986829,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2607272,"kb_used_data":1657952,"kb_used_omap":0,"kb_used_meta":949312,"kb_avail":91137880,"statfs":{"total":95995035648,"available":93325189120,"internally_reserved":0,"allocated":1697742848,"data_stored":1695950405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":972095488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":2.0099999999999998,"15min":2.8170000000000002},"min":{"1min":0.32600000000000001,"5min":0.246,"15min":0.20699999999999999},"max":{"1min":6.8390000000000004,"5min":10.436,"15min":226.751},"last":7.3099999999999996},{"interface":"front","average":{"1min":1.492,"5min":1.5469999999999999,"15min":2.5990000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":4.6900000000000004,"5min":12.754,"15min":226.43299999999999},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.583,"15min":3.6789999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.0709999999999997,"5min":12.805,"15min":127.411},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.4689999999999999,"5min":2.1619999999999999,"15min":3.194},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.625,"5min":10.625,"15min":116.78100000000001},"last":7.3529999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:21:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2390000000000008,"5min":8.0739999999999998,"15min":6.0629999999999997},"min":{"1min":0.33600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":70.393000000000001,"5min":108.32899999999999,"15min":210.50299999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":8.7129999999999992,"5min":8.0129999999999999,"15min":6.1669999999999998},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":70.340000000000003,"5min":108.104,"15min":210.59},"last":7.5579999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.2999999999999998,"15min":4.4119999999999999},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.122999999999999,"5min":14.323,"15min":214.64099999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.391,"15min":4.8159999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":10.587,"5min":10.587,"15min":214.54900000000001},"last":7.3810000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.3410000000000002,"15min":1.8280000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":12.997999999999999,"5min":17.879999999999999,"15min":34.497999999999998},"last":3.3220000000000001},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.3050000000000002,"15min":2.0760000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":19.425000000000001,"5min":19.425000000000001,"15min":34.883000000000003},"last":7.6150000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:21:02 2023","interfaces":[{"interface":"back","average":{"1min":7.1280000000000001,"5min":3.952,"15min":2.1869999999999998},"min":{"1min":0.34899999999999998,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":67.697999999999993,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.50700000000000001},{"interface":"front","average":{"1min":8.2810000000000006,"5min":4.7009999999999996,"15min":2.4020000000000001},"min":{"1min":0.38600000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":67.596999999999994,"5min":67.596999999999994,"15min":67.596999999999994},"last":1.3380000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":3.3159999999999998,"15min":1.9710000000000001},"min":{"1min":0.38700000000000001,"5min":0.317,"15min":0.28100000000000003},"max":{"1min":15.050000000000001,"5min":17.817,"15min":18.352},"last":7.5899999999999999},{"interface":"front","average":{"1min":3.323,"5min":3.177,"15min":1.8819999999999999},"min":{"1min":0.441,"5min":0.307,"15min":0.307},"max":{"1min":16.332999999999998,"5min":30.117000000000001,"15min":30.117000000000001},"last":1.0209999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725193,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3298132,"kb_used_data":2337868,"kb_used_omap":0,"kb_used_meta":960256,"kb_avail":90447020,"statfs":{"total":95995035648,"available":92617748480,"internally_reserved":0,"allocated":2393976832,"data_stored":2392200069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":983302144},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":1.762,"15min":1.9730000000000001},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.217},"max":{"1min":12.558999999999999,"5min":12.558999999999999,"15min":82.280000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.617,"5min":1.8089999999999999,"15min":2.0840000000000001},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":9.407,"5min":9.407,"15min":85.236000000000004},"last":1.6100000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":2.2440000000000002,"15min":3.9900000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":18.859999999999999,"5min":18.859999999999999,"15min":233.77099999999999},"last":1.6279999999999999},{"interface":"front","average":{"1min":1.198,"5min":1.544,"15min":3.5899999999999999},"min":{"1min":0.27400000000000002,"5min":0.27000000000000002,"15min":0.251},"max":{"1min":7.5220000000000002,"5min":12.654999999999999,"15min":233.75200000000001},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":6.7320000000000002,"5min":4.5449999999999999,"15min":4.3639999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":82.072000000000003,"5min":160.749,"15min":231.34700000000001},"last":1.3029999999999999},{"interface":"front","average":{"1min":6.2869999999999999,"5min":4.5359999999999996,"15min":4.4699999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":82.099999999999994,"5min":161.18000000000001,"15min":231.381},"last":1.649}]},{"osd":3,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":2.1869999999999998,"15min":3.9049999999999998},"min":{"1min":0.36099999999999999,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":18.826000000000001,"5min":18.826000000000001,"15min":263.887},"last":4.2539999999999996},{"interface":"front","average":{"1min":3.0680000000000001,"5min":1.895,"15min":3.6440000000000001},"min":{"1min":0.45300000000000001,"5min":0.35799999999999998,"15min":0.30099999999999999},"max":{"1min":12.683999999999999,"5min":12.683999999999999,"15min":263.70100000000002},"last":1.212}]},{"osd":4,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":3.0649999999999999,"15min":1.7649999999999999},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":7.6299999999999999,"5min":77.804000000000002,"15min":77.804000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.7719999999999998,"5min":4.2880000000000003,"15min":2.1619999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":8.3810000000000002,"5min":77.825999999999993,"15min":77.825999999999993},"last":1.6950000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":3.9420000000000002,"5min":2.786,"15min":1.913},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":13.965999999999999,"5min":13.965999999999999,"15min":83.451999999999998},"last":2.956},{"interface":"front","average":{"1min":4.0800000000000001,"5min":3.0960000000000001,"15min":2.1920000000000002},"min":{"1min":0.34000000000000002,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":19.029,"5min":19.029,"15min":83.484999999999999},"last":2.9169999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:21:42 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":2.6309999999999998,"15min":1.792},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":10.654999999999999,"5min":10.654999999999999,"15min":10.654999999999999},"last":1.6739999999999999},{"interface":"front","average":{"1min":2.7160000000000002,"5min":2.2629999999999999,"15min":1.411},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":18.257999999999999,"5min":18.257999999999999,"15min":18.257999999999999},"last":9.9160000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1226702848,"data_stored":1226702848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1690046464,"data_stored":1690042392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2377388032,"data_stored":2377383960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2025852928,"data_stored":2025848856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":967507968,"data_stored":967507968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:22:04.395 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:22:04.454 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:04.521 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:04.635 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "MOSDPGPush(14.a 650/629 [PushOp(14:54aa9b9e:::benchmark_data_smithi121_684892_object54702:head, version: 588'3394, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:54aa9b9e:::benchmark_data_smithi121_684892_object54702:head@588'3394, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:22:04.632333+0000", 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 0.0023316560000000001, 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023859689999999999, 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "started", 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.636 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632333+0000", 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632333+0000", 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632332+0000", 2023-09-23T21:22:04.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632365+0000", 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2589999999999998e-05 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632385+0000", 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.016e-05 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:04.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632395+0000", 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0237999999999999e-05 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632469+0000", 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4196999999999995e-05 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:04.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:04.632477+0000", 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4340000000000003e-06 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 1 2023-09-23T21:22:04.640 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:22:04.661 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:04.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:04 smithi121 ceph-mon[211526]: failed to encode map e650 with expected crc 2023-09-23T21:22:04.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:04 smithi121 ceph-mon[219709]: failed to encode map e650 with expected crc 2023-09-23T21:22:04.673 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:04 smithi123 ceph-mon[193903]: failed to encode map e650 with expected crc 2023-09-23T21:22:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:04.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:04.775 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:04.818 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:04.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:22:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:22:04] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.43.0" 2023-09-23T21:22:05.043 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:05.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:05.138 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:05.154 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:05.173 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:22:05.185 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:05.194 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:22:05.221 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:22:05.241 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:22:05.266 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:05.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:05.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:05.336 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:22:05.370 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:05.370 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:22:05.408 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:22:05.439 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.441 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:22:05.458 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:05.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/347776928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:05 smithi121 ceph-mon[211526]: pgmap v1966: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 17720/93630 objects degraded (18.926%); 19507/93630 objects misplaced (20.834%); 11 MiB/s, 183 objects/s recovering 2023-09-23T21:22:05.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/347776928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:05.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:05 smithi121 ceph-mon[219709]: pgmap v1966: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 17720/93630 objects degraded (18.926%); 19507/93630 objects misplaced (20.834%); 11 MiB/s, 183 objects/s recovering 2023-09-23T21:22:05.697 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/347776928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:05.698 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:05 smithi123 ceph-mon[193903]: pgmap v1966: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 17720/93630 objects degraded (18.926%); 19507/93630 objects misplaced (20.834%); 11 MiB/s, 183 objects/s recovering 2023-09-23T21:22:05.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:05.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:05.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:05.725 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:22:05.817 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:05.844 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:06.484 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:22:06.666 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:22:06.793 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:06.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:06.900 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:06.953 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:07.059 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:07.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:07.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:07.063 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:06 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 17720/93630 objects degraded (18.926%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:07.064 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:06 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 17720/93630 objects degraded (18.926%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:07.078 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:06 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 17720/93630 objects degraded (18.926%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:07.164 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:07.208 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:07.257 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:07.289 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:07.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:07.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:07.393 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:07.437 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:07.619 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:07.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:07.724 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:07.755 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:07.800 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:07.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:07.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:07.910 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:07.944 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:08.009 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:08 smithi121 ceph-mon[211526]: pgmap v1967: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 17581/92313 objects degraded (19.045%); 19368/92313 objects misplaced (20.981%); 8.8 MiB/s, 187 objects/s recovering 2023-09-23T21:22:08.009 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:08 smithi121 ceph-mon[219709]: pgmap v1967: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 17581/92313 objects degraded (19.045%); 19368/92313 objects misplaced (20.981%); 8.8 MiB/s, 187 objects/s recovering 2023-09-23T21:22:08.039 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:22:08.039 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 17581/92313 objects degraded (19.045%), 9 pgs degraded, 2 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2584,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":650,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695504058,"num_remapped_pgs":12},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+degraded+remapped","count":7},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":30771,"data_bytes":2015673091,"bytes_used":14526214144,"bytes_avail":657439035392,"bytes_total":671965249536,"degraded_objects":17581,"degraded_total":92313,"degraded_ratio":0.19044988246509159,"misplaced_objects":19368,"misplaced_total":92313,"misplaced_ratio":0.20980793604367748,"recovering_objects_per_sec":187,"recovering_bytes_per_sec":9271916,"recovering_keys_per_sec":0,"num_objects_recovered":1876,"num_bytes_recovered":92798976,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":471},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":35,"modified":"2023-09-23T21:21:43.417977+0000","services":{}},"progress_events":{"432f78a9-9bfa-46a5-b9c3-f138454b372b":{"message":"Global Recovery Event (2m)\n [========================....] (remaining: 19s)","progress":0.87878787517547607,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:22:08.204 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:08.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:08.308 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:08.380 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:08 smithi123 ceph-mon[193903]: pgmap v1967: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 17581/92313 objects degraded (19.045%); 19368/92313 objects misplaced (20.981%); 8.8 MiB/s, 187 objects/s recovering 2023-09-23T21:22:08.389 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:08.498 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:08.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:08.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:08.657 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:08.848 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:22:08.879 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:08.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:08.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:08.985 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:09.032 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:09.075 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:22:09.075 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:22:09.282 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1390589435' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:22:09.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1390589435' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:22:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1390589435' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:22:09.548 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:09.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:09.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:09.567 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:22:09.567 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:22:09.653 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:09.708 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:09.816 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:09.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:09.924 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:09.936 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:10.018 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:10.140 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:22:10.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:10 smithi121 ceph-mon[211526]: pgmap v1968: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 16277/86679 objects degraded (18.778%); 18526/86679 objects misplaced (21.373%); 8.9 MiB/s, 189 objects/s recovering 2023-09-23T21:22:10.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:10 smithi121 ceph-mon[219709]: pgmap v1968: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 16277/86679 objects degraded (18.778%); 18526/86679 objects misplaced (21.373%); 8.9 MiB/s, 189 objects/s recovering 2023-09-23T21:22:10.169 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:10.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:10.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:10.346 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:10.447 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:10 smithi123 ceph-mon[193903]: pgmap v1968: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 16277/86679 objects degraded (18.778%); 18526/86679 objects misplaced (21.373%); 8.9 MiB/s, 189 objects/s recovering 2023-09-23T21:22:10.452 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:10.457 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:10.588 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:10.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:10.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:10.709 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:10.813 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:10.879 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:10.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:10.922 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:22:10.986 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:11.031 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:11.066 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:22:11.162 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/379375665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:11.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/379375665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:11.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/379375665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:11.502 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:11.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:11.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:11.606 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:11.664 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:11.802 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:11.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:11.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:11.912 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:11.946 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:12.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/379375665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T21:22:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: pgmap v1970: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 16277/86679 objects degraded (18.778%); 18526/86679 objects misplaced (21.373%); 8.8 MiB/s, 179 objects/s recovering 2023-09-23T21:22:12.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:12 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 16277/86679 objects degraded (18.778%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:12.268 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:12.270 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:12.270 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/379375665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:12.270 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: pgmap v1970: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 16277/86679 objects degraded (18.778%); 18526/86679 objects misplaced (21.373%); 8.8 MiB/s, 179 objects/s recovering 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 16277/86679 objects degraded (18.778%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/379375665' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T21:22:12.271 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.272 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.272 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: failed to encode map e651 with expected crc 2023-09-23T21:22:12.272 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: pgmap v1970: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 16277/86679 objects degraded (18.778%); 18526/86679 objects misplaced (21.373%); 8.8 MiB/s, 179 objects/s recovering 2023-09-23T21:22:12.272 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:12 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 16277/86679 objects degraded (18.778%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:12.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:12.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:12.373 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:12.454 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:12.744 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:12.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:12.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:12.849 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:12.892 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:13.217 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:22:13.217 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:22:13.217 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:22:13.217 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.218 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.219 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.219 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159372 12.d 12:b6c49147:::benchmark_data_smithi121_563784_object68969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 532.15012923400002, 2023-09-23T21:22:13.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.04656902, 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159372 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:22:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423456+0000", 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423455+0000", 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:22:13.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423460+0000", 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5589999999999996e-06 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423473+0000", 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2984e-05 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.584 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423479+0000", 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0290000000000002e-06 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423555+0000", 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5654000000000004e-05 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.423563+0000", 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4989999999999994e-06 2023-09-23T21:22:13.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432320+0000", 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.008756972 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432467+0000", 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014657299999999999 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:22:13.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.432861+0000", 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039391600000000001 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442657+0000", 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.009796674 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469909+0000", 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.027251573000000001 2023-09-23T21:22:13.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469942+0000", 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3423000000000002e-05 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.469983+0000", 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1127000000000001e-05 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.588 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470025+0000", 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1706999999999999e-05 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159374 12.d 12:bcbc6391:::benchmark_data_smithi121_563784_object68971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 532.14312231700001, 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0397036749999999, 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.589 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159374 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430463+0000", 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430462+0000", 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430471+0000", 2023-09-23T21:22:13.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8060000000000001e-06 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430485+0000", 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4771e-05 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430492+0000", 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2400000000000004e-06 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430543+0000", 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1551999999999997e-05 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.430554+0000", 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0668e-05 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436375+0000", 2023-09-23T21:22:13.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0058211749999999 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436460+0000", 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4902999999999997e-05 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.436540+0000", 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9652000000000002e-05 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447278+0000", 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010738077 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470065+0000", 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022787136999999999 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470102+0000", 2023-09-23T21:22:13.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6761000000000003e-05 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470134+0000", 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2085999999999997e-05 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470167+0000", 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3135e-05 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.596 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159390 12.d 12:b1ecff3e:::benchmark_data_smithi121_563784_object68987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 532.11933451200002, 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.016060076, 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:13.597 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159390 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454251+0000", 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454249+0000", 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454265+0000", 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5506999999999999e-05 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454282+0000", 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6966e-05 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454288+0000", 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4189999999999999e-06 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454345+0000", 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6897000000000001e-05 2023-09-23T21:22:13.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.454353+0000", 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2530000000000004e-06 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438507+0000", 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98415336099999995 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.601 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438592+0000", 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5316999999999996e-05 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.438669+0000", 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6503999999999998e-05 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447287+0000", 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0086180560000000007 2023-09-23T21:22:13.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470085+0000", 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.02279871 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470250+0000", 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016454100000000001 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470281+0000", 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0571000000000003e-05 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470311+0000", 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0451999999999999e-05 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159394 12.d 12:b55656ba:::benchmark_data_smithi121_563784_object68991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:22:13.604 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 532.11463420500002, 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.011503713, 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159394 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458951+0000", 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458950+0000", 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458958+0000", 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4789999999999996e-06 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458972+0000", 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.31e-05 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.607 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.458978+0000", 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6930000000000001e-06 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.459041+0000", 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3157999999999994e-05 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.459049+0000", 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0639999999999994e-06 2023-09-23T21:22:13.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440532+0000", 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98148217000000004 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440613+0000", 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0908000000000005e-05 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:22:13.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.440689+0000", 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6810000000000005e-05 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457467+0000", 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016777255000000001 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470372+0000", 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012905020999999999 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470399+0000", 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7229999999999998e-05 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470424+0000", 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5514e-05 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470455+0000", 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0576e-05 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159428 12.d 12:b4378ef8:::benchmark_data_smithi121_563784_object69025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 532.06713503799995, 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96414785400000003, 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.612 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159428 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506450+0000", 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506449+0000", 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506456+0000", 2023-09-23T21:22:13.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9489999999999999e-06 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506471+0000", 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5101e-05 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506478+0000", 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8709999999999999e-06 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506530+0000", 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2335e-05 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506539+0000", 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6319999999999997e-06 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442702+0000", 2023-09-23T21:22:13.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93616331100000005 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442785+0000", 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2917000000000006e-05 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.442859+0000", 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.4169000000000001e-05 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.457477+0000", 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014618133 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470489+0000", 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013011224 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470543+0000", 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4104999999999998e-05 2023-09-23T21:22:13.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470575+0000", 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1878000000000002e-05 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470598+0000", 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3657999999999999e-05 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.619 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159431 12.d 12:b9860c03:::benchmark_data_smithi121_563784_object69028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 532.06709295600001, 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96419124899999997, 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159431 2023-09-23T21:22:13.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506492+0000", 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506495+0000", 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6230000000000001e-06 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506501+0000", 2023-09-23T21:22:13.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.879e-06 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506503+0000", 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7319999999999998e-06 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506557+0000", 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3786000000000003e-05 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.623 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.506561+0000", 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7440000000000001e-06 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445001+0000", 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93843964400000002 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445133+0000", 2023-09-23T21:22:13.624 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013199800000000001 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.445286+0000", 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015309000000000001 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462371+0000", 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.017085625 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.625 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470516+0000", 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0081449550000000006 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470630+0000", 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113372 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470654+0000", 2023-09-23T21:22:13.626 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4085000000000001e-05 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470684+0000", 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0071000000000001e-05 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.36681.0:159432 12.d 12:b2bd39fd:::benchmark_data_smithi121_563784_object69029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e471)", 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:22:13.627 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 532.05112947700002, 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94835160100000004, 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.36681", 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 159432 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.628 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522456+0000", 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522454+0000", 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522466+0000", 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1997000000000001e-05 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522497+0000", 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1341999999999999e-05 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522511+0000", 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3417e-05 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522631+0000", 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000120647 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:21.522648+0000", 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7096999999999999e-05 2023-09-23T21:22:13.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447334+0000", 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92468556300000004 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447475+0000", 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014071400000000001 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,2", 2023-09-23T21:22:13.632 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.447625+0000", 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015043700000000001 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.462380+0000", 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014754949 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470712+0000", 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083322190000000001 2023-09-23T21:22:13.633 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470763+0000", 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0980999999999999e-05 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470789+0000", 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.6191999999999999e-05 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.634 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:13:22.470808+0000", 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8159999999999999e-05 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50819 13.0 13:07dbfe5f:::benchmark_data_smithi121_621349_object50818:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.686076+0000", 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.88750953900001, 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0142723499999999, 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.635 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50819 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686076+0000", 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686076+0000", 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686074+0000", 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686117+0000", 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3220999999999998e-05 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686133+0000", 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5795999999999999e-05 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686140+0000", 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0090000000000001e-06 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686202+0000", 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1588999999999996e-05 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op must wait for map", 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.686209+0000", 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6210000000000004e-06 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.691240+0000", 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0050300650000000002 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.691245+0000", 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3369999999999999e-06 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.694345+0000", 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.003100034 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.640 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.694506+0000", 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016077499999999999 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.694726+0000", 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000220781 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.641 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697158+0000", 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024318690000000001 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700181+0000", 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003023038 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700239+0000", 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7735999999999997e-05 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.642 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700304+0000", 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5214000000000003e-05 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700348+0000", 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3934999999999999e-05 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.643 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50835 13.0 13:070b5ba2:::benchmark_data_smithi121_621349_object50834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.697856+0000", 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.87572943999999, 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0027707880000001, 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50835 2023-09-23T21:22:13.644 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697856+0000", 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697856+0000", 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.645 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697855+0000", 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697929+0000", 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.326e-05 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697943+0000", 2023-09-23T21:22:13.646 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4089e-05 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.697948+0000", 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9479999999999999e-06 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.698013+0000", 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5320000000000005e-05 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.647 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.698024+0000", 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1396999999999999e-05 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697199+0000", 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99917456299999996 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697346+0000", 2023-09-23T21:22:13.648 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014690999999999999 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.697477+0000", 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013073499999999999 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700411+0000", 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002934716 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700485+0000", 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3169000000000004e-05 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700572+0000", 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.7449999999999998e-05 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700604+0000", 2023-09-23T21:22:13.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1837e-05 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700627+0000", 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2895e-05 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50857 13.0 13:0be955d9:::benchmark_data_smithi121_621349_object50856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:22:13.651 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.85919111200002, 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98640245000000004, 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50857 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:22:13.652 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714394+0000", 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714426+0000", 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2026999999999997e-05 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714439+0000", 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3606e-05 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714445+0000", 2023-09-23T21:22:13.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3550000000000004e-06 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715059+0000", 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00061476900000000004 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715067+0000", 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0080000000000002e-06 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.655 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.698491+0000", 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98342367900000005 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.698626+0000", 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013504700000000001 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.698783+0000", 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015678400000000001 2023-09-23T21:22:13.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700546+0000", 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0017630740000000001 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700637+0000", 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1219999999999997e-05 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.657 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700688+0000", 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0742999999999999e-05 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700760+0000", 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2452000000000004e-05 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700797+0000", 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6395e-05 2023-09-23T21:22:13.658 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50859 13.0 13:0bb990fe:::benchmark_data_smithi121_621349_object50858:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.85912534400001, 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.99097865500000004, 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.659 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50859 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:22:13.660 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714460+0000", 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714524+0000", 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4227999999999996e-05 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714532+0000", 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9489999999999996e-06 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.714536+0000", 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1350000000000002e-06 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715080+0000", 2023-09-23T21:22:13.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00054414899999999996 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.715084+0000", 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.647e-06 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.700850+0000", 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98576657599999995 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.663 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.701010+0000", 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015928399999999999 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.701101+0000", 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1631000000000004e-05 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705290+0000", 2023-09-23T21:22:13.664 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041886479999999997 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705297+0000", 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5680000000000002e-06 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705346+0000", 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8714000000000001e-05 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705395+0000", 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8738e-05 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705439+0000", 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3816999999999997e-05 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50889 13.0 13:00c0c20f:::benchmark_data_smithi121_621349_object50888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.736071+0000", 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.83751431000002, 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97526533100000001, 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50889 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.667 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736071+0000", 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736071+0000", 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736070+0000", 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736117+0000", 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6950000000000003e-05 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736129+0000", 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2414e-05 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736138+0000", 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6279999999999994e-06 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736189+0000", 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1014000000000003e-05 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.736198+0000", 2023-09-23T21:22:13.670 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.02e-06 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.702264+0000", 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96606645300000005 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.702350+0000", 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5798999999999996e-05 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.702460+0000", 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110215 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705301+0000", 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002840857 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705322+0000", 2023-09-23T21:22:13.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0720000000000002e-05 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711265+0000", 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005942854 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711305+0000", 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0679999999999997e-05 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711336+0000", 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0973000000000003e-05 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50894 13.0 13:058cc793:::benchmark_data_smithi121_621349_object50893:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.738626+0000", 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.83495972700001, 2023-09-23T21:22:13.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97285262100000003, 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50894 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738626+0000", 2023-09-23T21:22:13.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738626+0000", 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738625+0000", 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738676+0000", 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0964999999999997e-05 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738687+0000", 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1051e-05 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738691+0000", 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6890000000000003e-06 2023-09-23T21:22:13.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738738+0000", 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6381999999999997e-05 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.738744+0000", 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6100000000000002e-06 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.703672+0000", 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.96492752400000004 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.703794+0000", 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012215300000000001 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.703910+0000", 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011578000000000001 2023-09-23T21:22:13.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705496+0000", 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001585899 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711165+0000", 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005669716 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711382+0000", 2023-09-23T21:22:13.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021679199999999999 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711437+0000", 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4718e-05 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711478+0000", 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1434999999999999e-05 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.681 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50903 13.0 13:0f9f6d1d:::benchmark_data_smithi121_621349_object50902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.747607+0000", 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.82597840599999, 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97278066100000005, 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.682 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50903 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747607+0000", 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747607+0000", 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747606+0000", 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747684+0000", 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8508000000000001e-05 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747697+0000", 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.296e-05 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747704+0000", 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2200000000000003e-06 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747761+0000", 2023-09-23T21:22:13.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6390000000000001e-05 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.747771+0000", 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0530000000000001e-05 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705554+0000", 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95778294900000005 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705662+0000", 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000108287 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.705834+0000", 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017180200000000001 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711216+0000", 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053816890000000003 2023-09-23T21:22:13.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715936+0000", 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0047201630000000003 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720289+0000", 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0043527870000000003 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720343+0000", 2023-09-23T21:22:13.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3619000000000003e-05 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720388+0000", 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5161999999999999e-05 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50916 13.0 13:049e41a7:::benchmark_data_smithi121_621349_object50915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.757423+0000", 2023-09-23T21:22:13.689 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.81616282900001, 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.963065434, 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50916 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757423+0000", 2023-09-23T21:22:13.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757423+0000", 2023-09-23T21:22:13.694 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.694 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757421+0000", 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757474+0000", 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2800000000000003e-05 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757492+0000", 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7567e-05 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757498+0000", 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5259999999999999e-06 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757560+0000", 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1404999999999999e-05 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.757570+0000", 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.007e-05 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.707628+0000", 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.95005845799999999 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.707746+0000", 2023-09-23T21:22:13.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000117658 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.707840+0000", 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3846999999999998e-05 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713639+0000", 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0057993460000000004 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715960+0000", 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002320871 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720401+0000", 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0044407250000000004 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720452+0000", 2023-09-23T21:22:13.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1483000000000002e-05 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720488+0000", 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5890999999999999e-05 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50919 13.0 13:0850c3dd:::benchmark_data_smithi121_621349_object50918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.759629+0000", 2023-09-23T21:22:13.700 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.81395670000001, 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.97667352299999999, 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50919 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759629+0000", 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759629+0000", 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759627+0000", 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759690+0000", 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2829000000000004e-05 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759707+0000", 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7243000000000001e-05 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.759714+0000", 2023-09-23T21:22:13.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8120000000000002e-06 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.761268+0000", 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001554115 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.761279+0000", 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0654e-05 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.709282+0000", 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94800265699999997 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.709401+0000", 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119968 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.709570+0000", 2023-09-23T21:22:13.705 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016860200000000001 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713667+0000", 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040964399999999998 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715988+0000", 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023218010000000001 2023-09-23T21:22:13.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736189+0000", 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.020200722000000001 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736248+0000", 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8765999999999999e-05 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736302+0000", 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4419000000000002e-05 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50958 13.0 13:081dfd76:::benchmark_data_smithi121_621349_object50957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.791211+0000", 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.78237416899998, 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94518114399999997, 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.708 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50958 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791211+0000", 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791211+0000", 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791210+0000", 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791278+0000", 2023-09-23T21:22:13.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8035999999999997e-05 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791290+0000", 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1032e-05 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791296+0000", 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1240000000000002e-06 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.711 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791357+0000", 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1005000000000003e-05 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.791366+0000", 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3710000000000006e-06 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711515+0000", 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92014889099999997 2023-09-23T21:22:13.712 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711647+0000", 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013166 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.711798+0000", 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015160799999999999 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713692+0000", 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00189364 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.717934+0000", 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042422359999999999 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736312+0000", 2023-09-23T21:22:13.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.018377935000000001 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736356+0000", 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4456999999999998e-05 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736392+0000", 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5951000000000002e-05 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.715 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50971 13.0 13:0a7f1691:::benchmark_data_smithi121_621349_object50970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.77183731000002, 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.93478755099999999, 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.716 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50971 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.717 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801748+0000", 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801776+0000", 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8784000000000001e-05 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801784+0000", 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3610000000000003e-06 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801788+0000", 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.301e-06 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801839+0000", 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1397999999999998e-05 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.801848+0000", 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1140000000000006e-06 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713711+0000", 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.91186365400000002 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713824+0000", 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000112525 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.713961+0000", 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013691699999999999 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.715908+0000", 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001946925 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.720538+0000", 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0046308019999999998 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736447+0000", 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015908164999999998 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736497+0000", 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0773000000000003e-05 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736536+0000", 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8309000000000002e-05 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.723 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:50977 13.0 13:0e0af701:::benchmark_data_smithi121_621349_object50976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:56.808462+0000", 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 436.76512349400002, 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.92816345899999997, 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 50977 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808462+0000", 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808462+0000", 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808460+0000", 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808504+0000", 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3204000000000002e-05 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808518+0000", 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4809e-05 2023-09-23T21:22:13.726 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808525+0000", 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2759999999999997e-06 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808575+0000", 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0324999999999997e-05 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:56.808588+0000", 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2578999999999999e-05 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.716009+0000", 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90742122400000003 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.716109+0000", 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010050799999999999 2023-09-23T21:22:13.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6", 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.716204+0000", 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4594999999999998e-05 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.717906+0000", 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001702425 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.725391+0000", 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0074848570000000001 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736548+0000", 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01115709 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736590+0000", 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1406000000000003e-05 2023-09-23T21:22:13.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:57.736625+0000", 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5738999999999998e-05 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100279 14.f 14:f7019cd7:::benchmark_data_smithi121_684892_object14962:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.505286+0000", 2023-09-23T21:22:13.731 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 103.068298973, 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.691666796, 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100279 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.505286+0000", 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.505286+0000", 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.505285+0000", 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:22:13.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.505294+0000", 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4509999999999998e-06 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.505310+0000", 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5841999999999998e-05 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.505318+0000", 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1440000000000003e-06 2023-09-23T21:22:13.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.526731+0000", 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021412431999999999 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.526740+0000", 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.7079999999999992e-06 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.735 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.737314+0000", 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.210573833 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.738145+0000", 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00083062100000000003 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.926473+0000", 2023-09-23T21:22:13.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.18832794 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.926588+0000", 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000115024 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3,6", 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.926757+0000", 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016930600000000001 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.929219+0000", 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024619339999999998 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.932576+0000", 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003356871 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.978848+0000", 2023-09-23T21:22:13.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.046271944000000002 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.193406+0000", 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.21455879999999999 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.193461+0000", 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4302000000000003e-05 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:22:13.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.196953+0000", 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003492448 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:22:13.740 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:22:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:13.741 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:13.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:13.838 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:13.839 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:13.839 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/412877391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:13.842 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:13.887 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:13.887 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:13.887 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/412877391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:13.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:13.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:13.888 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/412877391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:13.909 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:14.098 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:14.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:14.206 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:14.291 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:14.436 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:14.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:14.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:14.522 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:22:14.549 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:14.651 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:14.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:14 smithi123 ceph-mon[193903]: pgmap v1971: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 16121/85221 objects degraded (18.917%); 18370/85221 objects misplaced (21.556%); 7.4 MiB/s, 190 objects/s recovering 2023-09-23T21:22:14.785 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:14 smithi121 ceph-mon[211526]: pgmap v1971: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 16121/85221 objects degraded (18.917%); 18370/85221 objects misplaced (21.556%); 7.4 MiB/s, 190 objects/s recovering 2023-09-23T21:22:14.786 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:22:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:22:14] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:22:14.786 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:14 smithi121 ceph-mon[219709]: pgmap v1971: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 16121/85221 objects degraded (18.917%); 18370/85221 objects misplaced (21.556%); 7.4 MiB/s, 190 objects/s recovering 2023-09-23T21:22:14.898 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:22:14.900 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:14.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:14.968 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:22:14.971 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:22:15.018 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:15.108 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:15.303 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:15.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:15.410 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:15.441 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:15.927 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:15.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:15.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:16.032 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:16.077 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:16.090 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:16.133 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1971,"stamp":"2023-09-23T21:22:13.432809+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1860745987,"num_objects":28407,"num_object_clones":0,"num_object_copies":85221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16121,"num_objects_misplaced":18370,"num_objects_unfound":0,"num_objects_dirty":28407,"num_whiteouts":0,"num_read":92,"num_read_kb":169,"num_write":142351,"num_write_kb":5460565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25593,"num_bytes_recovered":1524839781,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85887,"ondisk_log_size":85887,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13789672,"kb_used_data":7271408,"kb_used_omap":16,"kb_used_meta":6517679,"kb_avail":642426392,"statfs":{"total":671965249536,"available":657844625408,"internally_reserved":0,"allocated":7445921792,"data_stored":7428877177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6674104169},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-313655296,"num_objects":-4786,"num_object_clones":0,"num_object_copies":-14358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2685,"num_objects_misplaced":-2294,"num_objects_unfound":0,"num_objects_dirty":-4786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4786,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1902,"num_bytes_recovered":78053376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003154"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":225,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575702+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:21:54.575702+0000","last_peered":"2023-09-23T21:21:54.575702+0000","last_clean":"2023-09-23T21:21:54.575702+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:21:54.575702+0000","last_undegraded":"2023-09-23T21:21:54.575702+0000","last_fullsized":"2023-09-23T21:21:54.575702+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558684+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:21:54.558684+0000","last_peered":"2023-09-23T21:21:54.558684+0000","last_clean":"2023-09-23T21:21:54.558684+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:21:54.558684+0000","last_undegraded":"2023-09-23T21:21:54.558684+0000","last_fullsized":"2023-09-23T21:21:54.558684+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.988607+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:22:02.988607+0000","last_peered":"2023-09-23T21:22:02.988607+0000","last_clean":"2023-09-23T21:22:02.988607+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:22:02.988607+0000","last_undegraded":"2023-09-23T21:22:02.988607+0000","last_fullsized":"2023-09-23T21:22:02.988607+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":337,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575391+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:21:54.575391+0000","last_peered":"2023-09-23T21:21:54.575391+0000","last_clean":"2023-09-23T21:21:54.575391+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:21:54.575391+0000","last_undegraded":"2023-09-23T21:21:54.575391+0000","last_fullsized":"2023-09-23T21:21:54.575391+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":53,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558150+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:21:54.558150+0000","last_peered":"2023-09-23T21:21:54.558150+0000","last_clean":"2023-09-23T21:21:54.558150+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:21:54.558150+0000","last_undegraded":"2023-09-23T21:21:54.558150+0000","last_fullsized":"2023-09-23T21:21:54.558150+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":156,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.953458+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:22:02.953458+0000","last_peered":"2023-09-23T21:22:02.953458+0000","last_clean":"2023-09-23T21:22:02.953458+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:22:02.953458+0000","last_undegraded":"2023-09-23T21:22:02.953458+0000","last_fullsized":"2023-09-23T21:22:02.953458+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":57,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.584849+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:21:54.584849+0000","last_peered":"2023-09-23T21:21:54.584849+0000","last_clean":"2023-09-23T21:21:54.584849+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:21:54.584849+0000","last_undegraded":"2023-09-23T21:21:54.584849+0000","last_fullsized":"2023-09-23T21:21:54.584849+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558218+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:21:54.558218+0000","last_peered":"2023-09-23T21:21:54.558218+0000","last_clean":"2023-09-23T21:21:54.558218+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:21:54.558218+0000","last_undegraded":"2023-09-23T21:21:54.558218+0000","last_fullsized":"2023-09-23T21:21:54.558218+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":47,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.583589+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:21:54.583589+0000","last_peered":"2023-09-23T21:21:54.583589+0000","last_clean":"2023-09-23T21:21:54.583589+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:21:54.583589+0000","last_undegraded":"2023-09-23T21:21:54.583589+0000","last_fullsized":"2023-09-23T21:21:54.583589+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":44,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.560145+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:21:54.560145+0000","last_peered":"2023-09-23T21:21:54.560145+0000","last_clean":"2023-09-23T21:21:54.560145+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:21:54.560145+0000","last_undegraded":"2023-09-23T21:21:54.560145+0000","last_fullsized":"2023-09-23T21:21:54.560145+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":285,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575879+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:21:54.575879+0000","last_peered":"2023-09-23T21:21:54.575879+0000","last_clean":"2023-09-23T21:21:54.575879+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:21:54.575879+0000","last_undegraded":"2023-09-23T21:21:54.575879+0000","last_fullsized":"2023-09-23T21:21:54.575879+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.989904+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:22:02.989904+0000","last_peered":"2023-09-23T21:22:02.989904+0000","last_clean":"2023-09-23T21:22:02.989904+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:22:02.989904+0000","last_undegraded":"2023-09-23T21:22:02.989904+0000","last_fullsized":"2023-09-23T21:22:02.989904+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":381,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.568075+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:21:54.568075+0000","last_peered":"2023-09-23T21:21:54.568075+0000","last_clean":"2023-09-23T21:21:54.568075+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:21:54.568075+0000","last_undegraded":"2023-09-23T21:21:54.568075+0000","last_fullsized":"2023-09-23T21:21:54.568075+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":608,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.571158+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:21:54.571158+0000","last_peered":"2023-09-23T21:21:54.571158+0000","last_clean":"2023-09-23T21:21:54.571158+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:21:54.571158+0000","last_undegraded":"2023-09-23T21:21:54.571158+0000","last_fullsized":"2023-09-23T21:21:54.571158+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":233,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.576359+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:21:54.576359+0000","last_peered":"2023-09-23T21:21:54.576359+0000","last_clean":"2023-09-23T21:21:54.576359+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:21:54.576359+0000","last_undegraded":"2023-09-23T21:21:54.576359+0000","last_fullsized":"2023-09-23T21:21:54.576359+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":53,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558524+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:21:54.558524+0000","last_peered":"2023-09-23T21:21:54.558524+0000","last_clean":"2023-09-23T21:21:54.558524+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:21:54.558524+0000","last_undegraded":"2023-09-23T21:21:54.558524+0000","last_fullsized":"2023-09-23T21:21:54.558524+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":259,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.579419+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:21:54.579419+0000","last_peered":"2023-09-23T21:21:54.579419+0000","last_clean":"2023-09-23T21:21:54.579419+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:21:54.579419+0000","last_undegraded":"2023-09-23T21:21:54.579419+0000","last_fullsized":"2023-09-23T21:21:54.579419+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":267,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575996+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:21:54.575996+0000","last_peered":"2023-09-23T21:21:54.575996+0000","last_clean":"2023-09-23T21:21:54.575996+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:21:54.575996+0000","last_undegraded":"2023-09-23T21:21:54.575996+0000","last_fullsized":"2023-09-23T21:21:54.575996+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":227,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575456+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:21:54.575456+0000","last_peered":"2023-09-23T21:21:54.575456+0000","last_clean":"2023-09-23T21:21:54.575456+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:21:54.575456+0000","last_undegraded":"2023-09-23T21:21:54.575456+0000","last_fullsized":"2023-09-23T21:21:54.575456+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":44,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.559045+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:21:54.559045+0000","last_peered":"2023-09-23T21:21:54.559045+0000","last_clean":"2023-09-23T21:21:54.559045+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:21:54.559045+0000","last_undegraded":"2023-09-23T21:21:54.559045+0000","last_fullsized":"2023-09-23T21:21:54.559045+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":747,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.952447+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:22:02.952447+0000","last_peered":"2023-09-23T21:22:02.952447+0000","last_clean":"2023-09-23T21:22:02.952447+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:22:02.952447+0000","last_undegraded":"2023-09-23T21:22:02.952447+0000","last_fullsized":"2023-09-23T21:22:02.952447+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558868+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:21:54.558868+0000","last_peered":"2023-09-23T21:21:54.558868+0000","last_clean":"2023-09-23T21:21:54.558868+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:21:54.558868+0000","last_undegraded":"2023-09-23T21:21:54.558868+0000","last_fullsized":"2023-09-23T21:21:54.558868+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"651'8820","reported_seq":10644,"reported_epoch":651,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:11.357756+0000","last_change":"2023-09-23T21:21:01.215269+0000","last_active":"2023-09-23T21:22:11.357756+0000","last_peered":"2023-09-23T21:22:11.357756+0000","last_clean":"2023-09-23T21:19:28.877182+0000","last_became_active":"2023-09-23T21:21:01.190149+0000","last_became_peered":"2023-09-23T21:21:01.190149+0000","last_unstale":"2023-09-23T21:22:11.357756+0000","last_undegraded":"2023-09-23T21:21:00.607176+0000","last_fullsized":"2023-09-23T21:22:11.357756+0000","mapping_epoch":629,"log_start":"588'3267","ondisk_log_start":"588'3267","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:05.989327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1666,"num_objects_misplaced":1666,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8820,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":844,"num_bytes_recovered":55312384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,4,5],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,5,6","objects":986},{"shards":"4,6,7","objects":680}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":287,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.576159+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:21:54.576159+0000","last_peered":"2023-09-23T21:21:54.576159+0000","last_clean":"2023-09-23T21:21:54.576159+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:21:54.576159+0000","last_undegraded":"2023-09-23T21:21:54.576159+0000","last_fullsized":"2023-09-23T21:21:54.576159+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":47,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.573114+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:21:54.573114+0000","last_peered":"2023-09-23T21:21:54.573114+0000","last_clean":"2023-09-23T21:21:54.573114+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:21:54.573114+0000","last_undegraded":"2023-09-23T21:21:54.573114+0000","last_fullsized":"2023-09-23T21:21:54.573114+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"650'8953","reported_seq":10977,"reported_epoch":650,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:09.098051+0000","last_change":"2023-09-23T21:21:01.210342+0000","last_active":"2023-09-23T21:22:09.098051+0000","last_peered":"2023-09-23T21:22:09.098051+0000","last_clean":"2023-09-23T21:19:28.923531+0000","last_became_active":"2023-09-23T21:21:01.189847+0000","last_became_peered":"2023-09-23T21:21:01.189847+0000","last_unstale":"2023-09-23T21:22:09.098051+0000","last_undegraded":"2023-09-23T21:21:00.610088+0000","last_fullsized":"2023-09-23T21:22:09.098051+0000","mapping_epoch":629,"log_start":"590'3898","ondisk_log_start":"590'3898","created":573,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5055,"log_dups_size":0,"ondisk_log_size":5055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:58.686882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119209984,"num_objects":1819,"num_object_clones":0,"num_object_copies":5457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1819,"num_objects_misplaced":1819,"num_objects_unfound":0,"num_objects_dirty":1819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8953,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":938,"num_bytes_recovered":61341696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[4,6,5],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":1093},{"shards":"4,6,7","objects":726}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":301,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575465+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:21:54.575465+0000","last_peered":"2023-09-23T21:21:54.575465+0000","last_clean":"2023-09-23T21:21:54.575465+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:21:54.575465+0000","last_undegraded":"2023-09-23T21:21:54.575465+0000","last_fullsized":"2023-09-23T21:21:54.575465+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.988529+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:22:02.988529+0000","last_peered":"2023-09-23T21:22:02.988529+0000","last_clean":"2023-09-23T21:22:02.988529+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:22:02.988529+0000","last_undegraded":"2023-09-23T21:22:02.988529+0000","last_fullsized":"2023-09-23T21:22:02.988529+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"650'8771","reported_seq":14004,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:08.604820+0000","last_change":"2023-09-23T21:21:17.539683+0000","last_active":"2023-09-23T21:22:08.604820+0000","last_peered":"2023-09-23T21:22:08.604820+0000","last_clean":"2023-09-23T21:22:08.604820+0000","last_became_active":"2023-09-23T21:21:17.521633+0000","last_became_peered":"2023-09-23T21:21:17.521633+0000","last_unstale":"2023-09-23T21:22:08.604820+0000","last_undegraded":"2023-09-23T21:22:08.604820+0000","last_fullsized":"2023-09-23T21:22:08.604820+0000","mapping_epoch":636,"log_start":"589'3765","ondisk_log_start":"589'3765","created":573,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:05.844201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111214592,"num_objects":1697,"num_object_clones":0,"num_object_copies":5091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8771,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2555,"num_bytes_recovered":80478208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":273,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.576042+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:21:54.576042+0000","last_peered":"2023-09-23T21:21:54.576042+0000","last_clean":"2023-09-23T21:21:54.576042+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:21:54.576042+0000","last_undegraded":"2023-09-23T21:21:54.576042+0000","last_fullsized":"2023-09-23T21:21:54.576042+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.952644+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:22:02.952644+0000","last_peered":"2023-09-23T21:22:02.952644+0000","last_clean":"2023-09-23T21:22:02.952644+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:22:02.952644+0000","last_undegraded":"2023-09-23T21:22:02.952644+0000","last_fullsized":"2023-09-23T21:22:02.952644+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"650'8751","reported_seq":8853,"reported_epoch":650,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:08.673990+0000","last_change":"2023-09-23T21:20:59.644218+0000","last_active":"2023-09-23T21:22:08.673990+0000","last_peered":"2023-09-23T21:22:08.673990+0000","last_clean":"2023-09-23T21:20:30.430127+0000","last_became_active":"2023-09-23T21:20:59.632855+0000","last_became_peered":"2023-09-23T21:20:59.632855+0000","last_unstale":"2023-09-23T21:22:08.673990+0000","last_undegraded":"2023-09-23T21:22:08.673990+0000","last_fullsized":"2023-09-23T21:22:08.673990+0000","mapping_epoch":628,"log_start":"590'3825","ondisk_log_start":"590'3825","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4926,"log_dups_size":0,"ondisk_log_size":4926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:33.014367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121044992,"num_objects":1847,"num_object_clones":0,"num_object_copies":5541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3694,"num_objects_unfound":0,"num_objects_dirty":1847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8751,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1847}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":53,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558603+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:21:54.558603+0000","last_peered":"2023-09-23T21:21:54.558603+0000","last_clean":"2023-09-23T21:21:54.558603+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:21:54.558603+0000","last_undegraded":"2023-09-23T21:21:54.558603+0000","last_fullsized":"2023-09-23T21:21:54.558603+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.990940+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:22:02.990940+0000","last_peered":"2023-09-23T21:22:02.990940+0000","last_clean":"2023-09-23T21:22:02.990940+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:22:02.990940+0000","last_undegraded":"2023-09-23T21:22:02.990940+0000","last_fullsized":"2023-09-23T21:22:02.990940+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1076,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575280+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:21:54.575280+0000","last_peered":"2023-09-23T21:21:54.575280+0000","last_clean":"2023-09-23T21:21:54.575280+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:21:54.575280+0000","last_undegraded":"2023-09-23T21:21:54.575280+0000","last_fullsized":"2023-09-23T21:21:54.575280+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.6","version":"650'8915","reported_seq":14272,"reported_epoch":650,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:08.737772+0000","last_change":"2023-09-23T21:21:01.239919+0000","last_active":"2023-09-23T21:22:08.737772+0000","last_peered":"2023-09-23T21:22:08.737772+0000","last_clean":"2023-09-23T21:19:28.802255+0000","last_became_active":"2023-09-23T21:21:01.197858+0000","last_became_peered":"2023-09-23T21:21:01.197858+0000","last_unstale":"2023-09-23T21:22:08.737772+0000","last_undegraded":"2023-09-23T21:21:00.673065+0000","last_fullsized":"2023-09-23T21:22:08.737772+0000","mapping_epoch":629,"log_start":"590'3898","ondisk_log_start":"590'3898","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:39.315049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118685696,"num_objects":1811,"num_object_clones":0,"num_object_copies":5433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1811,"num_objects_misplaced":1811,"num_objects_unfound":0,"num_objects_dirty":1811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8915,"num_write_kb":343232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2620,"num_bytes_recovered":171638784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,4,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,4,6","objects":681},{"shards":"2,5,6","objects":1130}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":61,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.557989+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:21:54.557989+0000","last_peered":"2023-09-23T21:21:54.557989+0000","last_clean":"2023-09-23T21:21:54.557989+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:21:54.557989+0000","last_undegraded":"2023-09-23T21:21:54.557989+0000","last_fullsized":"2023-09-23T21:21:54.557989+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":408,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.568170+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:21:54.568170+0000","last_peered":"2023-09-23T21:21:54.568170+0000","last_clean":"2023-09-23T21:21:54.568170+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:21:54.568170+0000","last_undegraded":"2023-09-23T21:21:54.568170+0000","last_fullsized":"2023-09-23T21:21:54.568170+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":944,"reported_epoch":649,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:21:54.555624+0000","last_change":"2023-09-23T21:20:48.157415+0000","last_active":"2023-09-23T21:21:54.555624+0000","last_peered":"2023-09-23T21:21:54.555624+0000","last_clean":"2023-09-23T21:20:41.603469+0000","last_became_active":"2023-09-23T21:20:48.156886+0000","last_became_peered":"2023-09-23T21:20:48.156886+0000","last_unstale":"2023-09-23T21:21:54.555624+0000","last_undegraded":"2023-09-23T21:20:47.922781+0000","last_fullsized":"2023-09-23T21:20:47.921128+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:30.898986+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4,6","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.7","version":"650'9037","reported_seq":10241,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:08.869875+0000","last_change":"2023-09-23T21:20:59.594443+0000","last_active":"2023-09-23T21:22:08.869875+0000","last_peered":"2023-09-23T21:22:08.869875+0000","last_clean":"2023-09-23T21:22:08.869875+0000","last_became_active":"2023-09-23T21:20:48.149995+0000","last_became_peered":"2023-09-23T21:20:48.149995+0000","last_unstale":"2023-09-23T21:22:08.869875+0000","last_undegraded":"2023-09-23T21:22:08.869875+0000","last_fullsized":"2023-09-23T21:22:08.869875+0000","mapping_epoch":623,"log_start":"582'2665","ondisk_log_start":"582'2665","created":573,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":6372,"log_dups_size":0,"ondisk_log_size":6372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.278714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121569280,"num_objects":1855,"num_object_clones":0,"num_object_copies":5565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9037,"num_write_kb":348544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":146,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.952839+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:22:02.952839+0000","last_peered":"2023-09-23T21:22:02.952839+0000","last_clean":"2023-09-23T21:22:02.952839+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:22:02.952839+0000","last_undegraded":"2023-09-23T21:22:02.952839+0000","last_fullsized":"2023-09-23T21:22:02.952839+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":658,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.574069+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:21:54.574069+0000","last_peered":"2023-09-23T21:21:54.574069+0000","last_clean":"2023-09-23T21:21:54.574069+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:21:54.574069+0000","last_undegraded":"2023-09-23T21:21:54.574069+0000","last_fullsized":"2023-09-23T21:21:54.574069+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"651'8835","reported_seq":17854,"reported_epoch":651,"state":"active+recovering+remapped","last_fresh":"2023-09-23T21:22:11.439249+0000","last_change":"2023-09-23T21:20:59.612076+0000","last_active":"2023-09-23T21:22:11.439249+0000","last_peered":"2023-09-23T21:22:11.439249+0000","last_clean":"2023-09-23T21:19:28.911487+0000","last_became_active":"2023-09-23T21:20:32.169326+0000","last_became_peered":"2023-09-23T21:20:32.169326+0000","last_unstale":"2023-09-23T21:22:11.439249+0000","last_undegraded":"2023-09-23T21:22:11.439249+0000","last_fullsized":"2023-09-23T21:22:11.439249+0000","mapping_epoch":617,"log_start":"588'3368","ondisk_log_start":"588'3368","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5467,"log_dups_size":0,"ondisk_log_size":5467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:09.080183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107282432,"num_objects":1637,"num_object_clones":0,"num_object_copies":4911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8835,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4454,"num_bytes_recovered":237240320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,4],"avail_no_missing":["6","5","7"],"object_location_counts":[{"shards":"4,6,7","objects":1637}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":287,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.576794+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:21:54.576794+0000","last_peered":"2023-09-23T21:21:54.576794+0000","last_clean":"2023-09-23T21:21:54.576794+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:21:54.576794+0000","last_undegraded":"2023-09-23T21:21:54.576794+0000","last_fullsized":"2023-09-23T21:21:54.576794+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":695,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.568449+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:21:54.568449+0000","last_peered":"2023-09-23T21:21:54.568449+0000","last_clean":"2023-09-23T21:21:54.568449+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:21:54.568449+0000","last_undegraded":"2023-09-23T21:21:54.568449+0000","last_fullsized":"2023-09-23T21:21:54.568449+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":913,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.559965+0000","last_change":"2023-09-23T21:20:49.968107+0000","last_active":"2023-09-23T21:21:54.559965+0000","last_peered":"2023-09-23T21:21:54.559965+0000","last_clean":"2023-09-23T21:21:54.559965+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:21:54.559965+0000","last_undegraded":"2023-09-23T21:21:54.559965+0000","last_fullsized":"2023-09-23T21:21:54.559965+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":429358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.5","version":"650'9098","reported_seq":16371,"reported_epoch":650,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:09.101333+0000","last_change":"2023-09-23T21:20:59.554347+0000","last_active":"2023-09-23T21:22:09.101333+0000","last_peered":"2023-09-23T21:22:09.101333+0000","last_clean":"2023-09-23T21:19:28.864495+0000","last_became_active":"2023-09-23T21:20:32.158375+0000","last_became_peered":"2023-09-23T21:20:32.158375+0000","last_unstale":"2023-09-23T21:22:09.101333+0000","last_undegraded":"2023-09-23T21:20:59.551954+0000","last_fullsized":"2023-09-23T21:22:09.101333+0000","mapping_epoch":617,"log_start":"590'4092","ondisk_log_start":"590'4092","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:46.553959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122028032,"num_objects":1862,"num_object_clones":0,"num_object_copies":5586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1149,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":1862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9098,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3581,"num_bytes_recovered":234684416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"2,4,7","objects":750},{"shards":"4,5,7","objects":1112}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":53,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.557989+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:21:54.557989+0000","last_peered":"2023-09-23T21:21:54.557989+0000","last_clean":"2023-09-23T21:21:54.557989+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:21:54.557989+0000","last_undegraded":"2023-09-23T21:21:54.557989+0000","last_fullsized":"2023-09-23T21:21:54.557989+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.989404+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:22:02.989404+0000","last_peered":"2023-09-23T21:22:02.989404+0000","last_clean":"2023-09-23T21:22:02.989404+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:22:02.989404+0000","last_undegraded":"2023-09-23T21:22:02.989404+0000","last_fullsized":"2023-09-23T21:22:02.989404+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":733,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.953338+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:22:02.953338+0000","last_peered":"2023-09-23T21:22:02.953338+0000","last_clean":"2023-09-23T21:22:02.953338+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:22:02.953338+0000","last_undegraded":"2023-09-23T21:22:02.953338+0000","last_fullsized":"2023-09-23T21:22:02.953338+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"650'8845","reported_seq":9696,"reported_epoch":650,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:08.737870+0000","last_change":"2023-09-23T21:21:01.233673+0000","last_active":"2023-09-23T21:22:08.737870+0000","last_peered":"2023-09-23T21:22:08.737870+0000","last_clean":"2023-09-23T21:19:28.898608+0000","last_became_active":"2023-09-23T21:21:01.195228+0000","last_became_peered":"2023-09-23T21:21:01.195228+0000","last_unstale":"2023-09-23T21:22:08.737870+0000","last_undegraded":"2023-09-23T21:21:00.570455+0000","last_fullsized":"2023-09-23T21:22:08.737870+0000","mapping_epoch":629,"log_start":"590'3846","ondisk_log_start":"590'3846","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:03.164901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116719616,"num_objects":1781,"num_object_clones":0,"num_object_copies":5343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1781,"num_objects_misplaced":1781,"num_objects_unfound":0,"num_objects_dirty":1781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8845,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[2,7,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1064},{"shards":"2,6,7","objects":717}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.988855+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:22:02.988855+0000","last_peered":"2023-09-23T21:22:02.988855+0000","last_clean":"2023-09-23T21:22:02.988855+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:22:02.988855+0000","last_undegraded":"2023-09-23T21:22:02.988855+0000","last_fullsized":"2023-09-23T21:22:02.988855+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":37,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.584464+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:21:54.584464+0000","last_peered":"2023-09-23T21:21:54.584464+0000","last_clean":"2023-09-23T21:21:54.584464+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:21:54.584464+0000","last_undegraded":"2023-09-23T21:21:54.584464+0000","last_fullsized":"2023-09-23T21:21:54.584464+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.989979+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:22:02.989979+0000","last_peered":"2023-09-23T21:22:02.989979+0000","last_clean":"2023-09-23T21:22:02.989979+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:22:02.989979+0000","last_undegraded":"2023-09-23T21:22:02.989979+0000","last_fullsized":"2023-09-23T21:22:02.989979+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":925,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.555774+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:21:54.555774+0000","last_peered":"2023-09-23T21:21:54.555774+0000","last_clean":"2023-09-23T21:21:54.555774+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:21:54.555774+0000","last_undegraded":"2023-09-23T21:21:54.555774+0000","last_fullsized":"2023-09-23T21:21:54.555774+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"650'8789","reported_seq":8874,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:09.018748+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:22:09.018748+0000","last_peered":"2023-09-23T21:22:09.018748+0000","last_clean":"2023-09-23T21:22:09.018748+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:22:09.018748+0000","last_undegraded":"2023-09-23T21:22:09.018748+0000","last_fullsized":"2023-09-23T21:22:09.018748+0000","mapping_epoch":573,"log_start":"590'3787","ondisk_log_start":"590'3787","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5002,"log_dups_size":0,"ondisk_log_size":5002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114884608,"num_objects":1753,"num_object_clones":0,"num_object_copies":5259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8789,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.990164+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:22:02.990164+0000","last_peered":"2023-09-23T21:22:02.990164+0000","last_clean":"2023-09-23T21:22:02.990164+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:22:02.990164+0000","last_undegraded":"2023-09-23T21:22:02.990164+0000","last_fullsized":"2023-09-23T21:22:02.990164+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":563,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.571020+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:21:54.571020+0000","last_peered":"2023-09-23T21:21:54.571020+0000","last_clean":"2023-09-23T21:21:54.571020+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:21:54.571020+0000","last_undegraded":"2023-09-23T21:21:54.571020+0000","last_fullsized":"2023-09-23T21:21:54.571020+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.988854+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:22:02.988854+0000","last_peered":"2023-09-23T21:22:02.988854+0000","last_clean":"2023-09-23T21:22:02.988854+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:22:02.988854+0000","last_undegraded":"2023-09-23T21:22:02.988854+0000","last_fullsized":"2023-09-23T21:22:02.988854+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1036,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.579330+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:21:54.579330+0000","last_peered":"2023-09-23T21:21:54.579330+0000","last_clean":"2023-09-23T21:21:54.579330+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:21:54.579330+0000","last_undegraded":"2023-09-23T21:21:54.579330+0000","last_fullsized":"2023-09-23T21:21:54.579330+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.d","version":"651'8963","reported_seq":9124,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.427521+0000","last_change":"2023-09-23T21:21:22.585504+0000","last_active":"2023-09-23T21:22:11.427521+0000","last_peered":"2023-09-23T21:22:11.427521+0000","last_clean":"2023-09-23T21:22:11.427521+0000","last_became_active":"2023-09-23T21:21:22.568285+0000","last_became_peered":"2023-09-23T21:21:22.568285+0000","last_unstale":"2023-09-23T21:22:11.427521+0000","last_undegraded":"2023-09-23T21:22:11.427521+0000","last_fullsized":"2023-09-23T21:22:11.427521+0000","mapping_epoch":639,"log_start":"589'3448","ondisk_log_start":"589'3448","created":573,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:01.251062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111804440,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8963,"num_write_kb":341377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":679,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.556089+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:21:54.556089+0000","last_peered":"2023-09-23T21:21:54.556089+0000","last_clean":"2023-09-23T21:21:54.556089+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:21:54.556089+0000","last_undegraded":"2023-09-23T21:21:54.556089+0000","last_fullsized":"2023-09-23T21:21:54.556089+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.989439+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:22:02.989439+0000","last_peered":"2023-09-23T21:22:02.989439+0000","last_clean":"2023-09-23T21:22:02.989439+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:22:02.989439+0000","last_undegraded":"2023-09-23T21:22:02.989439+0000","last_fullsized":"2023-09-23T21:22:02.989439+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.989330+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:22:02.989330+0000","last_peered":"2023-09-23T21:22:02.989330+0000","last_clean":"2023-09-23T21:22:02.989330+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:22:02.989330+0000","last_undegraded":"2023-09-23T21:22:02.989330+0000","last_fullsized":"2023-09-23T21:22:02.989330+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"650'8887","reported_seq":11788,"reported_epoch":650,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:08.569886+0000","last_change":"2023-09-23T21:21:01.214995+0000","last_active":"2023-09-23T21:22:08.569886+0000","last_peered":"2023-09-23T21:22:08.569886+0000","last_clean":"2023-09-23T21:19:28.902983+0000","last_became_active":"2023-09-23T21:21:01.189928+0000","last_became_peered":"2023-09-23T21:21:01.189928+0000","last_unstale":"2023-09-23T21:22:08.569886+0000","last_undegraded":"2023-09-23T21:21:00.625306+0000","last_fullsized":"2023-09-23T21:22:08.569886+0000","mapping_epoch":629,"log_start":"583'2886","ondisk_log_start":"583'2886","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:45.309159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117506048,"num_objects":1793,"num_object_clones":0,"num_object_copies":5379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1793,"num_objects_misplaced":1793,"num_objects_unfound":0,"num_objects_dirty":1793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8887,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1379,"num_bytes_recovered":90374144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,2,5],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,5,7","objects":1063},{"shards":"2,6,7","objects":730}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":508,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:58.735837+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:21:58.735837+0000","last_peered":"2023-09-23T21:21:58.735837+0000","last_clean":"2023-09-23T21:21:58.735837+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:21:58.735837+0000","last_undegraded":"2023-09-23T21:21:58.735837+0000","last_fullsized":"2023-09-23T21:21:58.735837+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1037,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.576999+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:21:54.576999+0000","last_peered":"2023-09-23T21:21:54.576999+0000","last_clean":"2023-09-23T21:21:54.576999+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:21:54.576999+0000","last_undegraded":"2023-09-23T21:21:54.576999+0000","last_fullsized":"2023-09-23T21:21:54.576999+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":37,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.577723+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:21:54.577723+0000","last_peered":"2023-09-23T21:21:54.577723+0000","last_clean":"2023-09-23T21:21:54.577723+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:21:54.577723+0000","last_undegraded":"2023-09-23T21:21:54.577723+0000","last_fullsized":"2023-09-23T21:21:54.577723+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":22,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.952549+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:22:02.952549+0000","last_peered":"2023-09-23T21:22:02.952549+0000","last_clean":"2023-09-23T21:22:02.952549+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:22:02.952549+0000","last_undegraded":"2023-09-23T21:22:02.952549+0000","last_fullsized":"2023-09-23T21:22:02.952549+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":685,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.991025+0000","last_change":"2023-09-23T21:21:21.571375+0000","last_active":"2023-09-23T21:22:02.991025+0000","last_peered":"2023-09-23T21:22:02.991025+0000","last_clean":"2023-09-23T21:22:02.991025+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:22:02.991025+0000","last_undegraded":"2023-09-23T21:22:02.991025+0000","last_fullsized":"2023-09-23T21:22:02.991025+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.551943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.551943+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.c","version":"650'9013","reported_seq":9098,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:09.078229+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:22:09.078229+0000","last_peered":"2023-09-23T21:22:09.078229+0000","last_clean":"2023-09-23T21:22:09.078229+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:22:09.078229+0000","last_undegraded":"2023-09-23T21:22:09.078229+0000","last_fullsized":"2023-09-23T21:22:09.078229+0000","mapping_epoch":573,"log_start":"590'3923","ondisk_log_start":"590'3923","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117243904,"num_objects":1789,"num_object_clones":0,"num_object_copies":5367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9013,"num_write_kb":345664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":317,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.577079+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:21:54.577079+0000","last_peered":"2023-09-23T21:21:54.577079+0000","last_clean":"2023-09-23T21:21:54.577079+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:21:54.577079+0000","last_undegraded":"2023-09-23T21:21:54.577079+0000","last_fullsized":"2023-09-23T21:21:54.577079+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":122,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.576675+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:21:54.576675+0000","last_peered":"2023-09-23T21:21:54.576675+0000","last_clean":"2023-09-23T21:21:54.576675+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:21:54.576675+0000","last_undegraded":"2023-09-23T21:21:54.576675+0000","last_fullsized":"2023-09-23T21:21:54.576675+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.989575+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:22:02.989575+0000","last_peered":"2023-09-23T21:22:02.989575+0000","last_clean":"2023-09-23T21:22:02.989575+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:22:02.989575+0000","last_undegraded":"2023-09-23T21:22:02.989575+0000","last_fullsized":"2023-09-23T21:22:02.989575+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":713,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.953008+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:22:02.953008+0000","last_peered":"2023-09-23T21:22:02.953008+0000","last_clean":"2023-09-23T21:22:02.953008+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:22:02.953008+0000","last_undegraded":"2023-09-23T21:22:02.953008+0000","last_fullsized":"2023-09-23T21:22:02.953008+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"650'8706","reported_seq":14877,"reported_epoch":650,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:22:09.102221+0000","last_change":"2023-09-23T21:20:48.304799+0000","last_active":"2023-09-23T21:22:09.102221+0000","last_peered":"2023-09-23T21:22:09.102221+0000","last_clean":"2023-09-23T21:19:28.920085+0000","last_became_active":"2023-09-23T21:20:48.224556+0000","last_became_peered":"2023-09-23T21:20:48.224556+0000","last_unstale":"2023-09-23T21:22:09.102221+0000","last_undegraded":"2023-09-23T21:20:48.204030+0000","last_fullsized":"2023-09-23T21:20:47.933463+0000","mapping_epoch":623,"log_start":"589'3644","ondisk_log_start":"589'3644","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5062,"log_dups_size":0,"ondisk_log_size":5062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:02.563744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114294784,"num_objects":1744,"num_object_clones":0,"num_object_copies":5232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3217,"num_objects_misplaced":561,"num_objects_unfound":0,"num_objects_dirty":1744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8706,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3021,"num_bytes_recovered":197853184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[4,2],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"2,4,6","objects":734},{"shards":"4,5,6","objects":1010}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":257,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.575957+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:21:54.575957+0000","last_peered":"2023-09-23T21:21:54.575957+0000","last_clean":"2023-09-23T21:21:54.575957+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:21:54.575957+0000","last_undegraded":"2023-09-23T21:21:54.575957+0000","last_fullsized":"2023-09-23T21:21:54.575957+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.988782+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:22:02.988782+0000","last_peered":"2023-09-23T21:22:02.988782+0000","last_clean":"2023-09-23T21:22:02.988782+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:22:02.988782+0000","last_undegraded":"2023-09-23T21:22:02.988782+0000","last_fullsized":"2023-09-23T21:22:02.988782+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.573175+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:21:54.573175+0000","last_peered":"2023-09-23T21:21:54.573175+0000","last_clean":"2023-09-23T21:21:54.573175+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:21:54.573175+0000","last_undegraded":"2023-09-23T21:21:54.573175+0000","last_fullsized":"2023-09-23T21:21:54.573175+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":951,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.571628+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:21:54.571628+0000","last_peered":"2023-09-23T21:21:54.571628+0000","last_clean":"2023-09-23T21:21:54.571628+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:21:54.571628+0000","last_undegraded":"2023-09-23T21:21:54.571628+0000","last_fullsized":"2023-09-23T21:21:54.571628+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.a","version":"650'8840","reported_seq":15994,"reported_epoch":650,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:09.051343+0000","last_change":"2023-09-23T21:21:01.218382+0000","last_active":"2023-09-23T21:22:09.051343+0000","last_peered":"2023-09-23T21:22:09.051343+0000","last_clean":"2023-09-23T21:19:28.896530+0000","last_became_active":"2023-09-23T21:21:01.190094+0000","last_became_peered":"2023-09-23T21:21:01.190094+0000","last_unstale":"2023-09-23T21:22:09.051343+0000","last_undegraded":"2023-09-23T21:21:00.719860+0000","last_fullsized":"2023-09-23T21:22:09.051343+0000","mapping_epoch":629,"log_start":"590'3818","ondisk_log_start":"590'3818","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5022,"log_dups_size":0,"ondisk_log_size":5022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:06.621301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117309440,"num_objects":1790,"num_object_clones":0,"num_object_copies":5370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2884,"num_objects_misplaced":2700,"num_objects_unfound":0,"num_objects_dirty":1790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8840,"num_write_kb":340160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3507,"num_bytes_recovered":229703680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[4,5,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":1078},{"shards":"4,6,7","objects":712}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":347,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.576237+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:21:54.576237+0000","last_peered":"2023-09-23T21:21:54.576237+0000","last_clean":"2023-09-23T21:21:54.576237+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:21:54.576237+0000","last_undegraded":"2023-09-23T21:21:54.576237+0000","last_fullsized":"2023-09-23T21:21:54.576237+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.990371+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:22:02.990371+0000","last_peered":"2023-09-23T21:22:02.990371+0000","last_clean":"2023-09-23T21:22:02.990371+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:22:02.990371+0000","last_undegraded":"2023-09-23T21:22:02.990371+0000","last_fullsized":"2023-09-23T21:22:02.990371+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":62,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558066+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:21:54.558066+0000","last_peered":"2023-09-23T21:21:54.558066+0000","last_clean":"2023-09-23T21:21:54.558066+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:21:54.558066+0000","last_undegraded":"2023-09-23T21:21:54.558066+0000","last_fullsized":"2023-09-23T21:21:54.558066+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1044,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558446+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:21:54.558446+0000","last_peered":"2023-09-23T21:21:54.558446+0000","last_clean":"2023-09-23T21:21:54.558446+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:21:54.558446+0000","last_undegraded":"2023-09-23T21:21:54.558446+0000","last_fullsized":"2023-09-23T21:21:54.558446+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"650'9024","reported_seq":12333,"reported_epoch":650,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:08.693376+0000","last_change":"2023-09-23T21:21:22.582611+0000","last_active":"2023-09-23T21:22:08.693376+0000","last_peered":"2023-09-23T21:22:08.693376+0000","last_clean":"2023-09-23T21:19:48.922364+0000","last_became_active":"2023-09-23T21:21:22.566477+0000","last_became_peered":"2023-09-23T21:21:22.566477+0000","last_unstale":"2023-09-23T21:22:08.693376+0000","last_undegraded":"2023-09-23T21:22:08.693376+0000","last_fullsized":"2023-09-23T21:22:08.693376+0000","mapping_epoch":639,"log_start":"590'4056","ondisk_log_start":"590'4056","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4968,"log_dups_size":0,"ondisk_log_size":4968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:46.195361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120061952,"num_objects":1832,"num_object_clones":0,"num_object_copies":5496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1832,"num_objects_unfound":0,"num_objects_dirty":1832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9024,"num_write_kb":347392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104660992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":1832}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":557,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.953463+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:22:02.953463+0000","last_peered":"2023-09-23T21:22:02.953463+0000","last_clean":"2023-09-23T21:22:02.953463+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:22:02.953463+0000","last_undegraded":"2023-09-23T21:22:02.953463+0000","last_fullsized":"2023-09-23T21:22:02.953463+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":37,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.989498+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:22:02.989498+0000","last_peered":"2023-09-23T21:22:02.989498+0000","last_clean":"2023-09-23T21:22:02.989498+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:22:02.989498+0000","last_undegraded":"2023-09-23T21:22:02.989498+0000","last_fullsized":"2023-09-23T21:22:02.989498+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":37,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.572989+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:21:54.572989+0000","last_peered":"2023-09-23T21:21:54.572989+0000","last_clean":"2023-09-23T21:21:54.572989+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:21:54.572989+0000","last_undegraded":"2023-09-23T21:21:54.572989+0000","last_fullsized":"2023-09-23T21:21:54.572989+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":101,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.571093+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:21:54.571093+0000","last_peered":"2023-09-23T21:21:54.571093+0000","last_clean":"2023-09-23T21:21:54.571093+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:21:54.571093+0000","last_undegraded":"2023-09-23T21:21:54.571093+0000","last_fullsized":"2023-09-23T21:21:54.571093+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":742,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.556218+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:21:54.556218+0000","last_peered":"2023-09-23T21:21:54.556218+0000","last_clean":"2023-09-23T21:21:54.556218+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:21:54.556218+0000","last_undegraded":"2023-09-23T21:21:54.556218+0000","last_fullsized":"2023-09-23T21:21:54.556218+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":410,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.567924+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:21:54.567924+0000","last_peered":"2023-09-23T21:21:54.567924+0000","last_clean":"2023-09-23T21:21:54.567924+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:21:54.567924+0000","last_undegraded":"2023-09-23T21:21:54.567924+0000","last_fullsized":"2023-09-23T21:21:54.567924+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":253,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.570832+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:21:54.570832+0000","last_peered":"2023-09-23T21:21:54.570832+0000","last_clean":"2023-09-23T21:21:54.570832+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:21:54.570832+0000","last_undegraded":"2023-09-23T21:21:54.570832+0000","last_fullsized":"2023-09-23T21:21:54.570832+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":162,"reported_epoch":650,"state":"active+clean","last_fresh":"2023-09-23T21:22:02.952787+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:22:02.952787+0000","last_peered":"2023-09-23T21:22:02.952787+0000","last_clean":"2023-09-23T21:22:02.952787+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:22:02.952787+0000","last_undegraded":"2023-09-23T21:22:02.952787+0000","last_fullsized":"2023-09-23T21:22:02.952787+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":44,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.558149+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:21:54.558149+0000","last_peered":"2023-09-23T21:21:54.558149+0000","last_clean":"2023-09-23T21:21:54.558149+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:21:54.558149+0000","last_undegraded":"2023-09-23T21:21:54.558149+0000","last_fullsized":"2023-09-23T21:21:54.558149+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":62,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.559115+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:21:54.559115+0000","last_peered":"2023-09-23T21:21:54.559115+0000","last_clean":"2023-09-23T21:21:54.559115+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:21:54.559115+0000","last_undegraded":"2023-09-23T21:21:54.559115+0000","last_fullsized":"2023-09-23T21:21:54.559115+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":53,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.560068+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:21:54.560068+0000","last_peered":"2023-09-23T21:21:54.560068+0000","last_clean":"2023-09-23T21:21:54.560068+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:21:54.560068+0000","last_undegraded":"2023-09-23T21:21:54.560068+0000","last_fullsized":"2023-09-23T21:21:54.560068+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":638,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.556348+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:21:54.556348+0000","last_peered":"2023-09-23T21:21:54.556348+0000","last_clean":"2023-09-23T21:21:54.556348+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:21:54.556348+0000","last_undegraded":"2023-09-23T21:21:54.556348+0000","last_fullsized":"2023-09-23T21:21:54.556348+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":357,"reported_epoch":649,"state":"active+clean","last_fresh":"2023-09-23T21:21:54.567890+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:21:54.567890+0000","last_peered":"2023-09-23T21:21:54.567890+0000","last_clean":"2023-09-23T21:21:54.567890+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:21:54.567890+0000","last_undegraded":"2023-09-23T21:21:54.567890+0000","last_fullsized":"2023-09-23T21:21:54.567890+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1860042776,"num_objects":28383,"num_object_clones":0,"num_object_copies":85149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16120,"num_objects_misplaced":18370,"num_objects_unfound":0,"num_objects_dirty":28383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142247,"num_write_kb":5460097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25414,"num_bytes_recovered":1523253248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7333490688,"data_stored":7333478472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84061,"ondisk_log_size":84061,"up":48,"acting":47,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697707,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1550763,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3297280,"data_stored":3279520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201921,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1736556,"kb_used_data":842020,"kb_used_omap":0,"kb_used_meta":894528,"kb_avail":92008596,"statfs":{"total":95995035648,"available":94216802304,"internally_reserved":0,"allocated":862228480,"data_stored":860439552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915996672},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.605,"15min":2.3159999999999998},"min":{"1min":0.27300000000000002,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":6.6390000000000002,"5min":16.468,"15min":141.33199999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.4790000000000001,"5min":1.992,"15min":2.4780000000000002},"min":{"1min":0.29799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":12.962,"5min":12.962,"15min":141.31700000000001},"last":3.0710000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":2.0840000000000001,"15min":3.4689999999999999},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.24099999999999999},"max":{"1min":13.054,"5min":13.054,"15min":114.16500000000001},"last":3.044},{"interface":"front","average":{"1min":3.0009999999999999,"5min":1.9350000000000001,"15min":3.8170000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":10.881,"5min":16.289000000000001,"15min":117.637},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":8.1029999999999998,"5min":6.1260000000000003,"15min":5.5369999999999999},"min":{"1min":0.252,"5min":0.246,"15min":0.224},"max":{"1min":68.793999999999997,"5min":158.726,"15min":183.44200000000001},"last":3.0030000000000001},{"interface":"front","average":{"1min":7.2279999999999998,"5min":5.7489999999999997,"15min":5.7119999999999997},"min":{"1min":0.32800000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":63.264000000000003,"5min":158.70400000000001,"15min":183.46199999999999},"last":0.93200000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:21:15 2023","interfaces":[{"interface":"back","average":{"1min":4.3310000000000004,"5min":2.2050000000000001,"15min":4.1589999999999998},"min":{"1min":0.41099999999999998,"5min":0.33700000000000002,"15min":0.26700000000000002},"max":{"1min":30.667999999999999,"5min":30.667999999999999,"15min":135.66200000000001},"last":0.37},{"interface":"front","average":{"1min":3.8620000000000001,"5min":2.1019999999999999,"15min":4.383},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":30.798999999999999,"5min":30.798999999999999,"15min":135.47},"last":0.96599999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.1849999999999996,"5min":3.653,"15min":1.9690000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":13.821999999999999,"5min":31.695,"15min":31.695},"last":10.68},{"interface":"front","average":{"1min":6.8120000000000003,"5min":3.7869999999999999,"15min":2.0179999999999998},"min":{"1min":0.48499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":27.821999999999999,"5min":34.994,"15min":34.994},"last":6.6130000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":4.7350000000000003,"15min":2.46},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":10.712,"5min":156.30099999999999,"15min":156.30099999999999},"last":4.8739999999999997},{"interface":"front","average":{"1min":4.5389999999999997,"5min":4.8739999999999997,"15min":2.548},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":12.151,"5min":156.22,"15min":156.22},"last":4.7610000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":3.75,"15min":2.8140000000000001},"min":{"1min":0.45700000000000002,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":14.066000000000001,"5min":23.574999999999999,"15min":143.18700000000001},"last":10.372999999999999},{"interface":"front","average":{"1min":6.4370000000000003,"5min":3.242,"15min":2.528},"min":{"1min":0.504,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":21.081,"5min":21.081,"15min":142.70400000000001},"last":1.0009999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463542,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2607016,"kb_used_data":1706848,"kb_used_omap":0,"kb_used_meta":900032,"kb_avail":91138136,"statfs":{"total":95995035648,"available":93325451264,"internally_reserved":0,"allocated":1747812352,"data_stored":1746028289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921632768},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.78,"15min":2.9660000000000002},"min":{"1min":0.28699999999999998,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":12.313000000000001,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.1880000000000002,"15min":3.1960000000000002},"min":{"1min":0.27400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.968,"5min":11.968,"15min":149.303},"last":1.421}]},{"osd":1,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.4020000000000001,"15min":2.9990000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":12.013,"5min":16.126999999999999,"15min":129.37899999999999},"last":2.0819999999999999},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.8540000000000001,"15min":2.742},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":14.366,"5min":14.366,"15min":129.40100000000001},"last":2.0150000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":25.170999999999999,"5min":9.1639999999999997,"15min":7.6150000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.23400000000001},"last":2.5529999999999999},{"interface":"front","average":{"1min":25.001000000000001,"5min":9.6969999999999992,"15min":7.7549999999999999},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":199.27000000000001,"5min":199.27000000000001,"15min":199.27000000000001},"last":4.3140000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":2.1880000000000002,"15min":3.5640000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":6.516,"5min":14.411,"15min":137.875},"last":1.72},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.986,"15min":3.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":12.35,"5min":12.35,"15min":137.60900000000001},"last":2.2109999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":3.6909999999999998,"15min":2.5169999999999999},"min":{"1min":0.29099999999999998,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":20.408999999999999,"5min":20.408999999999999,"15min":55.494},"last":1.379},{"interface":"front","average":{"1min":4.5709999999999997,"5min":3.367,"15min":2.3159999999999998},"min":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.23999999999999999},"max":{"1min":12.864000000000001,"5min":14.622,"15min":55.305},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":3.1499999999999999,"15min":2.8180000000000001},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":12.051,"5min":12.975,"15min":222.90899999999999},"last":1.454},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.9969999999999999,"15min":2.7709999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28499999999999998},"max":{"1min":10.625999999999999,"5min":17.927,"15min":224.655},"last":9.7349999999999994}]},{"osd":7,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":4.8970000000000002,"15min":2.2599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36399999999999999,"15min":0.28000000000000003},"max":{"1min":12.114000000000001,"5min":165.96799999999999,"15min":165.96799999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":3.5030000000000001,"5min":5.4870000000000001,"15min":2.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.251},"max":{"1min":14.457000000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":0.44600000000000001}]}]},{"osd":1,"up_from":276,"seq":1185410973989,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643456,"kb_used_data":8184,"kb_used_omap":5,"kb_used_meta":635258,"kb_avail":93101696,"statfs":{"total":95995035648,"available":95336136704,"internally_reserved":0,"allocated":8380416,"data_stored":5290105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":650504931},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.38200000000000001,"15min":2.0009999999999999},"min":{"1min":0.22600000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":0.505,"5min":1.9450000000000001,"15min":108.47},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.40100000000000002,"15min":1.952},"min":{"1min":0.26200000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":109.532},"last":0.54300000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":7.6120000000000001,"5min":8.4109999999999996,"15min":7.8899999999999997},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.20200000000000001},"max":{"1min":72.966999999999999,"5min":149.35300000000001,"15min":171.35400000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":7.6459999999999999,"5min":8.5180000000000007,"15min":7.5659999999999998},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":74.450000000000003,"5min":147.50399999999999,"15min":170.69900000000001},"last":0.877}]},{"osd":3,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":0.55200000000000005,"15min":3.593},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":17.247,"5min":17.247,"15min":99.120999999999995},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.66000000000000003,"15min":4.1600000000000001},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.21299999999999999},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":150.38},"last":0.60999999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4770000000000001,"5min":3.2519999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.5800000000000001,"5min":174.84399999999999,"15min":174.84399999999999},"last":2.4980000000000002},{"interface":"front","average":{"1min":0.91000000000000003,"5min":3.0230000000000001,"15min":2.1850000000000001},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.28199999999999997},"max":{"1min":2.931,"5min":174.82400000000001,"15min":174.82400000000001},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":4.2670000000000003,"15min":2.7269999999999999},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.31900000000000001},"max":{"1min":3.8839999999999999,"5min":110.955,"15min":110.955},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.53,"5min":4.5679999999999996,"15min":2.774},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.29199999999999998},"max":{"1min":9.2300000000000004,"5min":110.89,"15min":110.89},"last":3.1019999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1989999999999998,"5min":3.1160000000000001,"15min":2.4329999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":7.7859999999999996,"5min":119.999,"15min":119.999},"last":2.6030000000000002},{"interface":"front","average":{"1min":1.788,"5min":3.1269999999999998,"15min":2.3370000000000002},"min":{"1min":0.40500000000000003,"5min":0.38300000000000001,"15min":0.31900000000000001},"max":{"1min":8.9550000000000001,"5min":122.328,"15min":122.328},"last":2.9830000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":4.8159999999999998,"15min":2.9239999999999999},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":10.648999999999999,"5min":285.47500000000002,"15min":285.47500000000002},"last":3.0169999999999999},{"interface":"front","average":{"1min":1.875,"5min":4.5430000000000001,"15min":2.7330000000000001},"min":{"1min":0.38600000000000001,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":10.664999999999999,"5min":285.48599999999999,"15min":285.48599999999999},"last":0.51500000000000001}]}]},{"osd":0,"up_from":0,"seq":395136991728,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781619,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2354312,"kb_used_data":1060864,"kb_used_omap":7,"kb_used_meta":1293432,"kb_avail":91390840,"statfs":{"total":95995035648,"available":93584220160,"internally_reserved":0,"allocated":1086324736,"data_stored":1083151629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1324474449},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.67400000000000004,"15min":1.929},"min":{"1min":0.19900000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":4.2880000000000003,"5min":8.0079999999999991,"15min":103.021},"last":2.6120000000000001},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.621,"15min":2.5099999999999998},"min":{"1min":0.253,"5min":0.218,"15min":0.218},"max":{"1min":4.6840000000000002,"5min":4.9820000000000002,"15min":103.154},"last":3.0840000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.66300000000000003,"15min":2.7160000000000002},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.19700000000000001},"max":{"1min":4.7350000000000003,"5min":5.2670000000000003,"15min":143.482},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.71999999999999997,"15min":2.7490000000000001},"min":{"1min":0.23899999999999999,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":1.6659999999999999,"5min":8.048,"15min":143.61000000000001},"last":3.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.70399999999999996,"15min":3.5369999999999999},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.7679999999999998,"5min":5.2939999999999996,"15min":143.50800000000001},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.77300000000000002,"15min":3.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.2670000000000003,"5min":8.2189999999999994,"15min":143.27799999999999},"last":3.294}]},{"osd":4,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":4.3929999999999998,"15min":3.3849999999999998},"min":{"1min":0.318,"5min":0.308,"15min":0.27800000000000002},"max":{"1min":6.8659999999999997,"5min":220.82900000000001,"15min":225.80799999999999},"last":2.0710000000000002},{"interface":"front","average":{"1min":2.6160000000000001,"5min":4.3259999999999996,"15min":3.9769999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":6.8029999999999999,"5min":220.84299999999999,"15min":225.79300000000001},"last":1.724}]},{"osd":5,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":4.3010000000000002,"15min":3.48},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":9.0579999999999998,"5min":220.63900000000001,"15min":220.63900000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.5049999999999999,"5min":4.3789999999999996,"15min":3.5089999999999999},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":4.8129999999999997,"5min":220.31999999999999,"15min":220.31999999999999},"last":0.34899999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":5.0289999999999999,"15min":3.3460000000000001},"min":{"1min":0.41099999999999998,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":5.1980000000000004,"5min":210.74700000000001,"15min":210.74700000000001},"last":0.5},{"interface":"front","average":{"1min":1.865,"5min":4.782,"15min":2.6890000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.8170000000000002,"5min":210.72900000000001,"15min":210.72900000000001},"last":2.2679999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.0169999999999999,"15min":1.954},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.5250000000000004,"5min":14.019,"15min":25.260000000000002},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.758,"15min":2.3279999999999998},"min":{"1min":0.41099999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.6200000000000001,"5min":7.5289999999999999,"15min":16.875},"last":0.76200000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160213,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":878008,"kb_used_data":9328,"kb_used_omap":2,"kb_used_meta":868669,"kb_avail":92867144,"statfs":{"total":95995035648,"available":95095955456,"internally_reserved":0,"allocated":9551872,"data_stored":5920584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":889517109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.504,"15min":1.7669999999999999},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":3.8780000000000001,"5min":4.7430000000000003,"15min":25.722999999999999},"last":1.571},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.51200000000000001,"15min":1.5580000000000001},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19500000000000001},"max":{"1min":5.0190000000000001,"5min":5.0190000000000001,"15min":18.277999999999999},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:33 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.504,"15min":6.5910000000000002},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8980000000000001,"5min":2.8980000000000001,"15min":237.072},"last":1.042},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.54200000000000004,"15min":6.5460000000000003},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":3.8359999999999999,"5min":3.8359999999999999,"15min":234.21899999999999},"last":1.986}]},{"osd":2,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":16.347999999999999,"5min":3.8279999999999998,"15min":5.6299999999999999},"min":{"1min":0.224,"5min":0.219,"15min":0.215},"max":{"1min":207.59999999999999,"5min":207.59999999999999,"15min":207.59999999999999},"last":1.766},{"interface":"front","average":{"1min":16.359999999999999,"5min":3.7570000000000001,"15min":5.2400000000000002},"min":{"1min":0.27800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":208.024,"5min":208.024,"15min":208.024},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":3.2440000000000002,"15min":2.609},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":9.5579999999999998,"5min":117.717,"15min":117.717},"last":0.42399999999999999},{"interface":"front","average":{"1min":3.141,"5min":3.7130000000000001,"15min":2.6579999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":9.0649999999999995,"5min":117.771,"15min":117.771},"last":1.6870000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":4.3209999999999997,"15min":2.9910000000000001},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":9.1910000000000007,"5min":123.458,"15min":123.458},"last":0.67900000000000005},{"interface":"front","average":{"1min":2.218,"5min":4.0739999999999998,"15min":2.8839999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":8.4670000000000005,"5min":123.292,"15min":123.292},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":5.0510000000000002,"15min":4.7670000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":7.7889999999999997,"5min":130.613,"15min":304.22399999999999},"last":0.495},{"interface":"front","average":{"1min":2.5990000000000002,"5min":5.5640000000000001,"15min":4.8979999999999997},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":8.8940000000000001,"5min":130.50999999999999,"15min":304.15800000000002},"last":1.9690000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.974,"15min":2.2080000000000002},"min":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":7.4939999999999998,"5min":19.298999999999999,"15min":44.253},"last":1.075},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.3380000000000001,"15min":2.5870000000000002},"min":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":9.1549999999999994,"5min":13.343,"15min":45.706000000000003},"last":0.80400000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986831,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2423156,"kb_used_data":1472428,"kb_used_omap":0,"kb_used_meta":950336,"kb_avail":91321996,"statfs":{"total":95995035648,"available":93513723904,"internally_reserved":0,"allocated":1507766272,"data_stored":1505969973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":973144064},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.173,"15min":3.0070000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":12.195,"5min":12.195,"15min":226.751},"last":4.0140000000000002},{"interface":"front","average":{"1min":2.161,"5min":1.5640000000000001,"15min":2.7040000000000002},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":8.641,"5min":12.754,"15min":226.43299999999999},"last":8.641}]},{"osd":1,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.583,"15min":3.6789999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.0709999999999997,"5min":12.805,"15min":127.411},"last":8.5980000000000008},{"interface":"front","average":{"1min":2.4689999999999999,"5min":2.1619999999999999,"15min":3.194},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.625,"5min":10.625,"15min":116.78100000000001},"last":3.988}]},{"osd":2,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":11.018000000000001,"5min":8.4890000000000008,"15min":6.7539999999999996},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.248},"max":{"1min":124.70099999999999,"5min":124.70099999999999,"15min":210.50299999999999},"last":5.2400000000000002},{"interface":"front","average":{"1min":7.9050000000000002,"5min":7.6079999999999997,"15min":6.6630000000000003},"min":{"1min":0.46000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":116.47,"5min":116.47,"15min":210.59},"last":3.9550000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.2999999999999998,"15min":4.4119999999999999},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.122999999999999,"5min":14.323,"15min":214.64099999999999},"last":5.2830000000000004},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.391,"15min":4.8159999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":10.587,"5min":10.587,"15min":214.54900000000001},"last":4.04}]},{"osd":5,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.3410000000000002,"15min":1.8280000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":12.997999999999999,"5min":17.879999999999999,"15min":34.497999999999998},"last":8.7880000000000003},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.3050000000000002,"15min":2.0760000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":19.425000000000001,"5min":19.425000000000001,"15min":34.883000000000003},"last":4.3769999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.681,"5min":4.1980000000000004,"15min":2.3980000000000001},"min":{"1min":0.36199999999999999,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":15.502000000000001,"5min":67.697999999999993,"15min":67.697999999999993},"last":9.4399999999999995},{"interface":"front","average":{"1min":5.4560000000000004,"5min":5.3390000000000004,"15min":2.726},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":18.762,"5min":67.596999999999994,"15min":67.596999999999994},"last":8.8640000000000008}]},{"osd":7,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":3.3159999999999998,"15min":1.9710000000000001},"min":{"1min":0.38700000000000001,"5min":0.317,"15min":0.28100000000000003},"max":{"1min":15.050000000000001,"5min":17.817,"15min":18.352},"last":4.5540000000000003},{"interface":"front","average":{"1min":3.323,"5min":3.177,"15min":1.8819999999999999},"min":{"1min":0.441,"5min":0.307,"15min":0.307},"max":{"1min":16.332999999999998,"5min":30.117000000000001,"15min":30.117000000000001},"last":9.4740000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725195,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3147168,"kb_used_data":2171736,"kb_used_omap":0,"kb_used_meta":975424,"kb_avail":90597984,"statfs":{"total":95995035648,"available":92772335616,"internally_reserved":0,"allocated":2223857664,"data_stored":2222077045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":998834176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.6899999999999999,"15min":2.0049999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":7.5119999999999996,"5min":12.558999999999999,"15min":82.280000000000001},"last":0.313},{"interface":"front","average":{"1min":2.194,"5min":1.8320000000000001,"15min":2.181},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":6.0430000000000001,"5min":9.407,"15min":85.236000000000004},"last":5.0540000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":2.2440000000000002,"15min":3.9900000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":18.859999999999999,"5min":18.859999999999999,"15min":233.77099999999999},"last":5.2869999999999999},{"interface":"front","average":{"1min":1.198,"5min":1.544,"15min":3.5899999999999999},"min":{"1min":0.27400000000000002,"5min":0.27000000000000002,"15min":0.251},"max":{"1min":7.5220000000000002,"5min":12.654999999999999,"15min":233.75200000000001},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":5.569,"5min":5.2720000000000002,"15min":4.665},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":75.441999999999993,"5min":160.749,"15min":231.34700000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":6.3460000000000001,"5min":5.3890000000000002,"15min":4.8200000000000003},"min":{"1min":0.35999999999999999,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":75.247,"5min":161.18000000000001,"15min":231.381},"last":4.9829999999999997}]},{"osd":3,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":2.278,"15min":4.0350000000000001},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":8.0980000000000008,"5min":18.826000000000001,"15min":263.887},"last":5.2409999999999997},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.7989999999999999,"15min":3.6779999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":6.1950000000000003,"5min":12.683999999999999,"15min":263.70100000000002},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":3.0649999999999999,"15min":1.7649999999999999},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":7.6299999999999999,"5min":77.804000000000002,"15min":77.804000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.7719999999999998,"5min":4.2880000000000003,"15min":2.1619999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":8.3810000000000002,"5min":77.825999999999993,"15min":77.825999999999993},"last":5.1429999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":4.4279999999999999,"15min":2.617},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":68.346000000000004,"5min":68.346000000000004,"15min":83.451999999999998},"last":9.1300000000000008},{"interface":"front","average":{"1min":11.348000000000001,"5min":4.758,"15min":2.907},"min":{"1min":0.40600000000000003,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":72.393000000000001,"5min":72.393000000000001,"15min":83.484999999999999},"last":5.3220000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:21:42 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":2.6309999999999998,"15min":1.792},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":10.654999999999999,"5min":10.654999999999999,"15min":10.654999999999999},"last":9.2769999999999992},{"interface":"front","average":{"1min":2.7160000000000002,"5min":2.2629999999999999,"15min":1.411},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":18.257999999999999,"5min":18.257999999999999,"15min":18.257999999999999},"last":0.53400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1068826624,"data_stored":1068826624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1481052160,"data_stored":1481048088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2203062272,"data_stored":2203058200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1727401984,"data_stored":1727397912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":853147648,"data_stored":853147648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:22:16.145 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:22:16.175 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:16.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:16.287 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:16.288 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:16.368 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:16.465 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:16.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:16.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:16.578 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:16.614 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:16.773 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:16 smithi121 ceph-mon[219709]: from='client.47852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:16.774 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:16 smithi121 ceph-mon[219709]: pgmap v1972: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 11807/79020 objects degraded (14.942%); 18966/79020 objects misplaced (24.002%); 7.7 MiB/s, 194 objects/s recovering 2023-09-23T21:22:16.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:16 smithi121 ceph-mon[211526]: from='client.47852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:16.774 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:16 smithi121 ceph-mon[211526]: pgmap v1972: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 11807/79020 objects degraded (14.942%); 18966/79020 objects misplaced (24.002%); 7.7 MiB/s, 194 objects/s recovering 2023-09-23T21:22:16.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:16 smithi123 ceph-mon[193903]: from='client.47852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:16.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:16 smithi123 ceph-mon[193903]: pgmap v1972: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 11807/79020 objects degraded (14.942%); 18966/79020 objects misplaced (24.002%); 7.7 MiB/s, 194 objects/s recovering 2023-09-23T21:22:17.092 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:17.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:17.201 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:17.248 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:17.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:17 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 11807/79020 objects degraded (14.942%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:17.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:17 smithi121 ceph-mon[211526]: pgmap v1973: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 11667/77532 objects degraded (15.048%); 20292/77532 objects misplaced (26.172%); 7.7 MiB/s, 172 objects/s recovering 2023-09-23T21:22:17.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:17 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 11807/79020 objects degraded (14.942%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:17.666 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:17 smithi121 ceph-mon[219709]: pgmap v1973: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 11667/77532 objects degraded (15.048%); 20292/77532 objects misplaced (26.172%); 7.7 MiB/s, 172 objects/s recovering 2023-09-23T21:22:17.728 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:17.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:17.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:17.833 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:17.885 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:17 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 11807/79020 objects degraded (14.942%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:17.885 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:17 smithi123 ceph-mon[193903]: pgmap v1973: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 11667/77532 objects degraded (15.048%); 20292/77532 objects misplaced (26.172%); 7.7 MiB/s, 172 objects/s recovering 2023-09-23T21:22:17.891 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:18.021 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:18.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:18.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:18.136 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:18.163 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:18.502 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:18.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:18.612 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:18.657 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:18.704 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:18.730 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:18.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:18.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:18.835 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:18.879 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:19.057 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:19.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:19.140 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:22:19.162 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:19.255 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:19.403 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:19.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:19.425 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:22:19.425 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:147747 14.2 e651/629 14:453630a4:::benchmark_data_smithi121_684892_object62430:head v 651'9074, mlcod=594'4563)", 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.422534+0000", 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.0029852009999999998, 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003023029, 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:19.426 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422534+0000", 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422534+0000", 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422533+0000", 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:19.427 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422542+0000", 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7879999999999995e-06 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422554+0000", 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1883e-05 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:19.428 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422562+0000", 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.1790000000000008e-06 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422623+0000", 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0736000000000003e-05 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.422646+0000", 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3272999999999999e-05 2023-09-23T21:22:19.429 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:19.430 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:19.430 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:19.430 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.430 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.430 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:147751 14.2 e651/629 14:42c4313f:::benchmark_data_smithi121_684892_object62434:head v 651'9075, mlcod=594'4563)", 2023-09-23T21:22:19.430 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.424557+0000", 2023-09-23T21:22:19.431 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 0.00096240099999999999, 2023-09-23T21:22:19.431 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00107693, 2023-09-23T21:22:19.431 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:19.431 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:22:19.431 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:19.433 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.433 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:19.433 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424557+0000", 2023-09-23T21:22:19.433 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:19.434 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.434 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.434 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:19.434 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424557+0000", 2023-09-23T21:22:19.434 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:19.434 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.434 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.435 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:19.435 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424555+0000", 2023-09-23T21:22:19.435 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:19.435 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.435 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.435 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:19.435 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424565+0000", 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7340000000000003e-06 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424576+0000", 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0845e-05 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.436 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.437 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:19.437 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424584+0000", 2023-09-23T21:22:19.437 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3180000000000007e-06 2023-09-23T21:22:19.437 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.437 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.437 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:19.437 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424626+0000", 2023-09-23T21:22:19.438 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2103e-05 2023-09-23T21:22:19.438 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:19.438 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:19.438 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:19.438 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.424642+0000", 2023-09-23T21:22:19.438 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5488000000000001e-05 2023-09-23T21:22:19.438 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:19.439 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:19.439 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:19.439 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:19.439 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2023-09-23T21:22:19.439 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 2 2023-09-23T21:22:19.439 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:22:19.516 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:19.534 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:19 smithi123 ceph-mon[193903]: pgmap v1974: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 10366/70380 objects degraded (14.729%); 18795/70380 objects misplaced (26.705%); 8.8 MiB/s, 189 objects/s recovering 2023-09-23T21:22:19.534 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:19 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3683191374' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:19.534 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:19 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:19.563 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:22:19.603 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:19.730 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:19.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:19.782 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:19 smithi121 ceph-mon[211526]: pgmap v1974: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 10366/70380 objects degraded (14.729%); 18795/70380 objects misplaced (26.705%); 8.8 MiB/s, 189 objects/s recovering 2023-09-23T21:22:19.782 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:19 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3683191374' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:19.783 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:19 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:19.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:19 smithi121 ceph-mon[219709]: pgmap v1974: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 10366/70380 objects degraded (14.729%); 18795/70380 objects misplaced (26.705%); 8.8 MiB/s, 189 objects/s recovering 2023-09-23T21:22:19.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:19 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3683191374' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:19.783 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:19 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:19.840 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:19.880 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:22:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[211526]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T21:22:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[211526]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[211526]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[211526]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:22:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[219709]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T21:22:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[219709]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[219709]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:20 smithi121 ceph-mon[219709]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:20 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:22:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:20 smithi123 ceph-mon[193903]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T21:22:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:20 smithi123 ceph-mon[193903]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:20 smithi123 ceph-mon[193903]: failed to encode map e652 with expected crc 2023-09-23T21:22:20.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:20 smithi123 ceph-mon[193903]: failed to encode map e652 with expected crc 2023-09-23T21:22:21.011 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:21.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:21.115 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:21.140 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:21.203 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:21.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:21.312 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:21.361 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:21.508 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:21.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:21.545 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:22:21.627 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:21.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:21 smithi121 ceph-mon[211526]: pgmap v1976: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 10366/70380 objects degraded (14.729%); 18795/70380 objects misplaced (26.705%); 8.8 MiB/s, 189 objects/s recovering 2023-09-23T21:22:21.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:21 smithi121 ceph-mon[219709]: pgmap v1976: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 10366/70380 objects degraded (14.729%); 18795/70380 objects misplaced (26.705%); 8.8 MiB/s, 189 objects/s recovering 2023-09-23T21:22:21.720 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:21.840 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:21.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:21.848 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:21 smithi123 ceph-mon[193903]: pgmap v1976: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 10366/70380 objects degraded (14.729%); 18795/70380 objects misplaced (26.705%); 8.8 MiB/s, 189 objects/s recovering 2023-09-23T21:22:21.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:21.923 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:21.949 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:22.031 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:22.202 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:22.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:22.306 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:22.348 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:22.574 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:22:22.575 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:22.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:22.614 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:22:22.641 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:22:22.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:22 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:22:22.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:22 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 10366/70380 objects degraded (14.729%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:22.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:22 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3811506382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:22.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:22 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:22:22.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:22 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 10366/70380 objects degraded (14.729%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:22.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:22 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3811506382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:22.670 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:22:22.679 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:22.712 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:22:22.713 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:22.746 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:22:22.755 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:22.789 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:22:22.831 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:22:22.831 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.832 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.832 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.832 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.832 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.832 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.832 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.832 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:22:22.861 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:22.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:22 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:22:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:22 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 10366/70380 objects degraded (14.729%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:22.877 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:22 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3811506382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:22.972 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:23.065 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:23.180 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:23.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:23.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:23.305 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:23.373 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:23.430 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:22:23.605 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:23.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:23.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:23.717 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:23.754 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:23 smithi121 ceph-mon[211526]: pgmap v1977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 10193/68814 objects degraded (14.812%); 18450/68814 objects misplaced (26.811%); 8.8 MiB/s, 152 objects/s recovering 2023-09-23T21:22:23.754 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:23 smithi121 ceph-mon[219709]: pgmap v1977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 10193/68814 objects degraded (14.812%); 18450/68814 objects misplaced (26.811%); 8.8 MiB/s, 152 objects/s recovering 2023-09-23T21:22:23.757 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:23.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:23 smithi123 ceph-mon[193903]: pgmap v1977: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 10193/68814 objects degraded (14.812%); 18450/68814 objects misplaced (26.811%); 8.8 MiB/s, 152 objects/s recovering 2023-09-23T21:22:24.280 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:24.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:24.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:24.391 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:24.444 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:24.710 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:24.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:24.819 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:24.856 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:22:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:22:24] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-09-23T21:22:24.860 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:24.922 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:24.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:24.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:25.027 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:25.063 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:25.598 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:25.843 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:25.849 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:25 smithi121 ceph-mon[211526]: pgmap v1978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 8995/62259 objects degraded (14.448%); 17073/62259 objects misplaced (27.423%); 9.3 MiB/s, 159 objects/s recovering 2023-09-23T21:22:25.849 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:25 smithi121 ceph-mon[219709]: pgmap v1978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 8995/62259 objects degraded (14.448%); 17073/62259 objects misplaced (27.423%); 9.3 MiB/s, 159 objects/s recovering 2023-09-23T21:22:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:25.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:25.951 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:25 smithi123 ceph-mon[193903]: pgmap v1978: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 8995/62259 objects degraded (14.448%); 17073/62259 objects misplaced (27.423%); 9.3 MiB/s, 159 objects/s recovering 2023-09-23T21:22:26.010 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:26.173 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:26.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:26.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:26.199 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:22:26.208 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:22:26.285 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:26.350 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:26.420 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:22:26.524 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:26.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:26.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:26.635 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:26.693 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:26.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:26 smithi123 ceph-mon[193903]: from='client.47864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:26.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:26 smithi121 ceph-mon[211526]: from='client.47864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:26 smithi121 ceph-mon[219709]: from='client.47864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:27.573 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:22:27.573 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:22:27.781 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:27.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:27.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:27.884 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:27.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:27 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 8995/62259 objects degraded (14.448%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:27.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:27 smithi121 ceph-mon[211526]: pgmap v1979: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 8820/60798 objects degraded (14.507%); 16755/60798 objects misplaced (27.558%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:22:27.916 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:27.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:27 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 8995/62259 objects degraded (14.448%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:27.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:27 smithi121 ceph-mon[219709]: pgmap v1979: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 8820/60798 objects degraded (14.507%); 16755/60798 objects misplaced (27.558%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:22:27.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:27.924 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:27 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 8995/62259 objects degraded (14.448%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:27 smithi123 ceph-mon[193903]: pgmap v1979: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 8820/60798 objects degraded (14.507%); 16755/60798 objects misplaced (27.558%); 9.3 MiB/s, 148 objects/s recovering 2023-09-23T21:22:27.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:28.010 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1978,"stamp":"2023-09-23T21:22:25.438151+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1359133443,"num_objects":20753,"num_object_clones":0,"num_object_copies":62259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8995,"num_objects_misplaced":17073,"num_objects_unfound":0,"num_objects_dirty":20753,"num_whiteouts":0,"num_read":92,"num_read_kb":169,"num_write":150005,"num_write_kb":5460565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27840,"num_bytes_recovered":1664890213,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86441,"ondisk_log_size":86441,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12520916,"kb_used_data":5961052,"kb_used_omap":16,"kb_used_meta":6559727,"kb_avail":643695148,"statfs":{"total":671965249536,"available":659143831552,"internally_reserved":0,"allocated":6104117248,"data_stored":6087007213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6717161321},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,4,8,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-366149632,"num_objects":-5587,"num_object_clones":0,"num_object_copies":-16761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2812,"num_objects_misplaced":-1893,"num_objects_unfound":0,"num_objects_dirty":-5587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5587,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1592,"num_bytes_recovered":97124352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004846"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":227,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.073037+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:22:11.073037+0000","last_peered":"2023-09-23T21:22:11.073037+0000","last_clean":"2023-09-23T21:22:11.073037+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:22:11.073037+0000","last_undegraded":"2023-09-23T21:22:11.073037+0000","last_fullsized":"2023-09-23T21:22:11.073037+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.063230+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:22:11.063230+0000","last_peered":"2023-09-23T21:22:11.063230+0000","last_clean":"2023-09-23T21:22:11.063230+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:22:11.063230+0000","last_undegraded":"2023-09-23T21:22:11.063230+0000","last_fullsized":"2023-09-23T21:22:11.063230+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164443+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:22:11.164443+0000","last_peered":"2023-09-23T21:22:11.164443+0000","last_clean":"2023-09-23T21:22:11.164443+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:22:11.164443+0000","last_undegraded":"2023-09-23T21:22:11.164443+0000","last_fullsized":"2023-09-23T21:22:11.164443+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":339,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080938+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:22:11.080938+0000","last_peered":"2023-09-23T21:22:11.080938+0000","last_clean":"2023-09-23T21:22:11.080938+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:22:11.080938+0000","last_undegraded":"2023-09-23T21:22:11.080938+0000","last_fullsized":"2023-09-23T21:22:11.080938+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":55,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.062816+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:22:11.062816+0000","last_peered":"2023-09-23T21:22:11.062816+0000","last_clean":"2023-09-23T21:22:11.062816+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:22:11.062816+0000","last_undegraded":"2023-09-23T21:22:11.062816+0000","last_fullsized":"2023-09-23T21:22:11.062816+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":157,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.077422+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:22:11.077422+0000","last_peered":"2023-09-23T21:22:11.077422+0000","last_clean":"2023-09-23T21:22:11.077422+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:22:11.077422+0000","last_undegraded":"2023-09-23T21:22:11.077422+0000","last_fullsized":"2023-09-23T21:22:11.077422+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":59,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.086331+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:22:11.086331+0000","last_peered":"2023-09-23T21:22:11.086331+0000","last_clean":"2023-09-23T21:22:11.086331+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:22:11.086331+0000","last_undegraded":"2023-09-23T21:22:11.086331+0000","last_fullsized":"2023-09-23T21:22:11.086331+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.062870+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:22:11.062870+0000","last_peered":"2023-09-23T21:22:11.062870+0000","last_clean":"2023-09-23T21:22:11.062870+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:22:11.062870+0000","last_undegraded":"2023-09-23T21:22:11.062870+0000","last_fullsized":"2023-09-23T21:22:11.062870+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":49,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.085080+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:22:11.085080+0000","last_peered":"2023-09-23T21:22:11.085080+0000","last_clean":"2023-09-23T21:22:11.085080+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:22:11.085080+0000","last_undegraded":"2023-09-23T21:22:11.085080+0000","last_fullsized":"2023-09-23T21:22:11.085080+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":46,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.063153+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:22:11.063153+0000","last_peered":"2023-09-23T21:22:11.063153+0000","last_clean":"2023-09-23T21:22:11.063153+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:22:11.063153+0000","last_undegraded":"2023-09-23T21:22:11.063153+0000","last_fullsized":"2023-09-23T21:22:11.063153+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":287,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081329+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:22:11.081329+0000","last_peered":"2023-09-23T21:22:11.081329+0000","last_clean":"2023-09-23T21:22:11.081329+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:22:11.081329+0000","last_undegraded":"2023-09-23T21:22:11.081329+0000","last_fullsized":"2023-09-23T21:22:11.081329+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":43,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.083818+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:22:11.083818+0000","last_peered":"2023-09-23T21:22:11.083818+0000","last_clean":"2023-09-23T21:22:11.083818+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:22:11.083818+0000","last_undegraded":"2023-09-23T21:22:11.083818+0000","last_fullsized":"2023-09-23T21:22:11.083818+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":383,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.079587+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:22:11.079587+0000","last_peered":"2023-09-23T21:22:11.079587+0000","last_clean":"2023-09-23T21:22:11.079587+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:22:11.079587+0000","last_undegraded":"2023-09-23T21:22:11.079587+0000","last_fullsized":"2023-09-23T21:22:11.079587+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":610,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.078072+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:22:11.078072+0000","last_peered":"2023-09-23T21:22:11.078072+0000","last_clean":"2023-09-23T21:22:11.078072+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:22:11.078072+0000","last_undegraded":"2023-09-23T21:22:11.078072+0000","last_fullsized":"2023-09-23T21:22:11.078072+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":235,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081199+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:22:11.081199+0000","last_peered":"2023-09-23T21:22:11.081199+0000","last_clean":"2023-09-23T21:22:11.081199+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:22:11.081199+0000","last_undegraded":"2023-09-23T21:22:11.081199+0000","last_fullsized":"2023-09-23T21:22:11.081199+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":55,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.063702+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:22:11.063702+0000","last_peered":"2023-09-23T21:22:11.063702+0000","last_clean":"2023-09-23T21:22:11.063702+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:22:11.063702+0000","last_undegraded":"2023-09-23T21:22:11.063702+0000","last_fullsized":"2023-09-23T21:22:11.063702+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":261,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080841+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:22:11.080841+0000","last_peered":"2023-09-23T21:22:11.080841+0000","last_clean":"2023-09-23T21:22:11.080841+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:22:11.080841+0000","last_undegraded":"2023-09-23T21:22:11.080841+0000","last_fullsized":"2023-09-23T21:22:11.080841+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":269,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081528+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:22:11.081528+0000","last_peered":"2023-09-23T21:22:11.081528+0000","last_clean":"2023-09-23T21:22:11.081528+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:22:11.081528+0000","last_undegraded":"2023-09-23T21:22:11.081528+0000","last_fullsized":"2023-09-23T21:22:11.081528+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":229,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080655+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:22:11.080655+0000","last_peered":"2023-09-23T21:22:11.080655+0000","last_clean":"2023-09-23T21:22:11.080655+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:22:11.080655+0000","last_undegraded":"2023-09-23T21:22:11.080655+0000","last_fullsized":"2023-09-23T21:22:11.080655+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":46,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.064039+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:22:11.064039+0000","last_peered":"2023-09-23T21:22:11.064039+0000","last_clean":"2023-09-23T21:22:11.064039+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:22:11.064039+0000","last_undegraded":"2023-09-23T21:22:11.064039+0000","last_fullsized":"2023-09-23T21:22:11.064039+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":748,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.075128+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:22:11.075128+0000","last_peered":"2023-09-23T21:22:11.075128+0000","last_clean":"2023-09-23T21:22:11.075128+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:22:11.075128+0000","last_undegraded":"2023-09-23T21:22:11.075128+0000","last_fullsized":"2023-09-23T21:22:11.075128+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":55,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.064360+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:22:11.064360+0000","last_peered":"2023-09-23T21:22:11.064360+0000","last_clean":"2023-09-23T21:22:11.064360+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:22:11.064360+0000","last_undegraded":"2023-09-23T21:22:11.064360+0000","last_fullsized":"2023-09-23T21:22:11.064360+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"652'9133","reported_seq":10958,"reported_epoch":652,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:21.380911+0000","last_change":"2023-09-23T21:21:01.215269+0000","last_active":"2023-09-23T21:22:21.380911+0000","last_peered":"2023-09-23T21:22:21.380911+0000","last_clean":"2023-09-23T21:19:28.877182+0000","last_became_active":"2023-09-23T21:21:01.190149+0000","last_became_peered":"2023-09-23T21:21:01.190149+0000","last_unstale":"2023-09-23T21:22:21.380911+0000","last_undegraded":"2023-09-23T21:21:00.607176+0000","last_fullsized":"2023-09-23T21:22:21.380911+0000","mapping_epoch":629,"log_start":"589'3667","ondisk_log_start":"589'3667","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:05.989327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1353,"num_objects_misplaced":1353,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9133,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":844,"num_bytes_recovered":55312384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,4,5],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,5,6","objects":673},{"shards":"4,6,7","objects":680}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":289,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081978+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:22:11.081978+0000","last_peered":"2023-09-23T21:22:11.081978+0000","last_clean":"2023-09-23T21:22:11.081978+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:22:11.081978+0000","last_undegraded":"2023-09-23T21:22:11.081978+0000","last_fullsized":"2023-09-23T21:22:11.081978+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084677+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:22:11.084677+0000","last_peered":"2023-09-23T21:22:11.084677+0000","last_clean":"2023-09-23T21:22:11.084677+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:22:11.084677+0000","last_undegraded":"2023-09-23T21:22:11.084677+0000","last_fullsized":"2023-09-23T21:22:11.084677+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"652'9473","reported_seq":11499,"reported_epoch":652,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:24.126632+0000","last_change":"2023-09-23T21:21:01.210342+0000","last_active":"2023-09-23T21:22:24.126632+0000","last_peered":"2023-09-23T21:22:24.126632+0000","last_clean":"2023-09-23T21:19:28.923531+0000","last_became_active":"2023-09-23T21:21:01.189847+0000","last_became_peered":"2023-09-23T21:21:01.189847+0000","last_unstale":"2023-09-23T21:22:24.126632+0000","last_undegraded":"2023-09-23T21:21:00.610088+0000","last_fullsized":"2023-09-23T21:22:24.126632+0000","mapping_epoch":629,"log_start":"593'4398","ondisk_log_start":"593'4398","created":573,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5075,"log_dups_size":0,"ondisk_log_size":5075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:58.686882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85131264,"num_objects":1299,"num_object_clones":0,"num_object_copies":3897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1299,"num_objects_misplaced":1299,"num_objects_unfound":0,"num_objects_dirty":1299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9473,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":938,"num_bytes_recovered":61341696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[4,6,5],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":573},{"shards":"4,6,7","objects":726}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":303,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081024+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:22:11.081024+0000","last_peered":"2023-09-23T21:22:11.081024+0000","last_clean":"2023-09-23T21:22:11.081024+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:22:11.081024+0000","last_undegraded":"2023-09-23T21:22:11.081024+0000","last_fullsized":"2023-09-23T21:22:11.081024+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164402+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:22:11.164402+0000","last_peered":"2023-09-23T21:22:11.164402+0000","last_clean":"2023-09-23T21:22:11.164402+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:22:11.164402+0000","last_undegraded":"2023-09-23T21:22:11.164402+0000","last_fullsized":"2023-09-23T21:22:11.164402+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"652'9251","reported_seq":14487,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:23.743943+0000","last_change":"2023-09-23T21:21:17.539683+0000","last_active":"2023-09-23T21:22:23.743943+0000","last_peered":"2023-09-23T21:22:23.743943+0000","last_clean":"2023-09-23T21:22:23.743943+0000","last_became_active":"2023-09-23T21:21:17.521633+0000","last_became_peered":"2023-09-23T21:21:17.521633+0000","last_unstale":"2023-09-23T21:22:23.743943+0000","last_undegraded":"2023-09-23T21:22:23.743943+0000","last_fullsized":"2023-09-23T21:22:23.743943+0000","mapping_epoch":636,"log_start":"593'4265","ondisk_log_start":"593'4265","created":573,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4986,"log_dups_size":0,"ondisk_log_size":4986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:05.844201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79757312,"num_objects":1217,"num_object_clones":0,"num_object_copies":3651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9251,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2555,"num_bytes_recovered":80478208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":275,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081610+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:22:11.081610+0000","last_peered":"2023-09-23T21:22:11.081610+0000","last_clean":"2023-09-23T21:22:11.081610+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:22:11.081610+0000","last_undegraded":"2023-09-23T21:22:11.081610+0000","last_fullsized":"2023-09-23T21:22:11.081610+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.075725+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:22:11.075725+0000","last_peered":"2023-09-23T21:22:11.075725+0000","last_clean":"2023-09-23T21:22:11.075725+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:22:11.075725+0000","last_undegraded":"2023-09-23T21:22:11.075725+0000","last_fullsized":"2023-09-23T21:22:11.075725+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"652'9268","reported_seq":9372,"reported_epoch":652,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:23.677717+0000","last_change":"2023-09-23T21:20:59.644218+0000","last_active":"2023-09-23T21:22:23.677717+0000","last_peered":"2023-09-23T21:22:23.677717+0000","last_clean":"2023-09-23T21:20:30.430127+0000","last_became_active":"2023-09-23T21:20:59.632855+0000","last_became_peered":"2023-09-23T21:20:59.632855+0000","last_unstale":"2023-09-23T21:22:23.677717+0000","last_undegraded":"2023-09-23T21:22:23.677717+0000","last_fullsized":"2023-09-23T21:22:23.677717+0000","mapping_epoch":628,"log_start":"593'4325","ondisk_log_start":"593'4325","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:33.014367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87162880,"num_objects":1330,"num_object_clones":0,"num_object_copies":3990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2660,"num_objects_unfound":0,"num_objects_dirty":1330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9268,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1330}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":55,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.063784+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:22:11.063784+0000","last_peered":"2023-09-23T21:22:11.063784+0000","last_clean":"2023-09-23T21:22:11.063784+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:22:11.063784+0000","last_undegraded":"2023-09-23T21:22:11.063784+0000","last_fullsized":"2023-09-23T21:22:11.063784+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164492+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:22:11.164492+0000","last_peered":"2023-09-23T21:22:11.164492+0000","last_clean":"2023-09-23T21:22:11.164492+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:22:11.164492+0000","last_undegraded":"2023-09-23T21:22:11.164492+0000","last_fullsized":"2023-09-23T21:22:11.164492+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1078,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080424+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:22:11.080424+0000","last_peered":"2023-09-23T21:22:11.080424+0000","last_clean":"2023-09-23T21:22:11.080424+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:22:11.080424+0000","last_undegraded":"2023-09-23T21:22:11.080424+0000","last_fullsized":"2023-09-23T21:22:11.080424+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.6","version":"652'9438","reported_seq":15841,"reported_epoch":652,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:23.811431+0000","last_change":"2023-09-23T21:21:01.239919+0000","last_active":"2023-09-23T21:22:23.811431+0000","last_peered":"2023-09-23T21:22:23.811431+0000","last_clean":"2023-09-23T21:19:28.802255+0000","last_became_active":"2023-09-23T21:21:01.197858+0000","last_became_peered":"2023-09-23T21:21:01.197858+0000","last_unstale":"2023-09-23T21:22:23.811431+0000","last_undegraded":"2023-09-23T21:21:00.673065+0000","last_fullsized":"2023-09-23T21:22:23.811431+0000","mapping_epoch":629,"log_start":"593'4498","ondisk_log_start":"593'4498","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4940,"log_dups_size":0,"ondisk_log_size":4940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:39.315049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84410368,"num_objects":1288,"num_object_clones":0,"num_object_copies":3864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1288,"num_objects_misplaced":1288,"num_objects_unfound":0,"num_objects_dirty":1288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9438,"num_write_kb":343232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3142,"num_bytes_recovered":205848576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,4,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,4,6","objects":680},{"shards":"2,5,6","objects":608}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":63,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.062999+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:22:11.062999+0000","last_peered":"2023-09-23T21:22:11.062999+0000","last_clean":"2023-09-23T21:22:11.062999+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:22:11.062999+0000","last_undegraded":"2023-09-23T21:22:11.062999+0000","last_fullsized":"2023-09-23T21:22:11.062999+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":410,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.079678+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:22:11.079678+0000","last_peered":"2023-09-23T21:22:11.079678+0000","last_clean":"2023-09-23T21:22:11.079678+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:22:11.079678+0000","last_undegraded":"2023-09-23T21:22:11.079678+0000","last_fullsized":"2023-09-23T21:22:11.079678+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":946,"reported_epoch":651,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:22:11.074692+0000","last_change":"2023-09-23T21:20:48.157415+0000","last_active":"2023-09-23T21:22:11.074692+0000","last_peered":"2023-09-23T21:22:11.074692+0000","last_clean":"2023-09-23T21:20:41.603469+0000","last_became_active":"2023-09-23T21:20:48.156886+0000","last_became_peered":"2023-09-23T21:20:48.156886+0000","last_unstale":"2023-09-23T21:22:11.074692+0000","last_undegraded":"2023-09-23T21:20:47.922781+0000","last_fullsized":"2023-09-23T21:20:47.921128+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:30.898986+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4,6","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.7","version":"652'9555","reported_seq":10761,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:23.865365+0000","last_change":"2023-09-23T21:20:59.594443+0000","last_active":"2023-09-23T21:22:23.865365+0000","last_peered":"2023-09-23T21:22:23.865365+0000","last_clean":"2023-09-23T21:22:23.865365+0000","last_became_active":"2023-09-23T21:20:48.149995+0000","last_became_peered":"2023-09-23T21:20:48.149995+0000","last_unstale":"2023-09-23T21:22:23.865365+0000","last_undegraded":"2023-09-23T21:22:23.865365+0000","last_fullsized":"2023-09-23T21:22:23.865365+0000","mapping_epoch":623,"log_start":"582'2665","ondisk_log_start":"582'2665","created":573,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.278714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87621632,"num_objects":1337,"num_object_clones":0,"num_object_copies":4011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9555,"num_write_kb":348544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":147,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.077221+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:22:11.077221+0000","last_peered":"2023-09-23T21:22:11.077221+0000","last_clean":"2023-09-23T21:22:11.077221+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:22:11.077221+0000","last_undegraded":"2023-09-23T21:22:11.077221+0000","last_fullsized":"2023-09-23T21:22:11.077221+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":660,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.083737+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:22:11.083737+0000","last_peered":"2023-09-23T21:22:11.083737+0000","last_clean":"2023-09-23T21:22:11.083737+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:22:11.083737+0000","last_undegraded":"2023-09-23T21:22:11.083737+0000","last_fullsized":"2023-09-23T21:22:11.083737+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"652'9178","reported_seq":18420,"reported_epoch":652,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:21.446835+0000","last_change":"2023-09-23T21:22:12.152044+0000","last_active":"2023-09-23T21:22:21.446835+0000","last_peered":"2023-09-23T21:22:21.446835+0000","last_clean":"2023-09-23T21:19:28.911487+0000","last_became_active":"2023-09-23T21:20:32.169326+0000","last_became_peered":"2023-09-23T21:20:32.169326+0000","last_unstale":"2023-09-23T21:22:21.446835+0000","last_undegraded":"2023-09-23T21:22:21.446835+0000","last_fullsized":"2023-09-23T21:22:21.446835+0000","mapping_epoch":617,"log_start":"589'3668","ondisk_log_start":"589'3668","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:09.080183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84803584,"num_objects":1294,"num_object_clones":0,"num_object_copies":3882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1294,"num_objects_unfound":0,"num_objects_dirty":1294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9178,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4564,"num_bytes_recovered":237240320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,4],"avail_no_missing":["6","4","5","7"],"object_location_counts":[{"shards":"4,6,7","objects":1294}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":289,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.075116+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:22:11.075116+0000","last_peered":"2023-09-23T21:22:11.075116+0000","last_clean":"2023-09-23T21:22:11.075116+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:22:11.075116+0000","last_undegraded":"2023-09-23T21:22:11.075116+0000","last_fullsized":"2023-09-23T21:22:11.075116+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":697,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084418+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:22:11.084418+0000","last_peered":"2023-09-23T21:22:11.084418+0000","last_clean":"2023-09-23T21:22:11.084418+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:22:11.084418+0000","last_undegraded":"2023-09-23T21:22:11.084418+0000","last_fullsized":"2023-09-23T21:22:11.084418+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":915,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.075169+0000","last_change":"2023-09-23T21:20:49.968107+0000","last_active":"2023-09-23T21:22:11.075169+0000","last_peered":"2023-09-23T21:22:11.075169+0000","last_clean":"2023-09-23T21:22:11.075169+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:22:11.075169+0000","last_undegraded":"2023-09-23T21:22:11.075169+0000","last_fullsized":"2023-09-23T21:22:11.075169+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":429358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.5","version":"652'9619","reported_seq":17936,"reported_epoch":652,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:24.132672+0000","last_change":"2023-09-23T21:20:59.554347+0000","last_active":"2023-09-23T21:22:24.132672+0000","last_peered":"2023-09-23T21:22:24.132672+0000","last_clean":"2023-09-23T21:19:28.864495+0000","last_became_active":"2023-09-23T21:20:32.158375+0000","last_became_peered":"2023-09-23T21:20:32.158375+0000","last_unstale":"2023-09-23T21:22:24.132672+0000","last_undegraded":"2023-09-23T21:20:59.551954+0000","last_fullsized":"2023-09-23T21:22:24.132672+0000","mapping_epoch":617,"log_start":"593'4592","ondisk_log_start":"593'4592","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5027,"log_dups_size":0,"ondisk_log_size":5027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:46.553959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87883776,"num_objects":1341,"num_object_clones":0,"num_object_copies":4023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":628,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":1341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9619,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4102,"num_bytes_recovered":268828672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"2,4,7","objects":750},{"shards":"4,5,7","objects":591}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":55,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.062727+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:22:11.062727+0000","last_peered":"2023-09-23T21:22:11.062727+0000","last_clean":"2023-09-23T21:22:11.062727+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:22:11.062727+0000","last_undegraded":"2023-09-23T21:22:11.062727+0000","last_fullsized":"2023-09-23T21:22:11.062727+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164862+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:22:11.164862+0000","last_peered":"2023-09-23T21:22:11.164862+0000","last_clean":"2023-09-23T21:22:11.164862+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:22:11.164862+0000","last_undegraded":"2023-09-23T21:22:11.164862+0000","last_fullsized":"2023-09-23T21:22:11.164862+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":734,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.076569+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:22:11.076569+0000","last_peered":"2023-09-23T21:22:11.076569+0000","last_clean":"2023-09-23T21:22:11.076569+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:22:11.076569+0000","last_undegraded":"2023-09-23T21:22:11.076569+0000","last_fullsized":"2023-09-23T21:22:11.076569+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"652'9349","reported_seq":10203,"reported_epoch":652,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:23.746623+0000","last_change":"2023-09-23T21:21:01.233673+0000","last_active":"2023-09-23T21:22:23.746623+0000","last_peered":"2023-09-23T21:22:23.746623+0000","last_clean":"2023-09-23T21:19:28.898608+0000","last_became_active":"2023-09-23T21:21:01.195228+0000","last_became_peered":"2023-09-23T21:21:01.195228+0000","last_unstale":"2023-09-23T21:22:23.746623+0000","last_undegraded":"2023-09-23T21:21:00.570455+0000","last_fullsized":"2023-09-23T21:22:23.746623+0000","mapping_epoch":629,"log_start":"593'4346","ondisk_log_start":"593'4346","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:03.164901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689472,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1277,"num_objects_misplaced":1277,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9349,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[2,7,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,5,7","objects":560},{"shards":"2,6,7","objects":717}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084250+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:22:11.084250+0000","last_peered":"2023-09-23T21:22:11.084250+0000","last_clean":"2023-09-23T21:22:11.084250+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:22:11.084250+0000","last_undegraded":"2023-09-23T21:22:11.084250+0000","last_fullsized":"2023-09-23T21:22:11.084250+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.085189+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:22:11.085189+0000","last_peered":"2023-09-23T21:22:11.085189+0000","last_clean":"2023-09-23T21:22:11.085189+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:22:11.085189+0000","last_undegraded":"2023-09-23T21:22:11.085189+0000","last_fullsized":"2023-09-23T21:22:11.085189+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.083824+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:22:11.083824+0000","last_peered":"2023-09-23T21:22:11.083824+0000","last_clean":"2023-09-23T21:22:11.083824+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:22:11.083824+0000","last_undegraded":"2023-09-23T21:22:11.083824+0000","last_fullsized":"2023-09-23T21:22:11.083824+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":927,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080797+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:22:11.080797+0000","last_peered":"2023-09-23T21:22:11.080797+0000","last_clean":"2023-09-23T21:22:11.080797+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:22:11.080797+0000","last_undegraded":"2023-09-23T21:22:11.080797+0000","last_fullsized":"2023-09-23T21:22:11.080797+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"652'9285","reported_seq":9370,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:24.126745+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:22:24.126745+0000","last_peered":"2023-09-23T21:22:24.126745+0000","last_clean":"2023-09-23T21:22:24.126745+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:22:24.126745+0000","last_undegraded":"2023-09-23T21:22:24.126745+0000","last_fullsized":"2023-09-23T21:22:24.126745+0000","mapping_epoch":573,"log_start":"593'4187","ondisk_log_start":"593'4187","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5098,"log_dups_size":0,"ondisk_log_size":5098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82378752,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9285,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084061+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:22:11.084061+0000","last_peered":"2023-09-23T21:22:11.084061+0000","last_clean":"2023-09-23T21:22:11.084061+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:22:11.084061+0000","last_undegraded":"2023-09-23T21:22:11.084061+0000","last_fullsized":"2023-09-23T21:22:11.084061+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":565,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.077965+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:22:11.077965+0000","last_peered":"2023-09-23T21:22:11.077965+0000","last_clean":"2023-09-23T21:22:11.077965+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:22:11.077965+0000","last_undegraded":"2023-09-23T21:22:11.077965+0000","last_fullsized":"2023-09-23T21:22:11.077965+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084182+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:22:11.084182+0000","last_peered":"2023-09-23T21:22:11.084182+0000","last_clean":"2023-09-23T21:22:11.084182+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:22:11.084182+0000","last_undegraded":"2023-09-23T21:22:11.084182+0000","last_fullsized":"2023-09-23T21:22:11.084182+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1038,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080758+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:22:11.080758+0000","last_peered":"2023-09-23T21:22:11.080758+0000","last_clean":"2023-09-23T21:22:11.080758+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:22:11.080758+0000","last_undegraded":"2023-09-23T21:22:11.080758+0000","last_fullsized":"2023-09-23T21:22:11.080758+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.d","version":"652'9325","reported_seq":9486,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:21.400606+0000","last_change":"2023-09-23T21:21:22.585504+0000","last_active":"2023-09-23T21:22:21.400606+0000","last_peered":"2023-09-23T21:22:21.400606+0000","last_clean":"2023-09-23T21:22:21.400606+0000","last_became_active":"2023-09-23T21:21:22.568285+0000","last_became_peered":"2023-09-23T21:21:22.568285+0000","last_unstale":"2023-09-23T21:22:21.400606+0000","last_undegraded":"2023-09-23T21:22:21.400606+0000","last_fullsized":"2023-09-23T21:22:21.400606+0000","mapping_epoch":639,"log_start":"590'3848","ondisk_log_start":"590'3848","created":573,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5477,"log_dups_size":0,"ondisk_log_size":5477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:01.251062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88080408,"num_objects":1345,"num_object_clones":0,"num_object_copies":4035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1345,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9325,"num_write_kb":341377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":681,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081112+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:22:11.081112+0000","last_peered":"2023-09-23T21:22:11.081112+0000","last_clean":"2023-09-23T21:22:11.081112+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:22:11.081112+0000","last_undegraded":"2023-09-23T21:22:11.081112+0000","last_fullsized":"2023-09-23T21:22:11.081112+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164127+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:22:11.164127+0000","last_peered":"2023-09-23T21:22:11.164127+0000","last_clean":"2023-09-23T21:22:11.164127+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:22:11.164127+0000","last_undegraded":"2023-09-23T21:22:11.164127+0000","last_fullsized":"2023-09-23T21:22:11.164127+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164795+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:22:11.164795+0000","last_peered":"2023-09-23T21:22:11.164795+0000","last_clean":"2023-09-23T21:22:11.164795+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:22:11.164795+0000","last_undegraded":"2023-09-23T21:22:11.164795+0000","last_fullsized":"2023-09-23T21:22:11.164795+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"652'9377","reported_seq":12280,"reported_epoch":652,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:23.518377+0000","last_change":"2023-09-23T21:21:01.214995+0000","last_active":"2023-09-23T21:22:23.518377+0000","last_peered":"2023-09-23T21:22:23.518377+0000","last_clean":"2023-09-23T21:19:28.902983+0000","last_became_active":"2023-09-23T21:21:01.189928+0000","last_became_peered":"2023-09-23T21:21:01.189928+0000","last_unstale":"2023-09-23T21:22:23.518377+0000","last_undegraded":"2023-09-23T21:21:00.625306+0000","last_fullsized":"2023-09-23T21:22:23.518377+0000","mapping_epoch":629,"log_start":"588'3286","ondisk_log_start":"588'3286","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":6091,"log_dups_size":0,"ondisk_log_size":6091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:45.309159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85393408,"num_objects":1303,"num_object_clones":0,"num_object_copies":3909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1303,"num_objects_misplaced":1303,"num_objects_unfound":0,"num_objects_dirty":1303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9377,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1379,"num_bytes_recovered":90374144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,2,5],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,5,7","objects":573},{"shards":"2,6,7","objects":730}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":510,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080901+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:22:11.080901+0000","last_peered":"2023-09-23T21:22:11.080901+0000","last_clean":"2023-09-23T21:22:11.080901+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:22:11.080901+0000","last_undegraded":"2023-09-23T21:22:11.080901+0000","last_fullsized":"2023-09-23T21:22:11.080901+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1039,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080801+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:22:11.080801+0000","last_peered":"2023-09-23T21:22:11.080801+0000","last_clean":"2023-09-23T21:22:11.080801+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:22:11.080801+0000","last_undegraded":"2023-09-23T21:22:11.080801+0000","last_fullsized":"2023-09-23T21:22:11.080801+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.085191+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:22:11.085191+0000","last_peered":"2023-09-23T21:22:11.085191+0000","last_clean":"2023-09-23T21:22:11.085191+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:22:11.085191+0000","last_undegraded":"2023-09-23T21:22:11.085191+0000","last_fullsized":"2023-09-23T21:22:11.085191+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":23,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.079228+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:22:11.079228+0000","last_peered":"2023-09-23T21:22:11.079228+0000","last_clean":"2023-09-23T21:22:11.079228+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:22:11.079228+0000","last_undegraded":"2023-09-23T21:22:11.079228+0000","last_fullsized":"2023-09-23T21:22:11.079228+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":686,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164611+0000","last_change":"2023-09-23T21:21:21.571375+0000","last_active":"2023-09-23T21:22:11.164611+0000","last_peered":"2023-09-23T21:22:11.164611+0000","last_clean":"2023-09-23T21:22:11.164611+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:22:11.164611+0000","last_undegraded":"2023-09-23T21:22:11.164611+0000","last_fullsized":"2023-09-23T21:22:11.164611+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.551943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.551943+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.c","version":"652'9538","reported_seq":9624,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:24.130442+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:22:24.130442+0000","last_peered":"2023-09-23T21:22:24.130442+0000","last_clean":"2023-09-23T21:22:24.130442+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:22:24.130442+0000","last_undegraded":"2023-09-23T21:22:24.130442+0000","last_fullsized":"2023-09-23T21:22:24.130442+0000","mapping_epoch":573,"log_start":"593'4523","ondisk_log_start":"593'4523","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5015,"log_dups_size":0,"ondisk_log_size":5015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82837504,"num_objects":1264,"num_object_clones":0,"num_object_copies":3792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9538,"num_write_kb":345664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":319,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.080562+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:22:11.080562+0000","last_peered":"2023-09-23T21:22:11.080562+0000","last_clean":"2023-09-23T21:22:11.080562+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:22:11.080562+0000","last_undegraded":"2023-09-23T21:22:11.080562+0000","last_fullsized":"2023-09-23T21:22:11.080562+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":124,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.075006+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:22:11.075006+0000","last_peered":"2023-09-23T21:22:11.075006+0000","last_clean":"2023-09-23T21:22:11.075006+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:22:11.075006+0000","last_undegraded":"2023-09-23T21:22:11.075006+0000","last_fullsized":"2023-09-23T21:22:11.075006+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084252+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:22:11.084252+0000","last_peered":"2023-09-23T21:22:11.084252+0000","last_clean":"2023-09-23T21:22:11.084252+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:22:11.084252+0000","last_undegraded":"2023-09-23T21:22:11.084252+0000","last_fullsized":"2023-09-23T21:22:11.084252+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":714,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.077715+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:22:11.077715+0000","last_peered":"2023-09-23T21:22:11.077715+0000","last_clean":"2023-09-23T21:22:11.077715+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:22:11.077715+0000","last_undegraded":"2023-09-23T21:22:11.077715+0000","last_fullsized":"2023-09-23T21:22:11.077715+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"652'9180","reported_seq":16504,"reported_epoch":652,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:22:24.121052+0000","last_change":"2023-09-23T21:22:12.172611+0000","last_active":"2023-09-23T21:22:24.121052+0000","last_peered":"2023-09-23T21:22:24.121052+0000","last_clean":"2023-09-23T21:19:28.920085+0000","last_became_active":"2023-09-23T21:20:48.224556+0000","last_became_peered":"2023-09-23T21:20:48.224556+0000","last_unstale":"2023-09-23T21:22:24.121052+0000","last_undegraded":"2023-09-23T21:22:24.121052+0000","last_fullsized":"2023-09-23T21:20:47.933463+0000","mapping_epoch":623,"log_start":"593'4144","ondisk_log_start":"593'4144","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:02.563744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230720,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1903,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9180,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3596,"num_bytes_recovered":235536384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[4,2],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"2,4","objects":101},{"shards":"2,4,6","objects":734},{"shards":"4,5,6","objects":435}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":259,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081455+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:22:11.081455+0000","last_peered":"2023-09-23T21:22:11.081455+0000","last_clean":"2023-09-23T21:22:11.081455+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:22:11.081455+0000","last_undegraded":"2023-09-23T21:22:11.081455+0000","last_fullsized":"2023-09-23T21:22:11.081455+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084109+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:22:11.084109+0000","last_peered":"2023-09-23T21:22:11.084109+0000","last_clean":"2023-09-23T21:22:11.084109+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:22:11.084109+0000","last_undegraded":"2023-09-23T21:22:11.084109+0000","last_fullsized":"2023-09-23T21:22:11.084109+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":49,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.086358+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:22:11.086358+0000","last_peered":"2023-09-23T21:22:11.086358+0000","last_clean":"2023-09-23T21:22:11.086358+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:22:11.086358+0000","last_undegraded":"2023-09-23T21:22:11.086358+0000","last_fullsized":"2023-09-23T21:22:11.086358+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":953,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.082374+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:22:11.082374+0000","last_peered":"2023-09-23T21:22:11.082374+0000","last_clean":"2023-09-23T21:22:11.082374+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:22:11.082374+0000","last_undegraded":"2023-09-23T21:22:11.082374+0000","last_fullsized":"2023-09-23T21:22:11.082374+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.a","version":"652'9359","reported_seq":17553,"reported_epoch":652,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:24.092881+0000","last_change":"2023-09-23T21:21:01.218382+0000","last_active":"2023-09-23T21:22:24.092881+0000","last_peered":"2023-09-23T21:22:24.092881+0000","last_clean":"2023-09-23T21:19:28.896530+0000","last_became_active":"2023-09-23T21:21:01.190094+0000","last_became_peered":"2023-09-23T21:21:01.190094+0000","last_unstale":"2023-09-23T21:22:24.092881+0000","last_undegraded":"2023-09-23T21:21:00.719860+0000","last_fullsized":"2023-09-23T21:22:24.092881+0000","mapping_epoch":629,"log_start":"593'4318","ondisk_log_start":"593'4318","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:06.621301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1846,"num_objects_misplaced":2700,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9359,"num_write_kb":340160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4026,"num_bytes_recovered":263716864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[4,5,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":559},{"shards":"4,6,7","objects":712}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":349,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.082051+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:22:11.082051+0000","last_peered":"2023-09-23T21:22:11.082051+0000","last_clean":"2023-09-23T21:22:11.082051+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:22:11.082051+0000","last_undegraded":"2023-09-23T21:22:11.082051+0000","last_fullsized":"2023-09-23T21:22:11.082051+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":34,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.164097+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:22:11.164097+0000","last_peered":"2023-09-23T21:22:11.164097+0000","last_clean":"2023-09-23T21:22:11.164097+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:22:11.164097+0000","last_undegraded":"2023-09-23T21:22:11.164097+0000","last_fullsized":"2023-09-23T21:22:11.164097+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.062783+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:22:11.062783+0000","last_peered":"2023-09-23T21:22:11.062783+0000","last_clean":"2023-09-23T21:22:11.062783+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:22:11.062783+0000","last_undegraded":"2023-09-23T21:22:11.062783+0000","last_fullsized":"2023-09-23T21:22:11.062783+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1046,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.063624+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:22:11.063624+0000","last_peered":"2023-09-23T21:22:11.063624+0000","last_clean":"2023-09-23T21:22:11.063624+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:22:11.063624+0000","last_undegraded":"2023-09-23T21:22:11.063624+0000","last_fullsized":"2023-09-23T21:22:11.063624+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"652'9573","reported_seq":12884,"reported_epoch":652,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:23.795190+0000","last_change":"2023-09-23T21:21:22.582611+0000","last_active":"2023-09-23T21:22:23.795190+0000","last_peered":"2023-09-23T21:22:23.795190+0000","last_clean":"2023-09-23T21:19:48.922364+0000","last_became_active":"2023-09-23T21:21:22.566477+0000","last_became_peered":"2023-09-23T21:21:22.566477+0000","last_unstale":"2023-09-23T21:22:23.795190+0000","last_undegraded":"2023-09-23T21:22:23.795190+0000","last_fullsized":"2023-09-23T21:22:23.795190+0000","mapping_epoch":639,"log_start":"593'4556","ondisk_log_start":"593'4556","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:46.195361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84082688,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1283,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9573,"num_write_kb":347392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104660992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6,7],"avail_no_missing":["2","4","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":1283}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":558,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.077498+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:22:11.077498+0000","last_peered":"2023-09-23T21:22:11.077498+0000","last_clean":"2023-09-23T21:22:11.077498+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:22:11.077498+0000","last_undegraded":"2023-09-23T21:22:11.077498+0000","last_fullsized":"2023-09-23T21:22:11.077498+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":38,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084173+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:22:11.084173+0000","last_peered":"2023-09-23T21:22:11.084173+0000","last_clean":"2023-09-23T21:22:11.084173+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:22:11.084173+0000","last_undegraded":"2023-09-23T21:22:11.084173+0000","last_fullsized":"2023-09-23T21:22:11.084173+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":39,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.084826+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:22:11.084826+0000","last_peered":"2023-09-23T21:22:11.084826+0000","last_clean":"2023-09-23T21:22:11.084826+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:22:11.084826+0000","last_undegraded":"2023-09-23T21:22:11.084826+0000","last_fullsized":"2023-09-23T21:22:11.084826+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":103,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.077962+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:22:11.077962+0000","last_peered":"2023-09-23T21:22:11.077962+0000","last_clean":"2023-09-23T21:22:11.077962+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:22:11.077962+0000","last_undegraded":"2023-09-23T21:22:11.077962+0000","last_fullsized":"2023-09-23T21:22:11.077962+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":744,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.075477+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:22:11.075477+0000","last_peered":"2023-09-23T21:22:11.075477+0000","last_clean":"2023-09-23T21:22:11.075477+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:22:11.075477+0000","last_undegraded":"2023-09-23T21:22:11.075477+0000","last_fullsized":"2023-09-23T21:22:11.075477+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":412,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.079481+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:22:11.079481+0000","last_peered":"2023-09-23T21:22:11.079481+0000","last_clean":"2023-09-23T21:22:11.079481+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:22:11.079481+0000","last_undegraded":"2023-09-23T21:22:11.079481+0000","last_fullsized":"2023-09-23T21:22:11.079481+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":255,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.077782+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:22:11.077782+0000","last_peered":"2023-09-23T21:22:11.077782+0000","last_clean":"2023-09-23T21:22:11.077782+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:22:11.077782+0000","last_undegraded":"2023-09-23T21:22:11.077782+0000","last_fullsized":"2023-09-23T21:22:11.077782+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":163,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.079455+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:22:11.079455+0000","last_peered":"2023-09-23T21:22:11.079455+0000","last_clean":"2023-09-23T21:22:11.079455+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:22:11.079455+0000","last_undegraded":"2023-09-23T21:22:11.079455+0000","last_fullsized":"2023-09-23T21:22:11.079455+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":46,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.063181+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:22:11.063181+0000","last_peered":"2023-09-23T21:22:11.063181+0000","last_clean":"2023-09-23T21:22:11.063181+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:22:11.063181+0000","last_undegraded":"2023-09-23T21:22:11.063181+0000","last_fullsized":"2023-09-23T21:22:11.063181+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":64,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.064117+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:22:11.064117+0000","last_peered":"2023-09-23T21:22:11.064117+0000","last_clean":"2023-09-23T21:22:11.064117+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:22:11.064117+0000","last_undegraded":"2023-09-23T21:22:11.064117+0000","last_fullsized":"2023-09-23T21:22:11.064117+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":55,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.063098+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:22:11.063098+0000","last_peered":"2023-09-23T21:22:11.063098+0000","last_clean":"2023-09-23T21:22:11.063098+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:22:11.063098+0000","last_undegraded":"2023-09-23T21:22:11.063098+0000","last_fullsized":"2023-09-23T21:22:11.063098+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":640,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.081677+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:22:11.081677+0000","last_peered":"2023-09-23T21:22:11.081677+0000","last_clean":"2023-09-23T21:22:11.081677+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:22:11.081677+0000","last_undegraded":"2023-09-23T21:22:11.081677+0000","last_fullsized":"2023-09-23T21:22:11.081677+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":359,"reported_epoch":651,"state":"active+clean","last_fresh":"2023-09-23T21:22:11.073185+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:22:11.073185+0000","last_peered":"2023-09-23T21:22:11.073185+0000","last_clean":"2023-09-23T21:22:11.073185+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:22:11.073185+0000","last_undegraded":"2023-09-23T21:22:11.073185+0000","last_fullsized":"2023-09-23T21:22:11.073185+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1358430232,"num_objects":20729,"num_object_clones":0,"num_object_copies":62187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8994,"num_objects_misplaced":17073,"num_objects_unfound":0,"num_objects_dirty":20729,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149901,"num_write_kb":5460097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27661,"num_bytes_recovered":1663303680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5987250176,"data_stored":5987237960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84615,"ondisk_log_size":84615,"up":48,"acting":47,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697707,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1550763,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3297280,"data_stored":3279520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201924,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1567044,"kb_used_data":655100,"kb_used_omap":0,"kb_used_meta":911936,"kb_avail":92178108,"statfs":{"total":95995035648,"available":94390382592,"internally_reserved":0,"allocated":670822400,"data_stored":669025760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933822464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9470000000000001,"5min":1.9019999999999999,"15min":2.4689999999999999},"min":{"1min":0.252,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":10.522,"5min":16.468,"15min":141.33199999999999},"last":0.26400000000000001},{"interface":"front","average":{"1min":2.645,"5min":2.0790000000000002,"15min":2.6160000000000001},"min":{"1min":0.308,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.6389999999999993,"5min":12.962,"15min":141.31700000000001},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":2.0840000000000001,"15min":3.4689999999999999},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.24099999999999999},"max":{"1min":13.054,"5min":13.054,"15min":114.16500000000001},"last":4.8650000000000002},{"interface":"front","average":{"1min":3.0009999999999999,"5min":1.9350000000000001,"15min":3.8170000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":10.881,"5min":16.289000000000001,"15min":117.637},"last":7.343}]},{"osd":2,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":9.4480000000000004,"5min":7.5999999999999996,"15min":6.1239999999999997},"min":{"1min":0.39900000000000002,"5min":0.248,"15min":0.224},"max":{"1min":157.65700000000001,"5min":158.726,"15min":183.44200000000001},"last":4.827},{"interface":"front","average":{"1min":10.220000000000001,"5min":7.4779999999999998,"15min":6.3490000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":157.642,"5min":158.70400000000001,"15min":183.46199999999999},"last":3.4900000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.5070000000000001,"15min":4.3099999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.26700000000000002},"max":{"1min":10.547000000000001,"5min":30.667999999999999,"15min":135.66200000000001},"last":2.8660000000000001},{"interface":"front","average":{"1min":3.7970000000000002,"5min":2.5390000000000001,"15min":4.5810000000000004},"min":{"1min":0.32900000000000001,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":9.7919999999999998,"5min":30.798999999999999,"15min":135.47},"last":3.5270000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.1849999999999996,"5min":3.653,"15min":1.9690000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":13.821999999999999,"5min":31.695,"15min":31.695},"last":2.528},{"interface":"front","average":{"1min":6.8120000000000003,"5min":3.7869999999999999,"15min":2.0179999999999998},"min":{"1min":0.48499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":27.821999999999999,"5min":34.994,"15min":34.994},"last":13.840999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":4.7350000000000003,"15min":2.46},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":10.712,"5min":156.30099999999999,"15min":156.30099999999999},"last":1.45},{"interface":"front","average":{"1min":4.5389999999999997,"5min":4.8739999999999997,"15min":2.548},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":12.151,"5min":156.22,"15min":156.22},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":3.75,"15min":2.8140000000000001},"min":{"1min":0.45700000000000002,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":14.066000000000001,"5min":23.574999999999999,"15min":143.18700000000001},"last":11.007999999999999},{"interface":"front","average":{"1min":6.4370000000000003,"5min":3.242,"15min":2.528},"min":{"1min":0.504,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":21.081,"5min":21.081,"15min":142.70400000000001},"last":8.907}]}]},{"osd":6,"up_from":47,"seq":201863463544,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2388096,"kb_used_data":1488696,"kb_used_omap":0,"kb_used_meta":899392,"kb_avail":91357056,"statfs":{"total":95995035648,"available":93549625344,"internally_reserved":0,"allocated":1524424704,"data_stored":1522632929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920977408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.78,"15min":2.9660000000000002},"min":{"1min":0.28699999999999998,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":12.313000000000001,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.1880000000000002,"15min":3.1960000000000002},"min":{"1min":0.27400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.968,"5min":11.968,"15min":149.303},"last":4.3079999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7400000000000002,"5min":2.4020000000000001,"15min":2.9990000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":12.013,"5min":16.126999999999999,"15min":129.37899999999999},"last":4.3769999999999998},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.8540000000000001,"15min":2.742},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":14.366,"5min":14.366,"15min":129.40100000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":25.170999999999999,"5min":9.1639999999999997,"15min":7.6150000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.23400000000001},"last":1.3140000000000001},{"interface":"front","average":{"1min":25.001000000000001,"5min":9.6969999999999992,"15min":7.7549999999999999},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":199.27000000000001,"5min":199.27000000000001,"15min":199.27000000000001},"last":4.3499999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":2.1880000000000002,"15min":3.5640000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":6.516,"5min":14.411,"15min":137.875},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.986,"15min":3.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":12.35,"5min":12.35,"15min":137.60900000000001},"last":1.3380000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":3.6909999999999998,"15min":2.5169999999999999},"min":{"1min":0.29099999999999998,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":20.408999999999999,"5min":20.408999999999999,"15min":55.494},"last":4.4470000000000001},{"interface":"front","average":{"1min":4.5709999999999997,"5min":3.367,"15min":2.3159999999999998},"min":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.23999999999999999},"max":{"1min":12.864000000000001,"5min":14.622,"15min":55.305},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":3.1499999999999999,"15min":2.8180000000000001},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":12.051,"5min":12.975,"15min":222.90899999999999},"last":4.4160000000000004},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.9969999999999999,"15min":2.7709999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28499999999999998},"max":{"1min":10.625999999999999,"5min":17.927,"15min":224.655},"last":4.532}]},{"osd":7,"last update":"Sat Sep 23 21:21:37 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":4.8970000000000002,"15min":2.2599999999999998},"min":{"1min":0.36499999999999999,"5min":0.36399999999999999,"15min":0.28000000000000003},"max":{"1min":12.114000000000001,"5min":165.96799999999999,"15min":165.96799999999999},"last":1.365},{"interface":"front","average":{"1min":3.5030000000000001,"5min":5.4870000000000001,"15min":2.5609999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.251},"max":{"1min":14.457000000000001,"5min":165.82400000000001,"15min":165.82400000000001},"last":0.40600000000000003}]}]},{"osd":1,"up_from":276,"seq":1185410973991,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":655760,"kb_used_data":8200,"kb_used_omap":5,"kb_used_meta":647546,"kb_avail":93089392,"statfs":{"total":95995035648,"available":95323537408,"internally_reserved":0,"allocated":8396800,"data_stored":5298810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":663087843},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.38200000000000001,"15min":2.0009999999999999},"min":{"1min":0.22600000000000001,"5min":0.19600000000000001,"15min":0.19400000000000001},"max":{"1min":0.505,"5min":1.9450000000000001,"15min":108.47},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.40100000000000002,"15min":1.952},"min":{"1min":0.26200000000000001,"5min":0.216,"15min":0.19600000000000001},"max":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":109.532},"last":5.4409999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":7.6120000000000001,"5min":8.4109999999999996,"15min":7.8899999999999997},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.20200000000000001},"max":{"1min":72.966999999999999,"5min":149.35300000000001,"15min":171.35400000000001},"last":3.0089999999999999},{"interface":"front","average":{"1min":7.6459999999999999,"5min":8.5180000000000007,"15min":7.5659999999999998},"min":{"1min":0.22500000000000001,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":74.450000000000003,"5min":147.50399999999999,"15min":170.69900000000001},"last":1.206}]},{"osd":3,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":0.55200000000000005,"15min":3.593},"min":{"1min":0.25900000000000001,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":17.247,"5min":17.247,"15min":99.120999999999995},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.66000000000000003,"15min":4.1600000000000001},"min":{"1min":0.26200000000000001,"5min":0.25700000000000001,"15min":0.21299999999999999},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":150.38},"last":2.9580000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4770000000000001,"5min":3.2519999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.5800000000000001,"5min":174.84399999999999,"15min":174.84399999999999},"last":6.9089999999999998},{"interface":"front","average":{"1min":0.91000000000000003,"5min":3.0230000000000001,"15min":2.1850000000000001},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.28199999999999997},"max":{"1min":2.931,"5min":174.82400000000001,"15min":174.82400000000001},"last":1.181}]},{"osd":5,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":4.2670000000000003,"15min":2.7269999999999999},"min":{"1min":0.37,"5min":0.35499999999999998,"15min":0.31900000000000001},"max":{"1min":3.8839999999999999,"5min":110.955,"15min":110.955},"last":8.6329999999999991},{"interface":"front","average":{"1min":1.53,"5min":4.5679999999999996,"15min":2.774},"min":{"1min":0.33500000000000002,"5min":0.314,"15min":0.29199999999999998},"max":{"1min":9.2300000000000004,"5min":110.89,"15min":110.89},"last":6.9669999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1989999999999998,"5min":3.1160000000000001,"15min":2.4329999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":7.7859999999999996,"5min":119.999,"15min":119.999},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.788,"5min":3.1269999999999998,"15min":2.3370000000000002},"min":{"1min":0.40500000000000003,"5min":0.38300000000000001,"15min":0.31900000000000001},"max":{"1min":8.9550000000000001,"5min":122.328,"15min":122.328},"last":7.0090000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:21:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":4.8159999999999998,"15min":2.9239999999999999},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":10.648999999999999,"5min":285.47500000000002,"15min":285.47500000000002},"last":7.0430000000000001},{"interface":"front","average":{"1min":1.875,"5min":4.5430000000000001,"15min":2.7330000000000001},"min":{"1min":0.38600000000000001,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":10.664999999999999,"5min":285.48599999999999,"15min":285.48599999999999},"last":0.58699999999999997}]}]},{"osd":0,"up_from":0,"seq":395136991731,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781622,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2104292,"kb_used_data":806748,"kb_used_omap":7,"kb_used_meta":1297528,"kb_avail":91640860,"statfs":{"total":95995035648,"available":93840240640,"internally_reserved":0,"allocated":826109952,"data_stored":822925310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1328668753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.68899999999999995,"15min":1.9530000000000001},"min":{"1min":0.189,"5min":0.17899999999999999,"15min":0.17699999999999999},"max":{"1min":2.8700000000000001,"5min":5.0010000000000003,"15min":103.021},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.875,"5min":0.73699999999999999,"15min":2.54},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":4.5510000000000002,"5min":4.9820000000000002,"15min":103.154},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.66300000000000003,"15min":2.7160000000000002},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.19700000000000001},"max":{"1min":4.7350000000000003,"5min":5.2670000000000003,"15min":143.482},"last":0.443},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.71999999999999997,"15min":2.7490000000000001},"min":{"1min":0.23899999999999999,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":1.6659999999999999,"5min":8.048,"15min":143.61000000000001},"last":1.2370000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:21:24 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.70399999999999996,"15min":3.5369999999999999},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.7679999999999998,"5min":5.2939999999999996,"15min":143.50800000000001},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.77300000000000002,"15min":3.746},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.2670000000000003,"5min":8.2189999999999994,"15min":143.27799999999999},"last":0.34100000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0310000000000001,"15min":3.4900000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":5.6680000000000001,"5min":7.4880000000000004,"15min":225.80799999999999},"last":2.1720000000000002},{"interface":"front","average":{"1min":1.972,"5min":1.9419999999999999,"15min":4.0750000000000002},"min":{"1min":0.39300000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.6840000000000002,"5min":7.5010000000000003,"15min":225.79300000000001},"last":2.145}]},{"osd":5,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.51,"15min":3.5529999999999999},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":7.048,"5min":9.0579999999999998,"15min":220.63900000000001},"last":1.3740000000000001},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.6060000000000001,"15min":3.6269999999999998},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":8.0579999999999998,"5min":8.0579999999999998,"15min":220.31999999999999},"last":7.9370000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.5150000000000001,"15min":3.5049999999999999},"min":{"1min":0.5,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":10.148,"5min":42.363,"15min":210.74700000000001},"last":1.4590000000000001},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.4420000000000002,"15min":2.8319999999999999},"min":{"1min":0.57099999999999995,"5min":0.308,"15min":0.308},"max":{"1min":10.634,"5min":42.383000000000003,"15min":210.72900000000001},"last":2.6030000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":1.8660000000000001,"15min":2.0350000000000001},"min":{"1min":0.50900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.9500000000000002,"5min":7.9500000000000002,"15min":25.260000000000002},"last":2.8250000000000002},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.796,"15min":2.4319999999999999},"min":{"1min":0.39800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.1079999999999997,"5min":7.5289999999999999,"15min":16.875},"last":2.7330000000000001}]}]},{"osd":3,"up_from":148,"seq":635655160215,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":873944,"kb_used_data":9360,"kb_used_omap":2,"kb_used_meta":864573,"kb_avail":92871208,"statfs":{"total":95995035648,"available":95100116992,"internally_reserved":0,"allocated":9584640,"data_stored":5937994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":885322805},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.504,"15min":1.7669999999999999},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":3.8780000000000001,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.51200000000000001,"15min":1.5580000000000001},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19500000000000001},"max":{"1min":5.0190000000000001,"5min":5.0190000000000001,"15min":18.277999999999999},"last":0.629}]},{"osd":1,"last update":"Sat Sep 23 21:21:33 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.504,"15min":6.5910000000000002},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.8980000000000001,"5min":2.8980000000000001,"15min":237.072},"last":3.2450000000000001},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.54200000000000004,"15min":6.5460000000000003},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":3.8359999999999999,"5min":3.8359999999999999,"15min":234.21899999999999},"last":0.435}]},{"osd":2,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":16.347999999999999,"5min":3.8279999999999998,"15min":5.6299999999999999},"min":{"1min":0.224,"5min":0.219,"15min":0.215},"max":{"1min":207.59999999999999,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":16.359999999999999,"5min":3.7570000000000001,"15min":5.2400000000000002},"min":{"1min":0.27800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":208.024,"5min":208.024,"15min":208.024},"last":0.69399999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":3.2440000000000002,"15min":2.609},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":9.5579999999999998,"5min":117.717,"15min":117.717},"last":0.46999999999999997},{"interface":"front","average":{"1min":3.141,"5min":3.7130000000000001,"15min":2.6579999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":9.0649999999999995,"5min":117.771,"15min":117.771},"last":5.3250000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":4.3209999999999997,"15min":2.9910000000000001},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":9.1910000000000007,"5min":123.458,"15min":123.458},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.218,"5min":4.0739999999999998,"15min":2.8839999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":8.4670000000000005,"5min":123.292,"15min":123.292},"last":0.67500000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":5.0510000000000002,"15min":4.7670000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":7.7889999999999997,"5min":130.613,"15min":304.22399999999999},"last":0.497},{"interface":"front","average":{"1min":2.5990000000000002,"5min":5.5640000000000001,"15min":4.8979999999999997},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":8.8940000000000001,"5min":130.50999999999999,"15min":304.15800000000002},"last":0.76800000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.974,"15min":2.2080000000000002},"min":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":7.4939999999999998,"5min":19.298999999999999,"15min":44.253},"last":0.66100000000000003},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.3380000000000001,"15min":2.5870000000000002},"min":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":9.1549999999999994,"5min":13.343,"15min":45.706000000000003},"last":0.52700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986834,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2075084,"kb_used_data":1112452,"kb_used_omap":0,"kb_used_meta":962560,"kb_avail":91670068,"statfs":{"total":95995035648,"available":93870149632,"internally_reserved":0,"allocated":1139150848,"data_stored":1137346837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":985661440},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.173,"15min":3.0070000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":12.195,"5min":12.195,"15min":226.751},"last":5.0659999999999998},{"interface":"front","average":{"1min":2.161,"5min":1.5640000000000001,"15min":2.7040000000000002},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":8.641,"5min":12.754,"15min":226.43299999999999},"last":3.9780000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.583,"15min":3.6789999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":7.0709999999999997,"5min":12.805,"15min":127.411},"last":3.9369999999999998},{"interface":"front","average":{"1min":2.4689999999999999,"5min":2.1619999999999999,"15min":3.194},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.625,"5min":10.625,"15min":116.78100000000001},"last":5.1500000000000004}]},{"osd":2,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":11.018000000000001,"5min":8.4890000000000008,"15min":6.7539999999999996},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.248},"max":{"1min":124.70099999999999,"5min":124.70099999999999,"15min":210.50299999999999},"last":5.1630000000000003},{"interface":"front","average":{"1min":7.9050000000000002,"5min":7.6079999999999997,"15min":6.6630000000000003},"min":{"1min":0.46000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":116.47,"5min":116.47,"15min":210.59},"last":5.1200000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:21:34 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.2999999999999998,"15min":4.4119999999999999},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.122999999999999,"5min":14.323,"15min":214.64099999999999},"last":5.1959999999999997},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.391,"15min":4.8159999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":10.587,"5min":10.587,"15min":214.54900000000001},"last":5.226}]},{"osd":5,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.3410000000000002,"15min":1.8280000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":12.997999999999999,"5min":17.879999999999999,"15min":34.497999999999998},"last":6.9359999999999999},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.3050000000000002,"15min":2.0760000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":19.425000000000001,"5min":19.425000000000001,"15min":34.883000000000003},"last":5.1820000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.681,"5min":4.1980000000000004,"15min":2.3980000000000001},"min":{"1min":0.36199999999999999,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":15.502000000000001,"5min":67.697999999999993,"15min":67.697999999999993},"last":4.0199999999999996},{"interface":"front","average":{"1min":5.4560000000000004,"5min":5.3390000000000004,"15min":2.726},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":18.762,"5min":67.596999999999994,"15min":67.596999999999994},"last":12.066000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":3.3159999999999998,"15min":1.9710000000000001},"min":{"1min":0.38700000000000001,"5min":0.317,"15min":0.28100000000000003},"max":{"1min":15.050000000000001,"5min":17.817,"15min":18.352},"last":5.2039999999999997},{"interface":"front","average":{"1min":3.323,"5min":3.177,"15min":1.8819999999999999},"min":{"1min":0.441,"5min":0.307,"15min":0.307},"max":{"1min":16.332999999999998,"5min":30.117000000000001,"15min":30.117000000000001},"last":4.0039999999999996}]}]},{"osd":5,"up_from":39,"seq":167503725198,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2856696,"kb_used_data":1880496,"kb_used_omap":0,"kb_used_meta":976192,"kb_avail":90888456,"statfs":{"total":95995035648,"available":93069778944,"internally_reserved":0,"allocated":1925627904,"data_stored":1923839573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":999620608},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.6899999999999999,"15min":2.0049999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":7.5119999999999996,"5min":12.558999999999999,"15min":82.280000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.194,"5min":1.8320000000000001,"15min":2.181},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":6.0430000000000001,"5min":9.407,"15min":85.236000000000004},"last":0.28499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":2.2440000000000002,"15min":3.9900000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":18.859999999999999,"5min":18.859999999999999,"15min":233.77099999999999},"last":0.23100000000000001},{"interface":"front","average":{"1min":1.198,"5min":1.544,"15min":3.5899999999999999},"min":{"1min":0.27400000000000002,"5min":0.27000000000000002,"15min":0.251},"max":{"1min":7.5220000000000002,"5min":12.654999999999999,"15min":233.75200000000001},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":5.569,"5min":5.2720000000000002,"15min":4.665},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":75.441999999999993,"5min":160.749,"15min":231.34700000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":6.3460000000000001,"5min":5.3890000000000002,"15min":4.8200000000000003},"min":{"1min":0.35999999999999999,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":75.247,"5min":161.18000000000001,"15min":231.381},"last":0.25700000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":2.278,"15min":4.0350000000000001},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":8.0980000000000008,"5min":18.826000000000001,"15min":263.887},"last":3.843},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.7989999999999999,"15min":3.6779999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":6.1950000000000003,"5min":12.683999999999999,"15min":263.70100000000002},"last":0.57399999999999995}]},{"osd":4,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":3.0649999999999999,"15min":1.7649999999999999},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":7.6299999999999999,"5min":77.804000000000002,"15min":77.804000000000002},"last":4.2089999999999996},{"interface":"front","average":{"1min":2.7719999999999998,"5min":4.2880000000000003,"15min":2.1619999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":8.3810000000000002,"5min":77.825999999999993,"15min":77.825999999999993},"last":3.6760000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":4.4279999999999999,"15min":2.617},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":68.346000000000004,"5min":68.346000000000004,"15min":83.451999999999998},"last":0.316},{"interface":"front","average":{"1min":11.348000000000001,"5min":4.758,"15min":2.907},"min":{"1min":0.40600000000000003,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":72.393000000000001,"5min":72.393000000000001,"15min":83.484999999999999},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:21:42 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":2.6309999999999998,"15min":1.792},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":10.654999999999999,"5min":10.654999999999999,"15min":10.654999999999999},"last":3.6160000000000001},{"interface":"front","average":{"1min":2.7160000000000002,"5min":2.2629999999999999,"15min":1.411},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":18.257999999999999,"5min":18.257999999999999,"15min":18.257999999999999},"last":0.60199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":808124416,"data_stored":808124416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1115688960,"data_stored":1115684888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1907298304,"data_stored":1907294232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1504186368,"data_stored":1504182296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":651952128,"data_stored":651952128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:22:28.017 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:22:28.156 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:28.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:28.260 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:28.325 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:28.913 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:28.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:29.012 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:22:29.023 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:29.109 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:29.235 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:29.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:29.335 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:29.345 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:29.416 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:29 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:29.417 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:29 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:29.420 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:29.437 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:29.539 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:29.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:29.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:29.655 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:29.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:29 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:29.660 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:29 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:29.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:29 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:29.661 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:29 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:29.738 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:29.933 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:29.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:29.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:30.048 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:30.141 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:30.258 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:30.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:30.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:30.340 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:22:30.362 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:30.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:30 smithi121 ceph-mon[211526]: pgmap v1980: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 7517/53799 objects degraded (13.972%); 15274/53799 objects misplaced (28.391%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:30.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3220600750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:30 smithi121 ceph-mon[219709]: pgmap v1980: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 7517/53799 objects degraded (13.972%); 15274/53799 objects misplaced (28.391%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:30.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3220600750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:30.434 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:30 smithi123 ceph-mon[193903]: pgmap v1980: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 7517/53799 objects degraded (13.972%); 15274/53799 objects misplaced (28.391%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:30.434 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3220600750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:30.451 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:30.560 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:30.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:30.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:30.673 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:30.775 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:30.857 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:30.929 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:30.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:31.041 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:31.129 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:31.260 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:31.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:31.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:31.374 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:31.442 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:31 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:22:31.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:31 smithi123 ceph-mon[193903]: failed to encode map e653 with expected crc 2023-09-23T21:22:31.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:31 smithi123 ceph-mon[193903]: failed to encode map e653 with expected crc 2023-09-23T21:22:31.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:31 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3220600750' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:31.443 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:31 smithi123 ceph-mon[193903]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T21:22:31.445 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:22:31.445 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[211526]: failed to encode map e653 with expected crc 2023-09-23T21:22:31.445 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[211526]: failed to encode map e653 with expected crc 2023-09-23T21:22:31.445 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3220600750' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:31.445 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[211526]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T21:22:31.446 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:22:31.446 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[219709]: failed to encode map e653 with expected crc 2023-09-23T21:22:31.446 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[219709]: failed to encode map e653 with expected crc 2023-09-23T21:22:31.446 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3220600750' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:31.446 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:31 smithi121 ceph-mon[219709]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T21:22:31.459 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:31.572 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:31.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:31.688 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:31.773 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:31.912 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:31.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:31.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:32.023 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:32.084 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:32.291 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:32 smithi121 ceph-mon[211526]: pgmap v1982: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 7517/53799 objects degraded (13.972%); 15274/53799 objects misplaced (28.391%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:32.292 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 7517/53799 objects degraded (13.972%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:32.539 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:22:32.539 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:22:32.539 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:22:32.540 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:22:32.540 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:22:32.540 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:22:32.540 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:22:32.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:32 smithi121 ceph-mon[219709]: pgmap v1982: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 7517/53799 objects degraded (13.972%); 15274/53799 objects misplaced (28.391%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:32.602 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 7517/53799 objects degraded (13.972%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:32 smithi123 ceph-mon[193903]: pgmap v1982: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 7517/53799 objects degraded (13.972%); 15274/53799 objects misplaced (28.391%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:32.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 7517/53799 objects degraded (13.972%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:32.778 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:22:32.779 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:22:33.321 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:33 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3068562016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:33.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:33 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3068562016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:33.330 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:33.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:33.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:33.437 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:33.516 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:33.522 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:22:33.645 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:33.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:33.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:33.662 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:33 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3068562016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:33.762 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:33.798 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:33.859 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:33.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:33.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:33.964 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:34.063 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:34.189 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:34.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:34.303 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:34.343 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:34 smithi123 ceph-mon[193903]: pgmap v1983: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 7335/52251 objects degraded (14.038%); 14907/52251 objects misplaced (28.530%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:34.393 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:34.546 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:34.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:34.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:34.658 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:34 smithi121 ceph-mon[219709]: pgmap v1983: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 7335/52251 objects degraded (14.038%); 14907/52251 objects misplaced (28.530%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:34 smithi121 ceph-mon[211526]: pgmap v1983: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 7335/52251 objects degraded (14.038%); 14907/52251 objects misplaced (28.530%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T21:22:34.760 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:34.897 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:34.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:35.008 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:35.044 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:22:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:22:34] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2023-09-23T21:22:35.048 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:35.303 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:35.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:35.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:35.408 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:35.455 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:35.592 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:35.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:35.698 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:35.769 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:35.887 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:35.889 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:35.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:35.904 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151138 12.6 12:653b9060:::benchmark_data_smithi121_563784_object60735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.188555+0000", 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 570.70438683500004, 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41179959799999999, 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.905 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151138 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188555+0000", 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188555+0000", 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188554+0000", 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188560+0000", 2023-09-23T21:22:35.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8250000000000001e-06 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188572+0000", 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1725e-05 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188579+0000", 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7370000000000006e-06 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193786+0000", 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0052063190000000001 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193898+0000", 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011223899999999999 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193995+0000", 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6858e-05 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.599986+0000", 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.40599103199999997 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600200+0000", 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000213787 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600259+0000", 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9061999999999998e-05 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600317+0000", 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7729999999999998e-05 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600355+0000", 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8516999999999997e-05 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151146 12.7 12:ee3dd0cc:::benchmark_data_smithi121_563784_object60743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 570.70434326899999, 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.427544013, 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151146 2023-09-23T21:22:35.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188599+0000", 2023-09-23T21:22:35.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:35.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188602+0000", 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.971e-06 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188609+0000", 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1029999999999996e-06 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188615+0000", 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9649999999999996e-06 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.193889+0000", 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00527473 2023-09-23T21:22:35.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.194011+0000", 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121808 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T21:22:35.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.194126+0000", 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011511 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202346+0000", 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0082199960000000002 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202466+0000", 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119311 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600412+0000", 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39794612699999998 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.615972+0000", 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.015560289 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.616026+0000", 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4135999999999998e-05 2023-09-23T21:22:35.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.616143+0000", 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116902 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151147 12.d 12:b1facac4:::benchmark_data_smithi121_563784_object60744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 570.70431927100003, 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.41190085500000001, 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151147 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.923 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188623+0000", 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188625+0000", 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5409999999999999e-06 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188631+0000", 2023-09-23T21:22:35.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1040000000000003e-06 2023-09-23T21:22:35.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.188636+0000", 2023-09-23T21:22:35.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0479999999999998e-06 2023-09-23T21:22:35.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.189097+0000", 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000460932 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.189209+0000", 2023-09-23T21:22:35.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011203499999999999 2023-09-23T21:22:35.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T21:22:35.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.189324+0000", 2023-09-23T21:22:35.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011435299999999999 2023-09-23T21:22:35.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202379+0000", 2023-09-23T21:22:35.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.013055628 2023-09-23T21:22:35.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:35.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202388+0000", 2023-09-23T21:22:35.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9260000000000006e-06 2023-09-23T21:22:35.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600174+0000", 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.397786272 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600358+0000", 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000183852 2023-09-23T21:22:35.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600407+0000", 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8649000000000001e-05 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600524+0000", 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116909 2023-09-23T21:22:35.932 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "MOSDPGPushReply(12.7 467/461 [PushReplyOp(12:e9f18e90:::benchmark_data_smithi121_563784_object80001:head)])", 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 570.69048184200005, 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.397904694, 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202460+0000", 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202464+0000", 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7620000000000001e-06 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.202471+0000", 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7240000000000003e-06 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600012+0000", 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.397541533 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600067+0000", 2023-09-23T21:22:35.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4707999999999999e-05 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600365+0000", 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00029816199999999998 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:151146 12.7 e467/461 ondisk, result = 0)", 2023-09-23T21:22:35.937 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.204794+0000", 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 570.68814834700004, 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39562049399999999, 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204794+0000", 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204794+0000", 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204793+0000", 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.204799+0000", 2023-09-23T21:22:35.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1059999999999996e-06 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.599985+0000", 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39518625200000002 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600002+0000", 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6297999999999999e-05 2023-09-23T21:22:35.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600395+0000", 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00039376 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600404+0000", 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3699999999999995e-06 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600414+0000", 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0599e-05 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop_reply(client.36681.0:151138 12.6 e467/444 ondisk, result = 0)", 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:05.206919+0000", 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 570.68602357600003, 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39328686699999998, 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.942 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "started", 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206919+0000", 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206919+0000", 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206918+0000", 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.206924+0000", 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7509999999999997e-06 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600012+0000", 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.39308785499999999 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600021+0000", 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.3899999999999999e-06 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600187+0000", 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000165701 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600195+0000", 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4360000000000002e-06 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:05.600206+0000", 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0424999999999999e-05 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151879 12.7 12:ec8f3f31:::benchmark_data_smithi121_563784_object61476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.957213+0000", 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 568.93572923700003, 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27218313300000002, 2023-09-23T21:22:35.947 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151879 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957213+0000", 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957213+0000", 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957211+0000", 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957220+0000", 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2430000000000005e-06 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957233+0000", 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3592e-05 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957242+0000", 2023-09-23T21:22:35.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1509999999999993e-06 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962402+0000", 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051592410000000002 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962522+0000", 2023-09-23T21:22:35.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012 2023-09-23T21:22:35.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T21:22:35.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962634+0000", 2023-09-23T21:22:35.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000112538 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967748+0000", 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051140680000000003 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.971176+0000", 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003427491 2023-09-23T21:22:35.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.972496+0000", 2023-09-23T21:22:35.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001320203 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229210+0000", 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25671398000000001 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229277+0000", 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6853999999999994e-05 2023-09-23T21:22:35.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229396+0000", 2023-09-23T21:22:35.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000119454 2023-09-23T21:22:35.956 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.956 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151880 12.6 12:66a3ff02:::benchmark_data_smithi121_563784_object61477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.957616+0000", 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 568.935326261, 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27141181800000003, 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.957 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:35.958 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:35.958 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:35.958 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151880 2023-09-23T21:22:35.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.958 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957616+0000", 2023-09-23T21:22:35.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957616+0000", 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957615+0000", 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:35.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957619+0000", 2023-09-23T21:22:35.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0709999999999996e-06 2023-09-23T21:22:35.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957628+0000", 2023-09-23T21:22:35.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7819999999999999e-06 2023-09-23T21:22:35.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957635+0000", 2023-09-23T21:22:35.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7349999999999999e-06 2023-09-23T21:22:35.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.964 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957683+0000", 2023-09-23T21:22:35.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8510999999999997e-05 2023-09-23T21:22:35.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957798+0000", 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114957 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 0,7", 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.957904+0000", 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010575500000000001 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:35.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962235+0000", 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0043305610000000001 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967416+0000", 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051811959999999999 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228948+0000", 2023-09-23T21:22:35.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.261531658 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228992+0000", 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4322e-05 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229028+0000", 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5941e-05 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.967 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151882 12.d 12:bc206b0d:::benchmark_data_smithi121_563784_object61479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.962177+0000", 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 568.930764985, 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.27157668299999999, 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:35.968 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151882 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962177+0000", 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962177+0000", 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962175+0000", 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962184+0000", 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.6009999999999995e-06 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962197+0000", 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3162e-05 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962207+0000", 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.6590000000000002e-06 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962283+0000", 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6058999999999998e-05 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962438+0000", 2023-09-23T21:22:35.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000154976 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.962576+0000", 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000137601 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.971496+0000", 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0089202540000000007 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.976117+0000", 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0046207180000000002 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.980884+0000", 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047674350000000004 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.233602+0000", 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25271828000000002 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.233647+0000", 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4326999999999998e-05 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.233754+0000", 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010741700000000001 2023-09-23T21:22:35.975 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.36681.0:151887 12.5 e467/426 12:a0b219a3:::benchmark_data_smithi121_563784_object61484:head v 467'9404, mlcod=467'9400)", 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.967400+0000", 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 568.92554262099998, 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.26153186699999997, 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967400+0000", 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967400+0000", 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967399+0000", 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967402+0000", 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7010000000000001e-06 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967407+0000", 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4880000000000001e-06 2023-09-23T21:22:35.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.967410+0000", 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.709e-06 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.976250+0000", 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0088392139999999998 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.976277+0000", 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7022e-05 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228893+0000", 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25261600000000001 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228932+0000", 2023-09-23T21:22:35.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8902999999999997e-05 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151897 12.d 12:b8ef00ef:::benchmark_data_smithi121_563784_object61494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.982469+0000", 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 568.91047368299996, 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.263483784, 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.981 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151897 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982469+0000", 2023-09-23T21:22:35.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982469+0000", 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982467+0000", 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.982474+0000", 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0690000000000004e-06 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228881+0000", 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.24640680400000001 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.984 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228894+0000", 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3123e-05 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.228965+0000", 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1372999999999996e-05 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.985 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229134+0000", 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016843000000000001 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 2,3,7", 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.229277+0000", 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014363099999999999 2023-09-23T21:22:35.986 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.238499+0000", 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0092214940000000002 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.243619+0000", 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051197930000000001 2023-09-23T21:22:35.987 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.988 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.988 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.988 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.244134+0000", 2023-09-23T21:22:35.988 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00051562099999999996 2023-09-23T21:22:35.988 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.245785+0000", 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0016509020000000001 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.245842+0000", 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6743000000000002e-05 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.989 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.990 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.990 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.245952+0000", 2023-09-23T21:22:35.990 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000110326 2023-09-23T21:22:35.990 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.990 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:118107 14.b e635/628 14:d135d15f:::benchmark_data_smithi121_684892_object32790:head v 635'7486, mlcod=634'7484)", 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:21:15.499098+0000", 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 80.393844336000001, 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.063875460999999994, 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:35.991 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:35.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.992 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:35.992 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499098+0000", 2023-09-23T21:22:35.992 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.992 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.992 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:35.993 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499098+0000", 2023-09-23T21:22:35.993 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:35.993 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.993 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.993 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499098+0000", 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499103+0000", 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9590000000000003e-06 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:35.994 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499111+0000", 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7819999999999999e-06 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499118+0000", 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5950000000000004e-06 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499170+0000", 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1595999999999999e-05 2023-09-23T21:22:35.995 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.499192+0000", 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1965000000000001e-05 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.562935+0000", 2023-09-23T21:22:35.996 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.063743504000000006 2023-09-23T21:22:35.997 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.997 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.997 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:35.997 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.562973+0000", 2023-09-23T21:22:35.998 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.824e-05 2023-09-23T21:22:35.998 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.998 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:35.998 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:35.999 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:35.999 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:35.999 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:118112 14.1 e635/628 14:86c55bcf:::benchmark_data_smithi121_684892_object32795:head v 635'7318, mlcod=634'7316)", 2023-09-23T21:22:35.999 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:21:15.509955+0000", 2023-09-23T21:22:35.999 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 80.382987592999996, 2023-09-23T21:22:35.999 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.057193679999999997, 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.509955+0000", 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.000 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.509955+0000", 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.509954+0000", 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.001 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.509962+0000", 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8270000000000004e-06 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.562605+0000", 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.052642914999999998 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.002 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.562614+0000", 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8980000000000002e-06 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.562651+0000", 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.6925000000000003e-05 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.562673+0000", 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1976000000000001e-05 2023-09-23T21:22:36.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.567123+0000", 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.004450318 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:15.567148+0000", 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4966000000000001e-05 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.004 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:122142 14.8 e641/629 14:1160d723:::benchmark_data_smithi121_684892_object36825:head v 641'7560, mlcod=594'4596)", 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:21:26.585832+0000", 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 69.307109971000003, 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050229124999999999, 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.005 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585832+0000", 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585832+0000", 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.006 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585831+0000", 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585839+0000", 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9419999999999995e-06 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585850+0000", 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0968000000000001e-05 2023-09-23T21:22:36.007 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585858+0000", 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4440000000000002e-06 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585901+0000", 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3643000000000002e-05 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.008 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.585924+0000", 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2824e-05 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.636029+0000", 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050105192 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.009 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:21:26.636061+0000", 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2101000000000003e-05 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148063 14.6 e652/629 14:6a437867:::benchmark_data_smithi121_684892_object62746:head v 652'9298, mlcod=0'0)", 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.974321+0000", 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 15.918621085, 2023-09-23T21:22:36.010 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14833297100000001, 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974321+0000", 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.011 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974321+0000", 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974321+0000", 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974326+0000", 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6640000000000003e-06 2023-09-23T21:22:36.012 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974335+0000", 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3149999999999992e-06 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974340+0000", 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1900000000000003e-06 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.013 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974384+0000", 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3977000000000003e-05 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974398+0000", 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3913e-05 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.014 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.122623+0000", 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14822523200000001 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.122654+0000", 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1288000000000003e-05 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.015 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148073 14.8 e652/629 14:1265f9ff:::benchmark_data_smithi121_684892_object62756:head v 652'9201, mlcod=594'4596)", 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.975239+0000", 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 15.917703382999999, 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.147120059, 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.975239+0000", 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.016 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.975239+0000", 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.975238+0000", 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.017 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.975243+0000", 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.8570000000000003e-06 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117576+0000", 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.142333348 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.018 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117583+0000", 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.268e-06 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117899+0000", 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000316404 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117919+0000", 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0183e-05 2023-09-23T21:22:36.019 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.122319+0000", 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0043993000000000001 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.020 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.122359+0000", 2023-09-23T21:22:36.021 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0374999999999999e-05 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148075 14.f e652/629 14:f221d7dc:::benchmark_data_smithi121_684892_object62758:head v 652'9264, mlcod=594'4610)", 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.980604+0000", 2023-09-23T21:22:36.022 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 15.912338344, 2023-09-23T21:22:36.023 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14413788899999999, 2023-09-23T21:22:36.023 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.023 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.023 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.025 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.026 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.980604+0000", 2023-09-23T21:22:36.026 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.026 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.026 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.026 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.980604+0000", 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.980603+0000", 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.027 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.980609+0000", 2023-09-23T21:22:36.028 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.9669999999999998e-06 2023-09-23T21:22:36.028 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.028 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.028 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.028 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117517+0000", 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.13690819900000001 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117533+0000", 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5664000000000001e-05 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.029 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.119535+0000", 2023-09-23T21:22:36.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0020023089999999999 2023-09-23T21:22:36.030 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.030 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.030 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.030 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.119557+0000", 2023-09-23T21:22:36.030 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1954e-05 2023-09-23T21:22:36.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.031 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.031 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.124705+0000", 2023-09-23T21:22:36.031 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0051474219999999996 2023-09-23T21:22:36.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.031 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.032 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.032 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.124742+0000", 2023-09-23T21:22:36.032 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7094000000000001e-05 2023-09-23T21:22:36.032 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.032 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.033 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.033 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.033 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.033 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:155072 14.f e653/629 14:f7aaedc9:::benchmark_data_smithi121_684892_object69755:head v 653'9697, mlcod=594'4610)", 2023-09-23T21:22:36.033 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:32.496597+0000", 2023-09-23T21:22:36.034 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.3963448930000002, 2023-09-23T21:22:36.034 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.058432087000000001, 2023-09-23T21:22:36.034 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.034 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.034 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.034 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496597+0000", 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496597+0000", 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.035 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496597+0000", 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496604+0000", 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1289999999999998e-06 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.036 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496614+0000", 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0088000000000001e-05 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496621+0000", 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7090000000000002e-06 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.037 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.547448+0000", 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.050826235999999997 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.547478+0000", 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.0521999999999997e-05 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.038 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.554998+0000", 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0075196229999999996 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.555029+0000", 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1647999999999999e-05 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.039 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:155073 14.f e653/629 14:faa7bc5a:::benchmark_data_smithi121_684892_object69756:head v 653'9698, mlcod=594'4610)", 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:32.496637+0000", 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.3963047710000001, 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.058418598000000002, 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.040 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496637+0000", 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496637+0000", 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.041 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496637+0000", 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496641+0000", 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2610000000000004e-06 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.042 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496648+0000", 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6359999999999997e-06 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.496666+0000", 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7903000000000001e-05 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.043 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.550136+0000", 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.053470286999999998 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.550154+0000", 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7382999999999999e-05 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.044 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.555032+0000", 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048780439999999998 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.555056+0000", 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4403999999999999e-05 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.045 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:155081 14.3 e653/629 14:c3921f27:::benchmark_data_smithi121_684892_object69764:head v 653'9776, mlcod=594'4660)", 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:32.504404+0000", 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 3.3885386710000001, 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.051041448000000003, 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.046 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.504404+0000", 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.504404+0000", 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:36.047 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.504402+0000", 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.504413+0000", 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0241000000000001e-05 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.547452+0000", 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.043038856 2023-09-23T21:22:36.048 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.547464+0000", 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2251e-05 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.547764+0000", 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000299812 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.049 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.547787+0000", 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3102e-05 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.555417+0000", 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076302770000000004 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:36.050 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:32.555445+0000", 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8017999999999998e-05 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:36.051 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:22:36.051 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:36.056 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:22:36.056 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6084/45204 objects degraded (13.459%), 9 pgs degraded, 2 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2612,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":653,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695504058,"num_remapped_pgs":12},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":15068,"data_bytes":986561283,"bytes_used":11824230400,"bytes_avail":660141019136,"bytes_total":671965249536,"degraded_objects":6084,"degraded_total":45204,"degraded_ratio":0.13458985930448633,"misplaced_objects":13400,"misplaced_total":45204,"misplaced_ratio":0.29643394389877004,"recovering_objects_per_sec":156,"recovering_bytes_per_sec":10199076,"recovering_keys_per_sec":0,"num_objects_recovered":1565,"num_bytes_recovered":102039552,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":568},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":35,"modified":"2023-09-23T21:21:43.417977+0000","services":{}},"progress_events":{"432f78a9-9bfa-46a5-b9c3-f138454b372b":{"message":"Global Recovery Event (2m)\n [========================....] (remaining: 23s)","progress":0.87878787517547607,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:22:36.249 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:36.406 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:36.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:36.528 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:36.554 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:36.624 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:36.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:36.729 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:36.753 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:36 smithi123 ceph-mon[193903]: pgmap v1984: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 941 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 6084/45204 objects degraded (13.459%); 13400/45204 objects misplaced (29.643%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T21:22:36.753 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3902094776' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:22:36.759 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:36.858 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:36.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:36 smithi121 ceph-mon[211526]: pgmap v1984: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 941 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 6084/45204 objects degraded (13.459%); 13400/45204 objects misplaced (29.643%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T21:22:36.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3902094776' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:22:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:36 smithi121 ceph-mon[219709]: pgmap v1984: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 941 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 6084/45204 objects degraded (13.459%); 13400/45204 objects misplaced (29.643%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T21:22:36.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3902094776' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:22:36.970 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:37.015 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:37.235 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:37.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:37.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:37.339 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:37.375 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:37.398 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:22:37.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:37 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 6084/45204 objects degraded (13.459%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:37.665 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:37 smithi121 ceph-mon[211526]: pgmap v1985: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 906 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 5894/43515 objects degraded (13.545%); 13037/43515 objects misplaced (29.960%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T21:22:37.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:37 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 6084/45204 objects degraded (13.459%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:37.665 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:37 smithi121 ceph-mon[219709]: pgmap v1985: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 906 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 5894/43515 objects degraded (13.545%); 13037/43515 objects misplaced (29.960%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T21:22:37.859 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:22:37.860 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:22:37.913 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T21:22:37.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:37 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 6084/45204 objects degraded (13.459%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:37.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:37 smithi123 ceph-mon[193903]: pgmap v1985: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 906 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 5894/43515 objects degraded (13.545%); 13037/43515 objects misplaced (29.960%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T21:22:38.024 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:38.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:38.134 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:38.211 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:38.351 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:38.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:38.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:38.466 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:38.527 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:39.106 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:39.110 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.110 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.110 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:22:39.110 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:22:39.110 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:39.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:39.112 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.112 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.112 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:22:39.112 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:22:39.112 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:38 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.212 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:39.220 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:38 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:38 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:38 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:22:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:38 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:22:39.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:38 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:22:39.267 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:39.372 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:39.378 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:39.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:39.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:39.488 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:39.489 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:39.532 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:39.775 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:39.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:39.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:39.880 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:39.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:39 smithi121 ceph-mon[211526]: pgmap v1986: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 763 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4657/36645 objects degraded (12.708%); 11777/36645 objects misplaced (32.138%); 8.1 MiB/s, 148 objects/s recovering 2023-09-23T21:22:39.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:39 smithi121 ceph-mon[219709]: pgmap v1986: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 763 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4657/36645 objects degraded (12.708%); 11777/36645 objects misplaced (32.138%); 8.1 MiB/s, 148 objects/s recovering 2023-09-23T21:22:39.936 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:40.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:39 smithi123 ceph-mon[193903]: pgmap v1986: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 763 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4657/36645 objects degraded (12.708%); 11777/36645 objects misplaced (32.138%); 8.1 MiB/s, 148 objects/s recovering 2023-09-23T21:22:41.241 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:22:41.246 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:41.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:41.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:41.355 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:41.377 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:41 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2498578716' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:22:41.377 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:41 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2498578716' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:22:41.435 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:41.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:41 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2498578716' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2498578716' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: pgmap v1988: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 763 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4657/36645 objects degraded (12.708%); 11777/36645 objects misplaced (32.138%); 8.1 MiB/s, 148 objects/s recovering 2023-09-23T21:22:42.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 4657/36645 objects degraded (12.708%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2498578716' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: pgmap v1988: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 763 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4657/36645 objects degraded (12.708%); 11777/36645 objects misplaced (32.138%); 8.1 MiB/s, 148 objects/s recovering 2023-09-23T21:22:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:42 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 4657/36645 objects degraded (12.708%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2498578716' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:22:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T21:22:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: failed to encode map e654 with expected crc 2023-09-23T21:22:42.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: pgmap v1988: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 763 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4657/36645 objects degraded (12.708%); 11777/36645 objects misplaced (32.138%); 8.1 MiB/s, 148 objects/s recovering 2023-09-23T21:22:42.730 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:42 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 4657/36645 objects degraded (12.708%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:43.120 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:43.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:43.227 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:43.255 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:43.319 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:43.454 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:43.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:43.571 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:43.638 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:43.641 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:22:43.641 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [], 2023-09-23T21:22:43.641 INFO:teuthology.orchestra.run.smithi121.stdout: "num_ops": 0 2023-09-23T21:22:43.641 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:22:43.648 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:43.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:43.768 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:43.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:43.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:43.880 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:43.931 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:44.016 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:44.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:44.120 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:44.194 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:44.266 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:44 smithi121 ceph-mon[219709]: pgmap v1989: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 730 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4483/35094 objects degraded (12.774%); 11428/35094 objects misplaced (32.564%); 8.4 MiB/s, 153 objects/s recovering 2023-09-23T21:22:44.266 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3845880384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:44.288 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:44.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:44.355 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:22:44.356 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:22:44.356 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:22:44.363 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:22:44.364 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:22:44.364 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:22:44.364 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:22:44.364 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:22:44.365 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.365 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.365 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.365 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.365 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.365 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.365 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.366 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:22:44.370 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:22:44.378 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:22:44.397 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:44.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:44 smithi123 ceph-mon[193903]: pgmap v1989: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 730 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4483/35094 objects degraded (12.774%); 11428/35094 objects misplaced (32.564%); 8.4 MiB/s, 153 objects/s recovering 2023-09-23T21:22:44.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3845880384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:44.484 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:44.528 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:44.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:44.633 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:44 smithi121 ceph-mon[211526]: pgmap v1989: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 730 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 4483/35094 objects degraded (12.774%); 11428/35094 objects misplaced (32.564%); 8.4 MiB/s, 153 objects/s recovering 2023-09-23T21:22:44.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3845880384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:44.682 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:45.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:22:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:22:44] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T21:22:45.207 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:22:45.358 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:45.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:45.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:45.463 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:45.513 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:45.555 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:45.561 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:45 smithi123 ceph-mon[193903]: from='client.57760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:45.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:45.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:45 smithi121 ceph-mon[211526]: from='client.57760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:45.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:45 smithi121 ceph-mon[219709]: from='client.57760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:45.663 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:45.702 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:45.783 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T21:22:46.139 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1989,"stamp":"2023-09-23T21:22:43.445520+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":765704963,"num_objects":11698,"num_object_clones":0,"num_object_copies":35094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4483,"num_objects_misplaced":11428,"num_objects_unfound":0,"num_objects_dirty":11698,"num_whiteouts":0,"num_read":92,"num_read_kb":169,"num_write":159060,"num_write_kb":5460565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30172,"num_bytes_recovered":1805464933,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86896,"ondisk_log_size":86896,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10944952,"kb_used_data":4326784,"kb_used_omap":16,"kb_used_meta":6617839,"kb_avail":645271112,"statfs":{"total":671965249536,"available":660757618688,"internally_reserved":0,"allocated":4430626816,"data_stored":4413482110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6776668009},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,7,7,5,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374800384,"num_objects":-5719,"num_object_clones":0,"num_object_copies":-17157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2852,"num_objects_misplaced":-3479,"num_objects_unfound":0,"num_objects_dirty":-5719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5719,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1535,"num_bytes_recovered":88342528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003783"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":228,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.566849+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:22:19.566849+0000","last_peered":"2023-09-23T21:22:19.566849+0000","last_clean":"2023-09-23T21:22:19.566849+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:22:19.566849+0000","last_undegraded":"2023-09-23T21:22:19.566849+0000","last_fullsized":"2023-09-23T21:22:19.566849+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":65,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.549005+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:22:19.549005+0000","last_peered":"2023-09-23T21:22:19.549005+0000","last_clean":"2023-09-23T21:22:19.549005+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:22:19.549005+0000","last_undegraded":"2023-09-23T21:22:19.549005+0000","last_fullsized":"2023-09-23T21:22:19.549005+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345766+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:22:30.345766+0000","last_peered":"2023-09-23T21:22:30.345766+0000","last_clean":"2023-09-23T21:22:30.345766+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:22:30.345766+0000","last_undegraded":"2023-09-23T21:22:30.345766+0000","last_fullsized":"2023-09-23T21:22:30.345766+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":340,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.586597+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:22:19.586597+0000","last_peered":"2023-09-23T21:22:19.586597+0000","last_clean":"2023-09-23T21:22:19.586597+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:22:19.586597+0000","last_undegraded":"2023-09-23T21:22:19.586597+0000","last_fullsized":"2023-09-23T21:22:19.586597+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":56,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.548550+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:22:19.548550+0000","last_peered":"2023-09-23T21:22:19.548550+0000","last_clean":"2023-09-23T21:22:19.548550+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:22:19.548550+0000","last_undegraded":"2023-09-23T21:22:19.548550+0000","last_fullsized":"2023-09-23T21:22:19.548550+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":159,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.282348+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:22:30.282348+0000","last_peered":"2023-09-23T21:22:30.282348+0000","last_clean":"2023-09-23T21:22:30.282348+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:22:30.282348+0000","last_undegraded":"2023-09-23T21:22:30.282348+0000","last_fullsized":"2023-09-23T21:22:30.282348+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":60,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.572883+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:22:19.572883+0000","last_peered":"2023-09-23T21:22:19.572883+0000","last_clean":"2023-09-23T21:22:19.572883+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:22:19.572883+0000","last_undegraded":"2023-09-23T21:22:19.572883+0000","last_fullsized":"2023-09-23T21:22:19.572883+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":74,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.548626+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:22:19.548626+0000","last_peered":"2023-09-23T21:22:19.548626+0000","last_clean":"2023-09-23T21:22:19.548626+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:22:19.548626+0000","last_undegraded":"2023-09-23T21:22:19.548626+0000","last_fullsized":"2023-09-23T21:22:19.548626+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":50,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.565042+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:22:19.565042+0000","last_peered":"2023-09-23T21:22:19.565042+0000","last_clean":"2023-09-23T21:22:19.565042+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:22:19.565042+0000","last_undegraded":"2023-09-23T21:22:19.565042+0000","last_fullsized":"2023-09-23T21:22:19.565042+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":47,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.548681+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:22:19.548681+0000","last_peered":"2023-09-23T21:22:19.548681+0000","last_clean":"2023-09-23T21:22:19.548681+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:22:19.548681+0000","last_undegraded":"2023-09-23T21:22:19.548681+0000","last_fullsized":"2023-09-23T21:22:19.548681+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":288,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.586906+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:22:19.586906+0000","last_peered":"2023-09-23T21:22:19.586906+0000","last_clean":"2023-09-23T21:22:19.586906+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:22:19.586906+0000","last_undegraded":"2023-09-23T21:22:19.586906+0000","last_fullsized":"2023-09-23T21:22:19.586906+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":45,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346121+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:22:30.346121+0000","last_peered":"2023-09-23T21:22:30.346121+0000","last_clean":"2023-09-23T21:22:30.346121+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:22:30.346121+0000","last_undegraded":"2023-09-23T21:22:30.346121+0000","last_fullsized":"2023-09-23T21:22:30.346121+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":384,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.571684+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:22:19.571684+0000","last_peered":"2023-09-23T21:22:19.571684+0000","last_clean":"2023-09-23T21:22:19.571684+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:22:19.571684+0000","last_undegraded":"2023-09-23T21:22:19.571684+0000","last_fullsized":"2023-09-23T21:22:19.571684+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":611,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.570088+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:22:19.570088+0000","last_peered":"2023-09-23T21:22:19.570088+0000","last_clean":"2023-09-23T21:22:19.570088+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:22:19.570088+0000","last_undegraded":"2023-09-23T21:22:19.570088+0000","last_fullsized":"2023-09-23T21:22:19.570088+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":236,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.582671+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:22:19.582671+0000","last_peered":"2023-09-23T21:22:19.582671+0000","last_clean":"2023-09-23T21:22:19.582671+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:22:19.582671+0000","last_undegraded":"2023-09-23T21:22:19.582671+0000","last_fullsized":"2023-09-23T21:22:19.582671+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":56,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.549026+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:22:19.549026+0000","last_peered":"2023-09-23T21:22:19.549026+0000","last_clean":"2023-09-23T21:22:19.549026+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:22:19.549026+0000","last_undegraded":"2023-09-23T21:22:19.549026+0000","last_fullsized":"2023-09-23T21:22:19.549026+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":262,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.576378+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:22:19.576378+0000","last_peered":"2023-09-23T21:22:19.576378+0000","last_clean":"2023-09-23T21:22:19.576378+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:22:19.576378+0000","last_undegraded":"2023-09-23T21:22:19.576378+0000","last_fullsized":"2023-09-23T21:22:19.576378+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":270,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.587037+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:22:19.587037+0000","last_peered":"2023-09-23T21:22:19.587037+0000","last_clean":"2023-09-23T21:22:19.587037+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:22:19.587037+0000","last_undegraded":"2023-09-23T21:22:19.587037+0000","last_fullsized":"2023-09-23T21:22:19.587037+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":230,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.580824+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:22:19.580824+0000","last_peered":"2023-09-23T21:22:19.580824+0000","last_clean":"2023-09-23T21:22:19.580824+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:22:19.580824+0000","last_undegraded":"2023-09-23T21:22:19.580824+0000","last_fullsized":"2023-09-23T21:22:19.580824+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":47,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.548930+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:22:19.548930+0000","last_peered":"2023-09-23T21:22:19.548930+0000","last_clean":"2023-09-23T21:22:19.548930+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:22:19.548930+0000","last_undegraded":"2023-09-23T21:22:19.548930+0000","last_fullsized":"2023-09-23T21:22:19.548930+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":750,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.281838+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:22:30.281838+0000","last_peered":"2023-09-23T21:22:30.281838+0000","last_clean":"2023-09-23T21:22:30.281838+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:22:30.281838+0000","last_undegraded":"2023-09-23T21:22:30.281838+0000","last_fullsized":"2023-09-23T21:22:30.281838+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":56,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.549182+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:22:19.549182+0000","last_peered":"2023-09-23T21:22:19.549182+0000","last_clean":"2023-09-23T21:22:19.549182+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:22:19.549182+0000","last_undegraded":"2023-09-23T21:22:19.549182+0000","last_fullsized":"2023-09-23T21:22:19.549182+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"654'9854","reported_seq":11779,"reported_epoch":654,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:41.467662+0000","last_change":"2023-09-23T21:21:01.215269+0000","last_active":"2023-09-23T21:22:41.467662+0000","last_peered":"2023-09-23T21:22:41.467662+0000","last_clean":"2023-09-23T21:19:28.877182+0000","last_became_active":"2023-09-23T21:21:01.190149+0000","last_became_peered":"2023-09-23T21:21:01.190149+0000","last_unstale":"2023-09-23T21:22:41.467662+0000","last_undegraded":"2023-09-23T21:21:00.607176+0000","last_fullsized":"2023-09-23T21:22:41.467662+0000","mapping_epoch":629,"log_start":"593'4367","ondisk_log_start":"593'4367","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:05.989327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41418752,"num_objects":632,"num_object_clones":0,"num_object_copies":1896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":632,"num_objects_misplaced":632,"num_objects_unfound":0,"num_objects_dirty":632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9854,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":893,"num_bytes_recovered":58523648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,4,5],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,5,6","objects":1},{"shards":"4,6,7","objects":631}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":290,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.585538+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:22:19.585538+0000","last_peered":"2023-09-23T21:22:19.585538+0000","last_clean":"2023-09-23T21:22:19.585538+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:22:19.585538+0000","last_undegraded":"2023-09-23T21:22:19.585538+0000","last_fullsized":"2023-09-23T21:22:19.585538+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":50,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.565343+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:22:19.565343+0000","last_peered":"2023-09-23T21:22:19.565343+0000","last_clean":"2023-09-23T21:22:19.565343+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:22:19.565343+0000","last_undegraded":"2023-09-23T21:22:19.565343+0000","last_fullsized":"2023-09-23T21:22:19.565343+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"653'10017","reported_seq":12044,"reported_epoch":653,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:39.064652+0000","last_change":"2023-09-23T21:21:01.210342+0000","last_active":"2023-09-23T21:22:39.064652+0000","last_peered":"2023-09-23T21:22:39.064652+0000","last_clean":"2023-09-23T21:19:28.923531+0000","last_became_active":"2023-09-23T21:21:01.189847+0000","last_became_peered":"2023-09-23T21:21:01.189847+0000","last_unstale":"2023-09-23T21:22:39.064652+0000","last_undegraded":"2023-09-23T21:21:00.610088+0000","last_fullsized":"2023-09-23T21:22:39.064652+0000","mapping_epoch":629,"log_start":"601'4998","ondisk_log_start":"601'4998","created":573,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5019,"log_dups_size":0,"ondisk_log_size":5019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:58.686882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49479680,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":755,"num_objects_misplaced":755,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10017,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":938,"num_bytes_recovered":61341696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[4,6,5],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":29},{"shards":"4,6,7","objects":726}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":304,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.581087+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:22:19.581087+0000","last_peered":"2023-09-23T21:22:19.581087+0000","last_clean":"2023-09-23T21:22:19.581087+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:22:19.581087+0000","last_undegraded":"2023-09-23T21:22:19.581087+0000","last_fullsized":"2023-09-23T21:22:19.581087+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345696+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:22:30.345696+0000","last_peered":"2023-09-23T21:22:30.345696+0000","last_clean":"2023-09-23T21:22:30.345696+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:22:30.345696+0000","last_undegraded":"2023-09-23T21:22:30.345696+0000","last_fullsized":"2023-09-23T21:22:30.345696+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"653'9765","reported_seq":15002,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:38.601211+0000","last_change":"2023-09-23T21:21:17.539683+0000","last_active":"2023-09-23T21:22:38.601211+0000","last_peered":"2023-09-23T21:22:38.601211+0000","last_clean":"2023-09-23T21:22:38.601211+0000","last_became_active":"2023-09-23T21:21:17.521633+0000","last_became_peered":"2023-09-23T21:21:17.521633+0000","last_unstale":"2023-09-23T21:22:38.601211+0000","last_undegraded":"2023-09-23T21:22:38.601211+0000","last_fullsized":"2023-09-23T21:22:38.601211+0000","mapping_epoch":636,"log_start":"600'4765","ondisk_log_start":"600'4765","created":573,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5000,"log_dups_size":0,"ondisk_log_size":5000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:05.844201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9765,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2555,"num_bytes_recovered":80478208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":276,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.587110+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:22:19.587110+0000","last_peered":"2023-09-23T21:22:19.587110+0000","last_clean":"2023-09-23T21:22:19.587110+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:22:19.587110+0000","last_undegraded":"2023-09-23T21:22:19.587110+0000","last_fullsized":"2023-09-23T21:22:19.587110+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":25,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.282518+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:22:30.282518+0000","last_peered":"2023-09-23T21:22:30.282518+0000","last_clean":"2023-09-23T21:22:30.282518+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:22:30.282518+0000","last_undegraded":"2023-09-23T21:22:30.282518+0000","last_fullsized":"2023-09-23T21:22:30.282518+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"653'9820","reported_seq":9926,"reported_epoch":653,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:38.864356+0000","last_change":"2023-09-23T21:20:59.644218+0000","last_active":"2023-09-23T21:22:38.864356+0000","last_peered":"2023-09-23T21:22:38.864356+0000","last_clean":"2023-09-23T21:20:30.430127+0000","last_became_active":"2023-09-23T21:20:59.632855+0000","last_became_peered":"2023-09-23T21:20:59.632855+0000","last_unstale":"2023-09-23T21:22:38.864356+0000","last_undegraded":"2023-09-23T21:22:38.864356+0000","last_fullsized":"2023-09-23T21:22:38.864356+0000","mapping_epoch":628,"log_start":"600'4825","ondisk_log_start":"600'4825","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4995,"log_dups_size":0,"ondisk_log_size":4995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:33.014367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50987008,"num_objects":778,"num_object_clones":0,"num_object_copies":2334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1556,"num_objects_unfound":0,"num_objects_dirty":778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9820,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":778}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":56,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.549099+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:22:19.549099+0000","last_peered":"2023-09-23T21:22:19.549099+0000","last_clean":"2023-09-23T21:22:19.549099+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:22:19.549099+0000","last_undegraded":"2023-09-23T21:22:19.549099+0000","last_fullsized":"2023-09-23T21:22:19.549099+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345739+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:22:30.345739+0000","last_peered":"2023-09-23T21:22:30.345739+0000","last_clean":"2023-09-23T21:22:30.345739+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:22:30.345739+0000","last_undegraded":"2023-09-23T21:22:30.345739+0000","last_fullsized":"2023-09-23T21:22:30.345739+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1079,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.580699+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:22:19.580699+0000","last_peered":"2023-09-23T21:22:19.580699+0000","last_clean":"2023-09-23T21:22:19.580699+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:22:19.580699+0000","last_undegraded":"2023-09-23T21:22:19.580699+0000","last_fullsized":"2023-09-23T21:22:19.580699+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.6","version":"653'10010","reported_seq":17559,"reported_epoch":653,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:38.859320+0000","last_change":"2023-09-23T21:21:01.239919+0000","last_active":"2023-09-23T21:22:38.859320+0000","last_peered":"2023-09-23T21:22:38.859320+0000","last_clean":"2023-09-23T21:19:28.802255+0000","last_became_active":"2023-09-23T21:21:01.197858+0000","last_became_peered":"2023-09-23T21:21:01.197858+0000","last_unstale":"2023-09-23T21:22:38.859320+0000","last_undegraded":"2023-09-23T21:21:00.673065+0000","last_fullsized":"2023-09-23T21:22:38.859320+0000","mapping_epoch":629,"log_start":"600'4998","ondisk_log_start":"600'4998","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5012,"log_dups_size":0,"ondisk_log_size":5012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:39.315049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46923776,"num_objects":716,"num_object_clones":0,"num_object_copies":2148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":716,"num_objects_misplaced":716,"num_objects_unfound":0,"num_objects_dirty":716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10010,"num_write_kb":343232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3714,"num_bytes_recovered":243335168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,4,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,4,6","objects":680},{"shards":"2,5,6","objects":36}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":64,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.550981+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:22:19.550981+0000","last_peered":"2023-09-23T21:22:19.550981+0000","last_clean":"2023-09-23T21:22:19.550981+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:22:19.550981+0000","last_undegraded":"2023-09-23T21:22:19.550981+0000","last_fullsized":"2023-09-23T21:22:19.550981+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":411,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.571767+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:22:19.571767+0000","last_peered":"2023-09-23T21:22:19.571767+0000","last_clean":"2023-09-23T21:22:19.571767+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:22:19.571767+0000","last_undegraded":"2023-09-23T21:22:19.571767+0000","last_fullsized":"2023-09-23T21:22:19.571767+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":947,"reported_epoch":652,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:22:19.551469+0000","last_change":"2023-09-23T21:20:48.157415+0000","last_active":"2023-09-23T21:22:19.551469+0000","last_peered":"2023-09-23T21:22:19.551469+0000","last_clean":"2023-09-23T21:20:41.603469+0000","last_became_active":"2023-09-23T21:20:48.156886+0000","last_became_peered":"2023-09-23T21:20:48.156886+0000","last_unstale":"2023-09-23T21:22:19.551469+0000","last_undegraded":"2023-09-23T21:20:47.922781+0000","last_fullsized":"2023-09-23T21:20:47.921128+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:30.898986+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4,6","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.7","version":"653'10094","reported_seq":11301,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:38.621327+0000","last_change":"2023-09-23T21:20:59.594443+0000","last_active":"2023-09-23T21:22:38.621327+0000","last_peered":"2023-09-23T21:22:38.621327+0000","last_clean":"2023-09-23T21:22:38.621327+0000","last_became_active":"2023-09-23T21:20:48.149995+0000","last_became_peered":"2023-09-23T21:20:48.149995+0000","last_unstale":"2023-09-23T21:22:38.621327+0000","last_undegraded":"2023-09-23T21:22:38.621327+0000","last_fullsized":"2023-09-23T21:22:38.621327+0000","mapping_epoch":623,"log_start":"582'2665","ondisk_log_start":"582'2665","created":573,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":7429,"log_dups_size":0,"ondisk_log_size":7429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.278714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10094,"num_write_kb":348544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":149,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.284120+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:22:30.284120+0000","last_peered":"2023-09-23T21:22:30.284120+0000","last_clean":"2023-09-23T21:22:30.284120+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:22:30.284120+0000","last_undegraded":"2023-09-23T21:22:30.284120+0000","last_fullsized":"2023-09-23T21:22:30.284120+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":661,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.573195+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:22:19.573195+0000","last_peered":"2023-09-23T21:22:19.573195+0000","last_clean":"2023-09-23T21:22:19.573195+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:22:19.573195+0000","last_undegraded":"2023-09-23T21:22:19.573195+0000","last_fullsized":"2023-09-23T21:22:19.573195+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"654'9854","reported_seq":19098,"reported_epoch":654,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:41.426500+0000","last_change":"2023-09-23T21:22:12.152044+0000","last_active":"2023-09-23T21:22:41.426500+0000","last_peered":"2023-09-23T21:22:41.426500+0000","last_clean":"2023-09-23T21:19:28.911487+0000","last_became_active":"2023-09-23T21:20:32.169326+0000","last_became_peered":"2023-09-23T21:20:32.169326+0000","last_unstale":"2023-09-23T21:22:41.426500+0000","last_undegraded":"2023-09-23T21:22:41.426500+0000","last_fullsized":"2023-09-23T21:22:41.426500+0000","mapping_epoch":617,"log_start":"593'4368","ondisk_log_start":"593'4368","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:09.080183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40501248,"num_objects":618,"num_object_clones":0,"num_object_copies":1854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":618,"num_objects_unfound":0,"num_objects_dirty":618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9854,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4564,"num_bytes_recovered":237240320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,4],"avail_no_missing":["6","4","5","7"],"object_location_counts":[{"shards":"4,6,7","objects":618}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":290,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.566569+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:22:19.566569+0000","last_peered":"2023-09-23T21:22:19.566569+0000","last_clean":"2023-09-23T21:22:19.566569+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:22:19.566569+0000","last_undegraded":"2023-09-23T21:22:19.566569+0000","last_fullsized":"2023-09-23T21:22:19.566569+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":698,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.571756+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:22:19.571756+0000","last_peered":"2023-09-23T21:22:19.571756+0000","last_clean":"2023-09-23T21:22:19.571756+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:22:19.571756+0000","last_undegraded":"2023-09-23T21:22:19.571756+0000","last_fullsized":"2023-09-23T21:22:19.571756+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":916,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.551287+0000","last_change":"2023-09-23T21:20:49.968107+0000","last_active":"2023-09-23T21:22:19.551287+0000","last_peered":"2023-09-23T21:22:19.551287+0000","last_clean":"2023-09-23T21:22:19.551287+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:22:19.551287+0000","last_undegraded":"2023-09-23T21:22:19.551287+0000","last_fullsized":"2023-09-23T21:22:19.551287+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":429358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.5","version":"653'10180","reported_seq":19620,"reported_epoch":653,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:39.171641+0000","last_change":"2023-09-23T21:20:59.554347+0000","last_active":"2023-09-23T21:22:39.171641+0000","last_peered":"2023-09-23T21:22:39.171641+0000","last_clean":"2023-09-23T21:19:28.864495+0000","last_became_active":"2023-09-23T21:20:32.158375+0000","last_became_peered":"2023-09-23T21:20:32.158375+0000","last_unstale":"2023-09-23T21:22:39.171641+0000","last_undegraded":"2023-09-23T21:20:59.551954+0000","last_fullsized":"2023-09-23T21:22:39.171641+0000","mapping_epoch":617,"log_start":"600'5092","ondisk_log_start":"600'5092","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:46.553959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51118080,"num_objects":780,"num_object_clones":0,"num_object_copies":2340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":67,"num_objects_misplaced":713,"num_objects_unfound":0,"num_objects_dirty":780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10180,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4663,"num_bytes_recovered":305594368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"2,4,7","objects":750},{"shards":"4,5,7","objects":30}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":56,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.548434+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:22:19.548434+0000","last_peered":"2023-09-23T21:22:19.548434+0000","last_clean":"2023-09-23T21:22:19.548434+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:22:19.548434+0000","last_undegraded":"2023-09-23T21:22:19.548434+0000","last_fullsized":"2023-09-23T21:22:19.548434+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346926+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:22:30.346926+0000","last_peered":"2023-09-23T21:22:30.346926+0000","last_clean":"2023-09-23T21:22:30.346926+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:22:30.346926+0000","last_undegraded":"2023-09-23T21:22:30.346926+0000","last_fullsized":"2023-09-23T21:22:30.346926+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":736,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.282193+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:22:30.282193+0000","last_peered":"2023-09-23T21:22:30.282193+0000","last_clean":"2023-09-23T21:22:30.282193+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:22:30.282193+0000","last_undegraded":"2023-09-23T21:22:30.282193+0000","last_fullsized":"2023-09-23T21:22:30.282193+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"653'9875","reported_seq":10730,"reported_epoch":653,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:38.869047+0000","last_change":"2023-09-23T21:21:01.233673+0000","last_active":"2023-09-23T21:22:38.869047+0000","last_peered":"2023-09-23T21:22:38.869047+0000","last_clean":"2023-09-23T21:19:28.898608+0000","last_became_active":"2023-09-23T21:21:01.195228+0000","last_became_peered":"2023-09-23T21:21:01.195228+0000","last_unstale":"2023-09-23T21:22:38.869047+0000","last_undegraded":"2023-09-23T21:21:00.570455+0000","last_fullsized":"2023-09-23T21:22:38.869047+0000","mapping_epoch":629,"log_start":"601'4946","ondisk_log_start":"601'4946","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4929,"log_dups_size":0,"ondisk_log_size":4929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:03.164901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49217536,"num_objects":751,"num_object_clones":0,"num_object_copies":2253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":751,"num_objects_misplaced":751,"num_objects_unfound":0,"num_objects_dirty":751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9875,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[2,7,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,5,7","objects":34},{"shards":"2,6,7","objects":717}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345855+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:22:30.345855+0000","last_peered":"2023-09-23T21:22:30.345855+0000","last_clean":"2023-09-23T21:22:30.345855+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:22:30.345855+0000","last_undegraded":"2023-09-23T21:22:30.345855+0000","last_fullsized":"2023-09-23T21:22:30.345855+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":40,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.572483+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:22:19.572483+0000","last_peered":"2023-09-23T21:22:19.572483+0000","last_clean":"2023-09-23T21:22:19.572483+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:22:19.572483+0000","last_undegraded":"2023-09-23T21:22:19.572483+0000","last_fullsized":"2023-09-23T21:22:19.572483+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346127+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:22:30.346127+0000","last_peered":"2023-09-23T21:22:30.346127+0000","last_clean":"2023-09-23T21:22:30.346127+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:22:30.346127+0000","last_undegraded":"2023-09-23T21:22:30.346127+0000","last_fullsized":"2023-09-23T21:22:30.346127+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":928,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.551170+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:22:19.551170+0000","last_peered":"2023-09-23T21:22:19.551170+0000","last_clean":"2023-09-23T21:22:19.551170+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:22:19.551170+0000","last_undegraded":"2023-09-23T21:22:19.551170+0000","last_fullsized":"2023-09-23T21:22:19.551170+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"653'9825","reported_seq":9913,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:39.153914+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:22:39.153914+0000","last_peered":"2023-09-23T21:22:39.153914+0000","last_clean":"2023-09-23T21:22:39.153914+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:22:39.153914+0000","last_undegraded":"2023-09-23T21:22:39.153914+0000","last_fullsized":"2023-09-23T21:22:39.153914+0000","mapping_epoch":573,"log_start":"600'4787","ondisk_log_start":"600'4787","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5038,"log_dups_size":0,"ondisk_log_size":5038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46989312,"num_objects":717,"num_object_clones":0,"num_object_copies":2151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9825,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346239+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:22:30.346239+0000","last_peered":"2023-09-23T21:22:30.346239+0000","last_clean":"2023-09-23T21:22:30.346239+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:22:30.346239+0000","last_undegraded":"2023-09-23T21:22:30.346239+0000","last_fullsized":"2023-09-23T21:22:30.346239+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":566,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.567514+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:22:19.567514+0000","last_peered":"2023-09-23T21:22:19.567514+0000","last_clean":"2023-09-23T21:22:19.567514+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:22:19.567514+0000","last_undegraded":"2023-09-23T21:22:19.567514+0000","last_fullsized":"2023-09-23T21:22:19.567514+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345886+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:22:30.345886+0000","last_peered":"2023-09-23T21:22:30.345886+0000","last_clean":"2023-09-23T21:22:30.345886+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:22:30.345886+0000","last_undegraded":"2023-09-23T21:22:30.345886+0000","last_fullsized":"2023-09-23T21:22:30.345886+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1039,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.582008+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:22:19.582008+0000","last_peered":"2023-09-23T21:22:19.582008+0000","last_clean":"2023-09-23T21:22:19.582008+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:22:19.582008+0000","last_undegraded":"2023-09-23T21:22:19.582008+0000","last_fullsized":"2023-09-23T21:22:19.582008+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.d","version":"654'10011","reported_seq":10175,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.454380+0000","last_change":"2023-09-23T21:21:22.585504+0000","last_active":"2023-09-23T21:22:41.454380+0000","last_peered":"2023-09-23T21:22:41.454380+0000","last_clean":"2023-09-23T21:22:41.454380+0000","last_became_active":"2023-09-23T21:21:22.568285+0000","last_became_peered":"2023-09-23T21:21:22.568285+0000","last_unstale":"2023-09-23T21:22:41.454380+0000","last_undegraded":"2023-09-23T21:22:41.454380+0000","last_fullsized":"2023-09-23T21:22:41.454380+0000","mapping_epoch":639,"log_start":"594'4548","ondisk_log_start":"594'4548","created":573,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5463,"log_dups_size":0,"ondisk_log_size":5463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:01.251062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43122712,"num_objects":659,"num_object_clones":0,"num_object_copies":1977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":659,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10011,"num_write_kb":341377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":682,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.551347+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:22:19.551347+0000","last_peered":"2023-09-23T21:22:19.551347+0000","last_clean":"2023-09-23T21:22:19.551347+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:22:19.551347+0000","last_undegraded":"2023-09-23T21:22:19.551347+0000","last_fullsized":"2023-09-23T21:22:19.551347+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345880+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:22:30.345880+0000","last_peered":"2023-09-23T21:22:30.345880+0000","last_clean":"2023-09-23T21:22:30.345880+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:22:30.345880+0000","last_undegraded":"2023-09-23T21:22:30.345880+0000","last_fullsized":"2023-09-23T21:22:30.345880+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346888+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:22:30.346888+0000","last_peered":"2023-09-23T21:22:30.346888+0000","last_clean":"2023-09-23T21:22:30.346888+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:22:30.346888+0000","last_undegraded":"2023-09-23T21:22:30.346888+0000","last_fullsized":"2023-09-23T21:22:30.346888+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"653'9909","reported_seq":12813,"reported_epoch":653,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:38.550161+0000","last_change":"2023-09-23T21:21:01.214995+0000","last_active":"2023-09-23T21:22:38.550161+0000","last_peered":"2023-09-23T21:22:38.550161+0000","last_clean":"2023-09-23T21:19:28.902983+0000","last_became_active":"2023-09-23T21:21:01.189928+0000","last_became_peered":"2023-09-23T21:21:01.189928+0000","last_unstale":"2023-09-23T21:22:38.550161+0000","last_undegraded":"2023-09-23T21:21:00.625306+0000","last_fullsized":"2023-09-23T21:22:38.550161+0000","mapping_epoch":629,"log_start":"590'3886","ondisk_log_start":"590'3886","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":6023,"log_dups_size":0,"ondisk_log_size":6023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:45.309159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50528256,"num_objects":771,"num_object_clones":0,"num_object_copies":2313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":771,"num_objects_misplaced":771,"num_objects_unfound":0,"num_objects_dirty":771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1379,"num_bytes_recovered":90374144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,2,5],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,5,7","objects":41},{"shards":"2,6,7","objects":730}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":511,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.551590+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:22:19.551590+0000","last_peered":"2023-09-23T21:22:19.551590+0000","last_clean":"2023-09-23T21:22:19.551590+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:22:19.551590+0000","last_undegraded":"2023-09-23T21:22:19.551590+0000","last_fullsized":"2023-09-23T21:22:19.551590+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1040,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.581929+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:22:19.581929+0000","last_peered":"2023-09-23T21:22:19.581929+0000","last_clean":"2023-09-23T21:22:19.581929+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:22:19.581929+0000","last_undegraded":"2023-09-23T21:22:19.581929+0000","last_fullsized":"2023-09-23T21:22:19.581929+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":40,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.565531+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:22:19.565531+0000","last_peered":"2023-09-23T21:22:19.565531+0000","last_clean":"2023-09-23T21:22:19.565531+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:22:19.565531+0000","last_undegraded":"2023-09-23T21:22:19.565531+0000","last_fullsized":"2023-09-23T21:22:19.565531+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":25,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.284122+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:22:30.284122+0000","last_peered":"2023-09-23T21:22:30.284122+0000","last_clean":"2023-09-23T21:22:30.284122+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:22:30.284122+0000","last_undegraded":"2023-09-23T21:22:30.284122+0000","last_fullsized":"2023-09-23T21:22:30.284122+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":688,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345749+0000","last_change":"2023-09-23T21:21:21.571375+0000","last_active":"2023-09-23T21:22:30.345749+0000","last_peered":"2023-09-23T21:22:30.345749+0000","last_clean":"2023-09-23T21:22:30.345749+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:22:30.345749+0000","last_undegraded":"2023-09-23T21:22:30.345749+0000","last_fullsized":"2023-09-23T21:22:30.345749+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.551943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.551943+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.c","version":"653'10059","reported_seq":10147,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:39.131513+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:22:39.131513+0000","last_peered":"2023-09-23T21:22:39.131513+0000","last_clean":"2023-09-23T21:22:39.131513+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:22:39.131513+0000","last_undegraded":"2023-09-23T21:22:39.131513+0000","last_fullsized":"2023-09-23T21:22:39.131513+0000","mapping_epoch":573,"log_start":"600'5023","ondisk_log_start":"600'5023","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48693248,"num_objects":743,"num_object_clones":0,"num_object_copies":2229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10059,"num_write_kb":345664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":320,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.586230+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:22:19.586230+0000","last_peered":"2023-09-23T21:22:19.586230+0000","last_clean":"2023-09-23T21:22:19.586230+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:22:19.586230+0000","last_undegraded":"2023-09-23T21:22:19.586230+0000","last_fullsized":"2023-09-23T21:22:19.586230+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":125,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.566438+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:22:19.566438+0000","last_peered":"2023-09-23T21:22:19.566438+0000","last_clean":"2023-09-23T21:22:19.566438+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:22:19.566438+0000","last_undegraded":"2023-09-23T21:22:19.566438+0000","last_fullsized":"2023-09-23T21:22:19.566438+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346046+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:22:30.346046+0000","last_peered":"2023-09-23T21:22:30.346046+0000","last_clean":"2023-09-23T21:22:30.346046+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:22:30.346046+0000","last_undegraded":"2023-09-23T21:22:30.346046+0000","last_fullsized":"2023-09-23T21:22:30.346046+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":716,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.282333+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:22:30.282333+0000","last_peered":"2023-09-23T21:22:30.282333+0000","last_clean":"2023-09-23T21:22:30.282333+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:22:30.282333+0000","last_undegraded":"2023-09-23T21:22:30.282333+0000","last_fullsized":"2023-09-23T21:22:30.282333+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"653'9688","reported_seq":18444,"reported_epoch":653,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T21:22:39.166227+0000","last_change":"2023-09-23T21:22:39.171133+0000","last_active":"2023-09-23T21:22:39.166227+0000","last_peered":"2023-09-23T21:22:39.166227+0000","last_clean":"2023-09-23T21:19:28.920085+0000","last_became_active":"2023-09-23T21:20:48.224556+0000","last_became_peered":"2023-09-23T21:20:48.224556+0000","last_unstale":"2023-09-23T21:22:39.166227+0000","last_undegraded":"2023-09-23T21:22:39.165321+0000","last_fullsized":"2023-09-23T21:20:47.933463+0000","mapping_epoch":623,"log_start":"600'4644","ondisk_log_start":"600'4644","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5044,"log_dups_size":0,"ondisk_log_size":5044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:02.563744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49938432,"num_objects":762,"num_object_clones":0,"num_object_copies":2286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1468,"num_objects_unfound":0,"num_objects_dirty":762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9688,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4218,"num_bytes_recovered":264044544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[4,2],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"2,4","objects":28},{"shards":"2,4,6","objects":734}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":260,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.582641+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:22:19.582641+0000","last_peered":"2023-09-23T21:22:19.582641+0000","last_clean":"2023-09-23T21:22:19.582641+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:22:19.582641+0000","last_undegraded":"2023-09-23T21:22:19.582641+0000","last_fullsized":"2023-09-23T21:22:19.582641+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.345687+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:22:30.345687+0000","last_peered":"2023-09-23T21:22:30.345687+0000","last_clean":"2023-09-23T21:22:30.345687+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:22:30.345687+0000","last_undegraded":"2023-09-23T21:22:30.345687+0000","last_fullsized":"2023-09-23T21:22:30.345687+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":50,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.565293+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:22:19.565293+0000","last_peered":"2023-09-23T21:22:19.565293+0000","last_clean":"2023-09-23T21:22:19.565293+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:22:19.565293+0000","last_undegraded":"2023-09-23T21:22:19.565293+0000","last_fullsized":"2023-09-23T21:22:19.565293+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":954,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.571906+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:22:19.571906+0000","last_peered":"2023-09-23T21:22:19.571906+0000","last_clean":"2023-09-23T21:22:19.571906+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:22:19.571906+0000","last_undegraded":"2023-09-23T21:22:19.571906+0000","last_fullsized":"2023-09-23T21:22:19.571906+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.a","version":"653'9887","reported_seq":19138,"reported_epoch":653,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:39.116903+0000","last_change":"2023-09-23T21:21:01.218382+0000","last_active":"2023-09-23T21:22:39.116903+0000","last_peered":"2023-09-23T21:22:39.116903+0000","last_clean":"2023-09-23T21:19:28.896530+0000","last_became_active":"2023-09-23T21:21:01.190094+0000","last_became_peered":"2023-09-23T21:21:01.190094+0000","last_unstale":"2023-09-23T21:22:39.116903+0000","last_undegraded":"2023-09-23T21:21:00.719860+0000","last_fullsized":"2023-09-23T21:22:39.116903+0000","mapping_epoch":629,"log_start":"600'4818","ondisk_log_start":"600'4818","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5069,"log_dups_size":0,"ondisk_log_size":5069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:06.621301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48693248,"num_objects":743,"num_object_clones":0,"num_object_copies":2229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":790,"num_objects_misplaced":2700,"num_objects_unfound":0,"num_objects_dirty":743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9887,"num_write_kb":340160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4554,"num_bytes_recovered":298319872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[4,5,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,5,6","objects":31},{"shards":"4,6,7","objects":712}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":350,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.593107+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:22:19.593107+0000","last_peered":"2023-09-23T21:22:19.593107+0000","last_clean":"2023-09-23T21:22:19.593107+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:22:19.593107+0000","last_undegraded":"2023-09-23T21:22:19.593107+0000","last_fullsized":"2023-09-23T21:22:19.593107+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":36,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346146+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:22:30.346146+0000","last_peered":"2023-09-23T21:22:30.346146+0000","last_clean":"2023-09-23T21:22:30.346146+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:22:30.346146+0000","last_undegraded":"2023-09-23T21:22:30.346146+0000","last_fullsized":"2023-09-23T21:22:30.346146+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.340869+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:22:30.340869+0000","last_peered":"2023-09-23T21:22:30.340869+0000","last_clean":"2023-09-23T21:22:30.340869+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:22:30.340869+0000","last_undegraded":"2023-09-23T21:22:30.340869+0000","last_fullsized":"2023-09-23T21:22:30.340869+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1048,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.341078+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:22:30.341078+0000","last_peered":"2023-09-23T21:22:30.341078+0000","last_clean":"2023-09-23T21:22:30.341078+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:22:30.341078+0000","last_undegraded":"2023-09-23T21:22:30.341078+0000","last_fullsized":"2023-09-23T21:22:30.341078+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"653'10108","reported_seq":13420,"reported_epoch":653,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:38.872644+0000","last_change":"2023-09-23T21:21:22.582611+0000","last_active":"2023-09-23T21:22:38.872644+0000","last_peered":"2023-09-23T21:22:38.872644+0000","last_clean":"2023-09-23T21:19:48.922364+0000","last_became_active":"2023-09-23T21:21:22.566477+0000","last_became_peered":"2023-09-23T21:21:22.566477+0000","last_unstale":"2023-09-23T21:22:38.872644+0000","last_undegraded":"2023-09-23T21:22:38.872644+0000","last_fullsized":"2023-09-23T21:22:38.872644+0000","mapping_epoch":639,"log_start":"601'5156","ondisk_log_start":"601'5156","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4952,"log_dups_size":0,"ondisk_log_size":4952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:46.195361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49020928,"num_objects":748,"num_object_clones":0,"num_object_copies":2244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":748,"num_objects_unfound":0,"num_objects_dirty":748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10108,"num_write_kb":347392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104660992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6,7],"avail_no_missing":["2","4","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":748}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":560,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.282423+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:22:30.282423+0000","last_peered":"2023-09-23T21:22:30.282423+0000","last_clean":"2023-09-23T21:22:30.282423+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:22:30.282423+0000","last_undegraded":"2023-09-23T21:22:30.282423+0000","last_fullsized":"2023-09-23T21:22:30.282423+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":40,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.346059+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:22:30.346059+0000","last_peered":"2023-09-23T21:22:30.346059+0000","last_clean":"2023-09-23T21:22:30.346059+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:22:30.346059+0000","last_undegraded":"2023-09-23T21:22:30.346059+0000","last_fullsized":"2023-09-23T21:22:30.346059+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":40,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.565053+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:22:19.565053+0000","last_peered":"2023-09-23T21:22:19.565053+0000","last_clean":"2023-09-23T21:22:19.565053+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:22:19.565053+0000","last_undegraded":"2023-09-23T21:22:19.565053+0000","last_fullsized":"2023-09-23T21:22:19.565053+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":104,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.570018+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:22:19.570018+0000","last_peered":"2023-09-23T21:22:19.570018+0000","last_clean":"2023-09-23T21:22:19.570018+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:22:19.570018+0000","last_undegraded":"2023-09-23T21:22:19.570018+0000","last_fullsized":"2023-09-23T21:22:19.570018+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":745,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.551699+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:22:19.551699+0000","last_peered":"2023-09-23T21:22:19.551699+0000","last_clean":"2023-09-23T21:22:19.551699+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:22:19.551699+0000","last_undegraded":"2023-09-23T21:22:19.551699+0000","last_fullsized":"2023-09-23T21:22:19.551699+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":413,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.567368+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:22:19.567368+0000","last_peered":"2023-09-23T21:22:19.567368+0000","last_clean":"2023-09-23T21:22:19.567368+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:22:19.567368+0000","last_undegraded":"2023-09-23T21:22:19.567368+0000","last_fullsized":"2023-09-23T21:22:19.567368+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":256,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.569813+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:22:19.569813+0000","last_peered":"2023-09-23T21:22:19.569813+0000","last_clean":"2023-09-23T21:22:19.569813+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:22:19.569813+0000","last_undegraded":"2023-09-23T21:22:19.569813+0000","last_fullsized":"2023-09-23T21:22:19.569813+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":165,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.284226+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:22:30.284226+0000","last_peered":"2023-09-23T21:22:30.284226+0000","last_clean":"2023-09-23T21:22:30.284226+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:22:30.284226+0000","last_undegraded":"2023-09-23T21:22:30.284226+0000","last_fullsized":"2023-09-23T21:22:30.284226+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.551397+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:22:19.551397+0000","last_peered":"2023-09-23T21:22:19.551397+0000","last_clean":"2023-09-23T21:22:19.551397+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:22:19.551397+0000","last_undegraded":"2023-09-23T21:22:19.551397+0000","last_fullsized":"2023-09-23T21:22:19.551397+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":66,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.341422+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:22:30.341422+0000","last_peered":"2023-09-23T21:22:30.341422+0000","last_clean":"2023-09-23T21:22:30.341422+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:22:30.341422+0000","last_undegraded":"2023-09-23T21:22:30.341422+0000","last_fullsized":"2023-09-23T21:22:30.341422+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":56,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.548654+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:22:19.548654+0000","last_peered":"2023-09-23T21:22:19.548654+0000","last_clean":"2023-09-23T21:22:19.548654+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:22:19.548654+0000","last_undegraded":"2023-09-23T21:22:19.548654+0000","last_fullsized":"2023-09-23T21:22:19.548654+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":642,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.357248+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:22:30.357248+0000","last_peered":"2023-09-23T21:22:30.357248+0000","last_clean":"2023-09-23T21:22:30.357248+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:22:30.357248+0000","last_undegraded":"2023-09-23T21:22:30.357248+0000","last_fullsized":"2023-09-23T21:22:30.357248+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":360,"reported_epoch":652,"state":"active+clean","last_fresh":"2023-09-23T21:22:19.565388+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:22:19.565388+0000","last_peered":"2023-09-23T21:22:19.565388+0000","last_clean":"2023-09-23T21:22:19.565388+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:22:19.565388+0000","last_undegraded":"2023-09-23T21:22:19.565388+0000","last_fullsized":"2023-09-23T21:22:19.565388+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":765001752,"num_objects":11674,"num_object_clones":0,"num_object_copies":35022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4482,"num_objects_misplaced":11428,"num_objects_unfound":0,"num_objects_dirty":11674,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158956,"num_write_kb":5460097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29993,"num_bytes_recovered":1803878400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4317392896,"data_stored":4317380680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85070,"ondisk_log_size":85070,"up":48,"acting":47,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697707,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1550763,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3297280,"data_stored":3279520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201927,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1367696,"kb_used_data":450248,"kb_used_omap":0,"kb_used_meta":917440,"kb_avail":92377456,"statfs":{"total":95995035648,"available":94594514944,"internally_reserved":0,"allocated":461053952,"data_stored":459253456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":939458560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9470000000000001,"5min":1.9019999999999999,"15min":2.4689999999999999},"min":{"1min":0.252,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":10.522,"5min":16.468,"15min":141.33199999999999},"last":6.1920000000000002},{"interface":"front","average":{"1min":2.645,"5min":2.0790000000000002,"15min":2.6160000000000001},"min":{"1min":0.308,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.6389999999999993,"5min":12.962,"15min":141.31700000000001},"last":2.9969999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2109999999999999,"15min":3.593},"min":{"1min":0.36699999999999999,"5min":0.29599999999999999,"15min":0.24099999999999999},"max":{"1min":12.112,"5min":13.054,"15min":114.16500000000001},"last":3.339},{"interface":"front","average":{"1min":3.181,"5min":2.2530000000000001,"15min":3.9670000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22900000000000001},"max":{"1min":10.477,"5min":16.289000000000001,"15min":117.637},"last":6.1509999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":9.4480000000000004,"5min":7.5999999999999996,"15min":6.1239999999999997},"min":{"1min":0.39900000000000002,"5min":0.248,"15min":0.224},"max":{"1min":157.65700000000001,"5min":158.726,"15min":183.44200000000001},"last":3.375},{"interface":"front","average":{"1min":10.220000000000001,"5min":7.4779999999999998,"15min":6.3490000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":157.642,"5min":158.70400000000001,"15min":183.46199999999999},"last":0.372}]},{"osd":3,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.5070000000000001,"15min":4.3099999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.26700000000000002},"max":{"1min":10.547000000000001,"5min":30.667999999999999,"15min":135.66200000000001},"last":6.218},{"interface":"front","average":{"1min":3.7970000000000002,"5min":2.5390000000000001,"15min":4.5810000000000004},"min":{"1min":0.32900000000000001,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":9.7919999999999998,"5min":30.798999999999999,"15min":135.47},"last":2.0219999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.1849999999999996,"5min":3.653,"15min":1.9690000000000001},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":13.821999999999999,"5min":31.695,"15min":31.695},"last":3.2629999999999999},{"interface":"front","average":{"1min":6.8120000000000003,"5min":3.7869999999999999,"15min":2.0179999999999998},"min":{"1min":0.48499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":27.821999999999999,"5min":34.994,"15min":34.994},"last":7.5890000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":4.7350000000000003,"15min":2.46},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":10.712,"5min":156.30099999999999,"15min":156.30099999999999},"last":8.3399999999999999},{"interface":"front","average":{"1min":4.5389999999999997,"5min":4.8739999999999997,"15min":2.548},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":12.151,"5min":156.22,"15min":156.22},"last":3.0350000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":3.75,"15min":2.8140000000000001},"min":{"1min":0.45700000000000002,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":14.066000000000001,"5min":23.574999999999999,"15min":143.18700000000001},"last":3.0640000000000001},{"interface":"front","average":{"1min":6.4370000000000003,"5min":3.242,"15min":2.528},"min":{"1min":0.504,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":21.081,"5min":21.081,"15min":142.70400000000001},"last":7.29}]}]},{"osd":6,"up_from":47,"seq":201863463548,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1937740,"kb_used_data":1026692,"kb_used_omap":0,"kb_used_meta":910784,"kb_avail":91807412,"statfs":{"total":95995035648,"available":94010789888,"internally_reserved":0,"allocated":1051332608,"data_stored":1049536977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932642816},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,0,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.78,"15min":2.9660000000000002},"min":{"1min":0.28699999999999998,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":12.313000000000001,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.246},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.1880000000000002,"15min":3.1960000000000002},"min":{"1min":0.27400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.968,"5min":11.968,"15min":149.303},"last":5.3399999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":2.1040000000000001,"15min":2.9940000000000002},"min":{"1min":0.25700000000000001,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":5.3810000000000002,"5min":12.013,"15min":129.37899999999999},"last":5.3810000000000002},{"interface":"front","average":{"1min":3.149,"5min":2.0859999999999999,"15min":2.8250000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.6399999999999997,"5min":14.366,"15min":129.40100000000001},"last":1.8759999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":25.170999999999999,"5min":9.1639999999999997,"15min":7.6150000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.23400000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":25.001000000000001,"5min":9.6969999999999992,"15min":7.7549999999999999},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":199.27000000000001,"5min":199.27000000000001,"15min":199.27000000000001},"last":5.4109999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":2.1880000000000002,"15min":3.5640000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":6.516,"5min":14.411,"15min":137.875},"last":1.8320000000000001},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.986,"15min":3.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":12.35,"5min":12.35,"15min":137.60900000000001},"last":0.315}]},{"osd":4,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":3.6909999999999998,"15min":2.5169999999999999},"min":{"1min":0.29099999999999998,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":20.408999999999999,"5min":20.408999999999999,"15min":55.494},"last":6.2279999999999998},{"interface":"front","average":{"1min":4.5709999999999997,"5min":3.367,"15min":2.3159999999999998},"min":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.23999999999999999},"max":{"1min":12.864000000000001,"5min":14.622,"15min":55.305},"last":6.5640000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":3.1499999999999999,"15min":2.8180000000000001},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":12.051,"5min":12.975,"15min":222.90899999999999},"last":5.4379999999999997},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.9969999999999999,"15min":2.7709999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28499999999999998},"max":{"1min":10.625999999999999,"5min":17.927,"15min":224.655},"last":15.478}]},{"osd":7,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.7429999999999999,"15min":2.379},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.28000000000000003},"max":{"1min":12.380000000000001,"5min":12.380000000000001,"15min":165.96799999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":4.1890000000000001,"5min":3.6179999999999999,"15min":2.7690000000000001},"min":{"1min":0.40600000000000003,"5min":0.307,"15min":0.251},"max":{"1min":11.526999999999999,"5min":19.815999999999999,"15min":165.82400000000001},"last":5.5119999999999996}]}]},{"osd":1,"up_from":276,"seq":1185410973995,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643496,"kb_used_data":8224,"kb_used_omap":5,"kb_used_meta":635258,"kb_avail":93101656,"statfs":{"total":95995035648,"available":95336095744,"internally_reserved":0,"allocated":8421376,"data_stored":5315674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":650504931},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.39200000000000002,"15min":1.944},"min":{"1min":0.214,"5min":0.214,"15min":0.19400000000000001},"max":{"1min":2.3410000000000002,"5min":2.3410000000000002,"15min":108.47},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.339,"5min":0.59799999999999998,"15min":1.952},"min":{"1min":0.23899999999999999,"5min":0.23499999999999999,"15min":0.19600000000000001},"max":{"1min":6.0149999999999997,"5min":6.0149999999999997,"15min":109.532},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":13.051,"5min":6.5170000000000003,"15min":8.6799999999999997},"min":{"1min":0.23200000000000001,"5min":0.22800000000000001,"15min":0.20200000000000001},"max":{"1min":130.78899999999999,"5min":130.78899999999999,"15min":171.35400000000001},"last":0.315},{"interface":"front","average":{"1min":12.999000000000001,"5min":6.5540000000000003,"15min":8.3290000000000006},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":133.751,"5min":133.751,"15min":170.69900000000001},"last":0.29099999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.60699999999999998,"15min":3.5550000000000002},"min":{"1min":0.27000000000000002,"5min":0.254,"15min":0.23799999999999999},"max":{"1min":3.2450000000000001,"5min":17.247,"15min":99.120999999999995},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.76200000000000001,"15min":4.0960000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":3.754,"5min":10.949999999999999,"15min":150.38},"last":0.35699999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":1.829,"15min":2.371},"min":{"1min":0.27400000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":6.9089999999999998,"5min":11.034000000000001,"15min":174.84399999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.3170000000000002,"5min":1.5,"15min":2.2749999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":8.5109999999999992,"5min":11.057,"15min":174.82400000000001},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":2.6440000000000001,"15min":2.7730000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.31900000000000001},"max":{"1min":8.6329999999999991,"5min":107.98,"15min":110.955},"last":2.0110000000000001},{"interface":"front","average":{"1min":3.0750000000000002,"5min":3.169,"15min":2.8999999999999999},"min":{"1min":0.36699999999999999,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":6.9669999999999996,"5min":107.961,"15min":110.89},"last":1.4430000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":3.6160000000000001,"15min":2.5960000000000001},"min":{"1min":0.42199999999999999,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":20.265999999999998,"5min":119.999,"15min":119.999},"last":1.496},{"interface":"front","average":{"1min":3.5409999999999999,"5min":3.5710000000000002,"15min":2.4980000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31900000000000001},"max":{"1min":20.361999999999998,"5min":122.328,"15min":122.328},"last":0.96799999999999997}]},{"osd":7,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5979999999999999,"5min":2.1389999999999998,"15min":3.0009999999999999},"min":{"1min":0.39800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.0430000000000001,"5min":10.648999999999999,"15min":285.47500000000002},"last":1.377},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.726,"15min":2.754},"min":{"1min":0.46999999999999997,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":5.165,"5min":10.664999999999999,"15min":285.48599999999999},"last":1.359}]}]},{"osd":0,"up_from":0,"seq":395136991734,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781625,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1850288,"kb_used_data":526504,"kb_used_omap":7,"kb_used_meta":1323768,"kb_avail":91894864,"statfs":{"total":95995035648,"available":94100340736,"internally_reserved":0,"allocated":539140096,"data_stored":535951598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1355538513},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.68899999999999995,"15min":1.9530000000000001},"min":{"1min":0.189,"5min":0.17899999999999999,"15min":0.17699999999999999},"max":{"1min":2.8700000000000001,"5min":5.0010000000000003,"15min":103.021},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.875,"5min":0.73699999999999999,"15min":2.54},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":4.5510000000000002,"5min":4.9820000000000002,"15min":103.154},"last":0.313}]},{"osd":1,"last update":"Sat Sep 23 21:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.66300000000000003,"15min":2.7160000000000002},"min":{"1min":0.22800000000000001,"5min":0.216,"15min":0.19700000000000001},"max":{"1min":4.7350000000000003,"5min":5.2670000000000003,"15min":143.482},"last":2.27},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.71999999999999997,"15min":2.7490000000000001},"min":{"1min":0.23899999999999999,"5min":0.214,"15min":0.20799999999999999},"max":{"1min":1.6659999999999999,"5min":8.048,"15min":143.61000000000001},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.79600000000000004,"15min":3.5499999999999998},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.798,"5min":5.2939999999999996,"15min":143.50800000000001},"last":2.1659999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.77600000000000002,"15min":3.77},"min":{"1min":0.34100000000000003,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.3290000000000002,"5min":5.6790000000000003,"15min":143.27799999999999},"last":0.51800000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0310000000000001,"15min":3.4900000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":5.6680000000000001,"5min":7.4880000000000004,"15min":225.80799999999999},"last":3.9910000000000001},{"interface":"front","average":{"1min":1.972,"5min":1.9419999999999999,"15min":4.0750000000000002},"min":{"1min":0.39300000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.6840000000000002,"5min":7.5010000000000003,"15min":225.79300000000001},"last":2.0110000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.51,"15min":3.5529999999999999},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":7.048,"5min":9.0579999999999998,"15min":220.63900000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.6060000000000001,"15min":3.6269999999999998},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":8.0579999999999998,"5min":8.0579999999999998,"15min":220.31999999999999},"last":3.0699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.5150000000000001,"15min":3.5049999999999999},"min":{"1min":0.5,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":10.148,"5min":42.363,"15min":210.74700000000001},"last":2.2429999999999999},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.4420000000000002,"15min":2.8319999999999999},"min":{"1min":0.57099999999999995,"5min":0.308,"15min":0.308},"max":{"1min":10.634,"5min":42.383000000000003,"15min":210.72900000000001},"last":3.9550000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":1.8660000000000001,"15min":2.0350000000000001},"min":{"1min":0.50900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.9500000000000002,"5min":7.9500000000000002,"15min":25.260000000000002},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.796,"15min":2.4319999999999999},"min":{"1min":0.39800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.1079999999999997,"5min":7.5289999999999999,"15min":16.875},"last":5.6500000000000004}]}]},{"osd":3,"up_from":148,"seq":635655160219,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":883816,"kb_used_data":9376,"kb_used_omap":2,"kb_used_meta":874429,"kb_avail":92861336,"statfs":{"total":95995035648,"available":95090008064,"internally_reserved":0,"allocated":9601024,"data_stored":5946699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":895415349},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.504,"15min":1.7669999999999999},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":3.8780000000000001,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.51200000000000001,"15min":1.5580000000000001},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19500000000000001},"max":{"1min":5.0190000000000001,"5min":5.0190000000000001,"15min":18.277999999999999},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:22:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.70299999999999996,"15min":6.5880000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":8.3079999999999998,"5min":8.3079999999999998,"15min":237.072},"last":0.308},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.67000000000000004,"15min":6.4690000000000003},"min":{"1min":0.24299999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":234.21899999999999},"last":0.25600000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":16.347999999999999,"5min":3.8279999999999998,"15min":5.6299999999999999},"min":{"1min":0.224,"5min":0.219,"15min":0.215},"max":{"1min":207.59999999999999,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.443},{"interface":"front","average":{"1min":16.359999999999999,"5min":3.7570000000000001,"15min":5.2400000000000002},"min":{"1min":0.27800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":208.024,"5min":208.024,"15min":208.024},"last":0.27200000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":3.2440000000000002,"15min":2.609},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":9.5579999999999998,"5min":117.717,"15min":117.717},"last":1.181},{"interface":"front","average":{"1min":3.141,"5min":3.7130000000000001,"15min":2.6579999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":9.0649999999999995,"5min":117.771,"15min":117.771},"last":1.3560000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":4.3209999999999997,"15min":2.9910000000000001},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":9.1910000000000007,"5min":123.458,"15min":123.458},"last":1.0349999999999999},{"interface":"front","average":{"1min":2.218,"5min":4.0739999999999998,"15min":2.8839999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":8.4670000000000005,"5min":123.292,"15min":123.292},"last":1.5880000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":5.0510000000000002,"15min":4.7670000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":7.7889999999999997,"5min":130.613,"15min":304.22399999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.5990000000000002,"5min":5.5640000000000001,"15min":4.8979999999999997},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":8.8940000000000001,"5min":130.50999999999999,"15min":304.15800000000002},"last":0.84899999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.974,"15min":2.2080000000000002},"min":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":7.4939999999999998,"5min":19.298999999999999,"15min":44.253},"last":0.874},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.3380000000000001,"15min":2.5870000000000002},"min":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":9.1549999999999994,"5min":13.343,"15min":45.706000000000003},"last":0.65200000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986837,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1708888,"kb_used_data":734096,"kb_used_omap":0,"kb_used_meta":974784,"kb_avail":92036264,"statfs":{"total":95995035648,"available":94245134336,"internally_reserved":0,"allocated":751714304,"data_stored":749906437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":998178816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.173,"15min":3.0070000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":12.195,"5min":12.195,"15min":226.751},"last":0.23200000000000001},{"interface":"front","average":{"1min":2.161,"5min":1.5640000000000001,"15min":2.7040000000000002},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":8.641,"5min":12.754,"15min":226.43299999999999},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.802,"15min":3.0470000000000002},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":16.434999999999999,"5min":16.434999999999999,"15min":119.55800000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":3.3610000000000002,"5min":2.2730000000000001,"15min":3.0920000000000001},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.108000000000001,"5min":12.108000000000001,"15min":116.78100000000001},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":11.018000000000001,"5min":8.4890000000000008,"15min":6.7539999999999996},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.248},"max":{"1min":124.70099999999999,"5min":124.70099999999999,"15min":210.50299999999999},"last":0.879},{"interface":"front","average":{"1min":7.9050000000000002,"5min":7.6079999999999997,"15min":6.6630000000000003},"min":{"1min":0.46000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":116.47,"5min":116.47,"15min":210.59},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":2.6520000000000001,"15min":4.5990000000000002},"min":{"1min":0.314,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":12.973000000000001,"5min":14.323,"15min":214.64099999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.665,"5min":2.5110000000000001,"15min":4.9649999999999999},"min":{"1min":0.38300000000000001,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":13.673999999999999,"5min":13.673999999999999,"15min":214.54900000000001},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.3410000000000002,"15min":1.8280000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":12.997999999999999,"5min":17.879999999999999,"15min":34.497999999999998},"last":0.34799999999999998},{"interface":"front","average":{"1min":4.7830000000000004,"5min":3.3050000000000002,"15min":2.0760000000000001},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.23799999999999999},"max":{"1min":19.425000000000001,"5min":19.425000000000001,"15min":34.883000000000003},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.681,"5min":4.1980000000000004,"15min":2.3980000000000001},"min":{"1min":0.36199999999999999,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":15.502000000000001,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.54300000000000004},{"interface":"front","average":{"1min":5.4560000000000004,"5min":5.3390000000000004,"15min":2.726},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":18.762,"5min":67.596999999999994,"15min":67.596999999999994},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:21:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":3.3159999999999998,"15min":1.9710000000000001},"min":{"1min":0.38700000000000001,"5min":0.317,"15min":0.28100000000000003},"max":{"1min":15.050000000000001,"5min":17.817,"15min":18.352},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.323,"5min":3.177,"15min":1.8819999999999999},"min":{"1min":0.441,"5min":0.307,"15min":0.307},"max":{"1min":16.332999999999998,"5min":30.117000000000001,"15min":30.117000000000001},"last":0.58999999999999997}]}]},{"osd":5,"up_from":39,"seq":167503725201,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2553028,"kb_used_data":1571644,"kb_used_omap":0,"kb_used_meta":981376,"kb_avail":91192124,"statfs":{"total":95995035648,"available":93380734976,"internally_reserved":0,"allocated":1609363456,"data_stored":1607571269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1004929024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,4,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.6899999999999999,"15min":2.0049999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":7.5119999999999996,"5min":12.558999999999999,"15min":82.280000000000001},"last":2.5790000000000002},{"interface":"front","average":{"1min":2.194,"5min":1.8320000000000001,"15min":2.181},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":6.0430000000000001,"5min":9.407,"15min":85.236000000000004},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0490000000000004,"5min":2.2440000000000002,"15min":3.9900000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.23599999999999999},"max":{"1min":18.859999999999999,"5min":18.859999999999999,"15min":233.77099999999999},"last":0.255},{"interface":"front","average":{"1min":1.198,"5min":1.544,"15min":3.5899999999999999},"min":{"1min":0.27400000000000002,"5min":0.27000000000000002,"15min":0.251},"max":{"1min":7.5220000000000002,"5min":12.654999999999999,"15min":233.75200000000001},"last":2.4630000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":5.569,"5min":5.2720000000000002,"15min":4.665},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":75.441999999999993,"5min":160.749,"15min":231.34700000000001},"last":123.444},{"interface":"front","average":{"1min":6.3460000000000001,"5min":5.3890000000000002,"15min":4.8200000000000003},"min":{"1min":0.35999999999999999,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":75.247,"5min":161.18000000000001,"15min":231.381},"last":124.874}]},{"osd":3,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":2.278,"15min":4.0350000000000001},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":8.0980000000000008,"5min":18.826000000000001,"15min":263.887},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.7989999999999999,"15min":3.6779999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":6.1950000000000003,"5min":12.683999999999999,"15min":263.70100000000002},"last":2.548}]},{"osd":4,"last update":"Sat Sep 23 21:21:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":3.0649999999999999,"15min":1.7649999999999999},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":7.6299999999999999,"5min":77.804000000000002,"15min":77.804000000000002},"last":2.5259999999999998},{"interface":"front","average":{"1min":2.7719999999999998,"5min":4.2880000000000003,"15min":2.1619999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":8.3810000000000002,"5min":77.825999999999993,"15min":77.825999999999993},"last":3.3330000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":4.4279999999999999,"15min":2.617},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":68.346000000000004,"5min":68.346000000000004,"15min":83.451999999999998},"last":0.34200000000000003},{"interface":"front","average":{"1min":11.348000000000001,"5min":4.758,"15min":2.907},"min":{"1min":0.40600000000000003,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":72.393000000000001,"5min":72.393000000000001,"15min":83.484999999999999},"last":0.40100000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:21:42 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":2.6309999999999998,"15min":1.792},"min":{"1min":0.36299999999999999,"5min":0.33500000000000002,"15min":0.30099999999999999},"max":{"1min":10.654999999999999,"5min":10.654999999999999,"15min":10.654999999999999},"last":2.4990000000000001},{"interface":"front","average":{"1min":2.7160000000000002,"5min":2.2629999999999999,"15min":1.411},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":18.257999999999999,"5min":18.257999999999999,"15min":18.257999999999999},"last":2.5649999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":523108352,"data_stored":523108352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":730009600,"data_stored":730005528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1586171904,"data_stored":1586167832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1038290944,"data_stored":1038286872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":439812096,"data_stored":439812096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:22:46.165 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:46.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:46.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:46.272 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:46.330 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:46.372 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:46.412 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:46 smithi121 ceph-mon[219709]: pgmap v1990: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 590 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 3744/28362 objects degraded (13.201%); 9650/28362 objects misplaced (34.024%); 7.6 MiB/s, 157 objects/s recovering 2023-09-23T21:22:46.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:46 smithi121 ceph-mon[211526]: pgmap v1990: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 590 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 3744/28362 objects degraded (13.201%); 9650/28362 objects misplaced (34.024%); 7.6 MiB/s, 157 objects/s recovering 2023-09-23T21:22:46.413 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:46.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:46.440 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:46.520 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:46.565 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:46.730 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:46.731 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:46 smithi123 ceph-mon[193903]: pgmap v1990: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 590 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 3744/28362 objects degraded (13.201%); 9650/28362 objects misplaced (34.024%); 7.6 MiB/s, 157 objects/s recovering 2023-09-23T21:22:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:46.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:46.833 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:46.887 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:47.165 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:47.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:47.269 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:47.289 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:47 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 3744/28362 objects degraded (13.201%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:47.305 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:47.605 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:47 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 3744/28362 objects degraded (13.201%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:47.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:47 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 3744/28362 objects degraded (13.201%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:47.918 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:47.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:48.023 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:48.112 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:48.169 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:22:48.283 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:48.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:48.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:48.402 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:48.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:48 smithi123 ceph-mon[193903]: pgmap v1991: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 559 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 3575/26874 objects degraded (13.303%); 9316/26874 objects misplaced (34.665%); 8.7 MiB/s, 174 objects/s recovering 2023-09-23T21:22:48.492 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:48.594 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:48 smithi121 ceph-mon[219709]: pgmap v1991: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 559 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 3575/26874 objects degraded (13.303%); 9316/26874 objects misplaced (34.665%); 8.7 MiB/s, 174 objects/s recovering 2023-09-23T21:22:48.596 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:48 smithi121 ceph-mon[211526]: pgmap v1991: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 559 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 3575/26874 objects degraded (13.303%); 9316/26874 objects misplaced (34.665%); 8.7 MiB/s, 174 objects/s recovering 2023-09-23T21:22:48.605 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:48.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:48.719 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:48.770 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:48.786 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:48.937 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:48.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:49.046 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:49.090 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:49.147 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:22:49.348 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:49.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:49.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:49.460 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:49.566 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:49.613 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:22:49.613 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:22:49.659 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:49.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:49.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:49.763 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:49.872 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:49 smithi123 ceph-mon[193903]: pgmap v1992: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 420 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2828/20226 objects degraded (13.982%); 7214/20226 objects misplaced (35.667%); 10 MiB/s, 194 objects/s recovering 2023-09-23T21:22:49.886 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:49.908 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:49 smithi121 ceph-mon[219709]: pgmap v1992: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 420 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2828/20226 objects degraded (13.982%); 7214/20226 objects misplaced (35.667%); 10 MiB/s, 194 objects/s recovering 2023-09-23T21:22:49.909 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:49 smithi121 ceph-mon[211526]: pgmap v1992: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 420 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2828/20226 objects degraded (13.982%); 7214/20226 objects misplaced (35.667%); 10 MiB/s, 194 objects/s recovering 2023-09-23T21:22:50.024 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:50.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:50.142 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:50.182 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:50.569 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3262850210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:50.569 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:50 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:50.576 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:22:50.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3262850210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:50.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:50 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:50.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3262850210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:50.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:50 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:22:51.180 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T21:22:51.562 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:51.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:51.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:51.694 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:51.814 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:51.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:51.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:51.866 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[211526]: pgmap v1994: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 420 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2828/20226 objects degraded (13.982%); 7214/20226 objects misplaced (35.667%); 10 MiB/s, 194 objects/s recovering 2023-09-23T21:22:51.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:51.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:51.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T21:22:51.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.868 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:51 smithi121 ceph-mon[219709]: pgmap v1994: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 420 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2828/20226 objects degraded (13.982%); 7214/20226 objects misplaced (35.667%); 10 MiB/s, 194 objects/s recovering 2023-09-23T21:22:51.923 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:51.975 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:51.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:22:51.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:22:51.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T21:22:51.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: failed to encode map e655 with expected crc 2023-09-23T21:22:51.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:51 smithi123 ceph-mon[193903]: pgmap v1994: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 420 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2828/20226 objects degraded (13.982%); 7214/20226 objects misplaced (35.667%); 10 MiB/s, 194 objects/s recovering 2023-09-23T21:22:52.016 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:52.074 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:52.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:52.167 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:52.183 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:52.244 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:52.359 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:52.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:52.475 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:22:52.524 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:52.540 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:22:52.581 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:22:52.584 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:52 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 2828/20226 objects degraded (13.982%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:52.584 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3188272433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:52.618 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:22:52.652 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:22:52.671 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:22:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:52.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:22:52.690 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:22:52.732 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:22:52.777 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:52.778 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:22:52.805 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:52 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 2828/20226 objects degraded (13.982%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:52.806 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3188272433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:52.806 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:52 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 2828/20226 objects degraded (13.982%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:52.806 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3188272433' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:22:52.809 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:22:52.810 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.810 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.810 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.810 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.810 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.811 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.811 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.811 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:22:52.852 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:52.960 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:52.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:22:53.071 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:53.113 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:53.887 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:53 smithi121 ceph-mon[211526]: pgmap v1995: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 388 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2671/18666 objects degraded (14.309%); 6890/18666 objects misplaced (36.912%); 11 MiB/s, 205 objects/s recovering 2023-09-23T21:22:53.887 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:53 smithi121 ceph-mon[219709]: pgmap v1995: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 388 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2671/18666 objects degraded (14.309%); 6890/18666 objects misplaced (36.912%); 11 MiB/s, 205 objects/s recovering 2023-09-23T21:22:53.939 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:22:53.957 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:53.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:53.984 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:53 smithi123 ceph-mon[193903]: pgmap v1995: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 388 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 2671/18666 objects degraded (14.309%); 6890/18666 objects misplaced (36.912%); 11 MiB/s, 205 objects/s recovering 2023-09-23T21:22:54.065 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:54.124 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:54.168 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:54.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:54.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:54.274 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:54.328 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:54.441 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:54.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:54.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:54.553 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:54.658 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:54.813 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:54.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:54.932 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:54.967 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:55.044 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:55.049 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:22:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:22:54] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T21:22:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:55.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:55.151 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:55.221 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:55.276 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:55.298 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:55.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:55.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:55.403 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:55.449 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:55.560 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:55 smithi121 ceph-mon[219709]: pgmap v1996: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 241 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1811/11604 objects degraded (15.607%); 4419/11604 objects misplaced (38.082%); 14 MiB/s, 226 objects/s recovering 2023-09-23T21:22:55.632 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:55.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:55.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:55.656 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.36681.0:151904 12.3 12:ce488415:::benchmark_data_smithi121_563784_object61501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:13:06.987815+0000", 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 588.66691761000004, 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.25388111800000002, 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:22:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.36681", 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/255170782", 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 151904 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987815+0000", 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:22:55.658 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987815+0000", 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987814+0000", 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987820+0000", 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7259999999999999e-06 2023-09-23T21:22:55.659 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987831+0000", 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0838e-05 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987839+0000", 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0339999999999997e-06 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:22:55.660 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.987885+0000", 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6505999999999998e-05 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.988001+0000", 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000116183 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.988114+0000", 2023-09-23T21:22:55.661 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000113099 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.992852+0000", 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0047377039999999997 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:06.995655+0000", 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028032199999999999 2023-09-23T21:22:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241616+0000", 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.245961033 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241659+0000", 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2435999999999998e-05 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:22:55.663 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:13:07.241696+0000", 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7279e-05 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:22:55.664 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:22:55.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:55.793 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:55.832 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:55 smithi121 ceph-mon[211526]: pgmap v1996: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 241 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1811/11604 objects degraded (15.607%); 4419/11604 objects misplaced (38.082%); 14 MiB/s, 226 objects/s recovering 2023-09-23T21:22:55.889 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:55.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:55.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:55.908 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:55 smithi123 ceph-mon[193903]: pgmap v1996: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 241 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1811/11604 objects degraded (15.607%); 4419/11604 objects misplaced (38.082%); 14 MiB/s, 226 objects/s recovering 2023-09-23T21:22:55.922 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:22:56.009 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:22:56.113 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:56.292 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:22:56.312 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:22:56.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:56 smithi121 ceph-mon[211526]: from='client.37932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:56 smithi121 ceph-mon[219709]: from='client.37932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:56.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:56 smithi123 ceph-mon[193903]: from='client.37932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:22:57.305 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:22:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:57.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:22:57.410 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:22:57.450 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:57.564 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:57 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1811/11604 objects degraded (15.607%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:57.564 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:57 smithi121 ceph-mon[211526]: pgmap v1997: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 210 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1630/10116 objects degraded (16.113%); 4079/10116 objects misplaced (40.322%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:22:57.564 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:57.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:57 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1811/11604 objects degraded (15.607%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:57.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:57 smithi121 ceph-mon[219709]: pgmap v1997: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 210 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1630/10116 objects degraded (16.113%); 4079/10116 objects misplaced (40.322%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:22:57.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:57.891 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T21:22:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:57 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1811/11604 objects degraded (15.607%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:22:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:57 smithi123 ceph-mon[193903]: pgmap v1997: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 210 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 1630/10116 objects degraded (16.113%); 4079/10116 objects misplaced (40.322%); 14 MiB/s, 227 objects/s recovering 2023-09-23T21:22:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:22:58.240 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:22:58.256 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":1996,"stamp":"2023-09-23T21:22:55.449052+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":252558083,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1811,"num_objects_misplaced":4419,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":92,"num_read_kb":169,"num_write":166890,"num_write_kb":5460565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33243,"num_bytes_recovered":1988638053,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87426,"ondisk_log_size":87426,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":369,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9653520,"kb_used_data":2997016,"kb_used_omap":16,"kb_used_meta":6656111,"kb_avail":646562544,"statfs":{"total":671965249536,"available":662080045056,"internally_reserved":0,"allocated":3068944384,"data_stored":3051745760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6815858537},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,9,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-366084096,"num_objects":-5586,"num_object_clones":0,"num_object_copies":-16758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1933,"num_objects_misplaced":-5231,"num_objects_unfound":0,"num_objects_dirty":-5586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5586,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2262,"num_bytes_recovered":141754368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003051"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":230,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.254596+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:22:41.254596+0000","last_peered":"2023-09-23T21:22:41.254596+0000","last_clean":"2023-09-23T21:22:41.254596+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:22:41.254596+0000","last_undegraded":"2023-09-23T21:22:41.254596+0000","last_fullsized":"2023-09-23T21:22:41.254596+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":66,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.341380+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:22:30.341380+0000","last_peered":"2023-09-23T21:22:30.341380+0000","last_clean":"2023-09-23T21:22:30.341380+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:22:30.341380+0000","last_undegraded":"2023-09-23T21:22:30.341380+0000","last_fullsized":"2023-09-23T21:22:30.341380+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.249152+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:22:41.249152+0000","last_peered":"2023-09-23T21:22:41.249152+0000","last_clean":"2023-09-23T21:22:41.249152+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:22:41.249152+0000","last_undegraded":"2023-09-23T21:22:41.249152+0000","last_fullsized":"2023-09-23T21:22:41.249152+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":342,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.244663+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:22:41.244663+0000","last_peered":"2023-09-23T21:22:41.244663+0000","last_clean":"2023-09-23T21:22:41.244663+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:22:41.244663+0000","last_undegraded":"2023-09-23T21:22:41.244663+0000","last_fullsized":"2023-09-23T21:22:41.244663+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":57,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.340949+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:22:30.340949+0000","last_peered":"2023-09-23T21:22:30.340949+0000","last_clean":"2023-09-23T21:22:30.340949+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:22:30.340949+0000","last_undegraded":"2023-09-23T21:22:30.340949+0000","last_fullsized":"2023-09-23T21:22:30.340949+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":160,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255088+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:22:41.255088+0000","last_peered":"2023-09-23T21:22:41.255088+0000","last_clean":"2023-09-23T21:22:41.255088+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:22:41.255088+0000","last_undegraded":"2023-09-23T21:22:41.255088+0000","last_fullsized":"2023-09-23T21:22:41.255088+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.245638+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:22:41.245638+0000","last_peered":"2023-09-23T21:22:41.245638+0000","last_clean":"2023-09-23T21:22:41.245638+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:22:41.245638+0000","last_undegraded":"2023-09-23T21:22:41.245638+0000","last_fullsized":"2023-09-23T21:22:41.245638+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":76,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.236856+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:22:41.236856+0000","last_peered":"2023-09-23T21:22:41.236856+0000","last_clean":"2023-09-23T21:22:41.236856+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:22:41.236856+0000","last_undegraded":"2023-09-23T21:22:41.236856+0000","last_fullsized":"2023-09-23T21:22:41.236856+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":52,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.244900+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:22:41.244900+0000","last_peered":"2023-09-23T21:22:41.244900+0000","last_clean":"2023-09-23T21:22:41.244900+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:22:41.244900+0000","last_undegraded":"2023-09-23T21:22:41.244900+0000","last_fullsized":"2023-09-23T21:22:41.244900+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":49,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.237076+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:22:41.237076+0000","last_peered":"2023-09-23T21:22:41.237076+0000","last_clean":"2023-09-23T21:22:41.237076+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:22:41.237076+0000","last_undegraded":"2023-09-23T21:22:41.237076+0000","last_fullsized":"2023-09-23T21:22:41.237076+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":290,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.244544+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:22:41.244544+0000","last_peered":"2023-09-23T21:22:41.244544+0000","last_clean":"2023-09-23T21:22:41.244544+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:22:41.244544+0000","last_undegraded":"2023-09-23T21:22:41.244544+0000","last_fullsized":"2023-09-23T21:22:41.244544+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.250613+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:22:41.250613+0000","last_peered":"2023-09-23T21:22:41.250613+0000","last_clean":"2023-09-23T21:22:41.250613+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:22:41.250613+0000","last_undegraded":"2023-09-23T21:22:41.250613+0000","last_fullsized":"2023-09-23T21:22:41.250613+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":386,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255113+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:22:41.255113+0000","last_peered":"2023-09-23T21:22:41.255113+0000","last_clean":"2023-09-23T21:22:41.255113+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:22:41.255113+0000","last_undegraded":"2023-09-23T21:22:41.255113+0000","last_fullsized":"2023-09-23T21:22:41.255113+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":613,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.256046+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:22:41.256046+0000","last_peered":"2023-09-23T21:22:41.256046+0000","last_clean":"2023-09-23T21:22:41.256046+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:22:41.256046+0000","last_undegraded":"2023-09-23T21:22:41.256046+0000","last_fullsized":"2023-09-23T21:22:41.256046+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":238,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.240012+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:22:41.240012+0000","last_peered":"2023-09-23T21:22:41.240012+0000","last_clean":"2023-09-23T21:22:41.240012+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:22:41.240012+0000","last_undegraded":"2023-09-23T21:22:41.240012+0000","last_fullsized":"2023-09-23T21:22:41.240012+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":58,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.237155+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:22:41.237155+0000","last_peered":"2023-09-23T21:22:41.237155+0000","last_clean":"2023-09-23T21:22:41.237155+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:22:41.237155+0000","last_undegraded":"2023-09-23T21:22:41.237155+0000","last_fullsized":"2023-09-23T21:22:41.237155+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":264,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.239665+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:22:41.239665+0000","last_peered":"2023-09-23T21:22:41.239665+0000","last_clean":"2023-09-23T21:22:41.239665+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:22:41.239665+0000","last_undegraded":"2023-09-23T21:22:41.239665+0000","last_fullsized":"2023-09-23T21:22:41.239665+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":272,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.244755+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:22:41.244755+0000","last_peered":"2023-09-23T21:22:41.244755+0000","last_clean":"2023-09-23T21:22:41.244755+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:22:41.244755+0000","last_undegraded":"2023-09-23T21:22:41.244755+0000","last_fullsized":"2023-09-23T21:22:41.244755+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":232,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.239568+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:22:41.239568+0000","last_peered":"2023-09-23T21:22:41.239568+0000","last_clean":"2023-09-23T21:22:41.239568+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:22:41.239568+0000","last_undegraded":"2023-09-23T21:22:41.239568+0000","last_fullsized":"2023-09-23T21:22:41.239568+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":48,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.341408+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:22:30.341408+0000","last_peered":"2023-09-23T21:22:30.341408+0000","last_clean":"2023-09-23T21:22:30.341408+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:22:30.341408+0000","last_undegraded":"2023-09-23T21:22:30.341408+0000","last_fullsized":"2023-09-23T21:22:30.341408+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":751,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.251385+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:22:41.251385+0000","last_peered":"2023-09-23T21:22:41.251385+0000","last_clean":"2023-09-23T21:22:41.251385+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:22:41.251385+0000","last_undegraded":"2023-09-23T21:22:41.251385+0000","last_fullsized":"2023-09-23T21:22:41.251385+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":57,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.341667+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:22:30.341667+0000","last_peered":"2023-09-23T21:22:30.341667+0000","last_clean":"2023-09-23T21:22:30.341667+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:22:30.341667+0000","last_undegraded":"2023-09-23T21:22:30.341667+0000","last_fullsized":"2023-09-23T21:22:30.341667+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"655'10180","reported_seq":12755,"reported_epoch":655,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:51.500232+0000","last_change":"2023-09-23T21:21:01.215269+0000","last_active":"2023-09-23T21:22:51.500232+0000","last_peered":"2023-09-23T21:22:51.500232+0000","last_clean":"2023-09-23T21:19:28.877182+0000","last_became_active":"2023-09-23T21:21:01.190149+0000","last_became_peered":"2023-09-23T21:21:01.190149+0000","last_unstale":"2023-09-23T21:22:51.500232+0000","last_undegraded":"2023-09-23T21:21:00.607176+0000","last_fullsized":"2023-09-23T21:22:51.500232+0000","mapping_epoch":629,"log_start":"600'4667","ondisk_log_start":"600'4667","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:05.989327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20054016,"num_objects":306,"num_object_clones":0,"num_object_copies":918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":306,"num_objects_misplaced":306,"num_objects_unfound":0,"num_objects_dirty":306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10180,"num_write_kb":335552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1219,"num_bytes_recovered":79888384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,4,5],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,5,6","objects":1},{"shards":"4,6,7","objects":305}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":292,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.246199+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:22:41.246199+0000","last_peered":"2023-09-23T21:22:41.246199+0000","last_clean":"2023-09-23T21:22:41.246199+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:22:41.246199+0000","last_undegraded":"2023-09-23T21:22:41.246199+0000","last_fullsized":"2023-09-23T21:22:41.246199+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.243854+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:22:41.243854+0000","last_peered":"2023-09-23T21:22:41.243854+0000","last_clean":"2023-09-23T21:22:41.243854+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:22:41.243854+0000","last_undegraded":"2023-09-23T21:22:41.243854+0000","last_fullsized":"2023-09-23T21:22:41.243854+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"655'10521","reported_seq":13500,"reported_epoch":655,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:54.083961+0000","last_change":"2023-09-23T21:21:01.210342+0000","last_active":"2023-09-23T21:22:54.083961+0000","last_peered":"2023-09-23T21:22:54.083961+0000","last_clean":"2023-09-23T21:19:28.923531+0000","last_became_active":"2023-09-23T21:21:01.189847+0000","last_became_peered":"2023-09-23T21:21:01.189847+0000","last_unstale":"2023-09-23T21:22:54.083961+0000","last_undegraded":"2023-09-23T21:21:00.610088+0000","last_fullsized":"2023-09-23T21:22:54.083961+0000","mapping_epoch":629,"log_start":"612'5498","ondisk_log_start":"612'5498","created":573,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5023,"log_dups_size":0,"ondisk_log_size":5023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:58.686882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16449536,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":251,"num_objects_misplaced":251,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10521,"num_write_kb":344704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1413,"num_bytes_recovered":92471296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[4,6,5],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,6,7","objects":251}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":306,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.244755+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:22:41.244755+0000","last_peered":"2023-09-23T21:22:41.244755+0000","last_clean":"2023-09-23T21:22:41.244755+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:22:41.244755+0000","last_undegraded":"2023-09-23T21:22:41.244755+0000","last_fullsized":"2023-09-23T21:22:41.244755+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.249066+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:22:41.249066+0000","last_peered":"2023-09-23T21:22:41.249066+0000","last_clean":"2023-09-23T21:22:41.249066+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:22:41.249066+0000","last_undegraded":"2023-09-23T21:22:41.249066+0000","last_fullsized":"2023-09-23T21:22:41.249066+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"655'10253","reported_seq":15492,"reported_epoch":655,"state":"active+clean","last_fresh":"2023-09-23T21:22:53.872730+0000","last_change":"2023-09-23T21:21:17.539683+0000","last_active":"2023-09-23T21:22:53.872730+0000","last_peered":"2023-09-23T21:22:53.872730+0000","last_clean":"2023-09-23T21:22:53.872730+0000","last_became_active":"2023-09-23T21:21:17.521633+0000","last_became_peered":"2023-09-23T21:21:17.521633+0000","last_unstale":"2023-09-23T21:22:53.872730+0000","last_undegraded":"2023-09-23T21:22:53.872730+0000","last_fullsized":"2023-09-23T21:22:53.872730+0000","mapping_epoch":636,"log_start":"607'5265","ondisk_log_start":"607'5265","created":573,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4988,"log_dups_size":0,"ondisk_log_size":4988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:05.844201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14090240,"num_objects":215,"num_object_clones":0,"num_object_copies":645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10253,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2555,"num_bytes_recovered":80478208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":278,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.244840+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:22:41.244840+0000","last_peered":"2023-09-23T21:22:41.244840+0000","last_clean":"2023-09-23T21:22:41.244840+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:22:41.244840+0000","last_undegraded":"2023-09-23T21:22:41.244840+0000","last_fullsized":"2023-09-23T21:22:41.244840+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":26,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.251103+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:22:41.251103+0000","last_peered":"2023-09-23T21:22:41.251103+0000","last_clean":"2023-09-23T21:22:41.251103+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:22:41.251103+0000","last_undegraded":"2023-09-23T21:22:41.251103+0000","last_fullsized":"2023-09-23T21:22:41.251103+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"655'10356","reported_seq":10464,"reported_epoch":655,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:53.786063+0000","last_change":"2023-09-23T21:20:59.644218+0000","last_active":"2023-09-23T21:22:53.786063+0000","last_peered":"2023-09-23T21:22:53.786063+0000","last_clean":"2023-09-23T21:20:30.430127+0000","last_became_active":"2023-09-23T21:20:59.632855+0000","last_became_peered":"2023-09-23T21:20:59.632855+0000","last_unstale":"2023-09-23T21:22:53.786063+0000","last_undegraded":"2023-09-23T21:22:53.786063+0000","last_fullsized":"2023-09-23T21:22:53.786063+0000","mapping_epoch":628,"log_start":"612'5425","ondisk_log_start":"612'5425","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4931,"log_dups_size":0,"ondisk_log_size":4931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:33.014367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15859712,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":484,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10356,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[2,6,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":242}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":58,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.237231+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:22:41.237231+0000","last_peered":"2023-09-23T21:22:41.237231+0000","last_clean":"2023-09-23T21:22:41.237231+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:22:41.237231+0000","last_undegraded":"2023-09-23T21:22:41.237231+0000","last_fullsized":"2023-09-23T21:22:41.237231+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.251165+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:22:41.251165+0000","last_peered":"2023-09-23T21:22:41.251165+0000","last_clean":"2023-09-23T21:22:41.251165+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:22:41.251165+0000","last_undegraded":"2023-09-23T21:22:41.251165+0000","last_fullsized":"2023-09-23T21:22:41.251165+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1081,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.239401+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:22:41.239401+0000","last_peered":"2023-09-23T21:22:41.239401+0000","last_clean":"2023-09-23T21:22:41.239401+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:22:41.239401+0000","last_undegraded":"2023-09-23T21:22:41.239401+0000","last_fullsized":"2023-09-23T21:22:41.239401+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.6","version":"655'10515","reported_seq":18138,"reported_epoch":655,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:53.876795+0000","last_change":"2023-09-23T21:21:01.239919+0000","last_active":"2023-09-23T21:22:53.876795+0000","last_peered":"2023-09-23T21:22:53.876795+0000","last_clean":"2023-09-23T21:19:28.802255+0000","last_became_active":"2023-09-23T21:21:01.197858+0000","last_became_peered":"2023-09-23T21:21:01.197858+0000","last_unstale":"2023-09-23T21:22:53.876795+0000","last_undegraded":"2023-09-23T21:21:00.673065+0000","last_fullsized":"2023-09-23T21:22:53.876795+0000","mapping_epoch":629,"log_start":"612'5498","ondisk_log_start":"612'5498","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:39.315049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13828096,"num_objects":211,"num_object_clones":0,"num_object_copies":633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":211,"num_objects_misplaced":211,"num_objects_unfound":0,"num_objects_dirty":211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10515,"num_write_kb":343232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3750,"num_bytes_recovered":245694464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,4,6],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,4,6","objects":211}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":66,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.237240+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:22:41.237240+0000","last_peered":"2023-09-23T21:22:41.237240+0000","last_clean":"2023-09-23T21:22:41.237240+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:22:41.237240+0000","last_undegraded":"2023-09-23T21:22:41.237240+0000","last_fullsized":"2023-09-23T21:22:41.237240+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":413,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255155+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:22:41.255155+0000","last_peered":"2023-09-23T21:22:41.255155+0000","last_clean":"2023-09-23T21:22:41.255155+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:22:41.255155+0000","last_undegraded":"2023-09-23T21:22:41.255155+0000","last_fullsized":"2023-09-23T21:22:41.255155+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":949,"reported_epoch":654,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T21:22:41.256260+0000","last_change":"2023-09-23T21:20:48.157415+0000","last_active":"2023-09-23T21:22:41.256260+0000","last_peered":"2023-09-23T21:22:41.256260+0000","last_clean":"2023-09-23T21:20:41.603469+0000","last_became_active":"2023-09-23T21:20:48.156886+0000","last_became_peered":"2023-09-23T21:20:48.156886+0000","last_unstale":"2023-09-23T21:22:41.256260+0000","last_undegraded":"2023-09-23T21:20:47.922781+0000","last_fullsized":"2023-09-23T21:20:47.921128+0000","mapping_epoch":623,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":619,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:30.898986+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4,6","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.7","version":"655'10654","reported_seq":11863,"reported_epoch":655,"state":"active+clean","last_fresh":"2023-09-23T21:22:53.906893+0000","last_change":"2023-09-23T21:20:59.594443+0000","last_active":"2023-09-23T21:22:53.906893+0000","last_peered":"2023-09-23T21:22:53.906893+0000","last_clean":"2023-09-23T21:22:53.906893+0000","last_became_active":"2023-09-23T21:20:48.149995+0000","last_became_peered":"2023-09-23T21:20:48.149995+0000","last_unstale":"2023-09-23T21:22:53.906893+0000","last_undegraded":"2023-09-23T21:22:53.906893+0000","last_fullsized":"2023-09-23T21:22:53.906893+0000","mapping_epoch":623,"log_start":"583'2865","ondisk_log_start":"583'2865","created":573,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":7789,"log_dups_size":0,"ondisk_log_size":7789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.278714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10654,"num_write_kb":348544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":543,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":150,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.252647+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:22:41.252647+0000","last_peered":"2023-09-23T21:22:41.252647+0000","last_clean":"2023-09-23T21:22:41.252647+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:22:41.252647+0000","last_undegraded":"2023-09-23T21:22:41.252647+0000","last_fullsized":"2023-09-23T21:22:41.252647+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":663,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255784+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:22:41.255784+0000","last_peered":"2023-09-23T21:22:41.255784+0000","last_clean":"2023-09-23T21:22:41.255784+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:22:41.255784+0000","last_undegraded":"2023-09-23T21:22:41.255784+0000","last_fullsized":"2023-09-23T21:22:41.255784+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"655'10186","reported_seq":19431,"reported_epoch":655,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:51.502779+0000","last_change":"2023-09-23T21:22:12.152044+0000","last_active":"2023-09-23T21:22:51.502779+0000","last_peered":"2023-09-23T21:22:51.502779+0000","last_clean":"2023-09-23T21:19:28.911487+0000","last_became_active":"2023-09-23T21:20:32.169326+0000","last_became_peered":"2023-09-23T21:20:32.169326+0000","last_unstale":"2023-09-23T21:22:51.502779+0000","last_undegraded":"2023-09-23T21:22:51.502779+0000","last_fullsized":"2023-09-23T21:22:51.502779+0000","mapping_epoch":617,"log_start":"600'4668","ondisk_log_start":"600'4668","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5518,"log_dups_size":0,"ondisk_log_size":5518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:09.080183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18743296,"num_objects":286,"num_object_clones":0,"num_object_copies":858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":286,"num_objects_unfound":0,"num_objects_dirty":286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10186,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4564,"num_bytes_recovered":237240320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,4],"avail_no_missing":["6","4","5","7"],"object_location_counts":[{"shards":"4,6,7","objects":286}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":292,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255738+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:22:41.255738+0000","last_peered":"2023-09-23T21:22:41.255738+0000","last_clean":"2023-09-23T21:22:41.255738+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:22:41.255738+0000","last_undegraded":"2023-09-23T21:22:41.255738+0000","last_fullsized":"2023-09-23T21:22:41.255738+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":700,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.258286+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:22:41.258286+0000","last_peered":"2023-09-23T21:22:41.258286+0000","last_clean":"2023-09-23T21:22:41.258286+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:22:41.258286+0000","last_undegraded":"2023-09-23T21:22:41.258286+0000","last_fullsized":"2023-09-23T21:22:41.258286+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":918,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.256866+0000","last_change":"2023-09-23T21:20:49.968107+0000","last_active":"2023-09-23T21:22:41.256866+0000","last_peered":"2023-09-23T21:22:41.256866+0000","last_clean":"2023-09-23T21:22:41.256866+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:22:41.256866+0000","last_undegraded":"2023-09-23T21:22:41.256866+0000","last_fullsized":"2023-09-23T21:22:41.256866+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":429358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.5","version":"655'10749","reported_seq":20251,"reported_epoch":655,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:54.026580+0000","last_change":"2023-09-23T21:20:59.554347+0000","last_active":"2023-09-23T21:22:54.026580+0000","last_peered":"2023-09-23T21:22:54.026580+0000","last_clean":"2023-09-23T21:19:28.864495+0000","last_became_active":"2023-09-23T21:20:32.158375+0000","last_became_peered":"2023-09-23T21:20:32.158375+0000","last_unstale":"2023-09-23T21:22:54.026580+0000","last_undegraded":"2023-09-23T21:20:59.551954+0000","last_fullsized":"2023-09-23T21:22:54.026580+0000","mapping_epoch":617,"log_start":"612'5692","ondisk_log_start":"612'5692","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:46.553959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13828096,"num_objects":211,"num_object_clones":0,"num_object_copies":633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37,"num_objects_misplaced":174,"num_objects_unfound":0,"num_objects_dirty":211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10749,"num_write_kb":350720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4693,"num_bytes_recovered":307560448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,2],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"2,4,7","objects":211}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":58,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.236759+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:22:41.236759+0000","last_peered":"2023-09-23T21:22:41.236759+0000","last_clean":"2023-09-23T21:22:41.236759+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:22:41.236759+0000","last_undegraded":"2023-09-23T21:22:41.236759+0000","last_fullsized":"2023-09-23T21:22:41.236759+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.249529+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:22:41.249529+0000","last_peered":"2023-09-23T21:22:41.249529+0000","last_clean":"2023-09-23T21:22:41.249529+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:22:41.249529+0000","last_undegraded":"2023-09-23T21:22:41.249529+0000","last_fullsized":"2023-09-23T21:22:41.249529+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":737,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.254948+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:22:41.254948+0000","last_peered":"2023-09-23T21:22:41.254948+0000","last_clean":"2023-09-23T21:22:41.254948+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:22:41.254948+0000","last_undegraded":"2023-09-23T21:22:41.254948+0000","last_fullsized":"2023-09-23T21:22:41.254948+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"655'10370","reported_seq":12150,"reported_epoch":655,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:53.856153+0000","last_change":"2023-09-23T21:21:01.233673+0000","last_active":"2023-09-23T21:22:53.856153+0000","last_peered":"2023-09-23T21:22:53.856153+0000","last_clean":"2023-09-23T21:19:28.898608+0000","last_became_active":"2023-09-23T21:21:01.195228+0000","last_became_peered":"2023-09-23T21:21:01.195228+0000","last_unstale":"2023-09-23T21:22:53.856153+0000","last_undegraded":"2023-09-23T21:21:00.570455+0000","last_fullsized":"2023-09-23T21:22:53.856153+0000","mapping_epoch":629,"log_start":"612'5446","ondisk_log_start":"612'5446","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":4924,"log_dups_size":0,"ondisk_log_size":4924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:03.164901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16777216,"num_objects":256,"num_object_clones":0,"num_object_copies":768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":256,"num_objects_misplaced":256,"num_objects_unfound":0,"num_objects_dirty":256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10370,"num_write_kb":340032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":825,"num_bytes_recovered":53936128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[2,7,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,6,7","objects":256}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.249626+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:22:41.249626+0000","last_peered":"2023-09-23T21:22:41.249626+0000","last_clean":"2023-09-23T21:22:41.249626+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:22:41.249626+0000","last_undegraded":"2023-09-23T21:22:41.249626+0000","last_fullsized":"2023-09-23T21:22:41.249626+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":42,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.245454+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:22:41.245454+0000","last_peered":"2023-09-23T21:22:41.245454+0000","last_clean":"2023-09-23T21:22:41.245454+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:22:41.245454+0000","last_undegraded":"2023-09-23T21:22:41.245454+0000","last_fullsized":"2023-09-23T21:22:41.245454+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.250682+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:22:41.250682+0000","last_peered":"2023-09-23T21:22:41.250682+0000","last_clean":"2023-09-23T21:22:41.250682+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:22:41.250682+0000","last_undegraded":"2023-09-23T21:22:41.250682+0000","last_fullsized":"2023-09-23T21:22:41.250682+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":930,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.256705+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:22:41.256705+0000","last_peered":"2023-09-23T21:22:41.256705+0000","last_clean":"2023-09-23T21:22:41.256705+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:22:41.256705+0000","last_undegraded":"2023-09-23T21:22:41.256705+0000","last_fullsized":"2023-09-23T21:22:41.256705+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"655'10320","reported_seq":10410,"reported_epoch":655,"state":"active+clean","last_fresh":"2023-09-23T21:22:54.084436+0000","last_change":"2023-09-23T21:18:18.455283+0000","last_active":"2023-09-23T21:22:54.084436+0000","last_peered":"2023-09-23T21:22:54.084436+0000","last_clean":"2023-09-23T21:22:54.084436+0000","last_became_active":"2023-09-23T21:18:18.454989+0000","last_became_peered":"2023-09-23T21:18:18.454989+0000","last_unstale":"2023-09-23T21:22:54.084436+0000","last_undegraded":"2023-09-23T21:22:54.084436+0000","last_fullsized":"2023-09-23T21:22:54.084436+0000","mapping_epoch":573,"log_start":"606'5287","ondisk_log_start":"606'5287","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5033,"log_dups_size":0,"ondisk_log_size":5033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:55.346980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14548992,"num_objects":222,"num_object_clones":0,"num_object_copies":666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10320,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.250861+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:22:41.250861+0000","last_peered":"2023-09-23T21:22:41.250861+0000","last_clean":"2023-09-23T21:22:41.250861+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:22:41.250861+0000","last_undegraded":"2023-09-23T21:22:41.250861+0000","last_fullsized":"2023-09-23T21:22:41.250861+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":568,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255902+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:22:41.255902+0000","last_peered":"2023-09-23T21:22:41.255902+0000","last_clean":"2023-09-23T21:22:41.255902+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:22:41.255902+0000","last_undegraded":"2023-09-23T21:22:41.255902+0000","last_fullsized":"2023-09-23T21:22:41.255902+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.249557+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:22:41.249557+0000","last_peered":"2023-09-23T21:22:41.249557+0000","last_clean":"2023-09-23T21:22:41.249557+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:22:41.249557+0000","last_undegraded":"2023-09-23T21:22:41.249557+0000","last_fullsized":"2023-09-23T21:22:41.249557+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1041,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.239694+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:22:41.239694+0000","last_peered":"2023-09-23T21:22:41.239694+0000","last_clean":"2023-09-23T21:22:41.239694+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:22:41.239694+0000","last_undegraded":"2023-09-23T21:22:41.239694+0000","last_fullsized":"2023-09-23T21:22:41.239694+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.d","version":"655'10369","reported_seq":10534,"reported_epoch":655,"state":"active+clean","last_fresh":"2023-09-23T21:22:51.457870+0000","last_change":"2023-09-23T21:21:22.585504+0000","last_active":"2023-09-23T21:22:51.457870+0000","last_peered":"2023-09-23T21:22:51.457870+0000","last_clean":"2023-09-23T21:22:51.457870+0000","last_became_active":"2023-09-23T21:21:22.568285+0000","last_became_peered":"2023-09-23T21:21:22.568285+0000","last_unstale":"2023-09-23T21:22:51.457870+0000","last_undegraded":"2023-09-23T21:22:51.457870+0000","last_fullsized":"2023-09-23T21:22:51.457870+0000","mapping_epoch":639,"log_start":"600'4848","ondisk_log_start":"600'4848","created":573,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:01.251062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19660824,"num_objects":301,"num_object_clones":0,"num_object_copies":903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":301,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10369,"num_write_kb":341377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":684,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.257037+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:22:41.257037+0000","last_peered":"2023-09-23T21:22:41.257037+0000","last_clean":"2023-09-23T21:22:41.257037+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:22:41.257037+0000","last_undegraded":"2023-09-23T21:22:41.257037+0000","last_fullsized":"2023-09-23T21:22:41.257037+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.252132+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:22:41.252132+0000","last_peered":"2023-09-23T21:22:41.252132+0000","last_clean":"2023-09-23T21:22:41.252132+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:22:41.252132+0000","last_undegraded":"2023-09-23T21:22:41.252132+0000","last_fullsized":"2023-09-23T21:22:41.252132+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.249463+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:22:41.249463+0000","last_peered":"2023-09-23T21:22:41.249463+0000","last_clean":"2023-09-23T21:22:41.249463+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:22:41.249463+0000","last_undegraded":"2023-09-23T21:22:41.249463+0000","last_fullsized":"2023-09-23T21:22:41.249463+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"655'10442","reported_seq":14334,"reported_epoch":655,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:53.657609+0000","last_change":"2023-09-23T21:21:01.214995+0000","last_active":"2023-09-23T21:22:53.657609+0000","last_peered":"2023-09-23T21:22:53.657609+0000","last_clean":"2023-09-23T21:19:28.902983+0000","last_became_active":"2023-09-23T21:21:01.189928+0000","last_became_peered":"2023-09-23T21:21:01.189928+0000","last_unstale":"2023-09-23T21:22:53.657609+0000","last_undegraded":"2023-09-23T21:21:00.625306+0000","last_fullsized":"2023-09-23T21:22:53.657609+0000","mapping_epoch":629,"log_start":"593'4386","ondisk_log_start":"593'4386","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":6056,"log_dups_size":0,"ondisk_log_size":6056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:45.309159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":238,"num_objects_misplaced":238,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10442,"num_write_kb":341760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1872,"num_bytes_recovered":122683392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,2,5],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,5,7","objects":1},{"shards":"2,6,7","objects":237}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":513,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.260196+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:22:41.260196+0000","last_peered":"2023-09-23T21:22:41.260196+0000","last_clean":"2023-09-23T21:22:41.260196+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:22:41.260196+0000","last_undegraded":"2023-09-23T21:22:41.260196+0000","last_fullsized":"2023-09-23T21:22:41.260196+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1042,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.239089+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:22:41.239089+0000","last_peered":"2023-09-23T21:22:41.239089+0000","last_clean":"2023-09-23T21:22:41.239089+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:22:41.239089+0000","last_undegraded":"2023-09-23T21:22:41.239089+0000","last_fullsized":"2023-09-23T21:22:41.239089+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":42,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.245047+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:22:41.245047+0000","last_peered":"2023-09-23T21:22:41.245047+0000","last_clean":"2023-09-23T21:22:41.245047+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:22:41.245047+0000","last_undegraded":"2023-09-23T21:22:41.245047+0000","last_fullsized":"2023-09-23T21:22:41.245047+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":26,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.251224+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:22:41.251224+0000","last_peered":"2023-09-23T21:22:41.251224+0000","last_clean":"2023-09-23T21:22:41.251224+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:22:41.251224+0000","last_undegraded":"2023-09-23T21:22:41.251224+0000","last_fullsized":"2023-09-23T21:22:41.251224+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":689,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.251243+0000","last_change":"2023-09-23T21:21:21.571375+0000","last_active":"2023-09-23T21:22:41.251243+0000","last_peered":"2023-09-23T21:22:41.251243+0000","last_clean":"2023-09-23T21:22:41.251243+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:22:41.251243+0000","last_undegraded":"2023-09-23T21:22:41.251243+0000","last_fullsized":"2023-09-23T21:22:41.251243+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.551943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.551943+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.c","version":"655'10590","reported_seq":10680,"reported_epoch":655,"state":"active+clean","last_fresh":"2023-09-23T21:22:54.066201+0000","last_change":"2023-09-23T21:18:18.461455+0000","last_active":"2023-09-23T21:22:54.066201+0000","last_peered":"2023-09-23T21:22:54.066201+0000","last_clean":"2023-09-23T21:22:54.066201+0000","last_became_active":"2023-09-23T21:18:18.461068+0000","last_became_peered":"2023-09-23T21:18:18.461068+0000","last_unstale":"2023-09-23T21:22:54.066201+0000","last_undegraded":"2023-09-23T21:22:54.066201+0000","last_fullsized":"2023-09-23T21:22:54.066201+0000","mapping_epoch":573,"log_start":"611'5523","ondisk_log_start":"611'5523","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:07.319383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13893632,"num_objects":212,"num_object_clones":0,"num_object_copies":636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10590,"num_write_kb":345664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":322,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.239157+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:22:41.239157+0000","last_peered":"2023-09-23T21:22:41.239157+0000","last_clean":"2023-09-23T21:22:41.239157+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:22:41.239157+0000","last_undegraded":"2023-09-23T21:22:41.239157+0000","last_fullsized":"2023-09-23T21:22:41.239157+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255608+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:22:41.255608+0000","last_peered":"2023-09-23T21:22:41.255608+0000","last_clean":"2023-09-23T21:22:41.255608+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:22:41.255608+0000","last_undegraded":"2023-09-23T21:22:41.255608+0000","last_fullsized":"2023-09-23T21:22:41.255608+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.252134+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:22:41.252134+0000","last_peered":"2023-09-23T21:22:41.252134+0000","last_clean":"2023-09-23T21:22:41.252134+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:22:41.252134+0000","last_undegraded":"2023-09-23T21:22:41.252134+0000","last_fullsized":"2023-09-23T21:22:41.252134+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":717,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.251621+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:22:41.251621+0000","last_peered":"2023-09-23T21:22:41.251621+0000","last_clean":"2023-09-23T21:22:41.251621+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:22:41.251621+0000","last_undegraded":"2023-09-23T21:22:41.251621+0000","last_fullsized":"2023-09-23T21:22:41.251621+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"655'10221","reported_seq":20730,"reported_epoch":655,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T21:22:54.192528+0000","last_change":"2023-09-23T21:22:40.011959+0000","last_active":"2023-09-23T21:22:54.192528+0000","last_peered":"2023-09-23T21:22:54.192528+0000","last_clean":"2023-09-23T21:19:28.920085+0000","last_became_active":"2023-09-23T21:20:48.224556+0000","last_became_peered":"2023-09-23T21:20:48.224556+0000","last_unstale":"2023-09-23T21:22:54.192528+0000","last_undegraded":"2023-09-23T21:22:40.011054+0000","last_fullsized":"2023-09-23T21:20:47.933463+0000","mapping_epoch":623,"log_start":"606'5144","ondisk_log_start":"606'5144","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5077,"log_dups_size":0,"ondisk_log_size":5077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:02.563744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15007744,"num_objects":229,"num_object_clones":0,"num_object_copies":687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":264,"num_objects_misplaced":280,"num_objects_unfound":0,"num_objects_dirty":229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10221,"num_write_kb":334400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4956,"num_bytes_recovered":294322176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[4,2],"avail_no_missing":["4","2","6","7"],"object_location_counts":[{"shards":"2,4","objects":-43},{"shards":"2,4,6","objects":8}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":262,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.244677+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:22:41.244677+0000","last_peered":"2023-09-23T21:22:41.244677+0000","last_clean":"2023-09-23T21:22:41.244677+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:22:41.244677+0000","last_undegraded":"2023-09-23T21:22:41.244677+0000","last_fullsized":"2023-09-23T21:22:41.244677+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.249489+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:22:41.249489+0000","last_peered":"2023-09-23T21:22:41.249489+0000","last_clean":"2023-09-23T21:22:41.249489+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:22:41.249489+0000","last_undegraded":"2023-09-23T21:22:41.249489+0000","last_fullsized":"2023-09-23T21:22:41.249489+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.245004+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:22:41.245004+0000","last_peered":"2023-09-23T21:22:41.245004+0000","last_clean":"2023-09-23T21:22:41.245004+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:22:41.245004+0000","last_undegraded":"2023-09-23T21:22:41.245004+0000","last_fullsized":"2023-09-23T21:22:41.245004+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":956,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255566+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:22:41.255566+0000","last_peered":"2023-09-23T21:22:41.255566+0000","last_clean":"2023-09-23T21:22:41.255566+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:22:41.255566+0000","last_undegraded":"2023-09-23T21:22:41.255566+0000","last_fullsized":"2023-09-23T21:22:41.255566+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.a","version":"655'10399","reported_seq":20676,"reported_epoch":655,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T21:22:54.120035+0000","last_change":"2023-09-23T21:21:01.218382+0000","last_active":"2023-09-23T21:22:54.120035+0000","last_peered":"2023-09-23T21:22:54.120035+0000","last_clean":"2023-09-23T21:19:28.896530+0000","last_became_active":"2023-09-23T21:21:01.190094+0000","last_became_peered":"2023-09-23T21:21:01.190094+0000","last_unstale":"2023-09-23T21:22:54.120035+0000","last_undegraded":"2023-09-23T21:21:00.719860+0000","last_fullsized":"2023-09-23T21:22:54.120035+0000","mapping_epoch":629,"log_start":"606'5318","ondisk_log_start":"606'5318","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5081,"log_dups_size":0,"ondisk_log_size":5081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:06.621301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15138816,"num_objects":231,"num_object_clones":0,"num_object_copies":693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":247,"num_objects_misplaced":1738,"num_objects_unfound":0,"num_objects_dirty":231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10399,"num_write_kb":340160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5066,"num_bytes_recovered":331874304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[4,5,7],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"4,6,7","objects":231}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":352,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.246297+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:22:41.246297+0000","last_peered":"2023-09-23T21:22:41.246297+0000","last_clean":"2023-09-23T21:22:41.246297+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:22:41.246297+0000","last_undegraded":"2023-09-23T21:22:41.246297+0000","last_fullsized":"2023-09-23T21:22:41.246297+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":37,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.252210+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:22:41.252210+0000","last_peered":"2023-09-23T21:22:41.252210+0000","last_clean":"2023-09-23T21:22:41.252210+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:22:41.252210+0000","last_undegraded":"2023-09-23T21:22:41.252210+0000","last_fullsized":"2023-09-23T21:22:41.252210+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:35.537712+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:22:35.537712+0000","last_peered":"2023-09-23T21:22:35.537712+0000","last_clean":"2023-09-23T21:22:35.537712+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:22:35.537712+0000","last_undegraded":"2023-09-23T21:22:35.537712+0000","last_fullsized":"2023-09-23T21:22:35.537712+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1049,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:35.537896+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:22:35.537896+0000","last_peered":"2023-09-23T21:22:35.537896+0000","last_clean":"2023-09-23T21:22:35.537896+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:22:35.537896+0000","last_undegraded":"2023-09-23T21:22:35.537896+0000","last_fullsized":"2023-09-23T21:22:35.537896+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"655'10661","reported_seq":13976,"reported_epoch":655,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T21:22:53.895601+0000","last_change":"2023-09-23T21:21:22.582611+0000","last_active":"2023-09-23T21:22:53.895601+0000","last_peered":"2023-09-23T21:22:53.895601+0000","last_clean":"2023-09-23T21:19:48.922364+0000","last_became_active":"2023-09-23T21:21:22.566477+0000","last_became_peered":"2023-09-23T21:21:22.566477+0000","last_unstale":"2023-09-23T21:22:53.895601+0000","last_undegraded":"2023-09-23T21:22:53.895601+0000","last_fullsized":"2023-09-23T21:22:53.895601+0000","mapping_epoch":639,"log_start":"612'5656","ondisk_log_start":"612'5656","created":573,"last_epoch_clean":574,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:18:17.424187+0000","last_clean_scrub_stamp":"2023-09-23T21:18:17.424187+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:46.195361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12779520,"num_objects":195,"num_object_clones":0,"num_object_copies":585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":195,"num_objects_unfound":0,"num_objects_dirty":195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10661,"num_write_kb":347392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104660992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6,7],"avail_no_missing":["2","4","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":195}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":561,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255161+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:22:41.255161+0000","last_peered":"2023-09-23T21:22:41.255161+0000","last_clean":"2023-09-23T21:22:41.255161+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:22:41.255161+0000","last_undegraded":"2023-09-23T21:22:41.255161+0000","last_fullsized":"2023-09-23T21:22:41.255161+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":41,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.252061+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:22:41.252061+0000","last_peered":"2023-09-23T21:22:41.252061+0000","last_clean":"2023-09-23T21:22:41.252061+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:22:41.252061+0000","last_undegraded":"2023-09-23T21:22:41.252061+0000","last_fullsized":"2023-09-23T21:22:41.252061+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.245154+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:22:41.245154+0000","last_peered":"2023-09-23T21:22:41.245154+0000","last_clean":"2023-09-23T21:22:41.245154+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:22:41.245154+0000","last_undegraded":"2023-09-23T21:22:41.245154+0000","last_fullsized":"2023-09-23T21:22:41.245154+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":106,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255978+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:22:41.255978+0000","last_peered":"2023-09-23T21:22:41.255978+0000","last_clean":"2023-09-23T21:22:41.255978+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:22:41.255978+0000","last_undegraded":"2023-09-23T21:22:41.255978+0000","last_fullsized":"2023-09-23T21:22:41.255978+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":747,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.263510+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:22:41.263510+0000","last_peered":"2023-09-23T21:22:41.263510+0000","last_clean":"2023-09-23T21:22:41.263510+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:22:41.263510+0000","last_undegraded":"2023-09-23T21:22:41.263510+0000","last_fullsized":"2023-09-23T21:22:41.263510+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":415,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255023+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:22:41.255023+0000","last_peered":"2023-09-23T21:22:41.255023+0000","last_clean":"2023-09-23T21:22:41.255023+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:22:41.255023+0000","last_undegraded":"2023-09-23T21:22:41.255023+0000","last_fullsized":"2023-09-23T21:22:41.255023+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":258,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.255697+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:22:41.255697+0000","last_peered":"2023-09-23T21:22:41.255697+0000","last_clean":"2023-09-23T21:22:41.255697+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:22:41.255697+0000","last_undegraded":"2023-09-23T21:22:41.255697+0000","last_fullsized":"2023-09-23T21:22:41.255697+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":166,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.251454+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:22:41.251454+0000","last_peered":"2023-09-23T21:22:41.251454+0000","last_clean":"2023-09-23T21:22:41.251454+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:22:41.251454+0000","last_undegraded":"2023-09-23T21:22:41.251454+0000","last_fullsized":"2023-09-23T21:22:41.251454+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":48,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.341315+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:22:30.341315+0000","last_peered":"2023-09-23T21:22:30.341315+0000","last_clean":"2023-09-23T21:22:30.341315+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:22:30.341315+0000","last_undegraded":"2023-09-23T21:22:30.341315+0000","last_fullsized":"2023-09-23T21:22:30.341315+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":67,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.537113+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:22:30.537113+0000","last_peered":"2023-09-23T21:22:30.537113+0000","last_clean":"2023-09-23T21:22:30.537113+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:22:30.537113+0000","last_undegraded":"2023-09-23T21:22:30.537113+0000","last_fullsized":"2023-09-23T21:22:30.537113+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":57,"reported_epoch":653,"state":"active+clean","last_fresh":"2023-09-23T21:22:30.341161+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:22:30.341161+0000","last_peered":"2023-09-23T21:22:30.341161+0000","last_clean":"2023-09-23T21:22:30.341161+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:22:30.341161+0000","last_undegraded":"2023-09-23T21:22:30.341161+0000","last_fullsized":"2023-09-23T21:22:30.341161+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":644,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.263015+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:22:41.263015+0000","last_peered":"2023-09-23T21:22:41.263015+0000","last_clean":"2023-09-23T21:22:41.263015+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:22:41.263015+0000","last_undegraded":"2023-09-23T21:22:41.263015+0000","last_fullsized":"2023-09-23T21:22:41.263015+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":362,"reported_epoch":654,"state":"active+clean","last_fresh":"2023-09-23T21:22:41.252880+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:22:41.252880+0000","last_peered":"2023-09-23T21:22:41.252880+0000","last_clean":"2023-09-23T21:22:41.252880+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:22:41.252880+0000","last_undegraded":"2023-09-23T21:22:41.252880+0000","last_fullsized":"2023-09-23T21:22:41.252880+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":251854872,"num_objects":3844,"num_object_clones":0,"num_object_copies":11532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1810,"num_objects_misplaced":4419,"num_objects_unfound":0,"num_objects_dirty":3844,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":166786,"num_write_kb":5460097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33064,"num_bytes_recovered":1987051520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2980982784,"data_stored":2980970568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85600,"ondisk_log_size":85600,"up":48,"acting":47,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697707,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1550763,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3297280,"data_stored":3279520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201930,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1122088,"kb_used_data":203616,"kb_used_omap":0,"kb_used_meta":918400,"kb_avail":92623064,"statfs":{"total":95995035648,"available":94846017536,"internally_reserved":0,"allocated":208502784,"data_stored":206694576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":940441600},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9470000000000001,"5min":1.9019999999999999,"15min":2.4689999999999999},"min":{"1min":0.252,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":10.522,"5min":16.468,"15min":141.33199999999999},"last":0.86899999999999999},{"interface":"front","average":{"1min":2.645,"5min":2.0790000000000002,"15min":2.6160000000000001},"min":{"1min":0.308,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.6389999999999993,"5min":12.962,"15min":141.31700000000001},"last":0.88500000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2109999999999999,"15min":3.593},"min":{"1min":0.36699999999999999,"5min":0.29599999999999999,"15min":0.24099999999999999},"max":{"1min":12.112,"5min":13.054,"15min":114.16500000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":3.181,"5min":2.2530000000000001,"15min":3.9670000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22900000000000001},"max":{"1min":10.477,"5min":16.289000000000001,"15min":117.637},"last":0.501}]},{"osd":2,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":9.4480000000000004,"5min":7.5999999999999996,"15min":6.1239999999999997},"min":{"1min":0.39900000000000002,"5min":0.248,"15min":0.224},"max":{"1min":157.65700000000001,"5min":158.726,"15min":183.44200000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":10.220000000000001,"5min":7.4779999999999998,"15min":6.3490000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":157.642,"5min":158.70400000000001,"15min":183.46199999999999},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.5070000000000001,"15min":4.3099999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.26700000000000002},"max":{"1min":10.547000000000001,"5min":30.667999999999999,"15min":135.66200000000001},"last":8.3559999999999999},{"interface":"front","average":{"1min":3.7970000000000002,"5min":2.5390000000000001,"15min":4.5810000000000004},"min":{"1min":0.32900000000000001,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":9.7919999999999998,"5min":30.798999999999999,"15min":135.47},"last":10.337}]},{"osd":4,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":3.931,"15min":2.2000000000000002},"min":{"1min":0.34499999999999997,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":10.68,"5min":31.695,"15min":31.695},"last":0.32000000000000001},{"interface":"front","average":{"1min":5.7519999999999998,"5min":4.2199999999999998,"15min":2.3690000000000002},"min":{"1min":0.32700000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.840999999999999,"5min":34.994,"15min":34.994},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9279999999999999,"5min":4.6680000000000001,"15min":2.7469999999999999},"min":{"1min":0.44,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":13.906000000000001,"5min":156.30099999999999,"15min":156.30099999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.137,"5min":4.7089999999999996,"15min":2.7269999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32200000000000001},"max":{"1min":8.2889999999999997,"5min":156.22,"15min":156.22},"last":0.38900000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.7549999999999999,"15min":2.9529999999999998},"min":{"1min":0.433,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":11.007999999999999,"5min":23.574999999999999,"15min":143.18700000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":3.8849999999999998,"5min":3.431,"15min":2.702},"min":{"1min":0.36199999999999999,"5min":0.35899999999999999,"15min":0.29099999999999998},"max":{"1min":9.8439999999999994,"5min":21.081,"15min":142.70400000000001},"last":0.63}]}]},{"osd":6,"up_from":47,"seq":201863463550,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722148,"kb_used_data":811804,"kb_used_omap":0,"kb_used_meta":910208,"kb_avail":92023004,"statfs":{"total":95995035648,"available":94231556096,"internally_reserved":0,"allocated":831287296,"data_stored":829483953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932052992},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.78,"15min":2.9660000000000002},"min":{"1min":0.28699999999999998,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":12.313000000000001,"5min":12.313000000000001,"15min":149.61600000000001},"last":2.2360000000000002},{"interface":"front","average":{"1min":2.5920000000000001,"5min":2.1880000000000002,"15min":3.1960000000000002},"min":{"1min":0.27400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.968,"5min":11.968,"15min":149.303},"last":0.312}]},{"osd":1,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":2.1040000000000001,"15min":2.9940000000000002},"min":{"1min":0.25700000000000001,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":5.3810000000000002,"5min":12.013,"15min":129.37899999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.149,"5min":2.0859999999999999,"15min":2.8250000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.6399999999999997,"5min":14.366,"15min":129.40100000000001},"last":3.2909999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":25.170999999999999,"5min":9.1639999999999997,"15min":7.6150000000000002},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":199.08000000000001,"5min":199.08000000000001,"15min":199.23400000000001},"last":2.3159999999999998},{"interface":"front","average":{"1min":25.001000000000001,"5min":9.6969999999999992,"15min":7.7549999999999999},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":199.27000000000001,"5min":199.27000000000001,"15min":199.27000000000001},"last":0.374}]},{"osd":3,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":2.1880000000000002,"15min":3.5640000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":6.516,"5min":14.411,"15min":137.875},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.6150000000000002,"5min":1.986,"15min":3.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.318,"15min":0.29799999999999999},"max":{"1min":12.35,"5min":12.35,"15min":137.60900000000001},"last":2.2789999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":3.6909999999999998,"15min":2.5169999999999999},"min":{"1min":0.29099999999999998,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":20.408999999999999,"5min":20.408999999999999,"15min":55.494},"last":6.8700000000000001},{"interface":"front","average":{"1min":4.5709999999999997,"5min":3.367,"15min":2.3159999999999998},"min":{"1min":0.34200000000000003,"5min":0.33400000000000002,"15min":0.23999999999999999},"max":{"1min":12.864000000000001,"5min":14.622,"15min":55.305},"last":4.0990000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":3.1499999999999999,"15min":2.8180000000000001},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":12.051,"5min":12.975,"15min":222.90899999999999},"last":6.9100000000000001},{"interface":"front","average":{"1min":2.9860000000000002,"5min":2.9969999999999999,"15min":2.7709999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28499999999999998},"max":{"1min":10.625999999999999,"5min":17.927,"15min":224.655},"last":3.3210000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.7429999999999999,"15min":2.379},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.28000000000000003},"max":{"1min":12.380000000000001,"5min":12.380000000000001,"15min":165.96799999999999},"last":3.1789999999999998},{"interface":"front","average":{"1min":4.1890000000000001,"5min":3.6179999999999999,"15min":2.7690000000000001},"min":{"1min":0.40600000000000003,"5min":0.307,"15min":0.251},"max":{"1min":11.526999999999999,"5min":19.815999999999999,"15min":165.82400000000001},"last":3.2000000000000002}]}]},{"osd":1,"up_from":276,"seq":1185410973997,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":655796,"kb_used_data":8236,"kb_used_omap":5,"kb_used_meta":647546,"kb_avail":93089356,"statfs":{"total":95995035648,"available":95323500544,"internally_reserved":0,"allocated":8433664,"data_stored":5324106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":663087843},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.39200000000000002,"15min":1.944},"min":{"1min":0.214,"5min":0.214,"15min":0.19400000000000001},"max":{"1min":2.3410000000000002,"5min":2.3410000000000002,"15min":108.47},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.339,"5min":0.59799999999999998,"15min":1.952},"min":{"1min":0.23899999999999999,"5min":0.23499999999999999,"15min":0.19600000000000001},"max":{"1min":6.0149999999999997,"5min":6.0149999999999997,"15min":109.532},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":13.051,"5min":6.5170000000000003,"15min":8.6799999999999997},"min":{"1min":0.23200000000000001,"5min":0.22800000000000001,"15min":0.20200000000000001},"max":{"1min":130.78899999999999,"5min":130.78899999999999,"15min":171.35400000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":12.999000000000001,"5min":6.5540000000000003,"15min":8.3290000000000006},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":133.751,"5min":133.751,"15min":170.69900000000001},"last":1.1519999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.60699999999999998,"15min":3.5550000000000002},"min":{"1min":0.27000000000000002,"5min":0.254,"15min":0.23799999999999999},"max":{"1min":3.2450000000000001,"5min":17.247,"15min":99.120999999999995},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.76200000000000001,"15min":4.0960000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":3.754,"5min":10.949999999999999,"15min":150.38},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":1.829,"15min":2.371},"min":{"1min":0.27400000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":6.9089999999999998,"5min":11.034000000000001,"15min":174.84399999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.3170000000000002,"5min":1.5,"15min":2.2749999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":8.5109999999999992,"5min":11.057,"15min":174.82400000000001},"last":0.74299999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":2.6440000000000001,"15min":2.7730000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.31900000000000001},"max":{"1min":8.6329999999999991,"5min":107.98,"15min":110.955},"last":4.2290000000000001},{"interface":"front","average":{"1min":3.0750000000000002,"5min":3.169,"15min":2.8999999999999999},"min":{"1min":0.36699999999999999,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":6.9669999999999996,"5min":107.961,"15min":110.89},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":3.6160000000000001,"15min":2.5960000000000001},"min":{"1min":0.42199999999999999,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":20.265999999999998,"5min":119.999,"15min":119.999},"last":1.0129999999999999},{"interface":"front","average":{"1min":3.5409999999999999,"5min":3.5710000000000002,"15min":2.4980000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31900000000000001},"max":{"1min":20.361999999999998,"5min":122.328,"15min":122.328},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5979999999999999,"5min":2.1389999999999998,"15min":3.0009999999999999},"min":{"1min":0.39800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.0430000000000001,"5min":10.648999999999999,"15min":285.47500000000002},"last":3.1930000000000001},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.726,"15min":2.754},"min":{"1min":0.46999999999999997,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":5.165,"5min":10.664999999999999,"15min":285.48599999999999},"last":3.1779999999999999}]}]},{"osd":0,"up_from":0,"seq":395136991737,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781628,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1503308,"kb_used_data":191812,"kb_used_omap":7,"kb_used_meta":1311480,"kb_avail":92241844,"statfs":{"total":95995035648,"available":94455648256,"internally_reserved":0,"allocated":196415488,"data_stored":193215455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1342955601},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.68899999999999995,"15min":1.9530000000000001},"min":{"1min":0.189,"5min":0.17899999999999999,"15min":0.17699999999999999},"max":{"1min":2.8700000000000001,"5min":5.0010000000000003,"15min":103.021},"last":3.2330000000000001},{"interface":"front","average":{"1min":0.875,"5min":0.73699999999999999,"15min":2.54},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":4.5510000000000002,"5min":4.9820000000000002,"15min":103.154},"last":6.6459999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.749,"15min":2.6970000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":3.2829999999999999,"5min":5.2670000000000003,"15min":143.482},"last":6.4470000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.71499999999999997,"15min":2.7320000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":3.27,"5min":5.4820000000000002,"15min":143.61000000000001},"last":0.73799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.79600000000000004,"15min":3.5499999999999998},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.798,"5min":5.2939999999999996,"15min":143.50800000000001},"last":6.4039999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.77600000000000002,"15min":3.77},"min":{"1min":0.34100000000000003,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.3290000000000002,"5min":5.6790000000000003,"15min":143.27799999999999},"last":0.95999999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0310000000000001,"15min":3.4900000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":5.6680000000000001,"5min":7.4880000000000004,"15min":225.80799999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.972,"5min":1.9419999999999999,"15min":4.0750000000000002},"min":{"1min":0.39300000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.6840000000000002,"5min":7.5010000000000003,"15min":225.79300000000001},"last":6.5780000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.51,"15min":3.5529999999999999},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":7.048,"5min":9.0579999999999998,"15min":220.63900000000001},"last":6.7960000000000003},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.6060000000000001,"15min":3.6269999999999998},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":8.0579999999999998,"5min":8.0579999999999998,"15min":220.31999999999999},"last":4.1100000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.5150000000000001,"15min":3.5049999999999999},"min":{"1min":0.5,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":10.148,"5min":42.363,"15min":210.74700000000001},"last":5.2030000000000003},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.4420000000000002,"15min":2.8319999999999999},"min":{"1min":0.57099999999999995,"5min":0.308,"15min":0.308},"max":{"1min":10.634,"5min":42.383000000000003,"15min":210.72900000000001},"last":2.2149999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":1.8660000000000001,"15min":2.0350000000000001},"min":{"1min":0.50900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.9500000000000002,"5min":7.9500000000000002,"15min":25.260000000000002},"last":0.99299999999999999},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.796,"15min":2.4319999999999999},"min":{"1min":0.39800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.1079999999999997,"5min":7.5289999999999999,"15min":16.875},"last":6.6150000000000002}]}]},{"osd":3,"up_from":148,"seq":635655160221,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896120,"kb_used_data":9392,"kb_used_omap":2,"kb_used_meta":886717,"kb_avail":92849032,"statfs":{"total":95995035648,"available":95077408768,"internally_reserved":0,"allocated":9617408,"data_stored":5955404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":907998261},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.504,"15min":1.7669999999999999},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":3.8780000000000001,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.51200000000000001,"15min":1.5580000000000001},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.19500000000000001},"max":{"1min":5.0190000000000001,"5min":5.0190000000000001,"15min":18.277999999999999},"last":0.24299999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:22:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.70299999999999996,"15min":6.5880000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":8.3079999999999998,"5min":8.3079999999999998,"15min":237.072},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.67000000000000004,"15min":6.4690000000000003},"min":{"1min":0.24299999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":234.21899999999999},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":16.347999999999999,"5min":3.8279999999999998,"15min":5.6299999999999999},"min":{"1min":0.224,"5min":0.219,"15min":0.215},"max":{"1min":207.59999999999999,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.89400000000000002},{"interface":"front","average":{"1min":16.359999999999999,"5min":3.7570000000000001,"15min":5.2400000000000002},"min":{"1min":0.27800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":208.024,"5min":208.024,"15min":208.024},"last":1.147}]},{"osd":4,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":3.2440000000000002,"15min":2.609},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":9.5579999999999998,"5min":117.717,"15min":117.717},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.141,"5min":3.7130000000000001,"15min":2.6579999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":9.0649999999999995,"5min":117.771,"15min":117.771},"last":2.964}]},{"osd":5,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.9540000000000002,"5min":4.3209999999999997,"15min":2.9910000000000001},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.27200000000000002},"max":{"1min":9.1910000000000007,"5min":123.458,"15min":123.458},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.218,"5min":4.0739999999999998,"15min":2.8839999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":8.4670000000000005,"5min":123.292,"15min":123.292},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":5.0510000000000002,"15min":4.7670000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":7.7889999999999997,"5min":130.613,"15min":304.22399999999999},"last":3.8540000000000001},{"interface":"front","average":{"1min":2.5990000000000002,"5min":5.5640000000000001,"15min":4.8979999999999997},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.32500000000000001},"max":{"1min":8.8940000000000001,"5min":130.50999999999999,"15min":304.15800000000002},"last":3.7810000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.974,"15min":2.2080000000000002},"min":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":7.4939999999999998,"5min":19.298999999999999,"15min":44.253},"last":0.497},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.3380000000000001,"15min":2.5870000000000002},"min":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":9.1549999999999994,"5min":13.343,"15min":45.706000000000003},"last":3.9910000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986840,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1321392,"kb_used_data":335464,"kb_used_omap":0,"kb_used_meta":985792,"kb_avail":92423760,"statfs":{"total":95995035648,"available":94641930240,"internally_reserved":0,"allocated":343515136,"data_stored":341699557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1009451008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.173,"15min":3.0070000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":12.195,"5min":12.195,"15min":226.751},"last":4.6319999999999997},{"interface":"front","average":{"1min":2.161,"5min":1.5640000000000001,"15min":2.7040000000000002},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":8.641,"5min":12.754,"15min":226.43299999999999},"last":0.75700000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.802,"15min":3.0470000000000002},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":16.434999999999999,"5min":16.434999999999999,"15min":119.55800000000001},"last":3.2360000000000002},{"interface":"front","average":{"1min":3.3610000000000002,"5min":2.2730000000000001,"15min":3.0920000000000001},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.108000000000001,"5min":12.108000000000001,"15min":116.78100000000001},"last":4.5609999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":11.018000000000001,"5min":8.4890000000000008,"15min":6.7539999999999996},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.248},"max":{"1min":124.70099999999999,"5min":124.70099999999999,"15min":210.50299999999999},"last":4.4349999999999996},{"interface":"front","average":{"1min":7.9050000000000002,"5min":7.6079999999999997,"15min":6.6630000000000003},"min":{"1min":0.46000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":116.47,"5min":116.47,"15min":210.59},"last":4.6689999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":2.6520000000000001,"15min":4.5990000000000002},"min":{"1min":0.314,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":12.973000000000001,"5min":14.323,"15min":214.64099999999999},"last":4.4710000000000001},{"interface":"front","average":{"1min":3.665,"5min":2.5110000000000001,"15min":4.9649999999999999},"min":{"1min":0.38300000000000001,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":13.673999999999999,"5min":13.673999999999999,"15min":214.54900000000001},"last":4.5999999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:22:43 2023","interfaces":[{"interface":"back","average":{"1min":3.149,"5min":3.4140000000000001,"15min":2.004},"min":{"1min":0.34799999999999998,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":13.315,"5min":17.879999999999999,"15min":34.497999999999998},"last":13.31},{"interface":"front","average":{"1min":3.581,"5min":3.234,"15min":2.2450000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.23799999999999999},"max":{"1min":8.7249999999999996,"5min":19.425000000000001,"15min":34.883000000000003},"last":4.8200000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.681,"5min":4.1980000000000004,"15min":2.3980000000000001},"min":{"1min":0.36199999999999999,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":15.502000000000001,"5min":67.697999999999993,"15min":67.697999999999993},"last":5.3179999999999996},{"interface":"front","average":{"1min":5.4560000000000004,"5min":5.3390000000000004,"15min":2.726},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":18.762,"5min":67.596999999999994,"15min":67.596999999999994},"last":4.5090000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:22:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6980000000000004,"5min":3.645,"15min":2.222},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.28100000000000003},"max":{"1min":16.998999999999999,"5min":17.817,"15min":18.352},"last":4.7629999999999999},{"interface":"front","average":{"1min":4.2640000000000002,"5min":3.4980000000000002,"15min":2.0760000000000001},"min":{"1min":0.36399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":16.664999999999999,"5min":30.117000000000001,"15min":30.117000000000001},"last":4.5830000000000002}]}]},{"osd":5,"up_from":39,"seq":167503725204,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2432668,"kb_used_data":1436692,"kb_used_omap":0,"kb_used_meta":995968,"kb_avail":91312484,"statfs":{"total":95995035648,"available":93503983616,"internally_reserved":0,"allocated":1471172608,"data_stored":1469372709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1019871232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.6899999999999999,"15min":2.0049999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":7.5119999999999996,"5min":12.558999999999999,"15min":82.280000000000001},"last":4.9950000000000001},{"interface":"front","average":{"1min":2.194,"5min":1.8320000000000001,"15min":2.181},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":6.0430000000000001,"5min":9.407,"15min":85.236000000000004},"last":9.3889999999999993}]},{"osd":1,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.1440000000000001,"15min":3.2610000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.9240000000000004,"5min":18.859999999999999,"15min":125.23399999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.754,"5min":1.8149999999999999,"15min":2.9039999999999999},"min":{"1min":0.32200000000000001,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":12.164,"5min":12.654999999999999,"15min":125.249},"last":4.9509999999999996}]},{"osd":2,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":5.569,"5min":5.2720000000000002,"15min":4.665},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":75.441999999999993,"5min":160.749,"15min":231.34700000000001},"last":5.0640000000000001},{"interface":"front","average":{"1min":6.3460000000000001,"5min":5.3890000000000002,"15min":4.8200000000000003},"min":{"1min":0.35999999999999999,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":75.247,"5min":161.18000000000001,"15min":231.381},"last":9.3460000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":2.278,"15min":4.0350000000000001},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":8.0980000000000008,"5min":18.826000000000001,"15min":263.887},"last":0.442},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.7989999999999999,"15min":3.6779999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":6.1950000000000003,"5min":12.683999999999999,"15min":263.70100000000002},"last":5.04}]},{"osd":4,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7229999999999999,"5min":3.4830000000000001,"15min":1.974},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":12.212,"5min":77.804000000000002,"15min":77.804000000000002},"last":5.0190000000000001},{"interface":"front","average":{"1min":6.274,"5min":4.8659999999999997,"15min":2.516},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":19.718,"5min":77.825999999999993,"15min":77.825999999999993},"last":13.545}]},{"osd":6,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":4.4279999999999999,"15min":2.617},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":68.346000000000004,"5min":68.346000000000004,"15min":83.451999999999998},"last":13.643000000000001},{"interface":"front","average":{"1min":11.348000000000001,"5min":4.758,"15min":2.907},"min":{"1min":0.40600000000000003,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":72.393000000000001,"5min":72.393000000000001,"15min":83.484999999999999},"last":5.8540000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:22:45 2023","interfaces":[{"interface":"back","average":{"1min":5.2160000000000002,"5min":3.0550000000000002,"15min":2.0569999999999999},"min":{"1min":0.60499999999999998,"5min":0.36299999999999999,"15min":0.30099999999999999},"max":{"1min":12.692,"5min":12.692,"15min":12.692},"last":9.4550000000000001},{"interface":"front","average":{"1min":4.0270000000000001,"5min":2.5819999999999999,"15min":1.6359999999999999},"min":{"1min":0.48799999999999999,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.236000000000001,"5min":18.257999999999999,"15min":18.257999999999999},"last":5.0839999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":180879360,"data_stored":180879360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":320933888,"data_stored":320929816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1462767616,"data_stored":1462763544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":816058368,"data_stored":816054296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":200343552,"data_stored":200343552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:22:58.263 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T21:22:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:58.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:22:58.365 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:22:58.397 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:58.451 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:58.575 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:22:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:58.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:22:58.685 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:22:58.743 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:58.864 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:22:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:58.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:22:58.972 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:22:59.046 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:59.128 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:59.172 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:22:59.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:59.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:22:59.210 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:22:59.267 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:22:59.283 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:22:59.365 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2115615128' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:59.365 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:22:59 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:59.394 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:59.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2115615128' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:59.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:22:59 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:59.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2115615128' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:59.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:22:59 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:22:59.497 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:22:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:59.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:22:59.624 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:22:59.744 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:22:59.894 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:22:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:22:59.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:00.005 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:00.065 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:00.170 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:00.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:00.288 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:00.328 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:00.374 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Removed 85314 objects 2023-09-23T21:23:00.374 INFO:tasks.radosbench.radosbench.0.smithi121.stdout:Clean up completed and total clean up time :182.937 2023-09-23T21:23:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:00 smithi123 ceph-mon[193903]: pgmap v1999: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 63 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 705/3084 objects degraded (22.860%); 2263/3084 objects misplaced (73.379%); 15 MiB/s, 280 objects/s recovering 2023-09-23T21:23:00.628 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-09-23T21:23:00.628 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-09-23T21:23:00.652 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:00.652 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:23:00.652 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T21:23:00.652 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.652 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.652 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[211526]: pgmap v1999: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 63 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 705/3084 objects degraded (22.860%); 2263/3084 objects misplaced (73.379%); 15 MiB/s, 280 objects/s recovering 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:23:00.653 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: failed to encode map e656 with expected crc 2023-09-23T21:23:00.654 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:00 smithi121 ceph-mon[219709]: pgmap v1999: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 63 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 705/3084 objects degraded (22.860%); 2263/3084 objects misplaced (73.379%); 15 MiB/s, 280 objects/s recovering 2023-09-23T21:23:01.138 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:01.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:01.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:01.242 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:01.268 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:01.312 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:01.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:01.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:01.417 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:01.446 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:01.493 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:01.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:01.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:01.597 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:01.627 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:01.671 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:01.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:01.776 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:01.807 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:01 smithi121 ceph-mon[219709]: pgmap v2000: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 63 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 705/3084 objects degraded (22.860%); 2263/3084 objects misplaced (73.379%); 13 MiB/s, 249 objects/s recovering 2023-09-23T21:23:01.807 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:01 smithi121 ceph-mon[211526]: pgmap v2000: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 63 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 705/3084 objects degraded (22.860%); 2263/3084 objects misplaced (73.379%); 13 MiB/s, 249 objects/s recovering 2023-09-23T21:23:01.811 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:01 smithi123 ceph-mon[193903]: pgmap v2000: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 63 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 705/3084 objects degraded (22.860%); 2263/3084 objects misplaced (73.379%); 13 MiB/s, 249 objects/s recovering 2023-09-23T21:23:02.010 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:02.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:02.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:02.115 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:02.144 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:02.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:23:02.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:23:02.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:23:02.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:23:02.703 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:23:02.719 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:23:02.738 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:23:02.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:23:02.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.769 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.770 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.770 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.770 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.770 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.770 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.770 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:23:02.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 705/3084 objects degraded (22.860%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[211526]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[211526]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[211526]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[211526]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 705/3084 objects degraded (22.860%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[219709]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[219709]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[219709]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:02 smithi121 ceph-mon[219709]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 705/3084 objects degraded (22.860%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T21:23:02.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:02 smithi123 ceph-mon[193903]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:02 smithi123 ceph-mon[193903]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T21:23:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:02 smithi123 ceph-mon[193903]: failed to encode map e657 with expected crc 2023-09-23T21:23:02.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:02 smithi123 ceph-mon[193903]: failed to encode map e657 with expected crc 2023-09-23T21:23:03.085 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:23:03.314 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:03.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:03.418 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:03.445 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:03.495 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:03.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:03.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:03.601 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:03.628 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:03.635 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:23:03.635 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:23:03.657 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[211526]: failed to encode map e657 with expected crc 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3768427681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[211526]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[211526]: failed to encode map e658 with expected crc 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[211526]: failed to encode map e658 with expected crc 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[211526]: pgmap v2003: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 39 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 580/1893 objects degraded (30.639%); 2011/1893 objects misplaced (106.233%); 9.3 MiB/s, 209 objects/s recovering 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[219709]: failed to encode map e657 with expected crc 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3768427681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:03.658 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[219709]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T21:23:03.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[219709]: failed to encode map e658 with expected crc 2023-09-23T21:23:03.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[219709]: failed to encode map e658 with expected crc 2023-09-23T21:23:03.659 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:03 smithi121 ceph-mon[219709]: pgmap v2003: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 39 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 580/1893 objects degraded (30.639%); 2011/1893 objects misplaced (106.233%); 9.3 MiB/s, 209 objects/s recovering 2023-09-23T21:23:03.680 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:03.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:03.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:03.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:03 smithi123 ceph-mon[193903]: failed to encode map e657 with expected crc 2023-09-23T21:23:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:03 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3768427681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:03 smithi123 ceph-mon[193903]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T21:23:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:03 smithi123 ceph-mon[193903]: failed to encode map e658 with expected crc 2023-09-23T21:23:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:03 smithi123 ceph-mon[193903]: failed to encode map e658 with expected crc 2023-09-23T21:23:03.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:03 smithi123 ceph-mon[193903]: pgmap v2003: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 87 active+clean; 39 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 580/1893 objects degraded (30.639%); 2011/1893 objects misplaced (106.233%); 9.3 MiB/s, 209 objects/s recovering 2023-09-23T21:23:03.786 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:03.807 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:03.822 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:04.267 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:23:04.267 INFO:teuthology.orchestra.run.smithi121.stdout:{"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 580/1893 objects degraded (30.639%), 9 pgs degraded, 2 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2640,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":659,"num_osds":8,"num_up_osds":8,"osd_up_since":1695502663,"num_in_osds":7,"osd_in_since":1695504058,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":631,"data_bytes":40483563,"bytes_used":9300615168,"bytes_avail":662664634368,"bytes_total":671965249536,"degraded_objects":580,"degraded_total":1893,"degraded_ratio":0.30639197041732702,"misplaced_objects":2011,"misplaced_total":1893,"misplaced_ratio":1.0623349181193873,"recovering_objects_per_sec":209,"recovering_bytes_per_sec":9707123,"recovering_keys_per_sec":0,"num_objects_recovered":1260,"num_bytes_recovered":58261504,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":456},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi121.sdlxlb","status":"up:active","gid":24578}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.121:8443/","prometheus":"http://172.21.15.121:9283/"}},"servicemap":{"epoch":35,"modified":"2023-09-23T21:21:43.417977+0000","services":{}},"progress_events":{"432f78a9-9bfa-46a5-b9c3-f138454b372b":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 27s)","progress":0.87878787517547607,"add_to_ceph_s":true},"b5766e4e-830b-481a-a8ef-26ada69d4160":{"message":"Upgrade to 18.2.0-460-gbb0daeff (11m)\n [============................] (remaining: 13m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T21:23:04.283 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:04.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:04.387 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:04.413 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:04.457 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:04.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:04.565 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:04.590 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:04.631 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:04.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T21:23:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: 14.9 continuing backfill to osd.3 from (612'5444,656'10450] MIN to 656'10450 2023-09-23T21:23:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1915035008' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:23:04.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:04 smithi123 ceph-mon[193903]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.736 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:04.763 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:04.767 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:04.778 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T21:23:04.807 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:04.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:04.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:04.902 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:23:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:23:04] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: 14.9 continuing backfill to osd.3 from (612'5444,656'10450] MIN to 656'10450 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.903 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1915035008' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[211526]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: failed to encode map e658 with expected crc 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: 14.9 continuing backfill to osd.3 from (612'5444,656'10450] MIN to 656'10450 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.904 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1915035008' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T21:23:04.905 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:04 smithi121 ceph-mon[219709]: failed to encode map e659 with expected crc 2023-09-23T21:23:04.913 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:04.939 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:04.984 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:04.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:05.091 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:05.118 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:05.162 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:05.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:05.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:05.294 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:05.437 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:05.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:05.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:05.540 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:05.561 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:05 smithi121 ceph-mon[211526]: failed to encode map e659 with expected crc 2023-09-23T21:23:05.562 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:05 smithi121 ceph-mon[211526]: pgmap v2005: 99 pgs: 1 remapped+peering, 1 active+recovering+remapped, 4 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 687 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 1292/72 objects misplaced (1794.444%); 3.6 MiB/s, 240 objects/s recovering 2023-09-23T21:23:05.569 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:05.807 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T21:23:05.807 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:23:05.839 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:23:05.857 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:05 smithi121 ceph-mon[219709]: failed to encode map e659 with expected crc 2023-09-23T21:23:05.857 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:05 smithi121 ceph-mon[219709]: pgmap v2005: 99 pgs: 1 remapped+peering, 1 active+recovering+remapped, 4 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 687 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 1292/72 objects misplaced (1794.444%); 3.6 MiB/s, 240 objects/s recovering 2023-09-23T21:23:05.969 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:05 smithi123 ceph-mon[193903]: failed to encode map e659 with expected crc 2023-09-23T21:23:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:05 smithi123 ceph-mon[193903]: pgmap v2005: 99 pgs: 1 remapped+peering, 1 active+recovering+remapped, 4 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 687 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 1292/72 objects misplaced (1794.444%); 3.6 MiB/s, 240 objects/s recovering 2023-09-23T21:23:06.050 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:06.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:06.154 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:06.181 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:06.224 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:06.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:06.329 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:06.367 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:07.352 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:07 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T21:23:07.352 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:07 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T21:23:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:07 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2023-09-23T21:23:07.763 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:07.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:07.868 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:07.895 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:07.938 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:07.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:07.959 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:07.995 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:08.030 INFO:teuthology.orchestra.run.smithi121.stderr:pool 'unique_pool_9' removed 2023-09-23T21:23:08.046 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:08.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1741636521' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:23:08.075 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:08 smithi121 ceph-mon[219709]: pgmap v2006: 99 pgs: 1 remapped+peering, 2 active+recovering+remapped, 4 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 88 active+clean; 687 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 1292/72 objects misplaced (1794.444%); 3.6 MiB/s, 334 objects/s recovering 2023-09-23T21:23:08.079 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1741636521' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:23:08.079 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:08 smithi121 ceph-mon[211526]: pgmap v2006: 99 pgs: 1 remapped+peering, 2 active+recovering+remapped, 4 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 88 active+clean; 687 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 1292/72 objects misplaced (1794.444%); 3.6 MiB/s, 334 objects/s recovering 2023-09-23T21:23:08.153 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:08.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1741636521' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T21:23:08.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:08 smithi123 ceph-mon[193903]: pgmap v2006: 99 pgs: 1 remapped+peering, 2 active+recovering+remapped, 4 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 88 active+clean; 687 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 1292/72 objects misplaced (1794.444%); 3.6 MiB/s, 334 objects/s recovering 2023-09-23T21:23:09.050 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T21:23:09.051 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T21:23:09.051 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T21:23:09.054 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T21:23:09.054 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T21:23:09.055 DEBUG:teuthology.orchestra.run.smithi121:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:23:09.057 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:09.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:09.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:09.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:23:09.133 INFO:teuthology.orchestra.run.smithi121.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T21:23:09.133 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:23:09.163 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:09.172 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T21:23:09.173 DEBUG:teuthology.orchestra.run.smithi121:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T21:23:09.230 INFO:tasks.workunit:timeout=3h 2023-09-23T21:23:09.231 INFO:tasks.workunit:cleanup=True 2023-09-23T21:23:09.231 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:23:09.241 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:09.372 INFO:tasks.workunit.client.0.smithi121.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T21:23:09.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-09-23T21:23:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[211526]: failed to encode map e660 with expected crc 2023-09-23T21:23:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1741636521' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:23:09.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[211526]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T21:23:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-09-23T21:23:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[219709]: failed to encode map e660 with expected crc 2023-09-23T21:23:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1741636521' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:23:09.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:09 smithi121 ceph-mon[219709]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T21:23:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:09 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2023-09-23T21:23:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:09 smithi123 ceph-mon[193903]: failed to encode map e660 with expected crc 2023-09-23T21:23:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1741636521' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T21:23:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:09 smithi123 ceph-mon[193903]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T21:23:09.649 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:23:09.650 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:23:10.027 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:23:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:10 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/214221598' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:10 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:10 smithi121 ceph-mon[211526]: pgmap v2008: 83 pgs: 83 active+clean; 687 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:10 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/214221598' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:10 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:10.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:10 smithi121 ceph-mon[219709]: pgmap v2008: 83 pgs: 83 active+clean; 687 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:10.439 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:10.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:10 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/214221598' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:10 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:10 smithi123 ceph-mon[193903]: pgmap v2008: 83 pgs: 83 active+clean; 687 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:10.542 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:10.572 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[219709]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[219709]: failed to encode map e661 with expected crc 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[219709]: failed to encode map e660 with expected crc 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[219709]: failed to encode map e660 with expected crc 2023-09-23T21:23:11.287 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:23:11.288 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:23:11.288 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:23:11.288 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[211526]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T21:23:11.288 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[211526]: failed to encode map e661 with expected crc 2023-09-23T21:23:11.288 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[211526]: failed to encode map e660 with expected crc 2023-09-23T21:23:11.288 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:11 smithi121 ceph-mon[211526]: failed to encode map e660 with expected crc 2023-09-23T21:23:11.303 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T21:23:11.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:11 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:23:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:11 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:23:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:11 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:23:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:11 smithi123 ceph-mon[193903]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T21:23:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:11 smithi123 ceph-mon[193903]: failed to encode map e661 with expected crc 2023-09-23T21:23:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:11 smithi123 ceph-mon[193903]: failed to encode map e660 with expected crc 2023-09-23T21:23:11.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:11 smithi123 ceph-mon[193903]: failed to encode map e660 with expected crc 2023-09-23T21:23:11.621 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:11.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:11.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:11.725 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:11.756 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:11.905 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:11.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:11.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:12.008 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:12.036 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:12.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:12 smithi121 ceph-mon[211526]: failed to encode map e661 with expected crc 2023-09-23T21:23:12.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:12 smithi121 ceph-mon[211526]: pgmap v2010: 83 pgs: 83 active+clean; 687 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:12 smithi121 ceph-mon[219709]: failed to encode map e661 with expected crc 2023-09-23T21:23:12.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:12 smithi121 ceph-mon[219709]: pgmap v2010: 83 pgs: 83 active+clean; 687 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:12 smithi123 ceph-mon[193903]: failed to encode map e661 with expected crc 2023-09-23T21:23:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:12 smithi123 ceph-mon[193903]: pgmap v2010: 83 pgs: 83 active+clean; 687 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:12.595 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:12.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:12.700 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:12.731 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:12.780 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:12.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:12.834 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:12.885 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:12.914 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:12.960 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:12.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:13.064 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:13.077 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:23:13.079 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:23:13.092 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:13.137 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:13.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:13.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:13.200 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:23:13.228 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:23:13.243 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:13.244 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[211526]: 1.a scrub starts 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[211526]: 1.a scrub ok 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[211526]: 1.3 scrub starts 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[211526]: 1.3 scrub ok 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[219709]: 1.a scrub starts 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[219709]: 1.a scrub ok 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[219709]: 1.3 scrub starts 2023-09-23T21:23:13.245 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[219709]: 1.3 scrub ok 2023-09-23T21:23:13.246 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:13.246 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:13.253 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:23:13.270 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:13.276 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:23:13.295 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:23:13.312 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:13.314 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:23:13.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:13 smithi123 ceph-mon[193903]: 1.a scrub starts 2023-09-23T21:23:13.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:13 smithi123 ceph-mon[193903]: 1.a scrub ok 2023-09-23T21:23:13.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:13 smithi123 ceph-mon[193903]: 1.3 scrub starts 2023-09-23T21:23:13.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:13 smithi123 ceph-mon[193903]: 1.3 scrub ok 2023-09-23T21:23:13.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:13 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:13.316 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:13.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:13.333 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:23:13.351 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.353 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:23:13.418 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:13.443 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:13.485 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:13.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:13.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:13.590 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:13.621 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:13.643 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:23:14.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[211526]: from='client.37962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:23:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2614574277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:14.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[211526]: pgmap v2011: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[219709]: from='client.37962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:23:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2614574277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:14.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:14 smithi121 ceph-mon[219709]: pgmap v2011: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:14.273 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:14.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:14.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:14.377 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:14.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:14 smithi123 ceph-mon[193903]: from='client.37962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:23:14.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2614574277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:14.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:14 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:14.406 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:14 smithi123 ceph-mon[193903]: pgmap v2011: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-09-23T21:23:14.409 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:14.458 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:14.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:14.566 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:14.593 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:14.680 INFO:tasks.workunit.client.0.smithi121.stderr:Updating files: 90% (10424/11502) Updating files: 91% (10467/11502) Updating files: 92% (10582/11502) Updating files: 93% (10697/11502) Updating files: 94% (10812/11502) Updating files: 95% (10927/11502) Updating files: 96% (11042/11502) Updating files: 97% (11157/11502) Updating files: 98% (11272/11502) Updating files: 99% (11387/11502) Updating files: 100% (11502/11502) Updating files: 100% (11502/11502), done. 2023-09-23T21:23:14.786 DEBUG:teuthology.orchestra.run.smithi121:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T21:23:14.912 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:23:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:23:14] "GET /metrics HTTP/1.1" 200 34254 "" "Prometheus/2.43.0" 2023-09-23T21:23:14.918 INFO:tasks.workunit.client.0.smithi121.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T21:23:15.096 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:23:15.096 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T21:23:15.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:15 smithi121 ceph-mon[211526]: pgmap v2012: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:15.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:15 smithi121 ceph-mon[219709]: pgmap v2012: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:15.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:15 smithi123 ceph-mon[193903]: pgmap v2012: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:16.382 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:16.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:16.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:16.486 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:16.519 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:16.567 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:16.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:16.673 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:16.702 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:16.746 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:16.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:16.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:16.854 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:16.882 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:16.923 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:16.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:17.027 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:17.056 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:17.097 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:17.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:17.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:17.203 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:17.235 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:17.278 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:17.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:17.381 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:17.411 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:17.450 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:17.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:17.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:17.558 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:17.592 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:18.118 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:18.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:18.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:18.146 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":2010,"stamp":"2023-09-23T21:23:11.453445+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":703211,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":104,"num_write_kb":468,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":180,"num_bytes_recovered":1849221,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":366,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8788480,"kb_used_data":2062152,"kb_used_omap":16,"kb_used_meta":6726255,"kb_avail":647427584,"statfs":{"total":671965249536,"available":662965846016,"internally_reserved":0,"allocated":2111643648,"data_stored":2094276195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6887685993},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-16580608,"num_objects":-253,"num_object_clones":0,"num_object_copies":-759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-145,"num_objects_misplaced":-1413,"num_objects_unfound":0,"num_objects_dirty":-253,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-170377,"num_write_kb":-5460097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-35697,"num_bytes_recovered":-2065498112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.491382"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":234,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967805+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:23:02.967805+0000","last_peered":"2023-09-23T21:23:02.967805+0000","last_clean":"2023-09-23T21:23:02.967805+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:23:02.967805+0000","last_undegraded":"2023-09-23T21:23:02.967805+0000","last_fullsized":"2023-09-23T21:23:02.967805+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.026958+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:23:08.026958+0000","last_peered":"2023-09-23T21:23:08.026958+0000","last_clean":"2023-09-23T21:23:08.026958+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:23:08.026958+0000","last_undegraded":"2023-09-23T21:23:08.026958+0000","last_fullsized":"2023-09-23T21:23:08.026958+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":40,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.223532+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:23:02.223532+0000","last_peered":"2023-09-23T21:23:02.223532+0000","last_clean":"2023-09-23T21:23:02.223532+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:23:02.223532+0000","last_undegraded":"2023-09-23T21:23:02.223532+0000","last_fullsized":"2023-09-23T21:23:02.223532+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":346,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.970080+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:23:02.970080+0000","last_peered":"2023-09-23T21:23:02.970080+0000","last_clean":"2023-09-23T21:23:02.970080+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:23:02.970080+0000","last_undegraded":"2023-09-23T21:23:02.970080+0000","last_fullsized":"2023-09-23T21:23:02.970080+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":65,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.023392+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:23:08.023392+0000","last_peered":"2023-09-23T21:23:08.023392+0000","last_clean":"2023-09-23T21:23:08.023392+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:23:08.023392+0000","last_undegraded":"2023-09-23T21:23:08.023392+0000","last_fullsized":"2023-09-23T21:23:08.023392+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":164,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968134+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:23:02.968134+0000","last_peered":"2023-09-23T21:23:02.968134+0000","last_clean":"2023-09-23T21:23:02.968134+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:23:02.968134+0000","last_undegraded":"2023-09-23T21:23:02.968134+0000","last_fullsized":"2023-09-23T21:23:02.968134+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.967024+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:23:03.967024+0000","last_peered":"2023-09-23T21:23:03.967024+0000","last_clean":"2023-09-23T21:23:03.967024+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:23:03.967024+0000","last_undegraded":"2023-09-23T21:23:03.967024+0000","last_fullsized":"2023-09-23T21:23:03.967024+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.023422+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:23:08.023422+0000","last_peered":"2023-09-23T21:23:08.023422+0000","last_clean":"2023-09-23T21:23:08.023422+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:23:08.023422+0000","last_undegraded":"2023-09-23T21:23:08.023422+0000","last_fullsized":"2023-09-23T21:23:08.023422+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":57,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966741+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:23:03.966741+0000","last_peered":"2023-09-23T21:23:03.966741+0000","last_clean":"2023-09-23T21:23:03.966741+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:23:03.966741+0000","last_undegraded":"2023-09-23T21:23:03.966741+0000","last_fullsized":"2023-09-23T21:23:03.966741+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.024718+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:23:08.024718+0000","last_peered":"2023-09-23T21:23:08.024718+0000","last_clean":"2023-09-23T21:23:08.024718+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:23:08.024718+0000","last_undegraded":"2023-09-23T21:23:08.024718+0000","last_fullsized":"2023-09-23T21:23:08.024718+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":294,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969525+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:23:02.969525+0000","last_peered":"2023-09-23T21:23:02.969525+0000","last_clean":"2023-09-23T21:23:02.969525+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:23:02.969525+0000","last_undegraded":"2023-09-23T21:23:02.969525+0000","last_fullsized":"2023-09-23T21:23:02.969525+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.223605+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:23:02.223605+0000","last_peered":"2023-09-23T21:23:02.223605+0000","last_clean":"2023-09-23T21:23:02.223605+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:23:02.223605+0000","last_undegraded":"2023-09-23T21:23:02.223605+0000","last_fullsized":"2023-09-23T21:23:02.223605+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":390,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967313+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:23:02.967313+0000","last_peered":"2023-09-23T21:23:02.967313+0000","last_clean":"2023-09-23T21:23:02.967313+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:23:02.967313+0000","last_undegraded":"2023-09-23T21:23:02.967313+0000","last_fullsized":"2023-09-23T21:23:02.967313+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":617,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967820+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:23:02.967820+0000","last_peered":"2023-09-23T21:23:02.967820+0000","last_clean":"2023-09-23T21:23:02.967820+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:23:02.967820+0000","last_undegraded":"2023-09-23T21:23:02.967820+0000","last_fullsized":"2023-09-23T21:23:02.967820+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":242,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969148+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:23:02.969148+0000","last_peered":"2023-09-23T21:23:02.969148+0000","last_clean":"2023-09-23T21:23:02.969148+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:23:02.969148+0000","last_undegraded":"2023-09-23T21:23:02.969148+0000","last_fullsized":"2023-09-23T21:23:02.969148+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":65,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.025417+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:23:08.025417+0000","last_peered":"2023-09-23T21:23:08.025417+0000","last_clean":"2023-09-23T21:23:08.025417+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:23:08.025417+0000","last_undegraded":"2023-09-23T21:23:08.025417+0000","last_fullsized":"2023-09-23T21:23:08.025417+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":268,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969395+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:23:02.969395+0000","last_peered":"2023-09-23T21:23:02.969395+0000","last_clean":"2023-09-23T21:23:02.969395+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:23:02.969395+0000","last_undegraded":"2023-09-23T21:23:02.969395+0000","last_fullsized":"2023-09-23T21:23:02.969395+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969737+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:23:02.969737+0000","last_peered":"2023-09-23T21:23:02.969737+0000","last_clean":"2023-09-23T21:23:02.969737+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:23:02.969737+0000","last_undegraded":"2023-09-23T21:23:02.969737+0000","last_fullsized":"2023-09-23T21:23:02.969737+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":236,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969144+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:23:02.969144+0000","last_peered":"2023-09-23T21:23:02.969144+0000","last_clean":"2023-09-23T21:23:02.969144+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:23:02.969144+0000","last_undegraded":"2023-09-23T21:23:02.969144+0000","last_fullsized":"2023-09-23T21:23:02.969144+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":56,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.026056+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:23:08.026056+0000","last_peered":"2023-09-23T21:23:08.026056+0000","last_clean":"2023-09-23T21:23:08.026056+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:23:08.026056+0000","last_undegraded":"2023-09-23T21:23:08.026056+0000","last_fullsized":"2023-09-23T21:23:08.026056+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":755,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.966890+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:23:02.966890+0000","last_peered":"2023-09-23T21:23:02.966890+0000","last_clean":"2023-09-23T21:23:02.966890+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:23:02.966890+0000","last_undegraded":"2023-09-23T21:23:02.966890+0000","last_fullsized":"2023-09-23T21:23:02.966890+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.024698+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:23:08.024698+0000","last_peered":"2023-09-23T21:23:08.024698+0000","last_clean":"2023-09-23T21:23:08.024698+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:23:08.024698+0000","last_undegraded":"2023-09-23T21:23:08.024698+0000","last_fullsized":"2023-09-23T21:23:08.024698+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":296,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.971274+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:23:02.971274+0000","last_peered":"2023-09-23T21:23:02.971274+0000","last_clean":"2023-09-23T21:23:02.971274+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:23:02.971274+0000","last_undegraded":"2023-09-23T21:23:02.971274+0000","last_fullsized":"2023-09-23T21:23:02.971274+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966236+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:23:03.966236+0000","last_peered":"2023-09-23T21:23:03.966236+0000","last_clean":"2023-09-23T21:23:03.966236+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:23:03.966236+0000","last_undegraded":"2023-09-23T21:23:03.966236+0000","last_fullsized":"2023-09-23T21:23:03.966236+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":310,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.970167+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:23:02.970167+0000","last_peered":"2023-09-23T21:23:02.970167+0000","last_clean":"2023-09-23T21:23:02.970167+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:23:02.970167+0000","last_undegraded":"2023-09-23T21:23:02.970167+0000","last_fullsized":"2023-09-23T21:23:02.970167+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":40,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.223466+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:23:02.223466+0000","last_peered":"2023-09-23T21:23:02.223466+0000","last_clean":"2023-09-23T21:23:02.223466+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:23:02.223466+0000","last_undegraded":"2023-09-23T21:23:02.223466+0000","last_fullsized":"2023-09-23T21:23:02.223466+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":282,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969821+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:23:02.969821+0000","last_peered":"2023-09-23T21:23:02.969821+0000","last_clean":"2023-09-23T21:23:02.969821+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:23:02.969821+0000","last_undegraded":"2023-09-23T21:23:02.969821+0000","last_fullsized":"2023-09-23T21:23:02.969821+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":30,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.965677+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:23:02.965677+0000","last_peered":"2023-09-23T21:23:02.965677+0000","last_clean":"2023-09-23T21:23:02.965677+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:23:02.965677+0000","last_undegraded":"2023-09-23T21:23:02.965677+0000","last_fullsized":"2023-09-23T21:23:02.965677+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":65,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.025501+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:23:08.025501+0000","last_peered":"2023-09-23T21:23:08.025501+0000","last_clean":"2023-09-23T21:23:08.025501+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:23:08.025501+0000","last_undegraded":"2023-09-23T21:23:08.025501+0000","last_fullsized":"2023-09-23T21:23:08.025501+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":40,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.224031+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:23:02.224031+0000","last_peered":"2023-09-23T21:23:02.224031+0000","last_clean":"2023-09-23T21:23:02.224031+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:23:02.224031+0000","last_undegraded":"2023-09-23T21:23:02.224031+0000","last_fullsized":"2023-09-23T21:23:02.224031+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1085,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968890+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:23:02.968890+0000","last_peered":"2023-09-23T21:23:02.968890+0000","last_clean":"2023-09-23T21:23:02.968890+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:23:02.968890+0000","last_undegraded":"2023-09-23T21:23:02.968890+0000","last_fullsized":"2023-09-23T21:23:02.968890+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":73,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.027628+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:23:08.027628+0000","last_peered":"2023-09-23T21:23:08.027628+0000","last_clean":"2023-09-23T21:23:08.027628+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:23:08.027628+0000","last_undegraded":"2023-09-23T21:23:08.027628+0000","last_fullsized":"2023-09-23T21:23:08.027628+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":417,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967344+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:23:02.967344+0000","last_peered":"2023-09-23T21:23:02.967344+0000","last_clean":"2023-09-23T21:23:02.967344+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:23:02.967344+0000","last_undegraded":"2023-09-23T21:23:02.967344+0000","last_fullsized":"2023-09-23T21:23:02.967344+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":967,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966055+0000","last_change":"2023-09-23T21:23:02.971380+0000","last_active":"2023-09-23T21:23:03.966055+0000","last_peered":"2023-09-23T21:23:03.966055+0000","last_clean":"2023-09-23T21:23:03.966055+0000","last_became_active":"2023-09-23T21:23:02.970961+0000","last_became_peered":"2023-09-23T21:23:02.970961+0000","last_unstale":"2023-09-23T21:23:03.966055+0000","last_undegraded":"2023-09-23T21:23:03.966055+0000","last_fullsized":"2023-09-23T21:23:03.966055+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:24.685033+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":154,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.966325+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:23:02.966325+0000","last_peered":"2023-09-23T21:23:02.966325+0000","last_clean":"2023-09-23T21:23:02.966325+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:23:02.966325+0000","last_undegraded":"2023-09-23T21:23:02.966325+0000","last_fullsized":"2023-09-23T21:23:02.966325+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":667,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968356+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:23:02.968356+0000","last_peered":"2023-09-23T21:23:02.968356+0000","last_clean":"2023-09-23T21:23:02.968356+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:23:02.968356+0000","last_undegraded":"2023-09-23T21:23:02.968356+0000","last_fullsized":"2023-09-23T21:23:02.968356+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":296,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.966885+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:23:02.966885+0000","last_peered":"2023-09-23T21:23:02.966885+0000","last_clean":"2023-09-23T21:23:02.966885+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:23:02.966885+0000","last_undegraded":"2023-09-23T21:23:02.966885+0000","last_fullsized":"2023-09-23T21:23:02.966885+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":704,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967610+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:23:02.967610+0000","last_peered":"2023-09-23T21:23:02.967610+0000","last_clean":"2023-09-23T21:23:02.967610+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:23:02.967610+0000","last_undegraded":"2023-09-23T21:23:02.967610+0000","last_fullsized":"2023-09-23T21:23:02.967610+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":923,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966315+0000","last_change":"2023-09-23T21:20:49.968107+0000","last_active":"2023-09-23T21:23:03.966315+0000","last_peered":"2023-09-23T21:23:03.966315+0000","last_clean":"2023-09-23T21:23:03.966315+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:23:03.966315+0000","last_undegraded":"2023-09-23T21:23:03.966315+0000","last_fullsized":"2023-09-23T21:23:03.966315+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T20:26:58.272483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":429358,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.023305+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:23:08.023305+0000","last_peered":"2023-09-23T21:23:08.023305+0000","last_clean":"2023-09-23T21:23:08.023305+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:23:08.023305+0000","last_undegraded":"2023-09-23T21:23:08.023305+0000","last_fullsized":"2023-09-23T21:23:08.023305+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":40,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.223984+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:23:02.223984+0000","last_peered":"2023-09-23T21:23:02.223984+0000","last_clean":"2023-09-23T21:23:02.223984+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:23:02.223984+0000","last_undegraded":"2023-09-23T21:23:02.223984+0000","last_fullsized":"2023-09-23T21:23:02.223984+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":741,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968040+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:23:02.968040+0000","last_peered":"2023-09-23T21:23:02.968040+0000","last_clean":"2023-09-23T21:23:02.968040+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:23:02.968040+0000","last_undegraded":"2023-09-23T21:23:02.968040+0000","last_fullsized":"2023-09-23T21:23:02.968040+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":41,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:04.651562+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:23:04.651562+0000","last_peered":"2023-09-23T21:23:04.651562+0000","last_clean":"2023-09-23T21:23:04.651562+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:23:04.651562+0000","last_undegraded":"2023-09-23T21:23:04.651562+0000","last_fullsized":"2023-09-23T21:23:04.651562+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":47,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.967018+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:23:03.967018+0000","last_peered":"2023-09-23T21:23:03.967018+0000","last_clean":"2023-09-23T21:23:03.967018+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:23:03.967018+0000","last_undegraded":"2023-09-23T21:23:03.967018+0000","last_fullsized":"2023-09-23T21:23:03.967018+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":41,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:04.651364+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:23:04.651364+0000","last_peered":"2023-09-23T21:23:04.651364+0000","last_clean":"2023-09-23T21:23:04.651364+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:23:04.651364+0000","last_undegraded":"2023-09-23T21:23:04.651364+0000","last_fullsized":"2023-09-23T21:23:04.651364+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":936,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.015569+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:23:08.015569+0000","last_peered":"2023-09-23T21:23:08.015569+0000","last_clean":"2023-09-23T21:23:08.015569+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:23:08.015569+0000","last_undegraded":"2023-09-23T21:23:08.015569+0000","last_fullsized":"2023-09-23T21:23:08.015569+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":41,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:04.651657+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:23:04.651657+0000","last_peered":"2023-09-23T21:23:04.651657+0000","last_clean":"2023-09-23T21:23:04.651657+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:23:04.651657+0000","last_undegraded":"2023-09-23T21:23:04.651657+0000","last_fullsized":"2023-09-23T21:23:04.651657+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":572,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967669+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:23:02.967669+0000","last_peered":"2023-09-23T21:23:02.967669+0000","last_clean":"2023-09-23T21:23:02.967669+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:23:02.967669+0000","last_undegraded":"2023-09-23T21:23:02.967669+0000","last_fullsized":"2023-09-23T21:23:02.967669+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":41,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:04.651458+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:23:04.651458+0000","last_peered":"2023-09-23T21:23:04.651458+0000","last_clean":"2023-09-23T21:23:04.651458+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:23:04.651458+0000","last_undegraded":"2023-09-23T21:23:04.651458+0000","last_fullsized":"2023-09-23T21:23:04.651458+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1045,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969313+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:23:02.969313+0000","last_peered":"2023-09-23T21:23:02.969313+0000","last_clean":"2023-09-23T21:23:02.969313+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:23:02.969313+0000","last_undegraded":"2023-09-23T21:23:02.969313+0000","last_fullsized":"2023-09-23T21:23:02.969313+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":689,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966792+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:23:03.966792+0000","last_peered":"2023-09-23T21:23:03.966792+0000","last_clean":"2023-09-23T21:23:03.966792+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:23:03.966792+0000","last_undegraded":"2023-09-23T21:23:03.966792+0000","last_fullsized":"2023-09-23T21:23:03.966792+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":41,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.391613+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:23:02.391613+0000","last_peered":"2023-09-23T21:23:02.391613+0000","last_clean":"2023-09-23T21:23:02.391613+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:23:02.391613+0000","last_undegraded":"2023-09-23T21:23:02.391613+0000","last_fullsized":"2023-09-23T21:23:02.391613+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":40,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.223924+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:23:02.223924+0000","last_peered":"2023-09-23T21:23:02.223924+0000","last_clean":"2023-09-23T21:23:02.223924+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:23:02.223924+0000","last_undegraded":"2023-09-23T21:23:02.223924+0000","last_fullsized":"2023-09-23T21:23:02.223924+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":518,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.967608+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:23:03.967608+0000","last_peered":"2023-09-23T21:23:03.967608+0000","last_clean":"2023-09-23T21:23:03.967608+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:23:03.967608+0000","last_undegraded":"2023-09-23T21:23:03.967608+0000","last_fullsized":"2023-09-23T21:23:03.967608+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1046,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968923+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:23:02.968923+0000","last_peered":"2023-09-23T21:23:02.968923+0000","last_clean":"2023-09-23T21:23:02.968923+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:23:02.968923+0000","last_undegraded":"2023-09-23T21:23:02.968923+0000","last_fullsized":"2023-09-23T21:23:02.968923+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":47,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966530+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:23:03.966530+0000","last_peered":"2023-09-23T21:23:03.966530+0000","last_clean":"2023-09-23T21:23:03.966530+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:23:03.966530+0000","last_undegraded":"2023-09-23T21:23:03.966530+0000","last_fullsized":"2023-09-23T21:23:03.966530+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":30,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967600+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:23:02.967600+0000","last_peered":"2023-09-23T21:23:02.967600+0000","last_clean":"2023-09-23T21:23:02.967600+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:23:02.967600+0000","last_undegraded":"2023-09-23T21:23:02.967600+0000","last_fullsized":"2023-09-23T21:23:02.967600+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":693,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:04.651584+0000","last_change":"2023-09-23T21:21:21.571375+0000","last_active":"2023-09-23T21:23:04.651584+0000","last_peered":"2023-09-23T21:23:04.651584+0000","last_clean":"2023-09-23T21:23:04.651584+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:23:04.651584+0000","last_undegraded":"2023-09-23T21:23:04.651584+0000","last_fullsized":"2023-09-23T21:23:04.651584+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.551943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.551943+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":326,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969008+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:23:02.969008+0000","last_peered":"2023-09-23T21:23:02.969008+0000","last_clean":"2023-09-23T21:23:02.969008+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:23:02.969008+0000","last_undegraded":"2023-09-23T21:23:02.969008+0000","last_fullsized":"2023-09-23T21:23:02.969008+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":131,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.966763+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:23:02.966763+0000","last_peered":"2023-09-23T21:23:02.966763+0000","last_clean":"2023-09-23T21:23:02.966763+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:23:02.966763+0000","last_undegraded":"2023-09-23T21:23:02.966763+0000","last_fullsized":"2023-09-23T21:23:02.966763+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":40,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.224061+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:23:02.224061+0000","last_peered":"2023-09-23T21:23:02.224061+0000","last_clean":"2023-09-23T21:23:02.224061+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:23:02.224061+0000","last_undegraded":"2023-09-23T21:23:02.224061+0000","last_fullsized":"2023-09-23T21:23:02.224061+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":721,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968498+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:23:02.968498+0000","last_peered":"2023-09-23T21:23:02.968498+0000","last_clean":"2023-09-23T21:23:02.968498+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:23:02.968498+0000","last_undegraded":"2023-09-23T21:23:02.968498+0000","last_fullsized":"2023-09-23T21:23:02.968498+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":266,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.969673+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:23:02.969673+0000","last_peered":"2023-09-23T21:23:02.969673+0000","last_clean":"2023-09-23T21:23:02.969673+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:23:02.969673+0000","last_undegraded":"2023-09-23T21:23:02.969673+0000","last_fullsized":"2023-09-23T21:23:02.969673+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":41,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:04.651247+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:23:04.651247+0000","last_peered":"2023-09-23T21:23:04.651247+0000","last_clean":"2023-09-23T21:23:04.651247+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:23:04.651247+0000","last_undegraded":"2023-09-23T21:23:04.651247+0000","last_fullsized":"2023-09-23T21:23:04.651247+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":57,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966407+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:23:03.966407+0000","last_peered":"2023-09-23T21:23:03.966407+0000","last_clean":"2023-09-23T21:23:03.966407+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:23:03.966407+0000","last_undegraded":"2023-09-23T21:23:03.966407+0000","last_fullsized":"2023-09-23T21:23:03.966407+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":960,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968035+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:23:02.968035+0000","last_peered":"2023-09-23T21:23:02.968035+0000","last_clean":"2023-09-23T21:23:02.968035+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:23:02.968035+0000","last_undegraded":"2023-09-23T21:23:02.968035+0000","last_fullsized":"2023-09-23T21:23:02.968035+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":356,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.971370+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:23:02.971370+0000","last_peered":"2023-09-23T21:23:02.971370+0000","last_clean":"2023-09-23T21:23:02.971370+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:23:02.971370+0000","last_undegraded":"2023-09-23T21:23:02.971370+0000","last_fullsized":"2023-09-23T21:23:02.971370+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.224136+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:23:02.224136+0000","last_peered":"2023-09-23T21:23:02.224136+0000","last_clean":"2023-09-23T21:23:02.224136+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:23:02.224136+0000","last_undegraded":"2023-09-23T21:23:02.224136+0000","last_fullsized":"2023-09-23T21:23:02.224136+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":74,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.023343+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:23:08.023343+0000","last_peered":"2023-09-23T21:23:08.023343+0000","last_clean":"2023-09-23T21:23:08.023343+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:23:08.023343+0000","last_undegraded":"2023-09-23T21:23:08.023343+0000","last_fullsized":"2023-09-23T21:23:08.023343+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1056,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.025341+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:23:08.025341+0000","last_peered":"2023-09-23T21:23:08.025341+0000","last_clean":"2023-09-23T21:23:08.025341+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:23:08.025341+0000","last_undegraded":"2023-09-23T21:23:08.025341+0000","last_fullsized":"2023-09-23T21:23:08.025341+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":565,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.968199+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:23:02.968199+0000","last_peered":"2023-09-23T21:23:02.968199+0000","last_clean":"2023-09-23T21:23:02.968199+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:23:02.968199+0000","last_undegraded":"2023-09-23T21:23:02.968199+0000","last_fullsized":"2023-09-23T21:23:02.968199+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":44,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.224013+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:23:02.224013+0000","last_peered":"2023-09-23T21:23:02.224013+0000","last_clean":"2023-09-23T21:23:02.224013+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:23:02.224013+0000","last_undegraded":"2023-09-23T21:23:02.224013+0000","last_fullsized":"2023-09-23T21:23:02.224013+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":47,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966137+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:23:03.966137+0000","last_peered":"2023-09-23T21:23:03.966137+0000","last_clean":"2023-09-23T21:23:03.966137+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:23:03.966137+0000","last_undegraded":"2023-09-23T21:23:03.966137+0000","last_fullsized":"2023-09-23T21:23:03.966137+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":110,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967746+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:23:02.967746+0000","last_peered":"2023-09-23T21:23:02.967746+0000","last_clean":"2023-09-23T21:23:02.967746+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:23:02.967746+0000","last_undegraded":"2023-09-23T21:23:02.967746+0000","last_fullsized":"2023-09-23T21:23:02.967746+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":752,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.966865+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:23:03.966865+0000","last_peered":"2023-09-23T21:23:03.966865+0000","last_clean":"2023-09-23T21:23:03.966865+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:23:03.966865+0000","last_undegraded":"2023-09-23T21:23:03.966865+0000","last_fullsized":"2023-09-23T21:23:03.966865+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":419,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967211+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:23:02.967211+0000","last_peered":"2023-09-23T21:23:02.967211+0000","last_clean":"2023-09-23T21:23:02.967211+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:23:02.967211+0000","last_undegraded":"2023-09-23T21:23:02.967211+0000","last_fullsized":"2023-09-23T21:23:02.967211+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":262,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967485+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:23:02.967485+0000","last_peered":"2023-09-23T21:23:02.967485+0000","last_clean":"2023-09-23T21:23:02.967485+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:23:02.967485+0000","last_undegraded":"2023-09-23T21:23:02.967485+0000","last_fullsized":"2023-09-23T21:23:02.967485+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":170,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.967852+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:23:02.967852+0000","last_peered":"2023-09-23T21:23:02.967852+0000","last_clean":"2023-09-23T21:23:02.967852+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:23:02.967852+0000","last_undegraded":"2023-09-23T21:23:02.967852+0000","last_fullsized":"2023-09-23T21:23:02.967852+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.028219+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:23:08.028219+0000","last_peered":"2023-09-23T21:23:08.028219+0000","last_clean":"2023-09-23T21:23:08.028219+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:23:08.028219+0000","last_undegraded":"2023-09-23T21:23:08.028219+0000","last_fullsized":"2023-09-23T21:23:08.028219+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":74,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.026143+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:23:08.026143+0000","last_peered":"2023-09-23T21:23:08.026143+0000","last_clean":"2023-09-23T21:23:08.026143+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:23:08.026143+0000","last_undegraded":"2023-09-23T21:23:08.026143+0000","last_fullsized":"2023-09-23T21:23:08.026143+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":65,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T21:23:08.024649+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:23:08.024649+0000","last_peered":"2023-09-23T21:23:08.024649+0000","last_clean":"2023-09-23T21:23:08.024649+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:23:08.024649+0000","last_undegraded":"2023-09-23T21:23:08.024649+0000","last_fullsized":"2023-09-23T21:23:08.024649+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":649,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T21:23:03.968497+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:23:03.968497+0000","last_peered":"2023-09-23T21:23:03.968497+0000","last_clean":"2023-09-23T21:23:03.968497+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:23:03.968497+0000","last_undegraded":"2023-09-23T21:23:03.968497+0000","last_fullsized":"2023-09-23T21:23:03.968497+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":366,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T21:23:02.966849+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:23:02.966849+0000","last_peered":"2023-09-23T21:23:02.966849+0000","last_clean":"2023-09-23T21:23:02.966849+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:23:02.966849+0000","last_undegraded":"2023-09-23T21:23:02.966849+0000","last_fullsized":"2023-09-23T21:23:02.966849+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":697707,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":91,"num_read_kb":168,"num_write":43,"num_write_kb":439,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1813451,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201933,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1026136,"kb_used_data":97232,"kb_used_omap":0,"kb_used_meta":928896,"kb_avail":92719016,"statfs":{"total":95995035648,"available":94944272384,"internally_reserved":0,"allocated":99565568,"data_stored":97741861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":951189504},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":2.9470000000000001,"5min":1.9019999999999999,"15min":2.4689999999999999},"min":{"1min":0.252,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":10.522,"5min":16.468,"15min":141.33199999999999},"last":0.441},{"interface":"front","average":{"1min":2.645,"5min":2.0790000000000002,"15min":2.6160000000000001},"min":{"1min":0.308,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.6389999999999993,"5min":12.962,"15min":141.31700000000001},"last":0.53300000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2109999999999999,"15min":3.593},"min":{"1min":0.36699999999999999,"5min":0.29599999999999999,"15min":0.24099999999999999},"max":{"1min":12.112,"5min":13.054,"15min":114.16500000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.181,"5min":2.2530000000000001,"15min":3.9670000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22900000000000001},"max":{"1min":10.477,"5min":16.289000000000001,"15min":117.637},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":9.4480000000000004,"5min":7.5999999999999996,"15min":6.1239999999999997},"min":{"1min":0.39900000000000002,"5min":0.248,"15min":0.224},"max":{"1min":157.65700000000001,"5min":158.726,"15min":183.44200000000001},"last":0.495},{"interface":"front","average":{"1min":10.220000000000001,"5min":7.4779999999999998,"15min":6.3490000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":157.642,"5min":158.70400000000001,"15min":183.46199999999999},"last":0.48299999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:22:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":2.5070000000000001,"15min":4.3099999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.26700000000000002},"max":{"1min":10.547000000000001,"5min":30.667999999999999,"15min":135.66200000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.7970000000000002,"5min":2.5390000000000001,"15min":4.5810000000000004},"min":{"1min":0.32900000000000001,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":9.7919999999999998,"5min":30.798999999999999,"15min":135.47},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":3.931,"15min":2.2000000000000002},"min":{"1min":0.34499999999999997,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":10.68,"5min":31.695,"15min":31.695},"last":0.38},{"interface":"front","average":{"1min":5.7519999999999998,"5min":4.2199999999999998,"15min":2.3690000000000002},"min":{"1min":0.32700000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.840999999999999,"5min":34.994,"15min":34.994},"last":0.40799999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9279999999999999,"5min":4.6680000000000001,"15min":2.7469999999999999},"min":{"1min":0.44,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":13.906000000000001,"5min":156.30099999999999,"15min":156.30099999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":3.137,"5min":4.7089999999999996,"15min":2.7269999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32200000000000001},"max":{"1min":8.2889999999999997,"5min":156.22,"15min":156.22},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.7549999999999999,"15min":2.9529999999999998},"min":{"1min":0.433,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":11.007999999999999,"5min":23.574999999999999,"15min":143.18700000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":3.8849999999999998,"5min":3.431,"15min":2.702},"min":{"1min":0.36199999999999999,"5min":0.35899999999999999,"15min":0.29099999999999998},"max":{"1min":9.8439999999999994,"5min":21.081,"15min":142.70400000000001},"last":0.35499999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463553,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1530640,"kb_used_data":608840,"kb_used_omap":0,"kb_used_meta":921792,"kb_avail":92214512,"statfs":{"total":95995035648,"available":94427660288,"internally_reserved":0,"allocated":623452160,"data_stored":621637390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":943915008},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.978,"5min":1.764,"15min":2.6539999999999999},"min":{"1min":0.246,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":5.7779999999999996,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.315},{"interface":"front","average":{"1min":1.488,"5min":1.909,"15min":2.8410000000000002},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.223},"max":{"1min":5.3399999999999999,"5min":11.968,"15min":149.303},"last":0.45400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":2.1040000000000001,"15min":2.9940000000000002},"min":{"1min":0.25700000000000001,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":5.3810000000000002,"5min":12.013,"15min":129.37899999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":3.149,"5min":2.0859999999999999,"15min":2.8250000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.6399999999999997,"5min":14.366,"15min":129.40100000000001},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":7.0149999999999997,"15min":7.0570000000000004},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":54.561,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":5.3129999999999997,"5min":7.351,"15min":7.2460000000000004},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":54.603999999999999,"5min":199.27000000000001,"15min":199.27000000000001},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.1920000000000002,"15min":3.641},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":7.6269999999999998,"5min":14.411,"15min":137.875},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.927,"15min":3.476},"min":{"1min":0.315,"5min":0.315,"15min":0.29799999999999999},"max":{"1min":5.8040000000000003,"5min":12.35,"15min":137.60900000000001},"last":0.441}]},{"osd":4,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":3.5609999999999999,"15min":2.6240000000000001},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.22600000000000001},"max":{"1min":12.218999999999999,"5min":20.408999999999999,"15min":55.494},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.972,"5min":3.528,"15min":2.516},"min":{"1min":0.255,"5min":0.255,"15min":0.23999999999999999},"max":{"1min":13.308999999999999,"5min":14.622,"15min":55.305},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.6739999999999999,"15min":2.8799999999999999},"min":{"1min":0.317,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":6.9100000000000001,"5min":12.051,"15min":222.90899999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":5.6890000000000001,"5min":3.3500000000000001,"15min":3.0619999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":15.478,"5min":17.927,"15min":224.655},"last":0.32800000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.7429999999999999,"15min":2.379},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.28000000000000003},"max":{"1min":12.380000000000001,"5min":12.380000000000001,"15min":165.96799999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":4.1890000000000001,"5min":3.6179999999999999,"15min":2.7690000000000001},"min":{"1min":0.40600000000000003,"5min":0.307,"15min":0.251},"max":{"1min":11.526999999999999,"5min":19.815999999999999,"15min":165.82400000000001},"last":0.432}]}]},{"osd":1,"up_from":276,"seq":1185410974000,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665716,"kb_used_data":8300,"kb_used_omap":5,"kb_used_meta":657402,"kb_avail":93079436,"statfs":{"total":95995035648,"available":95313342464,"internally_reserved":0,"allocated":8499200,"data_stored":5358978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":673180387},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.39200000000000002,"15min":1.944},"min":{"1min":0.214,"5min":0.214,"15min":0.19400000000000001},"max":{"1min":2.3410000000000002,"5min":2.3410000000000002,"15min":108.47},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.339,"5min":0.59799999999999998,"15min":1.952},"min":{"1min":0.23899999999999999,"5min":0.23499999999999999,"15min":0.19600000000000001},"max":{"1min":6.0149999999999997,"5min":6.0149999999999997,"15min":109.532},"last":0.371}]},{"osd":2,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":13.051,"5min":6.5170000000000003,"15min":8.6799999999999997},"min":{"1min":0.23200000000000001,"5min":0.22800000000000001,"15min":0.20200000000000001},"max":{"1min":130.78899999999999,"5min":130.78899999999999,"15min":171.35400000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":12.999000000000001,"5min":6.5540000000000003,"15min":8.3290000000000006},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":133.751,"5min":133.751,"15min":170.69900000000001},"last":0.35299999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.60699999999999998,"15min":3.5550000000000002},"min":{"1min":0.27000000000000002,"5min":0.254,"15min":0.23799999999999999},"max":{"1min":3.2450000000000001,"5min":17.247,"15min":99.120999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.76200000000000001,"15min":4.0960000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":3.754,"5min":10.949999999999999,"15min":150.38},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":1.829,"15min":2.371},"min":{"1min":0.27400000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":6.9089999999999998,"5min":11.034000000000001,"15min":174.84399999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.3170000000000002,"5min":1.5,"15min":2.2749999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":8.5109999999999992,"5min":11.057,"15min":174.82400000000001},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":2.6440000000000001,"15min":2.7730000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.31900000000000001},"max":{"1min":8.6329999999999991,"5min":107.98,"15min":110.955},"last":0.496},{"interface":"front","average":{"1min":3.0750000000000002,"5min":3.169,"15min":2.8999999999999999},"min":{"1min":0.36699999999999999,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":6.9669999999999996,"5min":107.961,"15min":110.89},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":3.6160000000000001,"15min":2.5960000000000001},"min":{"1min":0.42199999999999999,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":20.265999999999998,"5min":119.999,"15min":119.999},"last":0.57799999999999996},{"interface":"front","average":{"1min":3.5409999999999999,"5min":3.5710000000000002,"15min":2.4980000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31900000000000001},"max":{"1min":20.361999999999998,"5min":122.328,"15min":122.328},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5979999999999999,"5min":2.1389999999999998,"15min":3.0009999999999999},"min":{"1min":0.39800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.0430000000000001,"5min":10.648999999999999,"15min":285.47500000000002},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.726,"15min":2.754},"min":{"1min":0.46999999999999997,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":5.165,"5min":10.664999999999999,"15min":285.48599999999999},"last":0.55000000000000004}]}]},{"osd":0,"up_from":0,"seq":395136991740,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781631,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1380632,"kb_used_data":38800,"kb_used_omap":7,"kb_used_meta":1341816,"kb_avail":92364520,"statfs":{"total":95995035648,"available":94581268480,"internally_reserved":0,"allocated":39731200,"data_stored":36496749,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1374019665},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":21,"apply_latency_ms":21,"commit_latency_ns":21000000,"apply_latency_ns":21000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.68899999999999995,"15min":1.9530000000000001},"min":{"1min":0.189,"5min":0.17899999999999999,"15min":0.17699999999999999},"max":{"1min":2.8700000000000001,"5min":5.0010000000000003,"15min":103.021},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.875,"5min":0.73699999999999999,"15min":2.54},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":4.5510000000000002,"5min":4.9820000000000002,"15min":103.154},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.749,"15min":2.6970000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":3.2829999999999999,"5min":5.2670000000000003,"15min":143.482},"last":0.313},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.71499999999999997,"15min":2.7320000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":3.27,"5min":5.4820000000000002,"15min":143.61000000000001},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.79600000000000004,"15min":3.5499999999999998},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.798,"5min":5.2939999999999996,"15min":143.50800000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.77600000000000002,"15min":3.77},"min":{"1min":0.34100000000000003,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.3290000000000002,"5min":5.6790000000000003,"15min":143.27799999999999},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":2.0310000000000001,"15min":3.4900000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":5.6680000000000001,"5min":7.4880000000000004,"15min":225.80799999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.972,"5min":1.9419999999999999,"15min":4.0750000000000002},"min":{"1min":0.39300000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.6840000000000002,"5min":7.5010000000000003,"15min":225.79300000000001},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.51,"15min":3.5529999999999999},"min":{"1min":0.32700000000000001,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":7.048,"5min":9.0579999999999998,"15min":220.63900000000001},"last":0.434},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.6060000000000001,"15min":3.6269999999999998},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":8.0579999999999998,"5min":8.0579999999999998,"15min":220.31999999999999},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.5150000000000001,"15min":3.5049999999999999},"min":{"1min":0.5,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":10.148,"5min":42.363,"15min":210.74700000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.6749999999999998,"5min":2.4420000000000002,"15min":2.8319999999999999},"min":{"1min":0.57099999999999995,"5min":0.308,"15min":0.308},"max":{"1min":10.634,"5min":42.383000000000003,"15min":210.72900000000001},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":1.8660000000000001,"15min":2.0350000000000001},"min":{"1min":0.50900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.9500000000000002,"5min":7.9500000000000002,"15min":25.260000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.1099999999999999,"5min":1.796,"15min":2.4319999999999999},"min":{"1min":0.39800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.1079999999999997,"5min":7.5289999999999999,"15min":16.875},"last":0.52600000000000002}]}]},{"osd":3,"up_from":148,"seq":635655160225,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896216,"kb_used_data":9488,"kb_used_omap":2,"kb_used_meta":886717,"kb_avail":92848936,"statfs":{"total":95995035648,"available":95077310464,"internally_reserved":0,"allocated":9715712,"data_stored":6007537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":907998261},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.52700000000000002,"15min":1.633},"min":{"1min":0.23400000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":2.2909999999999999,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.1679999999999999,"5min":0.65000000000000002,"15min":1.51},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.19500000000000001},"max":{"1min":5.0720000000000001,"5min":5.0720000000000001,"15min":18.277999999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 23 21:22:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.70299999999999996,"15min":6.5880000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":8.3079999999999998,"5min":8.3079999999999998,"15min":237.072},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.67000000000000004,"15min":6.4690000000000003},"min":{"1min":0.24299999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":234.21899999999999},"last":0.42899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":3.907,"15min":5.4729999999999999},"min":{"1min":0.34699999999999998,"5min":0.219,"15min":0.215},"max":{"1min":5.4699999999999998,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.3919999999999999,"5min":3.915,"15min":5.1589999999999998},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":5.6050000000000004,"5min":208.024,"15min":208.024},"last":0.48299999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":3.3359999999999999,"15min":2.532},"min":{"1min":0.42299999999999999,"5min":0.27900000000000003,"15min":0.27600000000000002},"max":{"1min":6.4370000000000003,"5min":117.717,"15min":117.717},"last":0.505},{"interface":"front","average":{"1min":2.8820000000000001,"5min":3.7730000000000001,"15min":2.597},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":6.9870000000000001,"5min":117.771,"15min":117.771},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":2.9129999999999998,"15min":2.8769999999999998},"min":{"1min":0.41099999999999998,"5min":0.35399999999999998,"15min":0.27200000000000002},"max":{"1min":7.7119999999999997,"5min":100.277,"15min":123.458},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.7000000000000002,"15min":2.7639999999999998},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":6.4740000000000002,"5min":100.69,"15min":123.292},"last":0.622}]},{"osd":6,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":4.0679999999999996,"15min":4.8070000000000004},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":6.3890000000000002,"5min":130.613,"15min":304.22399999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.3700000000000001,"15min":4.8840000000000003},"min":{"1min":0.53600000000000003,"5min":0.33500000000000002,"15min":0.33100000000000002},"max":{"1min":5.5099999999999998,"5min":130.50999999999999,"15min":304.15800000000002},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":1.927,"15min":2.2240000000000002},"min":{"1min":0.497,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":8.0440000000000005,"5min":19.298999999999999,"15min":44.253},"last":0.63900000000000001},{"interface":"front","average":{"1min":2.6080000000000001,"5min":2.4380000000000002,"15min":2.6099999999999999},"min":{"1min":0.52700000000000002,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":10.441000000000001,"5min":13.343,"15min":45.706000000000003},"last":0.68200000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986843,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1132444,"kb_used_data":151636,"kb_used_omap":0,"kb_used_meta":980800,"kb_avail":92612708,"statfs":{"total":95995035648,"available":94835412992,"internally_reserved":0,"allocated":155275264,"data_stored":153448258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1004339200},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.173,"15min":3.0070000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":12.195,"5min":12.195,"15min":226.751},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.161,"5min":1.5640000000000001,"15min":2.7040000000000002},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":8.641,"5min":12.754,"15min":226.43299999999999},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.802,"15min":3.0470000000000002},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":16.434999999999999,"5min":16.434999999999999,"15min":119.55800000000001},"last":0.497},{"interface":"front","average":{"1min":3.3610000000000002,"5min":2.2730000000000001,"15min":3.0920000000000001},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.108000000000001,"5min":12.108000000000001,"15min":116.78100000000001},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":11.018000000000001,"5min":8.4890000000000008,"15min":6.7539999999999996},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.248},"max":{"1min":124.70099999999999,"5min":124.70099999999999,"15min":210.50299999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":7.9050000000000002,"5min":7.6079999999999997,"15min":6.6630000000000003},"min":{"1min":0.46000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":116.47,"5min":116.47,"15min":210.59},"last":0.51000000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":2.6520000000000001,"15min":4.5990000000000002},"min":{"1min":0.314,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":12.973000000000001,"5min":14.323,"15min":214.64099999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.665,"5min":2.5110000000000001,"15min":4.9649999999999999},"min":{"1min":0.38300000000000001,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":13.673999999999999,"5min":13.673999999999999,"15min":214.54900000000001},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 21:22:43 2023","interfaces":[{"interface":"back","average":{"1min":3.149,"5min":3.4140000000000001,"15min":2.004},"min":{"1min":0.34799999999999998,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":13.315,"5min":17.879999999999999,"15min":34.497999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":3.581,"5min":3.234,"15min":2.2450000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.23799999999999999},"max":{"1min":8.7249999999999996,"5min":19.425000000000001,"15min":34.883000000000003},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":3.681,"5min":4.1980000000000004,"15min":2.3980000000000001},"min":{"1min":0.36199999999999999,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":15.502000000000001,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.32900000000000001},{"interface":"front","average":{"1min":5.4560000000000004,"5min":5.3390000000000004,"15min":2.726},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":18.762,"5min":67.596999999999994,"15min":67.596999999999994},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:22:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6980000000000004,"5min":3.645,"15min":2.222},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.28100000000000003},"max":{"1min":16.998999999999999,"5min":17.817,"15min":18.352},"last":0.53200000000000003},{"interface":"front","average":{"1min":4.2640000000000002,"5min":3.4980000000000002,"15min":2.0760000000000001},"min":{"1min":0.36399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":16.664999999999999,"5min":30.117000000000001,"15min":30.117000000000001},"last":0.38200000000000001}]}]},{"osd":5,"up_from":39,"seq":167503725207,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2156696,"kb_used_data":1147856,"kb_used_omap":0,"kb_used_meta":1008832,"kb_avail":91588456,"statfs":{"total":95995035648,"available":93786578944,"internally_reserved":0,"allocated":1175404544,"data_stored":1173585422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1033043968},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.6899999999999999,"15min":2.0049999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":7.5119999999999996,"5min":12.558999999999999,"15min":82.280000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.194,"5min":1.8320000000000001,"15min":2.181},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":6.0430000000000001,"5min":9.407,"15min":85.236000000000004},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.1440000000000001,"15min":3.2610000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.9240000000000004,"5min":18.859999999999999,"15min":125.23399999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.754,"5min":1.8149999999999999,"15min":2.9039999999999999},"min":{"1min":0.32200000000000001,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":12.164,"5min":12.654999999999999,"15min":125.249},"last":0.28599999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":5.569,"5min":5.2720000000000002,"15min":4.665},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":75.441999999999993,"5min":160.749,"15min":231.34700000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":6.3460000000000001,"5min":5.3890000000000002,"15min":4.8200000000000003},"min":{"1min":0.35999999999999999,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":75.247,"5min":161.18000000000001,"15min":231.381},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":2.9649999999999999,"5min":2.278,"15min":4.0350000000000001},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":8.0980000000000008,"5min":18.826000000000001,"15min":263.887},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.7989999999999999,"15min":3.6779999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":6.1950000000000003,"5min":12.683999999999999,"15min":263.70100000000002},"last":0.55700000000000005}]},{"osd":4,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7229999999999999,"5min":3.4830000000000001,"15min":1.974},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":12.212,"5min":77.804000000000002,"15min":77.804000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":6.274,"5min":4.8659999999999997,"15min":2.516},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":19.718,"5min":77.825999999999993,"15min":77.825999999999993},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":11.112,"5min":4.4279999999999999,"15min":2.617},"min":{"1min":0.40000000000000002,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":68.346000000000004,"5min":68.346000000000004,"15min":83.451999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":11.348000000000001,"5min":4.758,"15min":2.907},"min":{"1min":0.40600000000000003,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":72.393000000000001,"5min":72.393000000000001,"15min":83.484999999999999},"last":0.39900000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:22:45 2023","interfaces":[{"interface":"back","average":{"1min":5.2160000000000002,"5min":3.0550000000000002,"15min":2.0569999999999999},"min":{"1min":0.60499999999999998,"5min":0.36299999999999999,"15min":0.30099999999999999},"max":{"1min":12.692,"5min":12.692,"15min":12.692},"last":0.44800000000000001},{"interface":"front","average":{"1min":4.0270000000000001,"5min":2.5819999999999999,"15min":1.6359999999999999},"min":{"1min":0.48799999999999999,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.236000000000001,"5min":18.257999999999999,"15min":18.257999999999999},"last":0.58099999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:23:18.223 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:17 smithi123 ceph-mon[193903]: pgmap v2013: 83 pgs: 83 active+clean; 582 KiB data, 8.2 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:18.240 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:18.252 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:18.296 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:18.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:18.352 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:17 smithi121 ceph-mon[211526]: pgmap v2013: 83 pgs: 83 active+clean; 582 KiB data, 8.2 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:18.352 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:17 smithi121 ceph-mon[219709]: pgmap v2013: 83 pgs: 83 active+clean; 582 KiB data, 8.2 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:18.395 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T21:23:18.403 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:18.431 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:18.592 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T21:23:18.631 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:23:18.632 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:23:18.632 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T21:23:18.674 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:23:18.674 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:23:18.674 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:23:18.674 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:23:18.674 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.674 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62391 13.10 13:0f3317a3:::benchmark_data_smithi121_621349_object62390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.786851+0000", 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.883241371, 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1861043069999999, 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62391 2023-09-23T21:23:18.675 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.786851+0000", 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.786851+0000", 2023-09-23T21:23:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.786851+0000", 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.786879+0000", 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7849999999999999e-05 2023-09-23T21:23:18.677 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.678 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.786886+0000", 2023-09-23T21:23:18.678 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7400000000000004e-06 2023-09-23T21:23:18.678 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.678 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.678 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.786892+0000", 2023-09-23T21:23:18.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.3469999999999998e-06 2023-09-23T21:23:18.679 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.679 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.679 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.679 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.790263+0000", 2023-09-23T21:23:18.679 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033711549999999998 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.790289+0000", 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6095999999999999e-05 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.680 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.792959+0000", 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0026694350000001 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.792970+0000", 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1911e-05 2023-09-23T21:23:18.681 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.970576+0000", 2023-09-23T21:23:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.177605089 2023-09-23T21:23:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.972956+0000", 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0023801849999999999 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.683 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62399 13.10 13:0998abad:::benchmark_data_smithi121_621349_object62398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.788462+0000", 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.88163083500001, 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2139024249999997, 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62399 2023-09-23T21:23:18.684 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.788462+0000", 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.788462+0000", 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.685 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.788461+0000", 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.788501+0000", 2023-09-23T21:23:18.686 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0321999999999998e-05 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.788513+0000", 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.15e-05 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.788520+0000", 2023-09-23T21:23:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.537e-06 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.790324+0000", 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001803834 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.790332+0000", 2023-09-23T21:23:18.688 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2549999999999997e-06 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793004+0000", 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.002672051 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793009+0000", 2023-09-23T21:23:18.689 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0069999999999997e-06 2023-09-23T21:23:18.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.690 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.690 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.980734+0000", 2023-09-23T21:23:18.690 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1877247870000001 2023-09-23T21:23:18.690 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.690 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for missing object", 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.984199+0000", 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0034650380000000001 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.002280+0000", 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.018081264 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.691 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.692 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.692 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.002364+0000", 2023-09-23T21:23:18.693 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4039e-05 2023-09-23T21:23:18.693 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.693 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.693 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62427 13.10 13:0e77fbc2:::benchmark_data_smithi121_621349_object62426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.807643+0000", 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.86245005000001, 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1954459740000001, 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.694 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62427 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807643+0000", 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807643+0000", 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807641+0000", 2023-09-23T21:23:18.696 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807669+0000", 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7712000000000002e-05 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807685+0000", 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.6319000000000001e-05 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.697 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807693+0000", 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4970000000000004e-06 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807745+0000", 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2642000000000002e-05 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.698 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.807753+0000", 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0679999999999997e-06 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793034+0000", 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98528059700000004 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.699 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793040+0000", 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7039999999999999e-06 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.984495+0000", 2023-09-23T21:23:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.191455258 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.987477+0000", 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029826430000000001 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.991963+0000", 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0044851500000000002 2023-09-23T21:23:18.701 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:11.996905+0000", 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049426410000000002 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.002400+0000", 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0054948829999999999 2023-09-23T21:23:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.002435+0000", 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4802999999999999e-05 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.002485+0000", 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9990000000000001e-05 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.703 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.003089+0000", 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000603777 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62445 13.10 13:0f3c11c9:::benchmark_data_smithi121_621349_object62444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.822705+0000", 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.84738811800003, 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3017362630000004, 2023-09-23T21:23:18.704 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62445 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822705+0000", 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822705+0000", 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822703+0000", 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.706 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822724+0000", 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.1569999999999998e-05 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822737+0000", 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2651e-05 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.707 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822745+0000", 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8609999999999998e-06 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822800+0000", 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4738999999999999e-05 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.822811+0000", 2023-09-23T21:23:18.708 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1291000000000001e-05 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793061+0000", 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.97024972200000004 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793065+0000", 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0280000000000002e-06 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.709 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.003121+0000", 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.210056383 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.003217+0000", 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5686999999999994e-05 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.710 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.003341+0000", 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000123731 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.123376+0000", 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.12003517900000001 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.123407+0000", 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1378999999999999e-05 2023-09-23T21:23:18.711 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.123416+0000", 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8680000000000005e-06 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.123458+0000", 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2134e-05 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.712 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.124441+0000", 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00098283399999999992 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62454 13.10 13:0d2add49:::benchmark_data_smithi121_621349_object62453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.826231+0000", 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.843861903, 2023-09-23T21:23:18.713 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2999532739999999, 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62454 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826231+0000", 2023-09-23T21:23:18.714 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826231+0000", 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826230+0000", 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.715 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826251+0000", 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0665999999999998e-05 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826282+0000", 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.1767999999999998e-05 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.716 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826290+0000", 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2269999999999996e-06 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826348+0000", 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8349e-05 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.717 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.826357+0000", 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4749999999999999e-06 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793089+0000", 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96673151300000004 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.718 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793097+0000", 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7370000000000006e-06 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.122251+0000", 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3291544929999999 2023-09-23T21:23:18.719 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.122421+0000", 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016989800000000001 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.122536+0000", 2023-09-23T21:23:18.720 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000115207 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.125481+0000", 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029442499999999998 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.721 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.125490+0000", 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7990000000000005e-06 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.125517+0000", 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6767999999999999e-05 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.722 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.125550+0000", 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2425000000000002e-05 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.126184+0000", 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00063456999999999999 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.723 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62455 13.10 13:0a947c3c:::benchmark_data_smithi121_621349_object62454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.827389+0000", 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.842704002, 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3006062390000004, 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.724 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62455 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827389+0000", 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827389+0000", 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.725 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827387+0000", 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827406+0000", 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9035999999999999e-05 2023-09-23T21:23:18.726 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827420+0000", 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3789e-05 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827428+0000", 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5750000000000003e-06 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.727 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827467+0000", 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9369000000000002e-05 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.827474+0000", 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5599999999999999e-06 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793120+0000", 2023-09-23T21:23:18.728 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.96564634299999996 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793129+0000", 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8210000000000008e-06 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.124477+0000", 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.331347799 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.729 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.124586+0000", 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000109789 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.124698+0000", 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011127400000000001 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.730 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.127182+0000", 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0024842409999999999 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.127215+0000", 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3175e-05 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.127223+0000", 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2369999999999992e-06 2023-09-23T21:23:18.731 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.127256+0000", 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3016999999999999e-05 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.127995+0000", 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00073860500000000003 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.732 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62470 13.10 13:0d783943:::benchmark_data_smithi121_621349_object62469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.837346+0000", 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.83274636499999, 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2923846259999996, 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62470 2023-09-23T21:23:18.733 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837346+0000", 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837346+0000", 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.734 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837345+0000", 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837367+0000", 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2396999999999999e-05 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837384+0000", 2023-09-23T21:23:18.735 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.7011e-05 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837392+0000", 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5789999999999998e-06 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837435+0000", 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2564000000000001e-05 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.736 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.837443+0000", 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7199999999999995e-06 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793153+0000", 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.95570954500000005 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793161+0000", 2023-09-23T21:23:18.737 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7829999999999999e-06 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.126217+0000", 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3330562160000001 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.126331+0000", 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00011412 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.738 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.126439+0000", 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000107613 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.128975+0000", 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025362029999999999 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.128984+0000", 2023-09-23T21:23:18.739 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.081e-06 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.129012+0000", 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8129000000000001e-05 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.129046+0000", 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.4062e-05 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.740 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.129731+0000", 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00068495399999999999 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62494 13.10 13:0803ec8f:::benchmark_data_smithi121_621349_object62493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.853438+0000", 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.816655126, 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2784656080000003, 2023-09-23T21:23:18.741 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62494 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853438+0000", 2023-09-23T21:23:18.742 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853438+0000", 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853437+0000", 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.743 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853452+0000", 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5315000000000002e-05 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853461+0000", 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5429999999999999e-06 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.744 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853467+0000", 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2299999999999999e-06 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853523+0000", 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6623999999999999e-05 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.853535+0000", 2023-09-23T21:23:18.745 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1858000000000001e-05 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793186+0000", 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.939650651 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793194+0000", 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.2339999999999994e-06 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.746 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.128034+0000", 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.334839536 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.128139+0000", 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000105479 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.747 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.128246+0000", 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000106672 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.130909+0000", 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026635019999999999 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.130977+0000", 2023-09-23T21:23:18.748 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7717000000000002e-05 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.130987+0000", 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0232e-05 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.131054+0000", 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.7297000000000003e-05 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.749 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.131903+0000", 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00084878299999999998 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62502 13.10 13:0d7384fc:::benchmark_data_smithi121_621349_object62501:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.857552+0000", 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.81254102499997, 2023-09-23T21:23:18.750 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2762725159999997, 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62502 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.751 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857552+0000", 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857552+0000", 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857550+0000", 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.752 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857569+0000", 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8686999999999999e-05 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857577+0000", 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3809999999999999e-06 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.753 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857583+0000", 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.2920000000000003e-06 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857622+0000", 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9530000000000003e-05 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.857630+0000", 2023-09-23T21:23:18.754 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.6040000000000004e-06 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793222+0000", 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.93559217400000005 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793233+0000", 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1008e-05 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.755 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.129770+0000", 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3365372959999999 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.129900+0000", 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012967499999999999 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.756 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.130041+0000", 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000140653 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.133022+0000", 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029813349999999999 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.133036+0000", 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4331999999999999e-05 2023-09-23T21:23:18.757 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.133059+0000", 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2719000000000001e-05 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.133092+0000", 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3011e-05 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.758 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.133824+0000", 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00073216899999999996 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62553 13.10 13:0c8fcf57:::benchmark_data_smithi121_621349_object62552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.904069+0000", 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.76602383099998, 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2319858640000003, 2023-09-23T21:23:18.759 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62553 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904069+0000", 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.760 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904069+0000", 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904068+0000", 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.761 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904093+0000", 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5262e-05 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904112+0000", 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9559e-05 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904121+0000", 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.49e-06 2023-09-23T21:23:18.762 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904186+0000", 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4647000000000006e-05 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.904199+0000", 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3324e-05 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.763 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793276+0000", 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.889077483 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793286+0000", 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.5629999999999998e-06 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.131933+0000", 2023-09-23T21:23:18.764 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3386472010000001 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.132048+0000", 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000114726 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.132172+0000", 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000123867 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.765 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.135130+0000", 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0029578320000000001 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.135185+0000", 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5047999999999999e-05 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.766 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.135198+0000", 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.348e-05 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.135263+0000", 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4938999999999996e-05 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.136055+0000", 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00079179700000000001 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.767 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62630 13.10 13:0871123c:::benchmark_data_smithi121_621349_object62629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.962871+0000", 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.70722197100002, 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.175211526, 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.768 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62630 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962871+0000", 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962871+0000", 2023-09-23T21:23:18.769 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962870+0000", 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962894+0000", 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4737999999999999e-05 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.770 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962906+0000", 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2038000000000001e-05 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962914+0000", 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3629999999999996e-06 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962958+0000", 2023-09-23T21:23:18.771 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4675000000000002e-05 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.962973+0000", 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4263999999999999e-05 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793313+0000", 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.83033993900000003 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.772 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793320+0000", 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8290000000000008e-06 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.133874+0000", 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3405530959999998 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.134048+0000", 2023-09-23T21:23:18.773 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00017461500000000001 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.134254+0000", 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020538699999999999 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.137254+0000", 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0030004459999999999 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.774 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.137311+0000", 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6580999999999999e-05 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.137368+0000", 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7167999999999999e-05 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.775 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.137428+0000", 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.0705999999999998e-05 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.138082+0000", 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00065386899999999996 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.776 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62645 13.10 13:0e7e57ac:::benchmark_data_smithi121_621349_object62644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.978252+0000", 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.69184111800001, 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1616705820000002, 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62645 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.777 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978252+0000", 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978252+0000", 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978250+0000", 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.778 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978308+0000", 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8054999999999999e-05 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978321+0000", 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2903e-05 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.779 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978327+0000", 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1519999999999997e-06 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978371+0000", 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3446999999999999e-05 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.780 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.978381+0000", 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0343000000000001e-05 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793346+0000", 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.81496463299999999 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793354+0000", 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.276e-06 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.781 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.136092+0000", 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3427379909999999 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.136251+0000", 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015940400000000001 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.782 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.136447+0000", 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000195458 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.139133+0000", 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.002686234 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.139147+0000", 2023-09-23T21:23:18.783 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3597999999999999e-05 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.139172+0000", 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5187000000000001e-05 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.139210+0000", 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7598000000000002e-05 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.784 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.139922+0000", 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000712698 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62649 13.10 13:0c668d39:::benchmark_data_smithi121_621349_object62648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:07.981309+0000", 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.68878344699999, 2023-09-23T21:23:18.785 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1607454910000001, 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62649 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981309+0000", 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.786 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981309+0000", 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981307+0000", 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.787 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981334+0000", 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7044000000000001e-05 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981355+0000", 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0352000000000001e-05 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981365+0000", 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0547000000000001e-05 2023-09-23T21:23:18.788 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981430+0000", 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5102000000000001e-05 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:07.981443+0000", 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2686e-05 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.789 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793380+0000", 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.81193705800000004 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793388+0000", 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4499999999999998e-06 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.138126+0000", 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3447382230000002 2023-09-23T21:23:18.790 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.138250+0000", 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012382300000000001 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.138384+0000", 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013430699999999999 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.791 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.141188+0000", 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028043299999999998 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.141244+0000", 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5788999999999997e-05 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.792 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.141257+0000", 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3329000000000001e-05 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.141323+0000", 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5388999999999993e-05 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.142055+0000", 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00073204800000000001 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.793 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62701 13.10 13:09b6ec41:::benchmark_data_smithi121_621349_object62700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:08.028259+0000", 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.64183363900003, 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1164232759999999, 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.794 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62701 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028259+0000", 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.795 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028259+0000", 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028258+0000", 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028319+0000", 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1618999999999994e-05 2023-09-23T21:23:18.796 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028333+0000", 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3854e-05 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028340+0000", 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4459999999999998e-06 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.797 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028394+0000", 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.4163000000000003e-05 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.028406+0000", 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1596999999999999e-05 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793406+0000", 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.76500098100000002 2023-09-23T21:23:18.798 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793415+0000", 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0239999999999997e-06 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.139974+0000", 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3465592210000001 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.799 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.140143+0000", 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00016895500000000001 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.140349+0000", 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00020594999999999999 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.143475+0000", 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.003126134 2023-09-23T21:23:18.800 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.143491+0000", 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5880999999999999e-05 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.143576+0000", 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5519000000000006e-05 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.801 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.143654+0000", 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.7974999999999998e-05 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.144682+0000", 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0010282609999999999 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.802 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62731 13.10 13:09e6e515:::benchmark_data_smithi121_621349_object62730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:08.066680+0000", 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.60341259699999, 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0832063239999998, 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62731 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.803 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066680+0000", 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066680+0000", 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.804 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066679+0000", 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066723+0000", 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3897999999999999e-05 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066732+0000", 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.5089999999999993e-06 2023-09-23T21:23:18.805 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066740+0000", 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8900000000000007e-06 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066790+0000", 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9873000000000002e-05 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.806 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.066800+0000", 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0135e-05 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793440+0000", 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.72664024000000005 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793444+0000", 2023-09-23T21:23:18.807 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3080000000000001e-06 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.142088+0000", 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3486439049999999 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.142206+0000", 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000117874 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.808 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.142397+0000", 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019104199999999999 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.145750+0000", 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0033526519999999998 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.146686+0000", 2023-09-23T21:23:18.809 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00093644799999999999 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.148625+0000", 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001938768 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.148677+0000", 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1505999999999998e-05 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.810 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.149887+0000", 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001209987 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47003.0:62781 13.10 13:0d4c5084:::benchmark_data_smithi121_621349_object62780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:15:08.185481+0000", 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 490.484611422, 2023-09-23T21:23:18.811 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.9660198599999998, 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47003", 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 62781 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185481+0000", 2023-09-23T21:23:18.812 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185481+0000", 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185480+0000", 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.813 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185500+0000", 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.014e-05 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185510+0000", 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9939999999999995e-06 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185516+0000", 2023-09-23T21:23:18.814 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.6110000000000001e-06 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185560+0000", 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4459999999999998e-05 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.185568+0000", 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.525e-06 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.815 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793458+0000", 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.60789037800000001 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:08.793462+0000", 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0080000000000004e-06 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.144743+0000", 2023-09-23T21:23:18.816 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3512810100000001 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.144884+0000", 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00014055599999999999 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 5,7", 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.144991+0000", 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00010689 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.817 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.147753+0000", 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027629099999999999 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.148590+0000", 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00083647099999999998 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.818 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.150470+0000", 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001880278 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.150518+0000", 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7837999999999998e-05 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:15:12.151501+0000", 2023-09-23T21:23:18.819 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00098314800000000009 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47438.0:82773 14.b 14:db203776:::benchmark_data_smithi121_684892_object82772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:19:49.029471+0000", 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 209.64062224200001, 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.218842258, 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.820 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47438", 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 82773 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029471+0000", 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.821 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029471+0000", 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029469+0000", 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999981 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029498+0000", 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8887e-05 2023-09-23T21:23:18.822 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029521+0000", 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.3774999999999999e-05 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029532+0000", 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0662999999999999e-05 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.823 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029584+0000", 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1941000000000003e-05 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.029595+0000", 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1099e-05 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.824 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.928446+0000", 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.89885097899999999 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:49.928453+0000", 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.0469999999999996e-06 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.219101+0000", 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.290647358 2023-09-23T21:23:18.825 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.219316+0000", 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000215715 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.219549+0000", 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000232809 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.826 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.232456+0000", 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.012907167000000001 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.237346+0000", 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0048893900000000004 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.827 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.243222+0000", 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0058759540000000001 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.248132+0000", 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049108939999999998 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.248183+0000", 2023-09-23T21:23:18.828 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0504000000000003e-05 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:19:51.248313+0000", 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00012984499999999999 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47438.0:100267 14.8 14:1323d651:::benchmark_data_smithi121_684892_object14950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:23:18.829 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:30.471023+0000", 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 168.199069493, 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0101295050000001, 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47438", 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 100267 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.830 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.471023+0000", 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.471023+0000", 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.471022+0000", 2023-09-23T21:23:18.831 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.471027+0000", 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6500000000000004e-06 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.471037+0000", 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.7869999999999996e-06 2023-09-23T21:23:18.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.471044+0000", 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.357e-06 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.494028+0000", 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022983685 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.833 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.494032+0000", 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.941e-06 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.485418+0000", 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.99138636000000002 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.485422+0000", 2023-09-23T21:23:18.834 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5210000000000004e-06 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.408212+0000", 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92278930000000003 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for degraded object", 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.408845+0000", 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00063302499999999995 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.835 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.454465+0000", 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.045620274000000002 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.454587+0000", 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000121471 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.454740+0000", 2023-09-23T21:23:18.836 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015359600000000001 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.460305+0000", 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0055647099999999996 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.461593+0000", 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001287821 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.837 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.469284+0000", 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0076910859999999998 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.477934+0000", 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.008650072 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.838 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.477968+0000", 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3827e-05 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.481153+0000", 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0031851449999999999 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.839 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47438.0:100271 14.8 14:116d90ab:::benchmark_data_smithi121_684892_object14954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:30.485841+0000", 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 168.18425212400001, 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9669707279999999, 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47438", 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 100271 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.840 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.485841+0000", 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.485841+0000", 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.841 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.485840+0000", 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.485843+0000", 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.4310000000000001e-06 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.485848+0000", 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9089999999999999e-06 2023-09-23T21:23:18.842 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.485852+0000", 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4089999999999996e-06 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.497188+0000", 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.011335916 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.843 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.497203+0000", 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.4610000000000001e-05 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.485459+0000", 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98825666899999998 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.485464+0000", 2023-09-23T21:23:18.844 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3490000000000002e-06 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.408897+0000", 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92343379999999997 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for degraded object", 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.409462+0000", 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00056464099999999995 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.845 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.431870+0000", 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.022408052000000001 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.431942+0000", 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.1866999999999995e-05 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.432022+0000", 2023-09-23T21:23:18.846 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.9621000000000002e-05 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.441395+0000", 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0093736080000000003 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.441440+0000", 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4934e-05 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.847 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.441481+0000", 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0527e-05 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.450214+0000", 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0087328430000000005 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.450246+0000", 2023-09-23T21:23:18.848 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2879000000000001e-05 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.452811+0000", 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0025649929999999998 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.849 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47438.0:100273 14.8 14:1e60e88a:::benchmark_data_smithi121_684892_object14956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:30.489400+0000", 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 168.18069270500001, 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9649360520000001, 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47438", 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 100273 2023-09-23T21:23:18.850 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.489400+0000", 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.489400+0000", 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.851 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.489399+0000", 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.489405+0000", 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.4710000000000004e-06 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.489434+0000", 2023-09-23T21:23:18.852 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8245000000000001e-05 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.489446+0000", 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2892e-05 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.500091+0000", 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010644457 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.853 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.500107+0000", 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5699999999999999e-05 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.485477+0000", 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98536997999999998 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:18.854 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.485481+0000", 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0060000000000003e-06 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.409527+0000", 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.92404662000000004 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for degraded object", 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.410152+0000", 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00062456700000000003 2023-09-23T21:23:18.855 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.438996+0000", 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.028843687999999999 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.439132+0000", 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00013606499999999999 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.856 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.439316+0000", 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00018412600000000001 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.450151+0000", 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010835617000000001 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "op_commit", 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.450160+0000", 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.1330000000000005e-06 2023-09-23T21:23:18.857 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.450185+0000", 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5043000000000001e-05 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.452859+0000", 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0026732449999999999 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:18.858 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.452893+0000", 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3838999999999999e-05 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:32.454336+0000", 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0014435450000000001 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:18.859 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:23:18.860 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:23:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:18.861 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:18.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:18.862 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T21:23:18.862 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T21:23:18.963 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:18.990 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:19.117 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T21:23:19.119 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T21:23:19.119 DEBUG:teuthology.orchestra.run.smithi121:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T21:23:19.134 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:19.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:19.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:19.238 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:19.264 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:19.424 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:19.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:19.433 INFO:tasks.workunit.client.0.smithi121.stdout:Running main() from gmock_main.cc 2023-09-23T21:23:19.433 INFO:tasks.workunit.client.0.smithi121.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T21:23:19.433 INFO:tasks.workunit.client.0.smithi121.stdout:[----------] Global test environment set-up. 2023-09-23T21:23:19.433 INFO:tasks.workunit.client.0.smithi121.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T21:23:19.527 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:19.556 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:19.601 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:19.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:19.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:19.706 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:19.732 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:19.781 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:19.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:19.886 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:19.909 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:19 smithi123 ceph-mon[193903]: pgmap v2014: 83 pgs: 83 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:19.911 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:19.954 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:19.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:20.062 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:20.084 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:23:20.088 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:20.109 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:19 smithi121 ceph-mon[219709]: pgmap v2014: 83 pgs: 83 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:20.109 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:19 smithi121 ceph-mon[211526]: pgmap v2014: 83 pgs: 83 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:20.131 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:20.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:20.165 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T21:23:20.238 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:20.275 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:20.325 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:20.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:20.429 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:20.463 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:20.466 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:20.668 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:20.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:20.759 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T21:23:20.772 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:20.780 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_all_features (22 ms) 2023-09-23T21:23:20.780 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T21:23:20.801 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:20.842 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:20.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:20.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:20.909 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:20.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2657188039' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T21:23:20.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[211526]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: failed to encode map e662 with expected crc 2023-09-23T21:23:20.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2657188039' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:23:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:23:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T21:23:20.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:20 smithi121 ceph-mon[219709]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T21:23:20.948 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:20.973 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.copyup (193 ms) 2023-09-23T21:23:20.973 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T21:23:20.975 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:21.003 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.sparse_copyup (29 ms) 2023-09-23T21:23:21.003 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T21:23:21.018 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:21.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: failed to encode map e662 with expected crc 2023-09-23T21:23:21.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: failed to encode map e662 with expected crc 2023-09-23T21:23:21.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: failed to encode map e662 with expected crc 2023-09-23T21:23:21.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: failed to encode map e662 with expected crc 2023-09-23T21:23:21.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T21:23:21.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: failed to encode map e662 with expected crc 2023-09-23T21:23:21.022 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: failed to encode map e662 with expected crc 2023-09-23T21:23:21.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: failed to encode map e662 with expected crc 2023-09-23T21:23:21.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2657188039' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:23:21.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T21:23:21.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi121-760310-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T21:23:21.023 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:20 smithi123 ceph-mon[193903]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T21:23:21.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:21.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:21.042 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_and_set_id (39 ms) 2023-09-23T21:23:21.042 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T21:23:21.066 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.add_remove_child (24 ms) 2023-09-23T21:23:21.066 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T21:23:21.095 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.directory_methods (29 ms) 2023-09-23T21:23:21.095 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.create 2023-09-23T21:23:21.115 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.create (21 ms) 2023-09-23T21:23:21.115 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T21:23:21.117 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-09-23T21:23:21.117 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T21:23:21.119 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2023-09-23T21:23:21.119 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T21:23:21.121 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_create_timestamp (3 ms) 2023-09-23T21:23:21.121 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T21:23:21.124 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-09-23T21:23:21.124 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T21:23:21.124 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:21.125 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T21:23:21.125 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T21:23:21.129 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T21:23:21.129 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T21:23:21.133 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2023-09-23T21:23:21.134 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T21:23:21.138 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-09-23T21:23:21.138 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T21:23:21.151 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:21.152 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:23:21.155 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.protection_status (16 ms) 2023-09-23T21:23:21.155 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T21:23:21.163 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.snapshot_limits (9 ms) 2023-09-23T21:23:21.163 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T21:23:21.194 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:21.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:21.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:21.212 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.parents_v1 (49 ms) 2023-09-23T21:23:21.212 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T21:23:21.246 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.parents_v2 (34 ms) 2023-09-23T21:23:21.247 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T21:23:21.272 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.snapshots (25 ms) 2023-09-23T21:23:21.272 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T21:23:21.279 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2023-09-23T21:23:21.279 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T21:23:21.287 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-09-23T21:23:21.287 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T21:23:21.289 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-09-23T21:23:21.290 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T21:23:21.295 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.object_map_resize (6 ms) 2023-09-23T21:23:21.295 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T21:23:21.299 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:21.303 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2023-09-23T21:23:21.303 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T21:23:21.303 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-09-23T21:23:21.303 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T21:23:21.310 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-09-23T21:23:21.310 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T21:23:21.316 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.object_map_snap_remove (6 ms) 2023-09-23T21:23:21.317 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T21:23:21.322 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-23T21:23:21.322 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T21:23:21.332 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:21.359 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.metadata (38 ms) 2023-09-23T21:23:21.359 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T21:23:21.367 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-09-23T21:23:21.367 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T21:23:21.405 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.mirror (38 ms) 2023-09-23T21:23:21.405 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T21:23:21.420 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-09-23T21:23:21.421 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T21:23:21.739 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:21.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:21.776 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:23:21.844 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:21.871 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:21.917 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:21.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:22.024 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:22.050 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2726439888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[211526]: pgmap v2017: 115 pgs: 3 creating+peering, 29 unknown, 83 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2726439888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:23:22.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:23:22.052 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:21 smithi121 ceph-mon[219709]: pgmap v2017: 115 pgs: 3 creating+peering, 29 unknown, 83 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:22.053 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: failed to encode map e663 with expected crc 2023-09-23T21:23:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2726439888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:23:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:23:22.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:21 smithi123 ceph-mon[193903]: pgmap v2017: 115 pgs: 3 creating+peering, 29 unknown, 83 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:22.556 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:22.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:22.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:22.660 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:22.698 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:22.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:22.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[211526]: failed to encode map e664 with expected crc 2023-09-23T21:23:22.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:23:22.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[211526]: osdmap e664: 8 total, 8 up, 7 in 2023-09-23T21:23:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[219709]: failed to encode map e664 with expected crc 2023-09-23T21:23:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:23:22.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:22 smithi121 ceph-mon[219709]: osdmap e664: 8 total, 8 up, 7 in 2023-09-23T21:23:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:22 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:22 smithi123 ceph-mon[193903]: failed to encode map e664 with expected crc 2023-09-23T21:23:23.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:22 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:23:23.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:22 smithi123 ceph-mon[193903]: osdmap e664: 8 total, 8 up, 7 in 2023-09-23T21:23:23.369 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:23.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:23.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:23.474 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:23.501 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:23.548 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:23.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:23.653 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:23.680 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:23.727 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:23.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:23.784 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:23.834 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:23.860 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:23.905 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:23:23.906 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:23.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:23.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:23.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:23 smithi121 ceph-mon[211526]: failed to encode map e664 with expected crc 2023-09-23T21:23:23.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:23 smithi121 ceph-mon[211526]: pgmap v2019: 115 pgs: 3 creating+peering, 19 unknown, 93 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 1.3 MiB/s rd, 686 KiB/s wr, 23 op/s 2023-09-23T21:23:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:23 smithi121 ceph-mon[219709]: failed to encode map e664 with expected crc 2023-09-23T21:23:23.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:23 smithi121 ceph-mon[219709]: pgmap v2019: 115 pgs: 3 creating+peering, 19 unknown, 93 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 1.3 MiB/s rd, 686 KiB/s wr, 23 op/s 2023-09-23T21:23:23.929 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:23:23.952 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:23:23.976 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:23:24.012 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:23:24.013 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:24.022 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:23:24.037 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:23 smithi123 ceph-mon[193903]: failed to encode map e664 with expected crc 2023-09-23T21:23:24.037 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:23 smithi123 ceph-mon[193903]: pgmap v2019: 115 pgs: 3 creating+peering, 19 unknown, 93 active+clean; 582 KiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 1.3 MiB/s rd, 686 KiB/s wr, 23 op/s 2023-09-23T21:23:24.040 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:24.058 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:23:24.086 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:24.087 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:23:24.089 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.089 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.089 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.089 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.089 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.090 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:23:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:24.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:24.121 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:23:24.121 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:23:24.193 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:24.220 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:24.265 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:24.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:24.372 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:24.397 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.mirror_image_status (2976 ms) 2023-09-23T21:23:24.397 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T21:23:24.402 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:24.486 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.mirror_image_map (89 ms) 2023-09-23T21:23:24.486 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T21:23:24.493 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-09-23T21:23:24.493 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T21:23:24.679 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.mirror_snapshot (186 ms) 2023-09-23T21:23:24.680 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T21:23:24.682 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-09-23T21:23:24.682 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T21:23:24.684 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-09-23T21:23:24.684 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T21:23:24.687 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T21:23:24.687 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T21:23:24.693 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-09-23T21:23:24.694 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T21:23:24.696 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-23T21:23:24.697 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T21:23:24.701 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2023-09-23T21:23:24.701 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T21:23:24.703 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-23T21:23:24.703 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T21:23:24.706 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T21:23:24.707 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T21:23:24.709 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-09-23T21:23:24.709 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T21:23:24.713 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-09-23T21:23:24.713 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T21:23:24.716 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-09-23T21:23:24.716 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T21:23:24.719 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-09-23T21:23:24.719 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T21:23:24.721 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-09-23T21:23:24.721 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T21:23:24.723 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2023-09-23T21:23:24.723 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T21:23:24.724 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T21:23:24.724 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T21:23:24.727 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T21:23:24.727 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T21:23:24.730 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-23T21:23:24.730 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T21:23:24.732 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-09-23T21:23:24.732 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T21:23:24.735 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T21:23:24.735 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T21:23:24.751 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2023-09-23T21:23:24.751 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T21:23:24.943 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (191 ms) 2023-09-23T21:23:24.943 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T21:23:24.946 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-09-23T21:23:24.946 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T21:23:24.948 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T21:23:24.948 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T21:23:24.955 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-09-23T21:23:24.955 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T21:23:24.962 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-09-23T21:23:24.962 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T21:23:24.982 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2023-09-23T21:23:24.982 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T21:23:24.989 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.clone_parent_ns (7 ms) 2023-09-23T21:23:24.989 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T21:23:25.002 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2023-09-23T21:23:25.002 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T21:23:25.008 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-09-23T21:23:25.009 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T21:23:25.018 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-09-23T21:23:25.018 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T21:23:25.029 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2023-09-23T21:23:25.029 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T21:23:25.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:23:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:23:24] "GET /metrics HTTP/1.1" 200 34254 "" "Prometheus/2.43.0" 2023-09-23T21:23:25.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:24 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/868401232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:25.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:24 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/868401232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:25.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:24 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/868401232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:25.392 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T21:23:25.711 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:25.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:25.815 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:25.841 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:25 smithi121 ceph-mon[211526]: pgmap v2020: 115 pgs: 3 creating+peering, 112 active+clean; 582 KiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 1.25k op/s 2023-09-23T21:23:25.841 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:25 smithi121 ceph-mon[219709]: pgmap v2020: 115 pgs: 3 creating+peering, 112 active+clean; 582 KiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 1.25k op/s 2023-09-23T21:23:25.842 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:26.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:25 smithi123 ceph-mon[193903]: pgmap v2020: 115 pgs: 3 creating+peering, 112 active+clean; 582 KiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 1.25k op/s 2023-09-23T21:23:26.237 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:26.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:26.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:26.252 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T21:23:26.342 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:26.369 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:26.425 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:26.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:26.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:26.453 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:26.533 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:26.561 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:26.792 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1763 ms) 2023-09-23T21:23:26.792 INFO:tasks.workunit.client.0.smithi121.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T21:23:26.838 INFO:tasks.workunit.client.0.smithi121.stdout:[ OK ] TestClsRbd.sparsify (47 ms) 2023-09-23T21:23:26.855 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[219709]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.855 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[219709]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.855 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[219709]: osdmap e665: 8 total, 8 up, 7 in 2023-09-23T21:23:26.855 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[219709]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.856 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[219709]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[211526]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[211526]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[211526]: osdmap e665: 8 total, 8 up, 7 in 2023-09-23T21:23:26.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[211526]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:26 smithi121 ceph-mon[211526]: failed to encode map e665 with expected crc 2023-09-23T21:23:26.930 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:26.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:27.035 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:27.063 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:27.114 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:27.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:27.147 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:23:27.149 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:23:27.218 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:26 smithi123 ceph-mon[193903]: failed to encode map e665 with expected crc 2023-09-23T21:23:27.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:26 smithi123 ceph-mon[193903]: failed to encode map e665 with expected crc 2023-09-23T21:23:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:26 smithi123 ceph-mon[193903]: osdmap e665: 8 total, 8 up, 7 in 2023-09-23T21:23:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:26 smithi123 ceph-mon[193903]: failed to encode map e665 with expected crc 2023-09-23T21:23:27.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:26 smithi123 ceph-mon[193903]: failed to encode map e665 with expected crc 2023-09-23T21:23:27.246 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:27.592 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:23:27.800 INFO:tasks.workunit.client.0.smithi121.stdout:[----------] 70 tests from TestClsRbd (6080 ms total) 2023-09-23T21:23:27.801 INFO:tasks.workunit.client.0.smithi121.stdout: 2023-09-23T21:23:27.801 INFO:tasks.workunit.client.0.smithi121.stdout:[----------] Global test environment tear-down 2023-09-23T21:23:27.801 INFO:tasks.workunit.client.0.smithi121.stdout:[==========] 70 tests from 1 test suite ran. (8368 ms total) 2023-09-23T21:23:27.801 INFO:tasks.workunit.client.0.smithi121.stdout:[ PASSED ] 70 tests. 2023-09-23T21:23:27.804 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T21:23:27.804 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T21:23:27.832 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T21:23:27.832 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:23:27.894 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: osdmap e666: 8 total, 8 up, 7 in 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: from='client.47969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: pgmap v2023: 115 pgs: 115 active+clean; 582 KiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.2 MiB/s wr, 1.42k op/s 2023-09-23T21:23:27.896 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:27.897 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: osdmap e666: 8 total, 8 up, 7 in 2023-09-23T21:23:27.897 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: failed to encode map e666 with expected crc 2023-09-23T21:23:27.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: from='client.47969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:23:27.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: pgmap v2023: 115 pgs: 115 active+clean; 582 KiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.2 MiB/s wr, 1.42k op/s 2023-09-23T21:23:27.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:27.898 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:27.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:27.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:27.962 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:28.000 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:28.032 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:28.081 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: failed to encode map e666 with expected crc 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: failed to encode map e666 with expected crc 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: osdmap e666: 8 total, 8 up, 7 in 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: failed to encode map e666 with expected crc 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: failed to encode map e666 with expected crc 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: from='client.47969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: pgmap v2023: 115 pgs: 115 active+clean; 582 KiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 2.3 MiB/s rd, 1.2 MiB/s wr, 1.42k op/s 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:28.084 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:28.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:28.124 DEBUG:teuthology.parallel:result is None 2023-09-23T21:23:28.124 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:23:28.152 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:23:28.153 DEBUG:teuthology.orchestra.run.smithi121:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:23:28.186 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:28.218 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:28.344 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:23:28.344 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T21:23:28.344 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T21:23:28.344 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T21:23:28.344 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T21:23:28.344 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T21:23:28.345 DEBUG:teuthology.orchestra.run.smithi121:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:23:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:23:28.362 INFO:teuthology.orchestra.run.smithi121.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T21:23:28.362 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:23:28.419 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T21:23:28.420 DEBUG:teuthology.orchestra.run.smithi121:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T21:23:28.476 INFO:tasks.workunit:timeout=3h 2023-09-23T21:23:28.476 INFO:tasks.workunit:cleanup=True 2023-09-23T21:23:28.476 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:23:28.533 INFO:tasks.workunit.client.0.smithi121.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T21:23:28.690 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:28.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:28.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:28.794 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:28.824 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:28.901 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":2022,"stamp":"2023-09-23T21:23:26.789502+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":596246,"num_objects":65,"num_object_clones":0,"num_object_copies":195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65,"num_whiteouts":0,"num_read":5104,"num_read_kb":14036,"num_write":3253,"num_write_kb":7260,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":1151514,"num_keys_recovered":40,"num_objects_omap":47,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4149,"ondisk_log_size":4149,"up":345,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8326244,"kb_used_data":1609820,"kb_used_omap":16,"kb_used_meta":6716335,"kb_avail":647889820,"statfs":{"total":671965249536,"available":663439175680,"internally_reserved":0,"allocated":1648455680,"data_stored":1630944450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16535,"internal_metadata":6877527913},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":374,"num_objects":41,"num_object_clones":0,"num_object_copies":123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41,"num_whiteouts":0,"num_read":5104,"num_read_kb":14036,"num_write":3192,"num_write_kb":7231,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":33,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.034284"},"pg_stats":[{"pgid":"15.16","version":"663'22","reported_seq":42,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.368043+0000","last_change":"2023-09-23T21:23:20.776284+0000","last_active":"2023-09-23T21:23:21.368043+0000","last_peered":"2023-09-23T21:23:21.368043+0000","last_clean":"2023-09-23T21:23:21.368043+0000","last_became_active":"2023-09-23T21:23:20.776039+0000","last_became_peered":"2023-09-23T21:23:20.776039+0000","last_unstale":"2023-09-23T21:23:21.368043+0000","last_undegraded":"2023-09-23T21:23:21.368043+0000","last_fullsized":"2023-09-23T21:23:21.368043+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:00.883869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":23,"num_read_kb":44,"num_write":14,"num_write_kb":17,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":238,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769228+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:23:20.769228+0000","last_peered":"2023-09-23T21:23:20.769228+0000","last_clean":"2023-09-23T21:23:20.769228+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:23:20.769228+0000","last_undegraded":"2023-09-23T21:23:20.769228+0000","last_fullsized":"2023-09-23T21:23:20.769228+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":77,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757622+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:23:20.757622+0000","last_peered":"2023-09-23T21:23:20.757622+0000","last_clean":"2023-09-23T21:23:20.757622+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:23:20.757622+0000","last_undegraded":"2023-09-23T21:23:20.757622+0000","last_fullsized":"2023-09-23T21:23:20.757622+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.15","version":"663'1","reported_seq":12,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.290489+0000","last_change":"2023-09-23T21:23:20.774046+0000","last_active":"2023-09-23T21:23:21.290489+0000","last_peered":"2023-09-23T21:23:21.290489+0000","last_clean":"2023-09-23T21:23:21.290489+0000","last_became_active":"2023-09-23T21:23:20.773761+0000","last_became_peered":"2023-09-23T21:23:20.773761+0000","last_unstale":"2023-09-23T21:23:21.290489+0000","last_undegraded":"2023-09-23T21:23:21.290489+0000","last_fullsized":"2023-09-23T21:23:21.290489+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:05.222336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":45,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.763273+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:23:20.763273+0000","last_peered":"2023-09-23T21:23:20.763273+0000","last_clean":"2023-09-23T21:23:20.763273+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:23:20.763273+0000","last_undegraded":"2023-09-23T21:23:20.763273+0000","last_fullsized":"2023-09-23T21:23:20.763273+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":350,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762963+0000","last_change":"2023-09-23T21:20:59.516643+0000","last_active":"2023-09-23T21:23:20.762963+0000","last_peered":"2023-09-23T21:23:20.762963+0000","last_clean":"2023-09-23T21:23:20.762963+0000","last_became_active":"2023-09-23T21:20:59.516340+0000","last_became_peered":"2023-09-23T21:20:59.516340+0000","last_unstale":"2023-09-23T21:23:20.762963+0000","last_undegraded":"2023-09-23T21:23:20.762963+0000","last_fullsized":"2023-09-23T21:23:20.762963+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:47.245212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.14","version":"664'12","reported_seq":28,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:24.948729+0000","last_change":"2023-09-23T21:23:20.770757+0000","last_active":"2023-09-23T21:23:24.948729+0000","last_peered":"2023-09-23T21:23:24.948729+0000","last_clean":"2023-09-23T21:23:24.948729+0000","last_became_active":"2023-09-23T21:23:20.770478+0000","last_became_peered":"2023-09-23T21:23:20.770478+0000","last_unstale":"2023-09-23T21:23:24.948729+0000","last_undegraded":"2023-09-23T21:23:24.948729+0000","last_fullsized":"2023-09-23T21:23:24.948729+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:46:12.087445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":34,"num_read_kb":33,"num_write":11,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":68,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.758000+0000","last_change":"2023-09-23T21:20:59.515355+0000","last_active":"2023-09-23T21:23:20.758000+0000","last_peered":"2023-09-23T21:23:20.758000+0000","last_clean":"2023-09-23T21:23:20.758000+0000","last_became_active":"2023-09-23T21:20:59.515138+0000","last_became_peered":"2023-09-23T21:20:59.515138+0000","last_unstale":"2023-09-23T21:23:20.758000+0000","last_undegraded":"2023-09-23T21:23:20.758000+0000","last_fullsized":"2023-09-23T21:23:20.758000+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:33.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":170,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774725+0000","last_change":"2023-09-23T21:20:59.540523+0000","last_active":"2023-09-23T21:23:21.774725+0000","last_peered":"2023-09-23T21:23:21.774725+0000","last_clean":"2023-09-23T21:23:21.774725+0000","last_became_active":"2023-09-23T21:20:59.540067+0000","last_became_peered":"2023-09-23T21:20:59.540067+0000","last_unstale":"2023-09-23T21:23:21.774725+0000","last_undegraded":"2023-09-23T21:23:21.774725+0000","last_fullsized":"2023-09-23T21:23:21.774725+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:06.031358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.1b","version":"0'0","reported_seq":10,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.776982+0000","last_change":"2023-09-23T21:23:20.772888+0000","last_active":"2023-09-23T21:23:20.776982+0000","last_peered":"2023-09-23T21:23:20.776982+0000","last_clean":"2023-09-23T21:23:20.776982+0000","last_became_active":"2023-09-23T21:23:20.772600+0000","last_became_peered":"2023-09-23T21:23:20.772600+0000","last_unstale":"2023-09-23T21:23:20.776982+0000","last_undegraded":"2023-09-23T21:23:20.776982+0000","last_fullsized":"2023-09-23T21:23:20.776982+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:16.403049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.760581+0000","last_change":"2023-09-23T21:21:21.569477+0000","last_active":"2023-09-23T21:23:20.760581+0000","last_peered":"2023-09-23T21:23:20.760581+0000","last_clean":"2023-09-23T21:23:20.760581+0000","last_became_active":"2023-09-23T21:21:21.568975+0000","last_became_peered":"2023-09-23T21:21:21.568975+0000","last_unstale":"2023-09-23T21:23:20.760581+0000","last_undegraded":"2023-09-23T21:23:20.760581+0000","last_fullsized":"2023-09-23T21:23:20.760581+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:37.602093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757998+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:23:20.757998+0000","last_peered":"2023-09-23T21:23:20.757998+0000","last_clean":"2023-09-23T21:23:20.757998+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:23:20.757998+0000","last_undegraded":"2023-09-23T21:23:20.757998+0000","last_fullsized":"2023-09-23T21:23:20.757998+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.1a","version":"0'0","reported_seq":10,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.774367+0000","last_change":"2023-09-23T21:23:20.771719+0000","last_active":"2023-09-23T21:23:20.774367+0000","last_peered":"2023-09-23T21:23:20.774367+0000","last_clean":"2023-09-23T21:23:20.774367+0000","last_became_active":"2023-09-23T21:23:20.771443+0000","last_became_peered":"2023-09-23T21:23:20.771443+0000","last_unstale":"2023-09-23T21:23:20.774367+0000","last_undegraded":"2023-09-23T21:23:20.774367+0000","last_fullsized":"2023-09-23T21:23:20.774367+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:58.606520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.760119+0000","last_change":"2023-09-23T21:20:59.548536+0000","last_active":"2023-09-23T21:23:20.760119+0000","last_peered":"2023-09-23T21:23:20.760119+0000","last_clean":"2023-09-23T21:23:20.760119+0000","last_became_active":"2023-09-23T21:20:59.547411+0000","last_became_peered":"2023-09-23T21:20:59.547411+0000","last_unstale":"2023-09-23T21:23:20.760119+0000","last_undegraded":"2023-09-23T21:23:20.760119+0000","last_fullsized":"2023-09-23T21:23:20.760119+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:23.730254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":59,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757161+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:23:20.757161+0000","last_peered":"2023-09-23T21:23:20.757161+0000","last_clean":"2023-09-23T21:23:20.757161+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:23:20.757161+0000","last_undegraded":"2023-09-23T21:23:20.757161+0000","last_fullsized":"2023-09-23T21:23:20.757161+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.19","version":"663'7","reported_seq":19,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.288055+0000","last_change":"2023-09-23T21:23:20.772483+0000","last_active":"2023-09-23T21:23:21.288055+0000","last_peered":"2023-09-23T21:23:21.288055+0000","last_clean":"2023-09-23T21:23:21.288055+0000","last_became_active":"2023-09-23T21:23:20.772189+0000","last_became_peered":"2023-09-23T21:23:20.772189+0000","last_unstale":"2023-09-23T21:23:21.288055+0000","last_undegraded":"2023-09-23T21:23:21.288055+0000","last_fullsized":"2023-09-23T21:23:21.288055+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.525564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":13,"num_read_kb":10,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":298,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762430+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:23:20.762430+0000","last_peered":"2023-09-23T21:23:20.762430+0000","last_clean":"2023-09-23T21:23:20.762430+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:23:20.762430+0000","last_undegraded":"2023-09-23T21:23:20.762430+0000","last_fullsized":"2023-09-23T21:23:20.762430+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.846013+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:23:21.846013+0000","last_peered":"2023-09-23T21:23:21.846013+0000","last_clean":"2023-09-23T21:23:21.846013+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:23:21.846013+0000","last_undegraded":"2023-09-23T21:23:21.846013+0000","last_fullsized":"2023-09-23T21:23:21.846013+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.18","version":"664'18","reported_seq":60,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:24.707342+0000","last_change":"2023-09-23T21:23:20.767610+0000","last_active":"2023-09-23T21:23:24.707342+0000","last_peered":"2023-09-23T21:23:24.707342+0000","last_clean":"2023-09-23T21:23:24.707342+0000","last_became_active":"2023-09-23T21:23:20.767345+0000","last_became_peered":"2023-09-23T21:23:20.767345+0000","last_unstale":"2023-09-23T21:23:24.707342+0000","last_undegraded":"2023-09-23T21:23:24.707342+0000","last_fullsized":"2023-09-23T21:23:24.707342+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":18,"log_dups_size":0,"ondisk_log_size":18,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:09.745874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":88,"num_read_kb":84,"num_write":14,"num_write_kb":10,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":394,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769133+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:23:20.769133+0000","last_peered":"2023-09-23T21:23:20.769133+0000","last_clean":"2023-09-23T21:23:20.769133+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:23:20.769133+0000","last_undegraded":"2023-09-23T21:23:20.769133+0000","last_fullsized":"2023-09-23T21:23:20.769133+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":621,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.770498+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:23:20.770498+0000","last_peered":"2023-09-23T21:23:20.770498+0000","last_clean":"2023-09-23T21:23:20.770498+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:23:20.770498+0000","last_undegraded":"2023-09-23T21:23:20.770498+0000","last_fullsized":"2023-09-23T21:23:20.770498+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.1f","version":"0'0","reported_seq":11,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.779320+0000","last_change":"2023-09-23T21:23:20.777911+0000","last_active":"2023-09-23T21:23:20.779320+0000","last_peered":"2023-09-23T21:23:20.779320+0000","last_clean":"2023-09-23T21:23:20.779320+0000","last_became_active":"2023-09-23T21:23:20.777621+0000","last_became_peered":"2023-09-23T21:23:20.777621+0000","last_unstale":"2023-09-23T21:23:20.779320+0000","last_undegraded":"2023-09-23T21:23:20.779320+0000","last_fullsized":"2023-09-23T21:23:20.779320+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:19.965276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":246,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762076+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:23:20.762076+0000","last_peered":"2023-09-23T21:23:20.762076+0000","last_clean":"2023-09-23T21:23:20.762076+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:23:20.762076+0000","last_undegraded":"2023-09-23T21:23:20.762076+0000","last_fullsized":"2023-09-23T21:23:20.762076+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":68,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757137+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:23:20.757137+0000","last_peered":"2023-09-23T21:23:20.757137+0000","last_clean":"2023-09-23T21:23:20.757137+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:23:20.757137+0000","last_undegraded":"2023-09-23T21:23:20.757137+0000","last_fullsized":"2023-09-23T21:23:20.757137+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.1e","version":"663'7","reported_seq":19,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.771561+0000","last_change":"2023-09-23T21:23:20.772606+0000","last_active":"2023-09-23T21:23:21.771561+0000","last_peered":"2023-09-23T21:23:21.771561+0000","last_clean":"2023-09-23T21:23:21.771561+0000","last_became_active":"2023-09-23T21:23:20.772369+0000","last_became_peered":"2023-09-23T21:23:20.772369+0000","last_unstale":"2023-09-23T21:23:21.771561+0000","last_undegraded":"2023-09-23T21:23:21.771561+0000","last_fullsized":"2023-09-23T21:23:21.771561+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:37.391179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":4,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":272,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762166+0000","last_change":"2023-09-23T21:20:59.525934+0000","last_active":"2023-09-23T21:23:20.762166+0000","last_peered":"2023-09-23T21:23:20.762166+0000","last_clean":"2023-09-23T21:23:20.762166+0000","last_became_active":"2023-09-23T21:20:59.524870+0000","last_became_peered":"2023-09-23T21:20:59.524870+0000","last_unstale":"2023-09-23T21:23:20.762166+0000","last_undegraded":"2023-09-23T21:23:20.762166+0000","last_fullsized":"2023-09-23T21:23:20.762166+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:29.264937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":280,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762623+0000","last_change":"2023-09-23T21:20:46.945764+0000","last_active":"2023-09-23T21:23:20.762623+0000","last_peered":"2023-09-23T21:23:20.762623+0000","last_clean":"2023-09-23T21:23:20.762623+0000","last_became_active":"2023-09-23T21:20:46.945228+0000","last_became_peered":"2023-09-23T21:20:46.945228+0000","last_unstale":"2023-09-23T21:23:20.762623+0000","last_undegraded":"2023-09-23T21:23:20.762623+0000","last_fullsized":"2023-09-23T21:23:20.762623+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:43.608690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"15.1d","version":"663'2","reported_seq":12,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.215226+0000","last_change":"2023-09-23T21:23:20.772195+0000","last_active":"2023-09-23T21:23:21.215226+0000","last_peered":"2023-09-23T21:23:21.215226+0000","last_clean":"2023-09-23T21:23:21.215226+0000","last_became_active":"2023-09-23T21:23:20.771890+0000","last_became_peered":"2023-09-23T21:23:20.771890+0000","last_unstale":"2023-09-23T21:23:21.215226+0000","last_undegraded":"2023-09-23T21:23:21.215226+0000","last_fullsized":"2023-09-23T21:23:21.215226+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:23.325323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":240,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.761940+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:23:20.761940+0000","last_peered":"2023-09-23T21:23:20.761940+0000","last_clean":"2023-09-23T21:23:20.761940+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:23:20.761940+0000","last_undegraded":"2023-09-23T21:23:20.761940+0000","last_fullsized":"2023-09-23T21:23:20.761940+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":59,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757291+0000","last_change":"2023-09-23T21:20:31.498212+0000","last_active":"2023-09-23T21:23:20.757291+0000","last_peered":"2023-09-23T21:23:20.757291+0000","last_clean":"2023-09-23T21:23:20.757291+0000","last_became_active":"2023-09-23T21:20:31.497853+0000","last_became_peered":"2023-09-23T21:20:31.497853+0000","last_unstale":"2023-09-23T21:23:20.757291+0000","last_undegraded":"2023-09-23T21:23:20.757291+0000","last_fullsized":"2023-09-23T21:23:20.757291+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:10.198606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.1c","version":"0'0","reported_seq":11,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.846037+0000","last_change":"2023-09-23T21:23:20.774422+0000","last_active":"2023-09-23T21:23:21.846037+0000","last_peered":"2023-09-23T21:23:21.846037+0000","last_clean":"2023-09-23T21:23:21.846037+0000","last_became_active":"2023-09-23T21:23:20.774138+0000","last_became_peered":"2023-09-23T21:23:20.774138+0000","last_unstale":"2023-09-23T21:23:21.846037+0000","last_undegraded":"2023-09-23T21:23:21.846037+0000","last_fullsized":"2023-09-23T21:23:21.846037+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:54.643844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":761,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.773966+0000","last_change":"2023-09-23T21:20:47.062314+0000","last_active":"2023-09-23T21:23:21.773966+0000","last_peered":"2023-09-23T21:23:21.773966+0000","last_clean":"2023-09-23T21:23:21.773966+0000","last_became_active":"2023-09-23T21:20:46.923883+0000","last_became_peered":"2023-09-23T21:20:46.923883+0000","last_unstale":"2023-09-23T21:23:21.773966+0000","last_undegraded":"2023-09-23T21:23:21.773966+0000","last_fullsized":"2023-09-23T21:23:21.773966+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:50.713370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":68,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757445+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:23:20.757445+0000","last_peered":"2023-09-23T21:23:20.757445+0000","last_clean":"2023-09-23T21:23:20.757445+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:23:20.757445+0000","last_undegraded":"2023-09-23T21:23:20.757445+0000","last_fullsized":"2023-09-23T21:23:20.757445+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.3","version":"0'0","reported_seq":10,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.776423+0000","last_change":"2023-09-23T21:23:20.774275+0000","last_active":"2023-09-23T21:23:20.776423+0000","last_peered":"2023-09-23T21:23:20.776423+0000","last_clean":"2023-09-23T21:23:20.776423+0000","last_became_active":"2023-09-23T21:23:20.773803+0000","last_became_peered":"2023-09-23T21:23:20.773803+0000","last_unstale":"2023-09-23T21:23:20.776423+0000","last_undegraded":"2023-09-23T21:23:20.776423+0000","last_fullsized":"2023-09-23T21:23:20.776423+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:10.971900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":300,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.768418+0000","last_change":"2023-09-23T21:20:59.569107+0000","last_active":"2023-09-23T21:23:20.768418+0000","last_peered":"2023-09-23T21:23:20.768418+0000","last_clean":"2023-09-23T21:23:20.768418+0000","last_became_active":"2023-09-23T21:20:59.526694+0000","last_became_peered":"2023-09-23T21:20:59.526694+0000","last_unstale":"2023-09-23T21:23:20.768418+0000","last_undegraded":"2023-09-23T21:23:20.768418+0000","last_fullsized":"2023-09-23T21:23:20.768418+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:21.415832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.759205+0000","last_change":"2023-09-23T21:21:21.569355+0000","last_active":"2023-09-23T21:23:20.759205+0000","last_peered":"2023-09-23T21:23:20.759205+0000","last_clean":"2023-09-23T21:23:20.759205+0000","last_became_active":"2023-09-23T21:21:21.569044+0000","last_became_peered":"2023-09-23T21:21:21.569044+0000","last_unstale":"2023-09-23T21:23:20.759205+0000","last_undegraded":"2023-09-23T21:23:20.759205+0000","last_fullsized":"2023-09-23T21:23:20.759205+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:32.434591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.2","version":"663'1","reported_seq":13,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.771124+0000","last_change":"2023-09-23T21:23:20.775914+0000","last_active":"2023-09-23T21:23:21.771124+0000","last_peered":"2023-09-23T21:23:21.771124+0000","last_clean":"2023-09-23T21:23:21.771124+0000","last_became_active":"2023-09-23T21:23:20.772690+0000","last_became_peered":"2023-09-23T21:23:20.772690+0000","last_unstale":"2023-09-23T21:23:21.771124+0000","last_undegraded":"2023-09-23T21:23:21.771124+0000","last_fullsized":"2023-09-23T21:23:21.771124+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:53.782414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":314,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.763050+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:23:20.763050+0000","last_peered":"2023-09-23T21:23:20.763050+0000","last_clean":"2023-09-23T21:23:20.763050+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:23:20.763050+0000","last_undegraded":"2023-09-23T21:23:20.763050+0000","last_fullsized":"2023-09-23T21:23:20.763050+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.763106+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:23:20.763106+0000","last_peered":"2023-09-23T21:23:20.763106+0000","last_clean":"2023-09-23T21:23:20.763106+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:23:20.763106+0000","last_undegraded":"2023-09-23T21:23:20.763106+0000","last_fullsized":"2023-09-23T21:23:20.763106+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.1","version":"664'23","reported_seq":36,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:24.990452+0000","last_change":"2023-09-23T21:23:20.771764+0000","last_active":"2023-09-23T21:23:24.990452+0000","last_peered":"2023-09-23T21:23:24.990452+0000","last_clean":"2023-09-23T21:23:24.990452+0000","last_became_active":"2023-09-23T21:23:20.771049+0000","last_became_peered":"2023-09-23T21:23:20.771049+0000","last_unstale":"2023-09-23T21:23:24.990452+0000","last_undegraded":"2023-09-23T21:23:24.990452+0000","last_fullsized":"2023-09-23T21:23:24.990452+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:28.618920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":34,"num_read_kb":34,"num_write":22,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":286,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762801+0000","last_change":"2023-09-23T21:20:47.006569+0000","last_active":"2023-09-23T21:23:20.762801+0000","last_peered":"2023-09-23T21:23:20.762801+0000","last_clean":"2023-09-23T21:23:20.762801+0000","last_became_active":"2023-09-23T21:20:46.945406+0000","last_became_peered":"2023-09-23T21:20:46.945406+0000","last_unstale":"2023-09-23T21:23:20.762801+0000","last_undegraded":"2023-09-23T21:23:20.762801+0000","last_fullsized":"2023-09-23T21:23:20.762801+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:46.931874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":36,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.773998+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:23:21.773998+0000","last_peered":"2023-09-23T21:23:21.773998+0000","last_clean":"2023-09-23T21:23:21.773998+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:23:21.773998+0000","last_undegraded":"2023-09-23T21:23:21.773998+0000","last_fullsized":"2023-09-23T21:23:21.773998+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.0","version":"663'3","reported_seq":16,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.770832+0000","last_change":"2023-09-23T21:23:20.772064+0000","last_active":"2023-09-23T21:23:21.770832+0000","last_peered":"2023-09-23T21:23:21.770832+0000","last_clean":"2023-09-23T21:23:21.770832+0000","last_became_active":"2023-09-23T21:23:20.771721+0000","last_became_peered":"2023-09-23T21:23:20.771721+0000","last_unstale":"2023-09-23T21:23:21.770832+0000","last_undegraded":"2023-09-23T21:23:21.770832+0000","last_fullsized":"2023-09-23T21:23:21.770832+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:19.763454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":3,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757220+0000","last_change":"2023-09-23T21:20:46.924017+0000","last_active":"2023-09-23T21:23:20.757220+0000","last_peered":"2023-09-23T21:23:20.757220+0000","last_clean":"2023-09-23T21:23:20.757220+0000","last_became_active":"2023-09-23T21:20:46.923824+0000","last_became_peered":"2023-09-23T21:20:46.923824+0000","last_unstale":"2023-09-23T21:23:20.757220+0000","last_undegraded":"2023-09-23T21:23:20.757220+0000","last_fullsized":"2023-09-23T21:23:20.757220+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:03.066606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.770513+0000","last_change":"2023-09-23T21:20:59.521227+0000","last_active":"2023-09-23T21:23:20.770513+0000","last_peered":"2023-09-23T21:23:20.770513+0000","last_clean":"2023-09-23T21:23:20.770513+0000","last_became_active":"2023-09-23T21:20:59.520931+0000","last_became_peered":"2023-09-23T21:20:59.520931+0000","last_unstale":"2023-09-23T21:23:20.770513+0000","last_undegraded":"2023-09-23T21:23:20.770513+0000","last_fullsized":"2023-09-23T21:23:20.770513+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:30.804698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.7","version":"663'23","reported_seq":55,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.846717+0000","last_change":"2023-09-23T21:23:20.778311+0000","last_active":"2023-09-23T21:23:21.846717+0000","last_peered":"2023-09-23T21:23:21.846717+0000","last_clean":"2023-09-23T21:23:21.846717+0000","last_became_active":"2023-09-23T21:23:20.777876+0000","last_became_peered":"2023-09-23T21:23:20.777876+0000","last_unstale":"2023-09-23T21:23:21.846717+0000","last_undegraded":"2023-09-23T21:23:21.846717+0000","last_fullsized":"2023-09-23T21:23:21.846717+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:23.245655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":40,"num_read_kb":39,"num_write":17,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1089,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.761736+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:23:20.761736+0000","last_peered":"2023-09-23T21:23:20.761736+0000","last_clean":"2023-09-23T21:23:20.761736+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:23:20.761736+0000","last_undegraded":"2023-09-23T21:23:20.761736+0000","last_fullsized":"2023-09-23T21:23:20.761736+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":76,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757084+0000","last_change":"2023-09-23T21:20:59.610412+0000","last_active":"2023-09-23T21:23:20.757084+0000","last_peered":"2023-09-23T21:23:20.757084+0000","last_clean":"2023-09-23T21:23:20.757084+0000","last_became_active":"2023-09-23T21:20:59.526983+0000","last_became_peered":"2023-09-23T21:20:59.526983+0000","last_unstale":"2023-09-23T21:23:20.757084+0000","last_undegraded":"2023-09-23T21:23:20.757084+0000","last_fullsized":"2023-09-23T21:23:20.757084+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:01.125099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":421,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769222+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:23:20.769222+0000","last_peered":"2023-09-23T21:23:20.769222+0000","last_clean":"2023-09-23T21:23:20.769222+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:23:20.769222+0000","last_undegraded":"2023-09-23T21:23:20.769222+0000","last_fullsized":"2023-09-23T21:23:20.769222+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.6","version":"664'1896","reported_seq":1956,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:23.975167+0000","last_change":"2023-09-23T21:23:20.769211+0000","last_active":"2023-09-23T21:23:23.975167+0000","last_peered":"2023-09-23T21:23:23.975167+0000","last_clean":"2023-09-23T21:23:23.975167+0000","last_became_active":"2023-09-23T21:23:20.768913+0000","last_became_peered":"2023-09-23T21:23:20.768913+0000","last_unstale":"2023-09-23T21:23:23.975167+0000","last_undegraded":"2023-09-23T21:23:23.975167+0000","last_fullsized":"2023-09-23T21:23:23.975167+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":1896,"log_dups_size":0,"ondisk_log_size":1896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:23.700430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":3916,"num_read_kb":3883,"num_write":2811,"num_write_kb":2774,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":972,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.761120+0000","last_change":"2023-09-23T21:23:02.971380+0000","last_active":"2023-09-23T21:23:20.761120+0000","last_peered":"2023-09-23T21:23:20.761120+0000","last_clean":"2023-09-23T21:23:20.761120+0000","last_became_active":"2023-09-23T21:23:02.970961+0000","last_became_peered":"2023-09-23T21:23:02.970961+0000","last_unstale":"2023-09-23T21:23:20.761120+0000","last_undegraded":"2023-09-23T21:23:20.761120+0000","last_fullsized":"2023-09-23T21:23:20.761120+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:24.685033+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":160,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774157+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:23:21.774157+0000","last_peered":"2023-09-23T21:23:21.774157+0000","last_clean":"2023-09-23T21:23:21.774157+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:23:21.774157+0000","last_undegraded":"2023-09-23T21:23:21.774157+0000","last_fullsized":"2023-09-23T21:23:21.774157+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":671,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769123+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:23:20.769123+0000","last_peered":"2023-09-23T21:23:20.769123+0000","last_clean":"2023-09-23T21:23:20.769123+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:23:20.769123+0000","last_undegraded":"2023-09-23T21:23:20.769123+0000","last_fullsized":"2023-09-23T21:23:20.769123+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.5","version":"0'0","reported_seq":11,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774134+0000","last_change":"2023-09-23T21:23:20.776382+0000","last_active":"2023-09-23T21:23:21.774134+0000","last_peered":"2023-09-23T21:23:21.774134+0000","last_clean":"2023-09-23T21:23:21.774134+0000","last_became_active":"2023-09-23T21:23:20.776098+0000","last_became_peered":"2023-09-23T21:23:20.776098+0000","last_unstale":"2023-09-23T21:23:21.774134+0000","last_undegraded":"2023-09-23T21:23:21.774134+0000","last_fullsized":"2023-09-23T21:23:21.774134+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:14.254376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":300,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.770176+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:23:20.770176+0000","last_peered":"2023-09-23T21:23:20.770176+0000","last_clean":"2023-09-23T21:23:20.770176+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:23:20.770176+0000","last_undegraded":"2023-09-23T21:23:20.770176+0000","last_fullsized":"2023-09-23T21:23:20.770176+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":708,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769910+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:23:20.769910+0000","last_peered":"2023-09-23T21:23:20.769910+0000","last_clean":"2023-09-23T21:23:20.769910+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:23:20.769910+0000","last_undegraded":"2023-09-23T21:23:20.769910+0000","last_fullsized":"2023-09-23T21:23:20.769910+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.4","version":"0'0","reported_seq":11,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.846375+0000","last_change":"2023-09-23T21:23:20.768845+0000","last_active":"2023-09-23T21:23:21.846375+0000","last_peered":"2023-09-23T21:23:21.846375+0000","last_clean":"2023-09-23T21:23:21.846375+0000","last_became_active":"2023-09-23T21:23:20.768612+0000","last_became_peered":"2023-09-23T21:23:20.768612+0000","last_unstale":"2023-09-23T21:23:21.846375+0000","last_undegraded":"2023-09-23T21:23:21.846375+0000","last_fullsized":"2023-09-23T21:23:21.846375+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:44.236024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":940,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.759584+0000","last_change":"2023-09-23T21:23:11.789134+0000","last_active":"2023-09-23T21:23:20.759584+0000","last_peered":"2023-09-23T21:23:20.759584+0000","last_clean":"2023-09-23T21:23:20.759584+0000","last_became_active":"2023-09-23T21:20:49.967877+0000","last_became_peered":"2023-09-23T21:20:49.967877+0000","last_unstale":"2023-09-23T21:23:20.759584+0000","last_undegraded":"2023-09-23T21:23:20.759584+0000","last_fullsized":"2023-09-23T21:23:20.759584+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:23:11.788995+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:23:11.788995+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:28.771406+0000","scrub_duration":0.002393788,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":68,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757914+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:23:20.757914+0000","last_peered":"2023-09-23T21:23:20.757914+0000","last_clean":"2023-09-23T21:23:20.757914+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:23:20.757914+0000","last_undegraded":"2023-09-23T21:23:20.757914+0000","last_fullsized":"2023-09-23T21:23:20.757914+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.764614+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:23:20.764614+0000","last_peered":"2023-09-23T21:23:20.764614+0000","last_clean":"2023-09-23T21:23:20.764614+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:23:20.764614+0000","last_undegraded":"2023-09-23T21:23:20.764614+0000","last_fullsized":"2023-09-23T21:23:20.764614+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.9","version":"664'155","reported_seq":170,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:24.943539+0000","last_change":"2023-09-23T21:23:20.771936+0000","last_active":"2023-09-23T21:23:24.943539+0000","last_peered":"2023-09-23T21:23:24.943539+0000","last_clean":"2023-09-23T21:23:24.943539+0000","last_became_active":"2023-09-23T21:23:20.771501+0000","last_became_peered":"2023-09-23T21:23:20.771501+0000","last_unstale":"2023-09-23T21:23:24.943539+0000","last_undegraded":"2023-09-23T21:23:24.943539+0000","last_fullsized":"2023-09-23T21:23:24.943539+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":155,"log_dups_size":0,"ondisk_log_size":155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:56.590544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":421,"num_read_kb":1264,"num_write":155,"num_write_kb":155,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":747,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774670+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:23:21.774670+0000","last_peered":"2023-09-23T21:23:21.774670+0000","last_clean":"2023-09-23T21:23:21.774670+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:23:21.774670+0000","last_undegraded":"2023-09-23T21:23:21.774670+0000","last_fullsized":"2023-09-23T21:23:21.774670+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.763114+0000","last_change":"2023-09-23T21:20:59.523292+0000","last_active":"2023-09-23T21:23:20.763114+0000","last_peered":"2023-09-23T21:23:20.763114+0000","last_clean":"2023-09-23T21:23:20.763114+0000","last_became_active":"2023-09-23T21:20:59.522778+0000","last_became_peered":"2023-09-23T21:20:59.522778+0000","last_unstale":"2023-09-23T21:23:20.763114+0000","last_undegraded":"2023-09-23T21:23:20.763114+0000","last_fullsized":"2023-09-23T21:23:20.763114+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:11.024972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":51,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.760105+0000","last_change":"2023-09-23T21:20:46.932071+0000","last_active":"2023-09-23T21:23:20.760105+0000","last_peered":"2023-09-23T21:23:20.760105+0000","last_clean":"2023-09-23T21:23:20.760105+0000","last_became_active":"2023-09-23T21:20:46.931701+0000","last_became_peered":"2023-09-23T21:20:46.931701+0000","last_unstale":"2023-09-23T21:23:20.760105+0000","last_undegraded":"2023-09-23T21:23:20.760105+0000","last_fullsized":"2023-09-23T21:23:20.760105+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:26.349787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762847+0000","last_change":"2023-09-23T21:20:59.580891+0000","last_active":"2023-09-23T21:23:20.762847+0000","last_peered":"2023-09-23T21:23:20.762847+0000","last_clean":"2023-09-23T21:23:20.762847+0000","last_became_active":"2023-09-23T21:20:59.580624+0000","last_became_peered":"2023-09-23T21:20:59.580624+0000","last_unstale":"2023-09-23T21:23:20.762847+0000","last_undegraded":"2023-09-23T21:23:20.762847+0000","last_fullsized":"2023-09-23T21:23:20.762847+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:53.822970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.f","version":"664'30","reported_seq":55,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:24.701880+0000","last_change":"2023-09-23T21:23:20.777699+0000","last_active":"2023-09-23T21:23:24.701880+0000","last_peered":"2023-09-23T21:23:24.701880+0000","last_clean":"2023-09-23T21:23:24.701880+0000","last_became_active":"2023-09-23T21:23:20.777465+0000","last_became_peered":"2023-09-23T21:23:20.777465+0000","last_unstale":"2023-09-23T21:23:24.701880+0000","last_undegraded":"2023-09-23T21:23:24.701880+0000","last_fullsized":"2023-09-23T21:23:24.701880+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:37:20.454139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":51,"num_read_kb":8232,"num_write":29,"num_write_kb":4108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":940,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.760650+0000","last_change":"2023-09-23T21:20:49.970081+0000","last_active":"2023-09-23T21:23:20.760650+0000","last_peered":"2023-09-23T21:23:20.760650+0000","last_clean":"2023-09-23T21:23:20.760650+0000","last_became_active":"2023-09-23T21:20:49.969880+0000","last_became_peered":"2023-09-23T21:20:49.969880+0000","last_unstale":"2023-09-23T21:23:20.760650+0000","last_undegraded":"2023-09-23T21:23:20.760650+0000","last_fullsized":"2023-09-23T21:23:20.760650+0000","mapping_epoch":625,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:23.490205+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762996+0000","last_change":"2023-09-23T21:20:59.580665+0000","last_active":"2023-09-23T21:23:20.762996+0000","last_peered":"2023-09-23T21:23:20.762996+0000","last_clean":"2023-09-23T21:23:20.762996+0000","last_became_active":"2023-09-23T21:20:59.580213+0000","last_became_peered":"2023-09-23T21:20:59.580213+0000","last_unstale":"2023-09-23T21:23:20.762996+0000","last_undegraded":"2023-09-23T21:23:20.762996+0000","last_fullsized":"2023-09-23T21:23:20.762996+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:54.252754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":576,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.770333+0000","last_change":"2023-09-23T21:20:59.529263+0000","last_active":"2023-09-23T21:23:20.770333+0000","last_peered":"2023-09-23T21:23:20.770333+0000","last_clean":"2023-09-23T21:23:20.770333+0000","last_became_active":"2023-09-23T21:20:59.528797+0000","last_became_peered":"2023-09-23T21:20:59.528797+0000","last_unstale":"2023-09-23T21:23:20.770333+0000","last_undegraded":"2023-09-23T21:23:20.770333+0000","last_fullsized":"2023-09-23T21:23:20.770333+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:29.852599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.763040+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:23:20.763040+0000","last_peered":"2023-09-23T21:23:20.763040+0000","last_clean":"2023-09-23T21:23:20.763040+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:23:20.763040+0000","last_undegraded":"2023-09-23T21:23:20.763040+0000","last_fullsized":"2023-09-23T21:23:20.763040+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.c","version":"664'9","reported_seq":25,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:25.009677+0000","last_change":"2023-09-23T21:23:20.769553+0000","last_active":"2023-09-23T21:23:25.009677+0000","last_peered":"2023-09-23T21:23:25.009677+0000","last_clean":"2023-09-23T21:23:25.009677+0000","last_became_active":"2023-09-23T21:23:20.769156+0000","last_became_peered":"2023-09-23T21:23:20.769156+0000","last_unstale":"2023-09-23T21:23:25.009677+0000","last_undegraded":"2023-09-23T21:23:25.009677+0000","last_fullsized":"2023-09-23T21:23:25.009677+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:27.042535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":13,"num_read_kb":13,"num_write":6,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1049,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762073+0000","last_change":"2023-09-23T21:19:24.382854+0000","last_active":"2023-09-23T21:23:20.762073+0000","last_peered":"2023-09-23T21:23:20.762073+0000","last_clean":"2023-09-23T21:23:20.762073+0000","last_became_active":"2023-09-23T21:18:59.505370+0000","last_became_peered":"2023-09-23T21:18:59.505370+0000","last_unstale":"2023-09-23T21:23:20.762073+0000","last_undegraded":"2023-09-23T21:23:20.762073+0000","last_fullsized":"2023-09-23T21:23:20.762073+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:26.387622+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":693,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.760791+0000","last_change":"2023-09-23T21:20:59.539611+0000","last_active":"2023-09-23T21:23:20.760791+0000","last_peered":"2023-09-23T21:23:20.760791+0000","last_clean":"2023-09-23T21:23:20.760791+0000","last_became_active":"2023-09-23T21:20:59.539198+0000","last_became_peered":"2023-09-23T21:20:59.539198+0000","last_unstale":"2023-09-23T21:23:20.760791+0000","last_undegraded":"2023-09-23T21:23:20.760791+0000","last_fullsized":"2023-09-23T21:23:20.760791+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:23.311245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.765230+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:23:20.765230+0000","last_peered":"2023-09-23T21:23:20.765230+0000","last_clean":"2023-09-23T21:23:20.765230+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:23:20.765230+0000","last_undegraded":"2023-09-23T21:23:20.765230+0000","last_fullsized":"2023-09-23T21:23:20.765230+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.764540+0000","last_change":"2023-09-23T21:20:59.536980+0000","last_active":"2023-09-23T21:23:20.764540+0000","last_peered":"2023-09-23T21:23:20.764540+0000","last_clean":"2023-09-23T21:23:20.764540+0000","last_became_active":"2023-09-23T21:20:59.536669+0000","last_became_peered":"2023-09-23T21:20:59.536669+0000","last_unstale":"2023-09-23T21:23:20.764540+0000","last_undegraded":"2023-09-23T21:23:20.764540+0000","last_fullsized":"2023-09-23T21:23:20.764540+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:46.456127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.e","version":"663'9","reported_seq":48,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.771605+0000","last_change":"2023-09-23T21:23:20.771552+0000","last_active":"2023-09-23T21:23:21.771605+0000","last_peered":"2023-09-23T21:23:21.771605+0000","last_clean":"2023-09-23T21:23:21.771605+0000","last_became_active":"2023-09-23T21:23:20.771249+0000","last_became_peered":"2023-09-23T21:23:20.771249+0000","last_unstale":"2023-09-23T21:23:21.771605+0000","last_undegraded":"2023-09-23T21:23:21.771605+0000","last_fullsized":"2023-09-23T21:23:21.771605+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:06.623700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":32,"num_read_kb":52,"num_write":7,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":522,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.759996+0000","last_change":"2023-09-23T21:20:46.934314+0000","last_active":"2023-09-23T21:23:20.759996+0000","last_peered":"2023-09-23T21:23:20.759996+0000","last_clean":"2023-09-23T21:23:20.759996+0000","last_became_active":"2023-09-23T21:20:46.933899+0000","last_became_peered":"2023-09-23T21:20:46.933899+0000","last_unstale":"2023-09-23T21:23:20.759996+0000","last_undegraded":"2023-09-23T21:23:20.759996+0000","last_fullsized":"2023-09-23T21:23:20.759996+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:00:05.032476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1050,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.761691+0000","last_change":"2023-09-23T21:21:02.539530+0000","last_active":"2023-09-23T21:23:20.761691+0000","last_peered":"2023-09-23T21:23:20.761691+0000","last_clean":"2023-09-23T21:23:20.761691+0000","last_became_active":"2023-09-23T21:21:02.538960+0000","last_became_peered":"2023-09-23T21:21:02.538960+0000","last_unstale":"2023-09-23T21:23:20.761691+0000","last_undegraded":"2023-09-23T21:23:20.761691+0000","last_fullsized":"2023-09-23T21:23:20.761691+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:20.708044+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":51,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.759626+0000","last_change":"2023-09-23T21:20:46.934945+0000","last_active":"2023-09-23T21:23:20.759626+0000","last_peered":"2023-09-23T21:23:20.759626+0000","last_clean":"2023-09-23T21:23:20.759626+0000","last_became_active":"2023-09-23T21:20:46.934681+0000","last_became_peered":"2023-09-23T21:20:46.934681+0000","last_unstale":"2023-09-23T21:23:20.759626+0000","last_undegraded":"2023-09-23T21:23:20.759626+0000","last_fullsized":"2023-09-23T21:23:20.759626+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:04.141953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":36,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774400+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:23:21.774400+0000","last_peered":"2023-09-23T21:23:21.774400+0000","last_clean":"2023-09-23T21:23:21.774400+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:23:21.774400+0000","last_undegraded":"2023-09-23T21:23:21.774400+0000","last_fullsized":"2023-09-23T21:23:21.774400+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"15.d","version":"663'30","reported_seq":65,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.311376+0000","last_change":"2023-09-23T21:23:20.770387+0000","last_active":"2023-09-23T21:23:21.311376+0000","last_peered":"2023-09-23T21:23:21.311376+0000","last_clean":"2023-09-23T21:23:21.311376+0000","last_became_active":"2023-09-23T21:23:20.770077+0000","last_became_peered":"2023-09-23T21:23:20.770077+0000","last_unstale":"2023-09-23T21:23:21.311376+0000","last_undegraded":"2023-09-23T21:23:21.311376+0000","last_fullsized":"2023-09-23T21:23:21.311376+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:34.113345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":139,"num_read_kb":104,"num_write":29,"num_write_kb":24,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":710,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.770595+0000","last_change":"2023-09-23T21:23:12.024326+0000","last_active":"2023-09-23T21:23:20.770595+0000","last_peered":"2023-09-23T21:23:20.770595+0000","last_clean":"2023-09-23T21:23:20.770595+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:23:20.770595+0000","last_undegraded":"2023-09-23T21:23:20.770595+0000","last_fullsized":"2023-09-23T21:23:20.770595+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:23:12.024197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:23:12.024197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:18.945737+0000","scrub_duration":0.0017519600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":330,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.761679+0000","last_change":"2023-09-23T21:20:59.516633+0000","last_active":"2023-09-23T21:23:20.761679+0000","last_peered":"2023-09-23T21:23:20.761679+0000","last_clean":"2023-09-23T21:23:20.761679+0000","last_became_active":"2023-09-23T21:20:59.516343+0000","last_became_peered":"2023-09-23T21:20:59.516343+0000","last_unstale":"2023-09-23T21:23:20.761679+0000","last_undegraded":"2023-09-23T21:23:20.761679+0000","last_fullsized":"2023-09-23T21:23:20.761679+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:00.938309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":135,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769536+0000","last_change":"2023-09-23T21:20:46.966596+0000","last_active":"2023-09-23T21:23:20.769536+0000","last_peered":"2023-09-23T21:23:20.769536+0000","last_clean":"2023-09-23T21:23:20.769536+0000","last_became_active":"2023-09-23T21:20:46.966276+0000","last_became_peered":"2023-09-23T21:20:46.966276+0000","last_unstale":"2023-09-23T21:23:20.769536+0000","last_undegraded":"2023-09-23T21:23:20.769536+0000","last_fullsized":"2023-09-23T21:23:20.769536+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:55.003680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.766396+0000","last_change":"2023-09-23T21:20:59.524037+0000","last_active":"2023-09-23T21:23:20.766396+0000","last_peered":"2023-09-23T21:23:20.766396+0000","last_clean":"2023-09-23T21:23:20.766396+0000","last_became_active":"2023-09-23T21:20:59.523724+0000","last_became_peered":"2023-09-23T21:20:59.523724+0000","last_unstale":"2023-09-23T21:23:20.766396+0000","last_undegraded":"2023-09-23T21:23:20.766396+0000","last_fullsized":"2023-09-23T21:23:20.766396+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:05.266959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"15.8","version":"663'12","reported_seq":28,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.770831+0000","last_change":"2023-09-23T21:23:20.764366+0000","last_active":"2023-09-23T21:23:21.770831+0000","last_peered":"2023-09-23T21:23:21.770831+0000","last_clean":"2023-09-23T21:23:21.770831+0000","last_became_active":"2023-09-23T21:23:20.764075+0000","last_became_peered":"2023-09-23T21:23:20.764075+0000","last_unstale":"2023-09-23T21:23:21.770831+0000","last_undegraded":"2023-09-23T21:23:21.770831+0000","last_fullsized":"2023-09-23T21:23:21.770831+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:42.757689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":18,"num_read_kb":18,"num_write":9,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":727,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774497+0000","last_change":"2023-09-23T21:20:33.969338+0000","last_active":"2023-09-23T21:23:21.774497+0000","last_peered":"2023-09-23T21:23:21.774497+0000","last_clean":"2023-09-23T21:23:21.774497+0000","last_became_active":"2023-09-23T21:20:33.968894+0000","last_became_peered":"2023-09-23T21:20:33.968894+0000","last_unstale":"2023-09-23T21:23:21.774497+0000","last_undegraded":"2023-09-23T21:23:21.774497+0000","last_fullsized":"2023-09-23T21:23:21.774497+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:49.652752+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":270,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762548+0000","last_change":"2023-09-23T21:20:59.525577+0000","last_active":"2023-09-23T21:23:20.762548+0000","last_peered":"2023-09-23T21:23:20.762548+0000","last_clean":"2023-09-23T21:23:20.762548+0000","last_became_active":"2023-09-23T21:20:59.525262+0000","last_became_peered":"2023-09-23T21:20:59.525262+0000","last_unstale":"2023-09-23T21:23:20.762548+0000","last_undegraded":"2023-09-23T21:23:20.762548+0000","last_fullsized":"2023-09-23T21:23:20.762548+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:45.325125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.762964+0000","last_change":"2023-09-23T21:20:59.541616+0000","last_active":"2023-09-23T21:23:20.762964+0000","last_peered":"2023-09-23T21:23:20.762964+0000","last_clean":"2023-09-23T21:23:20.762964+0000","last_became_active":"2023-09-23T21:20:59.541325+0000","last_became_peered":"2023-09-23T21:20:59.541325+0000","last_unstale":"2023-09-23T21:23:20.762964+0000","last_undegraded":"2023-09-23T21:23:20.762964+0000","last_fullsized":"2023-09-23T21:23:20.762964+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:44.108004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":61,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.759669+0000","last_change":"2023-09-23T21:20:59.611577+0000","last_active":"2023-09-23T21:23:20.759669+0000","last_peered":"2023-09-23T21:23:20.759669+0000","last_clean":"2023-09-23T21:23:20.759669+0000","last_became_active":"2023-09-23T21:20:59.611279+0000","last_became_peered":"2023-09-23T21:20:59.611279+0000","last_unstale":"2023-09-23T21:23:20.759669+0000","last_undegraded":"2023-09-23T21:23:20.759669+0000","last_fullsized":"2023-09-23T21:23:20.759669+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:49.880939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.b","version":"663'37","reported_seq":73,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.771342+0000","last_change":"2023-09-23T21:23:20.776662+0000","last_active":"2023-09-23T21:23:21.771342+0000","last_peered":"2023-09-23T21:23:21.771342+0000","last_clean":"2023-09-23T21:23:21.771342+0000","last_became_active":"2023-09-23T21:23:20.776350+0000","last_became_peered":"2023-09-23T21:23:20.776350+0000","last_unstale":"2023-09-23T21:23:21.771342+0000","last_undegraded":"2023-09-23T21:23:21.771342+0000","last_fullsized":"2023-09-23T21:23:21.771342+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":37,"log_dups_size":0,"ondisk_log_size":37,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:32.630422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":215,"num_read_kb":174,"num_write":44,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":964,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769022+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:23:20.769022+0000","last_peered":"2023-09-23T21:23:20.769022+0000","last_clean":"2023-09-23T21:23:20.769022+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:23:20.769022+0000","last_undegraded":"2023-09-23T21:23:20.769022+0000","last_fullsized":"2023-09-23T21:23:20.769022+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":360,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.768479+0000","last_change":"2023-09-23T21:20:47.087756+0000","last_active":"2023-09-23T21:23:20.768479+0000","last_peered":"2023-09-23T21:23:20.768479+0000","last_clean":"2023-09-23T21:23:20.768479+0000","last_became_active":"2023-09-23T21:20:46.973099+0000","last_became_peered":"2023-09-23T21:20:46.973099+0000","last_unstale":"2023-09-23T21:23:20.768479+0000","last_undegraded":"2023-09-23T21:23:20.768479+0000","last_fullsized":"2023-09-23T21:23:20.768479+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.901489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":46,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.766528+0000","last_change":"2023-09-23T21:20:59.529142+0000","last_active":"2023-09-23T21:23:20.766528+0000","last_peered":"2023-09-23T21:23:20.766528+0000","last_clean":"2023-09-23T21:23:20.766528+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:23:20.766528+0000","last_undegraded":"2023-09-23T21:23:20.766528+0000","last_fullsized":"2023-09-23T21:23:20.766528+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757927+0000","last_change":"2023-09-23T21:20:59.514965+0000","last_active":"2023-09-23T21:23:20.757927+0000","last_peered":"2023-09-23T21:23:20.757927+0000","last_clean":"2023-09-23T21:23:20.757927+0000","last_became_active":"2023-09-23T21:20:59.514323+0000","last_became_peered":"2023-09-23T21:20:59.514323+0000","last_unstale":"2023-09-23T21:23:20.757927+0000","last_undegraded":"2023-09-23T21:23:20.757927+0000","last_fullsized":"2023-09-23T21:23:20.757927+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.a","version":"663'14","reported_seq":31,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.771303+0000","last_change":"2023-09-23T21:23:20.771292+0000","last_active":"2023-09-23T21:23:21.771303+0000","last_peered":"2023-09-23T21:23:21.771303+0000","last_clean":"2023-09-23T21:23:21.771303+0000","last_became_active":"2023-09-23T21:23:20.770994+0000","last_became_peered":"2023-09-23T21:23:20.770994+0000","last_unstale":"2023-09-23T21:23:21.771303+0000","last_undegraded":"2023-09-23T21:23:21.771303+0000","last_fullsized":"2023-09-23T21:23:21.771303+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:36.541235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":35,"num_write":10,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1059,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757055+0000","last_change":"2023-09-23T21:21:01.552181+0000","last_active":"2023-09-23T21:23:20.757055+0000","last_peered":"2023-09-23T21:23:20.757055+0000","last_clean":"2023-09-23T21:23:20.757055+0000","last_became_active":"2023-09-23T21:21:01.551806+0000","last_became_peered":"2023-09-23T21:21:01.551806+0000","last_unstale":"2023-09-23T21:23:20.757055+0000","last_undegraded":"2023-09-23T21:23:20.757055+0000","last_fullsized":"2023-09-23T21:23:20.757055+0000","mapping_epoch":630,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:18.167518+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":571,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774765+0000","last_change":"2023-09-23T21:12:51.653611+0000","last_active":"2023-09-23T21:23:21.774765+0000","last_peered":"2023-09-23T21:23:21.774765+0000","last_clean":"2023-09-23T21:23:21.774765+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:23:21.774765+0000","last_undegraded":"2023-09-23T21:23:21.774765+0000","last_fullsized":"2023-09-23T21:23:21.774765+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":50,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.766317+0000","last_change":"2023-09-23T21:20:59.560915+0000","last_active":"2023-09-23T21:23:20.766317+0000","last_peered":"2023-09-23T21:23:20.766317+0000","last_clean":"2023-09-23T21:23:20.766317+0000","last_became_active":"2023-09-23T21:20:59.528846+0000","last_became_peered":"2023-09-23T21:20:59.528846+0000","last_unstale":"2023-09-23T21:23:20.766317+0000","last_undegraded":"2023-09-23T21:23:20.766317+0000","last_fullsized":"2023-09-23T21:23:20.766317+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:14.369746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.759395+0000","last_change":"2023-09-23T21:20:46.963489+0000","last_active":"2023-09-23T21:23:20.759395+0000","last_peered":"2023-09-23T21:23:20.759395+0000","last_clean":"2023-09-23T21:23:20.759395+0000","last_became_active":"2023-09-23T21:20:46.961905+0000","last_became_peered":"2023-09-23T21:20:46.961905+0000","last_unstale":"2023-09-23T21:23:20.759395+0000","last_undegraded":"2023-09-23T21:23:20.759395+0000","last_fullsized":"2023-09-23T21:23:20.759395+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:21.027663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"15.17","version":"663'2","reported_seq":12,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.143428+0000","last_change":"2023-09-23T21:23:20.771564+0000","last_active":"2023-09-23T21:23:21.143428+0000","last_peered":"2023-09-23T21:23:21.143428+0000","last_clean":"2023-09-23T21:23:21.143428+0000","last_became_active":"2023-09-23T21:23:20.771238+0000","last_became_peered":"2023-09-23T21:23:20.771238+0000","last_unstale":"2023-09-23T21:23:21.143428+0000","last_undegraded":"2023-09-23T21:23:21.143428+0000","last_fullsized":"2023-09-23T21:23:21.143428+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:54.383604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":114,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.770417+0000","last_change":"2023-09-23T21:20:59.542345+0000","last_active":"2023-09-23T21:23:20.770417+0000","last_peered":"2023-09-23T21:23:20.770417+0000","last_clean":"2023-09-23T21:23:20.770417+0000","last_became_active":"2023-09-23T21:20:59.541973+0000","last_became_peered":"2023-09-23T21:20:59.541973+0000","last_unstale":"2023-09-23T21:23:20.770417+0000","last_undegraded":"2023-09-23T21:23:20.770417+0000","last_fullsized":"2023-09-23T21:23:20.770417+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.298716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":756,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.759549+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:23:20.759549+0000","last_peered":"2023-09-23T21:23:20.759549+0000","last_clean":"2023-09-23T21:23:20.759549+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:23:20.759549+0000","last_undegraded":"2023-09-23T21:23:20.759549+0000","last_fullsized":"2023-09-23T21:23:20.759549+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"15.10","version":"663'4","reported_seq":14,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.279500+0000","last_change":"2023-09-23T21:23:20.771402+0000","last_active":"2023-09-23T21:23:21.279500+0000","last_peered":"2023-09-23T21:23:21.279500+0000","last_clean":"2023-09-23T21:23:21.279500+0000","last_became_active":"2023-09-23T21:23:20.771099+0000","last_became_peered":"2023-09-23T21:23:20.771099+0000","last_unstale":"2023-09-23T21:23:21.279500+0000","last_undegraded":"2023-09-23T21:23:21.279500+0000","last_fullsized":"2023-09-23T21:23:21.279500+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:56.948010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":5,"num_read_kb":5,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":423,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769031+0000","last_change":"2023-09-23T21:20:59.615839+0000","last_active":"2023-09-23T21:23:20.769031+0000","last_peered":"2023-09-23T21:23:20.769031+0000","last_clean":"2023-09-23T21:23:20.769031+0000","last_became_active":"2023-09-23T21:20:59.543078+0000","last_became_peered":"2023-09-23T21:20:59.543078+0000","last_unstale":"2023-09-23T21:23:20.769031+0000","last_undegraded":"2023-09-23T21:23:20.769031+0000","last_fullsized":"2023-09-23T21:23:20.769031+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:19.278379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":266,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.770119+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:23:20.770119+0000","last_peered":"2023-09-23T21:23:20.770119+0000","last_clean":"2023-09-23T21:23:20.770119+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:23:20.770119+0000","last_undegraded":"2023-09-23T21:23:20.770119+0000","last_fullsized":"2023-09-23T21:23:20.770119+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"15.11","version":"663'5","reported_seq":18,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.770881+0000","last_change":"2023-09-23T21:23:20.776172+0000","last_active":"2023-09-23T21:23:21.770881+0000","last_peered":"2023-09-23T21:23:21.770881+0000","last_clean":"2023-09-23T21:23:21.770881+0000","last_became_active":"2023-09-23T21:23:20.773071+0000","last_became_peered":"2023-09-23T21:23:20.773071+0000","last_unstale":"2023-09-23T21:23:21.770881+0000","last_undegraded":"2023-09-23T21:23:21.770881+0000","last_fullsized":"2023-09-23T21:23:21.770881+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:56.116434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":176,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.774447+0000","last_change":"2023-09-23T21:20:47.083671+0000","last_active":"2023-09-23T21:23:21.774447+0000","last_peered":"2023-09-23T21:23:21.774447+0000","last_clean":"2023-09-23T21:23:21.774447+0000","last_became_active":"2023-09-23T21:20:46.950924+0000","last_became_peered":"2023-09-23T21:20:46.950924+0000","last_unstale":"2023-09-23T21:23:21.774447+0000","last_undegraded":"2023-09-23T21:23:21.774447+0000","last_fullsized":"2023-09-23T21:23:21.774447+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:27.909118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":59,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757518+0000","last_change":"2023-09-23T21:20:31.668925+0000","last_active":"2023-09-23T21:23:20.757518+0000","last_peered":"2023-09-23T21:23:20.757518+0000","last_clean":"2023-09-23T21:23:20.757518+0000","last_became_active":"2023-09-23T21:20:31.668636+0000","last_became_peered":"2023-09-23T21:20:31.668636+0000","last_unstale":"2023-09-23T21:23:20.757518+0000","last_undegraded":"2023-09-23T21:23:20.757518+0000","last_fullsized":"2023-09-23T21:23:20.757518+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:09.442390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.12","version":"663'1","reported_seq":13,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.122548+0000","last_change":"2023-09-23T21:23:20.766800+0000","last_active":"2023-09-23T21:23:21.122548+0000","last_peered":"2023-09-23T21:23:21.122548+0000","last_clean":"2023-09-23T21:23:21.122548+0000","last_became_active":"2023-09-23T21:23:20.766531+0000","last_became_peered":"2023-09-23T21:23:20.766531+0000","last_unstale":"2023-09-23T21:23:21.122548+0000","last_undegraded":"2023-09-23T21:23:21.122548+0000","last_fullsized":"2023-09-23T21:23:21.122548+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:47.723564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":77,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757398+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:23:20.757398+0000","last_peered":"2023-09-23T21:23:20.757398+0000","last_clean":"2023-09-23T21:23:20.757398+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:23:20.757398+0000","last_undegraded":"2023-09-23T21:23:20.757398+0000","last_fullsized":"2023-09-23T21:23:20.757398+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":68,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.757060+0000","last_change":"2023-09-23T21:20:46.926981+0000","last_active":"2023-09-23T21:23:20.757060+0000","last_peered":"2023-09-23T21:23:20.757060+0000","last_clean":"2023-09-23T21:23:20.757060+0000","last_became_active":"2023-09-23T21:20:46.926686+0000","last_became_peered":"2023-09-23T21:20:46.926686+0000","last_unstale":"2023-09-23T21:23:20.757060+0000","last_undegraded":"2023-09-23T21:23:20.757060+0000","last_fullsized":"2023-09-23T21:23:20.757060+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.029756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"15.13","version":"0'0","reported_seq":11,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-09-23T21:23:21.846109+0000","last_change":"2023-09-23T21:23:20.778584+0000","last_active":"2023-09-23T21:23:21.846109+0000","last_peered":"2023-09-23T21:23:21.846109+0000","last_clean":"2023-09-23T21:23:21.846109+0000","last_became_active":"2023-09-23T21:23:20.778287+0000","last_became_peered":"2023-09-23T21:23:20.778287+0000","last_unstale":"2023-09-23T21:23:21.846109+0000","last_undegraded":"2023-09-23T21:23:21.846109+0000","last_fullsized":"2023-09-23T21:23:21.846109+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":662,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T21:23:19.751244+0000","last_clean_scrub_stamp":"2023-09-23T21:23:19.751244+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:39.592693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":653,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.760322+0000","last_change":"2023-09-23T21:20:47.072347+0000","last_active":"2023-09-23T21:23:20.760322+0000","last_peered":"2023-09-23T21:23:20.760322+0000","last_clean":"2023-09-23T21:23:20.760322+0000","last_became_active":"2023-09-23T21:20:46.966023+0000","last_became_peered":"2023-09-23T21:20:46.966023+0000","last_unstale":"2023-09-23T21:23:20.760322+0000","last_undegraded":"2023-09-23T21:23:20.760322+0000","last_fullsized":"2023-09-23T21:23:20.760322+0000","mapping_epoch":622,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:28.360473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":370,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T21:23:20.769468+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:23:20.769468+0000","last_peered":"2023-09-23T21:23:20.769468+0000","last_clean":"2023-09-23T21:23:20.769468+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:23:20.769468+0000","last_undegraded":"2023-09-23T21:23:20.769468+0000","last_fullsized":"2023-09-23T21:23:20.769468+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":35770,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":15,"num_pg":32,"stat_sum":{"num_bytes":374,"num_objects":41,"num_object_clones":0,"num_object_copies":123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41,"num_whiteouts":0,"num_read":5104,"num_read_kb":14036,"num_write":3192,"num_write_kb":7231,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":33,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":86016,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2323,"ondisk_log_size":2323,"up":96,"acting":96,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201936,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":959000,"kb_used_data":43664,"kb_used_omap":0,"kb_used_meta":915328,"kb_avail":92786152,"statfs":{"total":95995035648,"available":95013019648,"internally_reserved":0,"allocated":44711936,"data_stored":42862795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":937295872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:20 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.883,"15min":2.5030000000000001},"min":{"1min":0.26400000000000001,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":6.1920000000000002,"5min":16.468,"15min":141.33199999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.016,"15min":2.6299999999999999},"min":{"1min":0.36399999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.6960000000000002,"5min":12.962,"15min":141.31700000000001},"last":0.65900000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2109999999999999,"15min":3.593},"min":{"1min":0.36699999999999999,"5min":0.29599999999999999,"15min":0.24099999999999999},"max":{"1min":12.112,"5min":13.054,"15min":114.16500000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":3.181,"5min":2.2530000000000001,"15min":3.9670000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22900000000000001},"max":{"1min":10.477,"5min":16.289000000000001,"15min":117.637},"last":0.53000000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:23:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":7.0869999999999997,"15min":5.4660000000000002},"min":{"1min":0.34899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":4.827,"5min":158.726,"15min":183.44200000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.5069999999999999,"5min":6.9669999999999996,"15min":5.7130000000000001},"min":{"1min":0.29699999999999999,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":7.9969999999999999,"5min":158.70400000000001,"15min":183.46199999999999},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:23:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":2.6160000000000001,"15min":4.2969999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.26700000000000002},"max":{"1min":8.3559999999999999,"5min":30.667999999999999,"15min":135.66200000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.2599999999999998,"5min":2.694,"15min":4.6020000000000003},"min":{"1min":0.33200000000000002,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":10.337,"5min":30.798999999999999,"15min":135.47},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0389999999999997,"5min":3.931,"15min":2.2000000000000002},"min":{"1min":0.34499999999999997,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":10.68,"5min":31.695,"15min":31.695},"last":0.32400000000000001},{"interface":"front","average":{"1min":5.7519999999999998,"5min":4.2199999999999998,"15min":2.3690000000000002},"min":{"1min":0.32700000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.840999999999999,"5min":34.994,"15min":34.994},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":4.9279999999999999,"5min":4.6680000000000001,"15min":2.7469999999999999},"min":{"1min":0.44,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":13.906000000000001,"5min":156.30099999999999,"15min":156.30099999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":3.137,"5min":4.7089999999999996,"15min":2.7269999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32200000000000001},"max":{"1min":8.2889999999999997,"5min":156.22,"15min":156.22},"last":0.40100000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:22:47 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.7549999999999999,"15min":2.9529999999999998},"min":{"1min":0.433,"5min":0.40600000000000003,"15min":0.313},"max":{"1min":11.007999999999999,"5min":23.574999999999999,"15min":143.18700000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.8849999999999998,"5min":3.431,"15min":2.702},"min":{"1min":0.36199999999999999,"5min":0.35899999999999999,"15min":0.29099999999999998},"max":{"1min":9.8439999999999994,"5min":21.081,"15min":142.70400000000001},"last":0.379}]}]},{"osd":6,"up_from":47,"seq":201863463557,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375816,"kb_used_data":462656,"kb_used_omap":0,"kb_used_meta":913152,"kb_avail":92369336,"statfs":{"total":95995035648,"available":94586200064,"internally_reserved":0,"allocated":473759744,"data_stored":471935266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":935067648},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.978,"5min":1.764,"15min":2.6539999999999999},"min":{"1min":0.246,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":5.7779999999999996,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.317},{"interface":"front","average":{"1min":1.488,"5min":1.909,"15min":2.8410000000000002},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.223},"max":{"1min":5.3399999999999999,"5min":11.968,"15min":149.303},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":2.1040000000000001,"15min":2.9940000000000002},"min":{"1min":0.25700000000000001,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":5.3810000000000002,"5min":12.013,"15min":129.37899999999999},"last":0.433},{"interface":"front","average":{"1min":3.149,"5min":2.0859999999999999,"15min":2.8250000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.6399999999999997,"5min":14.366,"15min":129.40100000000001},"last":0.247}]},{"osd":2,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":7.0149999999999997,"15min":7.0570000000000004},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":54.561,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":5.3129999999999997,"5min":7.351,"15min":7.2460000000000004},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":54.603999999999999,"5min":199.27000000000001,"15min":199.27000000000001},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.1920000000000002,"15min":3.641},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":7.6269999999999998,"5min":14.411,"15min":137.875},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.927,"15min":3.476},"min":{"1min":0.315,"5min":0.315,"15min":0.29799999999999999},"max":{"1min":5.8040000000000003,"5min":12.35,"15min":137.60900000000001},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":3.5609999999999999,"15min":2.6240000000000001},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.22600000000000001},"max":{"1min":12.218999999999999,"5min":20.408999999999999,"15min":55.494},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.972,"5min":3.528,"15min":2.516},"min":{"1min":0.255,"5min":0.255,"15min":0.23999999999999999},"max":{"1min":13.308999999999999,"5min":14.622,"15min":55.305},"last":0.376}]},{"osd":5,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.6739999999999999,"15min":2.8799999999999999},"min":{"1min":0.317,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":6.9100000000000001,"5min":12.051,"15min":222.90899999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":5.6890000000000001,"5min":3.3500000000000001,"15min":3.0619999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":15.478,"5min":17.927,"15min":224.655},"last":0.41799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.7429999999999999,"15min":2.379},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.28000000000000003},"max":{"1min":12.380000000000001,"5min":12.380000000000001,"15min":165.96799999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":4.1890000000000001,"5min":3.6179999999999999,"15min":2.7690000000000001},"min":{"1min":0.40600000000000003,"5min":0.307,"15min":0.251},"max":{"1min":11.526999999999999,"5min":19.815999999999999,"15min":165.82400000000001},"last":0.44700000000000001}]}]},{"osd":1,"up_from":276,"seq":1185410974003,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":665808,"kb_used_data":8376,"kb_used_omap":5,"kb_used_meta":657402,"kb_avail":93079344,"statfs":{"total":95995035648,"available":95313248256,"internally_reserved":0,"allocated":8577024,"data_stored":5401159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":673180387},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.39200000000000002,"15min":1.944},"min":{"1min":0.214,"5min":0.214,"15min":0.19400000000000001},"max":{"1min":2.3410000000000002,"5min":2.3410000000000002,"15min":108.47},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.339,"5min":0.59799999999999998,"15min":1.952},"min":{"1min":0.23899999999999999,"5min":0.23499999999999999,"15min":0.19600000000000001},"max":{"1min":6.0149999999999997,"5min":6.0149999999999997,"15min":109.532},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":13.051,"5min":6.5170000000000003,"15min":8.6799999999999997},"min":{"1min":0.23200000000000001,"5min":0.22800000000000001,"15min":0.20200000000000001},"max":{"1min":130.78899999999999,"5min":130.78899999999999,"15min":171.35400000000001},"last":0.439},{"interface":"front","average":{"1min":12.999000000000001,"5min":6.5540000000000003,"15min":8.3290000000000006},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":133.751,"5min":133.751,"15min":170.69900000000001},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.60699999999999998,"15min":3.5550000000000002},"min":{"1min":0.27000000000000002,"5min":0.254,"15min":0.23799999999999999},"max":{"1min":3.2450000000000001,"5min":17.247,"15min":99.120999999999995},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.76200000000000001,"15min":4.0960000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":3.754,"5min":10.949999999999999,"15min":150.38},"last":0.224}]},{"osd":4,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":1.829,"15min":2.371},"min":{"1min":0.27400000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":6.9089999999999998,"5min":11.034000000000001,"15min":174.84399999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.3170000000000002,"5min":1.5,"15min":2.2749999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":8.5109999999999992,"5min":11.057,"15min":174.82400000000001},"last":0.63}]},{"osd":5,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":2.6440000000000001,"15min":2.7730000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.31900000000000001},"max":{"1min":8.6329999999999991,"5min":107.98,"15min":110.955},"last":0.52600000000000002},{"interface":"front","average":{"1min":3.0750000000000002,"5min":3.169,"15min":2.8999999999999999},"min":{"1min":0.36699999999999999,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":6.9669999999999996,"5min":107.961,"15min":110.89},"last":0.41499999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":3.6160000000000001,"15min":2.5960000000000001},"min":{"1min":0.42199999999999999,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":20.265999999999998,"5min":119.999,"15min":119.999},"last":0.60699999999999998},{"interface":"front","average":{"1min":3.5409999999999999,"5min":3.5710000000000002,"15min":2.4980000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31900000000000001},"max":{"1min":20.361999999999998,"5min":122.328,"15min":122.328},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5979999999999999,"5min":2.1389999999999998,"15min":3.0009999999999999},"min":{"1min":0.39800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.0430000000000001,"5min":10.648999999999999,"15min":285.47500000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.726,"15min":2.754},"min":{"1min":0.46999999999999997,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":5.165,"5min":10.664999999999999,"15min":285.48599999999999},"last":0.67900000000000005}]}]},{"osd":0,"up_from":0,"seq":395136991743,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":129,"seq":554050781634,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351892,"kb_used_data":10060,"kb_used_omap":7,"kb_used_meta":1341816,"kb_avail":92393260,"statfs":{"total":95995035648,"available":94610698240,"internally_reserved":0,"allocated":10301440,"data_stored":7039727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1374019665},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":3.6030000000000002,"5min":1.2569999999999999,"15min":2.0089999999999999},"min":{"1min":0.193,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":51.968000000000004,"5min":51.968000000000004,"15min":103.021},"last":0.51900000000000002},{"interface":"front","average":{"1min":3.7919999999999998,"5min":1.3500000000000001,"15min":2.363},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":52.241999999999997,"5min":52.241999999999997,"15min":103.154},"last":0.67600000000000005}]},{"osd":1,"last update":"Sat Sep 23 21:22:39 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.749,"15min":2.6970000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19700000000000001},"max":{"1min":3.2829999999999999,"5min":5.2670000000000003,"15min":143.482},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.71499999999999997,"15min":2.7320000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":3.27,"5min":5.4820000000000002,"15min":143.61000000000001},"last":0.23300000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.79600000000000004,"15min":3.5499999999999998},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":4.798,"5min":5.2939999999999996,"15min":143.50800000000001},"last":1.202},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.77600000000000002,"15min":3.77},"min":{"1min":0.34100000000000003,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.3290000000000002,"5min":5.6790000000000003,"15min":143.27799999999999},"last":0.443}]},{"osd":4,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":2.0739999999999998,"15min":3.5049999999999999},"min":{"1min":0.312,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":8.5820000000000007,"5min":8.5820000000000007,"15min":225.80799999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.0129999999999999,"15min":3.9359999999999999},"min":{"1min":0.34999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.5690000000000008,"5min":8.5690000000000008,"15min":225.79300000000001},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":4.3879999999999999,"5min":2.0259999999999998,"15min":3.5550000000000002},"min":{"1min":0.34899999999999998,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":52.338000000000001,"5min":52.338000000000001,"15min":220.63900000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.603,"5min":1.601,"15min":3.5939999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":7.9370000000000003,"5min":8.0579999999999998,"15min":220.31999999999999},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":2.4199999999999999,"15min":3.472},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":5.2030000000000003,"5min":42.363,"15min":210.74700000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.665,"5min":2.3079999999999998,"15min":2.8159999999999998},"min":{"1min":0.34200000000000003,"5min":0.308,"15min":0.308},"max":{"1min":4.4050000000000002,"5min":42.383000000000003,"15min":210.72900000000001},"last":0.59799999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":5.0970000000000004,"5min":2.5299999999999998,"15min":2.2450000000000001},"min":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":52.121000000000002,"5min":52.121000000000002,"15min":52.121000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":6.0739999999999998,"5min":2.5289999999999999,"15min":2.5550000000000002},"min":{"1min":0.52600000000000002,"5min":0.38300000000000001,"15min":0.32700000000000001},"max":{"1min":52.308999999999997,"5min":52.308999999999997,"15min":52.308999999999997},"last":0.55700000000000005}]}]},{"osd":3,"up_from":148,"seq":635655160228,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906120,"kb_used_data":9536,"kb_used_omap":2,"kb_used_meta":896573,"kb_avail":92839032,"statfs":{"total":95995035648,"available":95067168768,"internally_reserved":0,"allocated":9764864,"data_stored":6041615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":918090805},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.52700000000000002,"15min":1.633},"min":{"1min":0.23400000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":2.2909999999999999,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.374},{"interface":"front","average":{"1min":1.1679999999999999,"5min":0.65000000000000002,"15min":1.51},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.19500000000000001},"max":{"1min":5.0720000000000001,"5min":5.0720000000000001,"15min":18.277999999999999},"last":0.245}]},{"osd":1,"last update":"Sat Sep 23 21:22:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.70299999999999996,"15min":6.5880000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":8.3079999999999998,"5min":8.3079999999999998,"15min":237.072},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.67000000000000004,"15min":6.4690000000000003},"min":{"1min":0.24299999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":234.21899999999999},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":3.907,"15min":5.4729999999999999},"min":{"1min":0.34699999999999998,"5min":0.219,"15min":0.215},"max":{"1min":5.4699999999999998,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.3919999999999999,"5min":3.915,"15min":5.1589999999999998},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":5.6050000000000004,"5min":208.024,"15min":208.024},"last":0.39800000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":3.3359999999999999,"15min":2.532},"min":{"1min":0.42299999999999999,"5min":0.27900000000000003,"15min":0.27600000000000002},"max":{"1min":6.4370000000000003,"5min":117.717,"15min":117.717},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.8820000000000001,"5min":3.7730000000000001,"15min":2.597},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":6.9870000000000001,"5min":117.771,"15min":117.771},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":2.9129999999999998,"15min":2.8769999999999998},"min":{"1min":0.41099999999999998,"5min":0.35399999999999998,"15min":0.27200000000000002},"max":{"1min":7.7119999999999997,"5min":100.277,"15min":123.458},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.7000000000000002,"15min":2.7639999999999998},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":6.4740000000000002,"5min":100.69,"15min":123.292},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":4.0679999999999996,"15min":4.8070000000000004},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":6.3890000000000002,"5min":130.613,"15min":304.22399999999999},"last":0.442},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.3700000000000001,"15min":4.8840000000000003},"min":{"1min":0.53600000000000003,"5min":0.33500000000000002,"15min":0.33100000000000002},"max":{"1min":5.5099999999999998,"5min":130.50999999999999,"15min":304.15800000000002},"last":0.433}]},{"osd":7,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":1.927,"15min":2.2240000000000002},"min":{"1min":0.497,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":8.0440000000000005,"5min":19.298999999999999,"15min":44.253},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.6080000000000001,"5min":2.4380000000000002,"15min":2.6099999999999999},"min":{"1min":0.52700000000000002,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":10.441000000000001,"5min":13.343,"15min":45.706000000000003},"last":0.47099999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986846,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1074844,"kb_used_data":97492,"kb_used_omap":0,"kb_used_meta":977344,"kb_avail":92670308,"statfs":{"total":95995035648,"available":94894395392,"internally_reserved":0,"allocated":99831808,"data_stored":97978976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1000800256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":2.286,"15min":3.0339999999999998},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":7.6589999999999998,"5min":12.195,"15min":226.751},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.762,"15min":2.702},"min":{"1min":0.33300000000000002,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":16.478000000000002,"5min":16.478000000000002,"15min":226.43299999999999},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.802,"15min":3.0470000000000002},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":16.434999999999999,"5min":16.434999999999999,"15min":119.55800000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":3.3610000000000002,"5min":2.2730000000000001,"15min":3.0920000000000001},"min":{"1min":0.29599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.108000000000001,"5min":12.108000000000001,"15min":116.78100000000001},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":7.1280000000000001,"15min":6.4390000000000001},"min":{"1min":0.36599999999999999,"5min":0.29499999999999998,"15min":0.248},"max":{"1min":5.1630000000000003,"5min":124.70099999999999,"15min":210.50299999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.0259999999999998,"5min":6.1630000000000003,"15min":6.4420000000000002},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":5.5860000000000003,"5min":116.47,"15min":210.59},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 21:22:35 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":2.6520000000000001,"15min":4.5990000000000002},"min":{"1min":0.314,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":12.973000000000001,"5min":14.323,"15min":214.64099999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.665,"5min":2.5110000000000001,"15min":4.9649999999999999},"min":{"1min":0.38300000000000001,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":13.673999999999999,"5min":13.673999999999999,"15min":214.54900000000001},"last":0.47099999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:22:43 2023","interfaces":[{"interface":"back","average":{"1min":3.149,"5min":3.4140000000000001,"15min":2.004},"min":{"1min":0.34799999999999998,"5min":0.30499999999999999,"15min":0.23200000000000001},"max":{"1min":13.315,"5min":17.879999999999999,"15min":34.497999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.581,"5min":3.234,"15min":2.2450000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.23799999999999999},"max":{"1min":8.7249999999999996,"5min":19.425000000000001,"15min":34.883000000000003},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":4.7309999999999999,"5min":3.9260000000000002,"15min":2.613},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":31.829999999999998,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.48899999999999999},{"interface":"front","average":{"1min":4.8250000000000002,"5min":5.2110000000000003,"15min":2.9580000000000002},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":26.898,"5min":67.596999999999994,"15min":67.596999999999994},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:22:43 2023","interfaces":[{"interface":"back","average":{"1min":4.6980000000000004,"5min":3.645,"15min":2.222},"min":{"1min":0.35599999999999998,"5min":0.33700000000000002,"15min":0.28100000000000003},"max":{"1min":16.998999999999999,"5min":17.817,"15min":18.352},"last":0.52800000000000002},{"interface":"front","average":{"1min":4.2640000000000002,"5min":3.4980000000000002,"15min":2.0760000000000001},"min":{"1min":0.36399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":16.664999999999999,"5min":30.117000000000001,"15min":30.117000000000001},"last":0.53500000000000003}]}]},{"osd":5,"up_from":39,"seq":167503725210,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1992764,"kb_used_data":978036,"kb_used_omap":0,"kb_used_meta":1014720,"kb_avail":91752388,"statfs":{"total":95995035648,"available":93954445312,"internally_reserved":0,"allocated":1001508864,"data_stored":999684912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1039073280},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":1.8799999999999999,"15min":2.0489999999999999},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":12.131,"5min":12.558999999999999,"15min":82.280000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":3.5099999999999998,"5min":2.1880000000000002,"15min":2.3159999999999998},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":12.641,"5min":12.641,"15min":85.236000000000004},"last":0.63500000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.1440000000000001,"15min":3.2610000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.9240000000000004,"5min":18.859999999999999,"15min":125.23399999999999},"last":0.67500000000000004},{"interface":"front","average":{"1min":2.754,"5min":1.8149999999999999,"15min":2.9039999999999999},"min":{"1min":0.32200000000000001,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":12.164,"5min":12.654999999999999,"15min":125.249},"last":0.72399999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":10.749000000000001,"5min":7.0819999999999999,"15min":4.3259999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":123.444,"5min":160.749,"15min":160.749},"last":0.65600000000000003},{"interface":"front","average":{"1min":11.595000000000001,"5min":7.3310000000000004,"15min":4.5289999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":124.874,"5min":161.18000000000001,"15min":161.18000000000001},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.4830000000000001,"15min":4.1299999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27800000000000002},"max":{"1min":12.028,"5min":18.826000000000001,"15min":263.887},"last":0.54000000000000004},{"interface":"front","average":{"1min":3.1850000000000001,"5min":2.0870000000000002,"15min":3.7799999999999998},"min":{"1min":0.38100000000000001,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":12.189,"5min":12.683999999999999,"15min":263.70100000000002},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7229999999999999,"5min":3.4830000000000001,"15min":1.974},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":12.212,"5min":77.804000000000002,"15min":77.804000000000002},"last":0.435},{"interface":"front","average":{"1min":6.274,"5min":4.8659999999999997,"15min":2.516},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":19.718,"5min":77.825999999999993,"15min":77.825999999999993},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":6.3650000000000002,"5min":5.173,"15min":2.972},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":19.675000000000001,"5min":68.346000000000004,"15min":83.451999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":3.2210000000000001,"5min":4.742,"15min":3.0499999999999998},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.29399999999999998},"max":{"1min":11.143000000000001,"5min":72.393000000000001,"15min":83.484999999999999},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:22:45 2023","interfaces":[{"interface":"back","average":{"1min":5.2160000000000002,"5min":3.0550000000000002,"15min":2.0569999999999999},"min":{"1min":0.60499999999999998,"5min":0.36299999999999999,"15min":0.30099999999999999},"max":{"1min":12.692,"5min":12.692,"15min":12.692},"last":0.52100000000000002},{"interface":"front","average":{"1min":4.0270000000000001,"5min":2.5819999999999999,"15min":1.6359999999999999},"min":{"1min":0.48799999999999999,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":12.236000000000001,"5min":18.257999999999999,"15min":18.257999999999999},"last":0.51100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":15,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:23:28.909 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:23:28.909 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:23:28.909 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-23T21:23:28.909 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-23T21:23:28.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:28 smithi121 ceph-mon[211526]: failed to encode map e667 with expected crc 2023-09-23T21:23:28.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:28 smithi121 ceph-mon[211526]: osdmap e667: 8 total, 8 up, 7 in 2023-09-23T21:23:28.968 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:28 smithi121 ceph-mon[211526]: failed to encode map e667 with expected crc 2023-09-23T21:23:28.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:28 smithi121 ceph-mon[219709]: failed to encode map e667 with expected crc 2023-09-23T21:23:28.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:28 smithi121 ceph-mon[219709]: osdmap e667: 8 total, 8 up, 7 in 2023-09-23T21:23:28.969 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:28 smithi121 ceph-mon[219709]: failed to encode map e667 with expected crc 2023-09-23T21:23:29.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:28 smithi123 ceph-mon[193903]: failed to encode map e667 with expected crc 2023-09-23T21:23:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:28 smithi123 ceph-mon[193903]: osdmap e667: 8 total, 8 up, 7 in 2023-09-23T21:23:29.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:28 smithi123 ceph-mon[193903]: failed to encode map e667 with expected crc 2023-09-23T21:23:29.813 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is set 2023-09-23T21:23:29.833 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:29.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:29.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:29.937 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:29.969 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:30.018 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:30.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:30.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:30.126 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:30.151 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:29 smithi123 ceph-mon[193903]: failed to encode map e667 with expected crc 2023-09-23T21:23:30.151 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:29 smithi123 ceph-mon[193903]: failed to encode map e667 with expected crc 2023-09-23T21:23:30.151 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:29 smithi123 ceph-mon[193903]: pgmap v2025: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:30.151 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3989823364' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:30.153 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:30.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[211526]: failed to encode map e667 with expected crc 2023-09-23T21:23:30.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[211526]: failed to encode map e667 with expected crc 2023-09-23T21:23:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[211526]: pgmap v2025: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:30.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3989823364' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[219709]: failed to encode map e667 with expected crc 2023-09-23T21:23:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[219709]: failed to encode map e667 with expected crc 2023-09-23T21:23:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[219709]: pgmap v2025: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:30.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3989823364' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:30.195 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:30.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:30.302 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:30.332 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:30.441 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:30.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:30.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:30.546 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:30.572 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:30.616 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:30.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:30.639 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:30.721 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:30.726 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:30.747 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:30.789 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:30.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:30.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:30.838 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:23:30.838 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:23:30.863 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[219709]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:30.863 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[219709]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.863 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[219709]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.863 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3989823364' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:23:30.863 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[219709]: osdmap e668: 8 total, 8 up, 7 in 2023-09-23T21:23:30.864 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[219709]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[211526]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:30.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[211526]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[211526]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3989823364' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:23:30.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[211526]: osdmap e668: 8 total, 8 up, 7 in 2023-09-23T21:23:30.864 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:30 smithi121 ceph-mon[211526]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.896 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:30.926 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:30.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:30 smithi123 ceph-mon[193903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:30.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:30 smithi123 ceph-mon[193903]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:30 smithi123 ceph-mon[193903]: failed to encode map e668 with expected crc 2023-09-23T21:23:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:30 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3989823364' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T21:23:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:30 smithi123 ceph-mon[193903]: osdmap e668: 8 total, 8 up, 7 in 2023-09-23T21:23:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:30 smithi123 ceph-mon[193903]: failed to encode map e668 with expected crc 2023-09-23T21:23:31.104 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:31.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:31.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:31.209 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:31.235 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:31.277 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:31.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:31.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:31.384 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:31.409 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:31.455 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:31.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:31.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:31.560 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:31.590 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:31.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:31 smithi121 ceph-mon[211526]: pgmap v2027: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:31 smithi121 ceph-mon[219709]: pgmap v2027: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:32.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:31 smithi123 ceph-mon[193903]: pgmap v2027: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:32.538 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:23:32.808 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:32.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:32.913 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:32.946 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T21:23:32.947 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:32.996 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:33.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:33.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:33.102 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:33.131 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:33.143 DEBUG:teuthology.orchestra.run.smithi121:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T21:23:33.161 INFO:tasks.workunit.client.0.smithi121.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T21:23:33.164 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:23:33.164 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T21:23:33.174 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:33.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:33.282 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:33.308 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:33.350 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:33.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:33.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:33.411 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T21:23:33.455 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:33.465 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T21:23:33.482 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:33.506 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T21:23:33.508 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:23:33.508 INFO:tasks.workunit.client.0.smithi121.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T21:23:33.523 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:33.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:33.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:33.550 INFO:tasks.workunit.client.0.smithi121.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T21:23:33.554 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2023-09-23T21:23:33.554 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T21:23:33.611 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T21:23:33.612 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T21:23:33.612 DEBUG:teuthology.orchestra.run.smithi121:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T21:23:33.630 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:33.671 INFO:tasks.workunit.client.0.smithi121.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T21:23:33.671 INFO:tasks.workunit.client.0.smithi121.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T21:23:33.672 INFO:tasks.workunit.client.0.smithi121.stderr:+ TMPDIR=/tmp/rbd_import_export_764439 2023-09-23T21:23:33.672 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm -rf /tmp/rbd_import_export_764439 2023-09-23T21:23:33.672 INFO:tasks.workunit.client.0.smithi121.stderr:+ mkdir /tmp/rbd_import_export_764439 2023-09-23T21:23:33.673 INFO:tasks.workunit.client.0.smithi121.stderr:+ trap 'rm -rf /tmp/rbd_import_export_764439' INT TERM EXIT 2023-09-23T21:23:33.673 INFO:tasks.workunit.client.0.smithi121.stderr:+ mkdir foo.764439 2023-09-23T21:23:33.674 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import foo.764439 foo.dir 2023-09-23T21:23:33.677 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:33.943 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: cannot import a directory 2023-09-23T21:23:33.944 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 0% complete...failed. 2023-09-23T21:23:33.944 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: import failed: (21) Is a directory 2023-09-23T21:23:33.946 INFO:tasks.workunit.client.0.smithi121.stderr:+ true 2023-09-23T21:23:33.946 INFO:tasks.workunit.client.0.smithi121.stderr:+ rmdir foo.764439 2023-09-23T21:23:33.946 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_764439/img bs=1k count=1 seek=10 2023-09-23T21:23:33.947 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records in 2023-09-23T21:23:33.947 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records out 2023-09-23T21:23:33.947 INFO:tasks.workunit.client.0.smithi121.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.4785e-05 s, 22.9 MB/s 2023-09-23T21:23:33.948 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_764439/img bs=1k count=10 seek=100 2023-09-23T21:23:33.948 INFO:tasks.workunit.client.0.smithi121.stderr:10+0 records in 2023-09-23T21:23:33.948 INFO:tasks.workunit.client.0.smithi121.stderr:10+0 records out 2023-09-23T21:23:33.948 INFO:tasks.workunit.client.0.smithi121.stderr:10240 bytes (10 kB, 10 KiB) copied, 4.9275e-05 s, 208 MB/s 2023-09-23T21:23:33.948 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_764439/img bs=1k count=100 seek=1000 2023-09-23T21:23:33.949 INFO:tasks.workunit.client.0.smithi121.stderr:70+1 records in 2023-09-23T21:23:33.949 INFO:tasks.workunit.client.0.smithi121.stderr:70+1 records out 2023-09-23T21:23:33.949 INFO:tasks.workunit.client.0.smithi121.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000161102 s, 447 MB/s 2023-09-23T21:23:33.949 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_764439/img bs=1k seek=10000 2023-09-23T21:23:33.950 INFO:tasks.workunit.client.0.smithi121.stderr:139+1 records in 2023-09-23T21:23:33.950 INFO:tasks.workunit.client.0.smithi121.stderr:139+1 records out 2023-09-23T21:23:33.950 INFO:tasks.workunit.client.0.smithi121.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000288569 s, 496 MB/s 2023-09-23T21:23:33.950 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_764439/img bs=1k seek=100000 2023-09-23T21:23:33.989 INFO:tasks.workunit.client.0.smithi121.stderr:70+1 records in 2023-09-23T21:23:33.989 INFO:tasks.workunit.client.0.smithi121.stderr:70+1 records out 2023-09-23T21:23:33.990 INFO:tasks.workunit.client.0.smithi121.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0386425 s, 1.9 MB/s 2023-09-23T21:23:33.990 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_764439/img bs=1k seek=1000000 2023-09-23T21:23:34.139 INFO:tasks.workunit.client.0.smithi121.stderr:189+1 records in 2023-09-23T21:23:34.139 INFO:tasks.workunit.client.0.smithi121.stderr:189+1 records out 2023-09-23T21:23:34.139 INFO:tasks.workunit.client.0.smithi121.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.148569 s, 1.3 MB/s 2023-09-23T21:23:34.139 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm testimg 2023-09-23T21:23:34.175 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 0% complete...failed. 2023-09-23T21:23:34.175 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T21:23:34.178 INFO:tasks.workunit.client.0.smithi121.stderr:+ true 2023-09-23T21:23:34.178 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format /tmp/rbd_import_export_764439/img testimg 2023-09-23T21:23:34.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:34 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:34.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:34 smithi121 ceph-mon[211526]: pgmap v2028: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:34 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:34.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:34 smithi121 ceph-mon[219709]: pgmap v2028: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:34.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:34 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:34.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:34 smithi123 ceph-mon[193903]: pgmap v2028: 83 pgs: 83 active+clean; 582 KiB data, 7.8 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:35.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:23:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:23:34] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.43.0" 2023-09-23T21:23:35.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[211526]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[211526]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T21:23:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[211526]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[211526]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[219709]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[219709]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T21:23:35.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[219709]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:35 smithi121 ceph-mon[219709]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:35 smithi123 ceph-mon[193903]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:35 smithi123 ceph-mon[193903]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T21:23:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:35 smithi123 ceph-mon[193903]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:35 smithi123 ceph-mon[193903]: failed to encode map e669 with expected crc 2023-09-23T21:23:35.960 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T21:23:35.961 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:23:35.964 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export testimg /tmp/rbd_import_export_764439/img2 2023-09-23T21:23:36.063 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:36.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:36.169 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:36.202 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:36.253 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:36.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:36.261 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:36.265 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export testimg - 2023-09-23T21:23:36.286 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:36.339 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:36.361 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:23:36.361 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:36.383 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: failed to encode map e669 with expected crc 2023-09-23T21:23:36.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T21:23:36.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: pgmap v2031: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:36.384 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/150582305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:36.392 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:23:36.406 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:36.416 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:23:36.441 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:23:36.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:23:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: failed to encode map e669 with expected crc 2023-09-23T21:23:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T21:23:36.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: pgmap v2031: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:36.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:36 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/150582305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:36.489 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5: {} 2023-09-23T21:23:36.509 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:23:36.527 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:36.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:36.545 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.546 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:23:36.633 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:36.662 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:36.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: failed to encode map e669 with expected crc 2023-09-23T21:23:36.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T21:23:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: failed to encode map e670 with expected crc 2023-09-23T21:23:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: pgmap v2031: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2023-09-23T21:23:36.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:36 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/150582305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:36.705 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:36.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:36.810 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:36.841 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:37.103 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:37.106 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:37.109 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm testimg 2023-09-23T21:23:37.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:37.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:37.195 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T21:23:37.198 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp /tmp/rbd_import_export_764439/img /tmp/rbd_import_export_764439/img2 2023-09-23T21:23:37.211 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:37.238 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:37.281 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:37.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:37.387 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:37.417 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:37.457 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:37.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:37.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:37.564 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:37.595 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:38.159 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp /tmp/rbd_import_export_764439/img /tmp/rbd_import_export_764439/img3 2023-09-23T21:23:38.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:37 smithi121 ceph-mon[219709]: pgmap v2032: 83 pgs: 83 active+clean; 4.2 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 86 KiB/s rd, 41 KiB/s wr, 13 op/s 2023-09-23T21:23:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:37 smithi121 ceph-mon[211526]: pgmap v2032: 83 pgs: 83 active+clean; 4.2 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 86 KiB/s rd, 41 KiB/s wr, 13 op/s 2023-09-23T21:23:38.166 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:38.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:38.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:37 smithi123 ceph-mon[193903]: pgmap v2032: 83 pgs: 83 active+clean; 4.2 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 86 KiB/s rd, 41 KiB/s wr, 13 op/s 2023-09-23T21:23:38.271 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:38.303 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:38.353 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:38.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:38.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:38.458 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:38.613 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img2 /tmp/rbd_import_export_764439/img3 2023-09-23T21:23:38.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:38 smithi121 conmon[211495]: 2023-09-23T21:23:38.358+0000 7fdc50c66700 -1 mon.a@0(leader).osd e670 definitely_dead 0 2023-09-23T21:23:38.767 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:38.992 INFO:teuthology.orchestra.run.smithi121.stderr:marked in osd.0. 2023-09-23T21:23:39.017 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:38 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3010747548' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T21:23:39.017 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:38 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:23:39.018 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:38 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3010747548' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T21:23:39.018 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:38 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:23:39.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:38 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3010747548' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T21:23:39.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:38 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:23:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[219709]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3010747548' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T21:23:40.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[219709]: osdmap e671: 8 total, 8 up, 8 in 2023-09-23T21:23:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[219709]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[219709]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[219709]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[219709]: pgmap v2034: 83 pgs: 10 peering, 1 unknown, 72 active+clean; 4.2 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T21:23:40.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[211526]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3010747548' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T21:23:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[211526]: osdmap e671: 8 total, 8 up, 8 in 2023-09-23T21:23:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[211526]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[211526]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[211526]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:39 smithi121 ceph-mon[211526]: pgmap v2034: 83 pgs: 10 peering, 1 unknown, 72 active+clean; 4.2 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T21:23:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:39 smithi123 ceph-mon[193903]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:39 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3010747548' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T21:23:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:39 smithi123 ceph-mon[193903]: osdmap e671: 8 total, 8 up, 8 in 2023-09-23T21:23:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:39 smithi123 ceph-mon[193903]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:39 smithi123 ceph-mon[193903]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:39 smithi123 ceph-mon[193903]: failed to encode map e671 with expected crc 2023-09-23T21:23:40.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:39 smithi123 ceph-mon[193903]: pgmap v2034: 83 pgs: 10 peering, 1 unknown, 72 active+clean; 4.2 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T21:23:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T21:23:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T21:23:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[211526]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.416 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:40 smithi121 ceph-mon[219709]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T21:23:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T21:23:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:41.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:40 smithi123 ceph-mon[193903]: failed to encode map e672 with expected crc 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: pgmap v2037: 83 pgs: 10 peering, 1 unknown, 72 active+clean; 4.2 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[211526]: failed to encode map e674 with expected crc 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: pgmap v2037: 83 pgs: 10 peering, 1 unknown, 72 active+clean; 4.2 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:23:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T21:23:42.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:41 smithi121 ceph-mon[219709]: failed to encode map e674 with expected crc 2023-09-23T21:23:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: failed to encode map e673 with expected crc 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: pgmap v2037: 83 pgs: 10 peering, 1 unknown, 72 active+clean; 4.2 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T21:23:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:41 smithi123 ceph-mon[193903]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[211526]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:43.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:43.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:42 smithi121 ceph-mon[219709]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T21:23:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: failed to encode map e674 with expected crc 2023-09-23T21:23:43.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:43.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:43.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:42 smithi123 ceph-mon[193903]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T21:23:44.254 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format - testimg 2023-09-23T21:23:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[219709]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[219709]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[219709]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[219709]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[219709]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[219709]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[219709]: pgmap v2040: 83 pgs: 1 active+undersized+remapped, 10 peering, 1 unknown, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T21:23:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[211526]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[211526]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[211526]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[211526]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[211526]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[211526]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:44 smithi121 ceph-mon[211526]: pgmap v2040: 83 pgs: 1 active+undersized+remapped, 10 peering, 1 unknown, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T21:23:44.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:44 smithi123 ceph-mon[193903]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:44 smithi123 ceph-mon[193903]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:44 smithi123 ceph-mon[193903]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:44 smithi123 ceph-mon[193903]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:44 smithi123 ceph-mon[193903]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:44 smithi123 ceph-mon[193903]: failed to encode map e675 with expected crc 2023-09-23T21:23:44.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:44 smithi123 ceph-mon[193903]: pgmap v2040: 83 pgs: 1 active+undersized+remapped, 10 peering, 1 unknown, 71 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T21:23:44.879 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:23:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:23:44] "GET /metrics HTTP/1.1" 200 34278 "" "Prometheus/2.43.0" 2023-09-23T21:23:44.951 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:23:44.951 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:23:44.954 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export testimg /tmp/rbd_import_export_764439/img2 2023-09-23T21:23:45.253 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:45.257 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export testimg - 2023-09-23T21:23:45.304 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:45.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:45.408 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:45.445 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:45.519 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-23T21:23:45.653 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:45.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:45.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:45.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:45.786 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:45 smithi121 ceph-mon[219709]: pgmap v2041: 83 pgs: 1 active+undersized+remapped, 82 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 105 B/s, 2 objects/s recovering 2023-09-23T21:23:45.787 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:45 smithi121 ceph-mon[219709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-09-23T21:23:45.787 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:45 smithi121 ceph-mon[211526]: pgmap v2041: 83 pgs: 1 active+undersized+remapped, 82 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 105 B/s, 2 objects/s recovering 2023-09-23T21:23:45.787 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:45 smithi121 ceph-mon[211526]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-09-23T21:23:45.789 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:45 smithi123 ceph-mon[193903]: pgmap v2041: 83 pgs: 1 active+undersized+remapped, 82 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 105 B/s, 2 objects/s recovering 2023-09-23T21:23:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:45 smithi123 ceph-mon[193903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-09-23T21:23:46.049 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:46.053 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm testimg 2023-09-23T21:23:46.136 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T21:23:46.139 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp /tmp/rbd_import_export_764439/img /tmp/rbd_import_export_764439/img2 2023-09-23T21:23:46.492 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:46.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:46.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:46.597 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:46.633 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:46.685 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:46.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:46.775 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp /tmp/rbd_import_export_764439/img /tmp/rbd_import_export_764439/img3 2023-09-23T21:23:46.793 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T21:23:46.820 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:46.864 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T21:23:46.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:46.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T21:23:46.970 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:46.996 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:47.038 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:47.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:47.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:47.144 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:47.170 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:47.208 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:23:47.211 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:47.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:47.232 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img /tmp/rbd_import_export_764439/img2 /tmp/rbd_import_export_764439/img3 2023-09-23T21:23:47.319 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:47.344 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:47.394 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:47.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:47.498 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:47.526 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:48.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:47 smithi123 ceph-mon[193903]: pgmap v2042: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 103 KiB/s rd, 30 KiB/s wr, 72 op/s; 88 B/s, 2 objects/s recovering 2023-09-23T21:23:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:47 smithi121 ceph-mon[211526]: pgmap v2042: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 103 KiB/s rd, 30 KiB/s wr, 72 op/s; 88 B/s, 2 objects/s recovering 2023-09-23T21:23:48.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:47 smithi121 ceph-mon[219709]: pgmap v2042: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 103 KiB/s rd, 30 KiB/s wr, 72 op/s; 88 B/s, 2 objects/s recovering 2023-09-23T21:23:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:49 smithi121 ceph-mon[211526]: pgmap v2043: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s; 71 B/s, 1 objects/s recovering 2023-09-23T21:23:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:49 smithi121 ceph-mon[219709]: pgmap v2043: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s; 71 B/s, 1 objects/s recovering 2023-09-23T21:23:49.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:49 smithi123 ceph-mon[193903]: pgmap v2043: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s; 71 B/s, 1 objects/s recovering 2023-09-23T21:23:50.154 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd help export 2023-09-23T21:23:50.155 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep -q export-format 2023-09-23T21:23:50.174 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_764439/img bs=1k count=10 seek=100 2023-09-23T21:23:50.175 INFO:tasks.workunit.client.0.smithi121.stderr:10+0 records in 2023-09-23T21:23:50.175 INFO:tasks.workunit.client.0.smithi121.stderr:10+0 records out 2023-09-23T21:23:50.175 INFO:tasks.workunit.client.0.smithi121.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.3521e-05 s, 161 MB/s 2023-09-23T21:23:50.175 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format /tmp/rbd_import_export_764439/img testimg 2023-09-23T21:23:50.302 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:23:50.302 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:23:50.305 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap create testimg@snap 2023-09-23T21:23:50.520 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7, 3, 5, 1, 0] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:23:50.520 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:23:50.520 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T21:23:50.520 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T21:23:50.521 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T21:23:50.624 INFO:tasks.workunit.client.0.smithi121.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T21:23:50.631 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T21:23:50.677 INFO:tasks.workunit.client.0.smithi121.stderr:++ rbd image-meta list testimg 2023-09-23T21:23:50.715 INFO:tasks.workunit.client.0.smithi121.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T21:23:50.715 INFO:tasks.workunit.client.0.smithi121.stderr: 2023-09-23T21:23:50.715 INFO:tasks.workunit.client.0.smithi121.stderr:Key Value 2023-09-23T21:23:50.715 INFO:tasks.workunit.client.0.smithi121.stderr:key1 value1' 2023-09-23T21:23:50.715 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:50.750 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:50.753 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_764439/img_v2 testimg_import 2023-09-23T21:23:50.837 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:23:50 smithi123 systemd[1]: Stopping Ceph osd.5 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:23:50.957 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:50.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:51.060 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:51.088 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:23:50 smithi123 conmon[122599]: 2023-09-23T21:23:50.835+0000 7fe65a2c5700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:23:51.088 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:23:50 smithi123 conmon[122599]: 2023-09-23T21:23:50.835+0000 7fe65a2c5700 -1 osd.5 676 *** Got signal Terminated *** 2023-09-23T21:23:51.088 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:23:50 smithi123 conmon[122599]: 2023-09-23T21:23:50.835+0000 7fe65a2c5700 -1 osd.5 676 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:23:51.091 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:51.142 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:51.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:51.210 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:51.245 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:51.275 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:51.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[219709]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[219709]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T21:23:51.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[219709]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[219709]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[219709]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[219709]: osd.5 marked itself down and dead 2023-09-23T21:23:51.575 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[219709]: pgmap v2045: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 158 KiB/s rd, 62 KiB/s wr, 58 op/s; 67 B/s, 0 objects/s recovering 2023-09-23T21:23:51.685 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:51.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:51.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:51.790 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:51.820 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:51.880 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2023-09-23T21:23:51.880 INFO:teuthology.orchestra.run.smithi121.stdout: "size": 20, 2023-09-23T21:23:51.880 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 600, 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "ops": [ 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_op(client.47438.0:100272 14.1 14.908983e1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:30.491811+0000", 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 201.38691160499999, 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.987100011, 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "reached pg", 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "client_info": { 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "client": "client.47438", 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:23:51.881 INFO:teuthology.orchestra.run.smithi121.stdout: "tid": 100272 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491811+0000", 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491811+0000", 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491810+0000", 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491816+0000", 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.212e-06 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.883 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491828+0000", 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1440000000000001e-05 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491837+0000", 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9379999999999998e-06 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491909+0000", 2023-09-23T21:23:51.884 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.2809e-05 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "waiting for peered", 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:30.491922+0000", 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2680000000000001e-05 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.478899+0000", 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.98697686900000003 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.885 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:31.478911+0000", 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2369e-05 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107205 14.8 e626/623 14:1c08e5d7:::benchmark_data_smithi121_684892_object21888:head v 626'6696, mlcod=594'4596)", 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.091697+0000", 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.78702593200001, 2023-09-23T21:23:51.886 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.11511460499999999, 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091697+0000", 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091697+0000", 2023-09-23T21:23:51.887 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091696+0000", 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091705+0000", 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8189999999999998e-06 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.888 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091717+0000", 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2295e-05 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091724+0000", 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8480000000000003e-06 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.889 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091766+0000", 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2113000000000002e-05 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.091784+0000", 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8300000000000001e-05 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.890 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093702+0000", 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.001917675 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206812+0000", 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.113109549 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.891 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107199 14.3 e626/622 14:cd3706fc:::benchmark_data_smithi121_684892_object21882:head v 626'6722, mlcod=594'4660)", 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.093648+0000", 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.785075052, 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.117245378, 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093648+0000", 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.892 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093648+0000", 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093647+0000", 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.893 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093656+0000", 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9919999999999996e-06 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093668+0000", 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2170999999999999e-05 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093676+0000", 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.5599999999999996e-06 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.894 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206793+0000", 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.113117598 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206821+0000", 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7557999999999999e-05 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.210863+0000", 2023-09-23T21:23:51.895 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00404251 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.210893+0000", 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9954000000000002e-05 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.897 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.898 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.898 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107203 14.3 e626/622 14:c71a4f12:::benchmark_data_smithi121_684892_object21886:head v 626'6723, mlcod=594'4660)", 2023-09-23T21:23:51.898 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.093692+0000", 2023-09-23T21:23:51.898 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.78503101300001, 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.117415886, 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093692+0000", 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.899 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093692+0000", 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093692+0000", 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.900 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093696+0000", 2023-09-23T21:23:51.901 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.3120000000000004e-06 2023-09-23T21:23:51.901 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.901 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.901 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.901 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093702+0000", 2023-09-23T21:23:51.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.7150000000000003e-06 2023-09-23T21:23:51.902 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.902 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.902 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.902 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093706+0000", 2023-09-23T21:23:51.902 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6310000000000002e-06 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.208912+0000", 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.115205981 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.208932+0000", 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9836000000000001e-05 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.903 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.211081+0000", 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0021487799999999999 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.211108+0000", 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6964000000000001e-05 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.904 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107193 14.9 e626/623 14:91e47c2b:::benchmark_data_smithi121_684892_object21876:head v 626'6569, mlcod=0'0)", 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.093711+0000", 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.78501212899999, 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.118416242, 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093711+0000", 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.905 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093711+0000", 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093711+0000", 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967296 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.906 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093714+0000", 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8260000000000003e-06 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.093719+0000", 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.1420000000000001e-06 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206763+0000", 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.113043138 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.907 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206960+0000", 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00019710100000000001 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206987+0000", 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6809e-05 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.908 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.212094+0000", 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00510736 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.212127+0000", 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.3099000000000003e-05 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107175 14.6 e626/617 14:6e25e7f6:::benchmark_data_smithi121_684892_object21858:head v 626'6721, mlcod=0'0)", 2023-09-23T21:23:51.909 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.100399+0000", 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.77832399499999, 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.111558003, 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.100399+0000", 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.910 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.100399+0000", 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.100398+0000", 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.100408+0000", 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0076e-05 2023-09-23T21:23:51.911 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206796+0000", 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10638829900000001 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206805+0000", 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9069999999999996e-06 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206892+0000", 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7281999999999995e-05 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206913+0000", 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0996999999999999e-05 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.211924+0000", 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0050110700000000003 2023-09-23T21:23:51.913 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.211957+0000", 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2687000000000001e-05 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107200 14.4 e626/617 14:2892d685:::benchmark_data_smithi121_684892_object21883:head v 626'6612, mlcod=0'0)", 2023-09-23T21:23:51.914 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.108452+0000", 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.77027088, 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.10236448300000001, 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.108452+0000", 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.915 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.108452+0000", 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.108451+0000", 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.108461+0000", 2023-09-23T21:23:51.916 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0647e-05 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206766+0000", 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.098305179000000006 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206780+0000", 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3368e-05 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.917 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206853+0000", 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3498999999999995e-05 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.206876+0000", 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2544e-05 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.918 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.210767+0000", 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0038911670000000001 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.210816+0000", 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.9499000000000002e-05 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.919 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107349 14.a e626/623 14:5e917dc8:::benchmark_data_smithi121_684892_object22032:head v 626'6677, mlcod=0'0)", 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.510269+0000", 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.36845358900001, 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.061025443999999998, 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510269+0000", 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.920 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510269+0000", 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510268+0000", 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510277+0000", 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.2660000000000007e-06 2023-09-23T21:23:51.921 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510289+0000", 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1334e-05 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510295+0000", 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.6710000000000002e-06 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.922 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510342+0000", 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.6113999999999999e-05 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.510364+0000", 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2089000000000001e-05 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.571249+0000", 2023-09-23T21:23:51.923 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.060885599999999998 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.571295+0000", 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.5510999999999998e-05 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107372 14.8 e626/623 14:17bb277f:::benchmark_data_smithi121_684892_object22055:head v 626'6708, mlcod=594'4596)", 2023-09-23T21:23:51.924 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.511295+0000", 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.367427502, 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.062366275999999998, 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511295+0000", 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.925 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511295+0000", 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511294+0000", 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511303+0000", 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.9109999999999999e-06 2023-09-23T21:23:51.926 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511315+0000", 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1456e-05 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511322+0000", 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.3889999999999999e-06 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.927 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511378+0000", 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.5945000000000001e-05 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511407+0000", 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.8969e-05 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.573623+0000", 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.062216004999999998 2023-09-23T21:23:51.928 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.573662+0000", 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8535999999999998e-05 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:107358 14.6 e626/617 14:6a11ef9a:::benchmark_data_smithi121_684892_object22041:head v 626'6738, mlcod=0'0)", 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:20:50.511481+0000", 2023-09-23T21:23:51.929 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 181.36724167, 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.063503740000000003, 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511481+0000", 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.930 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511481+0000", 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511481+0000", 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511487+0000", 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.1120000000000001e-06 2023-09-23T21:23:51.931 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511496+0000", 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.4679999999999998e-06 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.511503+0000", 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3559999999999998e-06 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.932 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.571419+0000", 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.059916913000000002 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.571439+0000", 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.9816000000000001e-05 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.574943+0000", 2023-09-23T21:23:51.933 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0035039340000000002 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:20:50.574985+0000", 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.1785000000000001e-05 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148062 14.2 e652/629 14:46e3b615:::benchmark_data_smithi121_684892_object62745:head v 652'9091, mlcod=594'4563)", 2023-09-23T21:23:51.934 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.963493+0000", 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 91.915230078999997, 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15110810499999999, 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963493+0000", 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.935 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963493+0000", 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963492+0000", 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963499+0000", 2023-09-23T21:23:51.936 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4440000000000002e-06 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963511+0000", 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1248e-05 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963518+0000", 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.4089999999999997e-06 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.937 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963561+0000", 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.2527999999999997e-05 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.963583+0000", 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2232999999999999e-05 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114563+0000", 2023-09-23T21:23:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15098049599999999 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114601+0000", 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7625e-05 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148067 14.1 e652/628 14:870bd909:::benchmark_data_smithi121_684892_object62750:head v 652'9154, mlcod=652'9152)", 2023-09-23T21:23:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.965419+0000", 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 91.913303955999993, 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15207411400000001, 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.965419+0000", 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.965419+0000", 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.965418+0000", 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.965426+0000", 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.8089999999999999e-06 2023-09-23T21:23:51.941 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.965438+0000", 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.1153e-05 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.965445+0000", 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 7.8480000000000001e-06 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.942 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114546+0000", 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.149100179 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114584+0000", 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8003000000000002e-05 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117454+0000", 2023-09-23T21:23:51.943 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0028705699999999998 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.117493+0000", 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8884000000000002e-05 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148070 14.2 e652/629 14:4b6fafd2:::benchmark_data_smithi121_684892_object62753:head v 652'9092, mlcod=594'4563)", 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.966922+0000", 2023-09-23T21:23:51.944 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 91.911801351999998, 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.15797661700000001, 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.966922+0000", 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.945 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.966922+0000", 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.966921+0000", 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.966927+0000", 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.5200000000000003e-06 2023-09-23T21:23:51.946 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114550+0000", 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14762289100000001 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114561+0000", 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0862e-05 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.947 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114625+0000", 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3683000000000007e-05 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114647+0000", 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2104e-05 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.124860+0000", 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010213165999999999 2023-09-23T21:23:51.948 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.124898+0000", 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.8013999999999999e-05 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148063 14.6 e652/629 14:6a437867:::benchmark_data_smithi121_684892_object62746:head v 652'9298, mlcod=0'0)", 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.974354+0000", 2023-09-23T21:23:51.949 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 91.904369083000006, 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14527108699999999, 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974354+0000", 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.950 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974354+0000", 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974353+0000", 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.974361+0000", 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.4100000000000008e-06 2023-09-23T21:23:51.951 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114547+0000", 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14018619600000001 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114561+0000", 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3091000000000001e-05 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.952 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114608+0000", 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7704999999999998e-05 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114631+0000", 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.2943999999999999e-05 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.119587+0000", 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0049558099999999997 2023-09-23T21:23:51.953 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.119625+0000", 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7907000000000001e-05 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:148056 14.a e652/629 14:547e8621:::benchmark_data_smithi121_684892_object62739:head v 652'9230, mlcod=0'0)", 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:19.982123+0000", 2023-09-23T21:23:51.954 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 91.896600163000002, 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.14280778099999999, 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.982123+0000", 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.982123+0000", 2023-09-23T21:23:51.955 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.982122+0000", 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:19.982130+0000", 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.0160000000000008e-06 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.956 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114565+0000", 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.132434617 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114573+0000", 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3850000000000002e-06 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.957 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114657+0000", 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.3832999999999995e-05 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.114677+0000", 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0678000000000001e-05 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.124901+0000", 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.010223916 2023-09-23T21:23:51.958 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:20.124931+0000", 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.9167999999999999e-05 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:155797 14.a e653/629 14:55fe9fe3:::benchmark_data_smithi121_684892_object70480:head v 653'9707, mlcod=0'0)", 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:33.833898+0000", 2023-09-23T21:23:51.959 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 78.044824750000004, 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.092200618999999998, 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833898+0000", 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833898+0000", 2023-09-23T21:23:51.960 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833897+0000", 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833907+0000", 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.8009999999999998e-06 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.961 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833920+0000", 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.3589e-05 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833929+0000", 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.388e-06 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.962 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833973+0000", 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.4870000000000002e-05 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.833992+0000", 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8496e-05 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.926058+0000", 2023-09-23T21:23:51.963 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.092066038000000003 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.926099+0000", 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.0741999999999997e-05 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47438.0:155810 14.4 e653/617 14:2dc932e6:::benchmark_data_smithi121_684892_object70493:head v 653'9589, mlcod=653'9586)", 2023-09-23T21:23:51.964 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:22:33.836803+0000", 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 78.041920028999996, 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.089970558000000006, 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.836803+0000", 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.965 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.836803+0000", 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.836802+0000", 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.966 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.836809+0000", 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.3659999999999997e-06 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.923740+0000", 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.086931199000000001 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.923758+0000", 2023-09-23T21:23:51.967 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8539999999999999e-05 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.923908+0000", 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.00015004800000000001 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.923967+0000", 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.8455999999999997e-05 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.968 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.926736+0000", 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.0027692039999999999 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:22:33.926773+0000", 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.735e-05 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.969 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47951.0:647 15.6 e663/662 15:67cae2ed:::rbd_mirroring:head v 663'153, mlcod=663'151)", 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:23:21.530593+0000", 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 30.348129891999999, 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.104229088, 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530593+0000", 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.970 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530593+0000", 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530593+0000", 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999995 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530598+0000", 2023-09-23T21:23:51.971 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 5.0200000000000002e-06 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530607+0000", 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 9.9510000000000001e-06 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530612+0000", 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4.7269999999999998e-06 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.972 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530649+0000", 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.7193000000000001e-05 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.530668+0000", 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.8159e-05 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.973 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.634797+0000", 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.104129042 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.634822+0000", 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.5482000000000001e-05 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47951.0:783 15.6 e663/662 15:67cae2ed:::rbd_mirroring:head v 663'289, mlcod=663'287)", 2023-09-23T21:23:51.974 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:23:21.752446+0000", 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 30.126277203000001, 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.088097635999999993, 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752446+0000", 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.975 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752446+0000", 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752445+0000", 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.999999 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752453+0000", 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 8.7530000000000007e-06 2023-09-23T21:23:51.976 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752469+0000", 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.5860000000000001e-05 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752479+0000", 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.0152999999999999e-05 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.977 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752548+0000", 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 6.8838999999999996e-05 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.752576+0000", 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.7804e-05 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.840508+0000", 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.087932072999999999 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.978 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:21.840543+0000", 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.5117000000000003e-05 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: "description": "osd_repop(client.47951.0:1567 15.6 e664/662 15:67cae2ed:::rbd_mirroring:head v 664'1073, mlcod=664'1071)", 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: "initiated_at": "2023-09-23T21:23:22.771149+0000", 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: "age": 29.107574095, 2023-09-23T21:23:51.979 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.11649757500000001, 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "type_data": { 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "events": [ 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "initiated", 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771149+0000", 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "throttled", 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771149+0000", 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0 2023-09-23T21:23:51.980 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "header_read", 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771147+0000", 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 4294967295.9999986 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "all_read", 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771160+0000", 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2167000000000001e-05 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "dispatched", 2023-09-23T21:23:51.981 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771180+0000", 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.0262999999999999e-05 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "queued_for_pg", 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771192+0000", 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 1.2072999999999999e-05 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "reached_pg", 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771324+0000", 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.000131848 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.982 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "started", 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.771357+0000", 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 3.2876999999999997e-05 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "commit_sent", 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.887620+0000", 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 0.116263006 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: { 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "event": "done", 2023-09-23T21:23:51.983 INFO:teuthology.orchestra.run.smithi121.stdout: "time": "2023-09-23T21:23:22.887646+0000", 2023-09-23T21:23:51.984 INFO:teuthology.orchestra.run.smithi121.stdout: "duration": 2.6761e-05 2023-09-23T21:23:51.984 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.984 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.984 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.984 INFO:teuthology.orchestra.run.smithi121.stdout: } 2023-09-23T21:23:51.984 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2023-09-23T21:23:51.984 INFO:teuthology.orchestra.run.smithi121.stdout:} 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:51 smithi123 ceph-mon[193903]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:51 smithi123 ceph-mon[193903]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:51 smithi123 ceph-mon[193903]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:51 smithi123 ceph-mon[193903]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:51 smithi123 ceph-mon[193903]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:51 smithi123 ceph-mon[193903]: osd.5 marked itself down and dead 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:51 smithi123 ceph-mon[193903]: pgmap v2045: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 158 KiB/s rd, 62 KiB/s wr, 58 op/s; 67 B/s, 0 objects/s recovering 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[211526]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.985 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[211526]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T21:23:51.986 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[211526]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.986 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[211526]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.986 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[211526]: failed to encode map e676 with expected crc 2023-09-23T21:23:51.986 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[211526]: osd.5 marked itself down and dead 2023-09-23T21:23:51.986 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:51 smithi121 ceph-mon[211526]: pgmap v2045: 83 pgs: 83 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 158 KiB/s rd, 62 KiB/s wr, 58 op/s; 67 B/s, 0 objects/s recovering 2023-09-23T21:23:52.015 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:52.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:52.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:52.160 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:52.585 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:23:52.595 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:23:52.598 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd info testimg_import 2023-09-23T21:23:52.630 INFO:tasks.workunit.client.0.smithi121.stdout:rbd image 'testimg_import': 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: size 110 KiB in 1 objects 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: order 22 (4 MiB objects) 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: snapshot_count: 1 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: id: 94da8f22e56c 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: block_name_prefix: rbd_data.94da8f22e56c 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: format: 2 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: op_features: 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: flags: 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: create_timestamp: Sat Sep 23 21:23:50 2023 2023-09-23T21:23:52.631 INFO:tasks.workunit.client.0.smithi121.stdout: access_timestamp: Sat Sep 23 21:23:50 2023 2023-09-23T21:23:52.632 INFO:tasks.workunit.client.0.smithi121.stdout: modify_timestamp: Sat Sep 23 21:23:50 2023 2023-09-23T21:23:52.633 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd info testimg_import@snap 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout:rbd image 'testimg_import': 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: size 110 KiB in 1 objects 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: order 22 (4 MiB objects) 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: snapshot_count: 1 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: id: 94da8f22e56c 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: block_name_prefix: rbd_data.94da8f22e56c 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: format: 2 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: op_features: 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: flags: 2023-09-23T21:23:52.665 INFO:tasks.workunit.client.0.smithi121.stdout: create_timestamp: Sat Sep 23 21:23:50 2023 2023-09-23T21:23:52.666 INFO:tasks.workunit.client.0.smithi121.stdout: access_timestamp: Sat Sep 23 21:23:50 2023 2023-09-23T21:23:52.666 INFO:tasks.workunit.client.0.smithi121.stdout: modify_timestamp: Sat Sep 23 21:23:50 2023 2023-09-23T21:23:52.666 INFO:tasks.workunit.client.0.smithi121.stdout: protected: False 2023-09-23T21:23:52.667 INFO:tasks.workunit.client.0.smithi121.stderr:++ rbd image-meta list testimg_import 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr: 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr:Key Value 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr:key1 value1' 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr: 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr:Key Value 2023-09-23T21:23:52.705 INFO:tasks.workunit.client.0.smithi121.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T21:23:52.706 INFO:tasks.workunit.client.0.smithi121.stderr: 2023-09-23T21:23:52.706 INFO:tasks.workunit.client.0.smithi121.stderr:Key Value 2023-09-23T21:23:52.706 INFO:tasks.workunit.client.0.smithi121.stderr:key1 value1' ']' 2023-09-23T21:23:52.706 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export testimg_import /tmp/rbd_import_export_764439/img_import 2023-09-23T21:23:52.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:23:52.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: osdmap e677: 8 total, 7 up, 8 in 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:52 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1676952025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:23:52.737 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete...done. 2023-09-23T21:23:52.741 INFO:tasks.workunit.client.0.smithi121.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_764439/img /tmp/rbd_import_export_764439/img_import 2023-09-23T21:23:52.741 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp -l /tmp/rbd_import_export_764439/img /tmp/rbd_import_export_764439/img_import 2023-09-23T21:23:52.741 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/img --format %b 2023-09-23T21:23:52.743 INFO:tasks.workunit.client.0.smithi121.stderr:+ origsize=24 2023-09-23T21:23:52.743 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/img_import --format %b 2023-09-23T21:23:52.744 INFO:tasks.workunit.client.0.smithi121.stderr:+ exportsize=224 2023-09-23T21:23:52.744 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=200 2023-09-23T21:23:52.744 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=200 2023-09-23T21:23:52.744 INFO:tasks.workunit.client.0.smithi121.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T21:23:52.744 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_764439/img_snap 2023-09-23T21:23:52.777 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete...done. 2023-09-23T21:23:52.780 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_764439/img_snap_import 2023-09-23T21:23:52.813 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete...done. 2023-09-23T21:23:52.816 INFO:tasks.workunit.client.0.smithi121.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_764439/img_snap /tmp/rbd_import_export_764439/img_snap_import 2023-09-23T21:23:52.816 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp -l /tmp/rbd_import_export_764439/img_snap /tmp/rbd_import_export_764439/img_snap_import 2023-09-23T21:23:52.817 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/img_snap --format %b 2023-09-23T21:23:52.818 INFO:tasks.workunit.client.0.smithi121.stderr:+ origsize=224 2023-09-23T21:23:52.818 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/img_snap_import --format %b 2023-09-23T21:23:52.819 INFO:tasks.workunit.client.0.smithi121.stderr:+ exportsize=224 2023-09-23T21:23:52.819 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:23:52.819 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:23:52.819 INFO:tasks.workunit.client.0.smithi121.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:23:52.819 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:52.820 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img_import 2023-09-23T21:23:52.820 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img_snap 2023-09-23T21:23:52.821 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img_snap_import 2023-09-23T21:23:52.821 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T21:23:52.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:23:52.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: osdmap e677: 8 total, 7 up, 8 in 2023-09-23T21:23:52.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1676952025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:23:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T21:23:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: osdmap e677: 8 total, 7 up, 8 in 2023-09-23T21:23:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:23:52.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:52 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1676952025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:23:53.228 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:23:52 smithi123 bash[449705]: ab39a1466c342c77cf9eb3a80e9aa27df1b9782e6860d9a6f51747a09a5f5f9c 2023-09-23T21:23:53.583 INFO:tasks.workunit.client.0.smithi121.stderr: Removing snap: 100% complete...done. 2023-09-23T21:23:53.591 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg_import 2023-09-23T21:23:53.674 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:23:53.676 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap rm testimg@snap 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1676952025' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: osdmap e678: 8 total, 7 up, 8 in 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[211526]: pgmap v2048: 83 pgs: 4 peering, 6 stale+active+clean, 73 active+clean; 692 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 125 KiB/s rd, 59 KiB/s wr, 25 op/s 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1676952025' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:23:53.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: osdmap e678: 8 total, 7 up, 8 in 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:53.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:53 smithi121 ceph-mon[219709]: pgmap v2048: 83 pgs: 4 peering, 6 stale+active+clean, 73 active+clean; 692 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 125 KiB/s rd, 59 KiB/s wr, 25 op/s 2023-09-23T21:23:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1676952025' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: osdmap e678: 8 total, 7 up, 8 in 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: failed to encode map e678 with expected crc 2023-09-23T21:23:53.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:53.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:53 smithi123 ceph-mon[193903]: pgmap v2048: 83 pgs: 4 peering, 6 stale+active+clean, 73 active+clean; 692 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 125 KiB/s rd, 59 KiB/s wr, 25 op/s 2023-09-23T21:23:54.012 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:54.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:54.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:54.113 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T21:23:54.116 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:54.149 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:54.246 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:54.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:54.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:54.350 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:54.376 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:54.596 INFO:tasks.workunit.client.0.smithi121.stderr: Removing snap: 100% complete...done. 2023-09-23T21:23:54.608 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm testimg 2023-09-23T21:23:54.700 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:23:54.702 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:54.702 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --order 20 /tmp/rbd_import_export_764439/img testimg 2023-09-23T21:23:54.704 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:54 smithi123 ceph-mon[193903]: failed to encode map e679 with expected crc 2023-09-23T21:23:54.704 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:54 smithi123 ceph-mon[193903]: osdmap e679: 8 total, 7 up, 8 in 2023-09-23T21:23:54.704 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:54 smithi123 ceph-mon[193903]: failed to encode map e679 with expected crc 2023-09-23T21:23:54.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:54.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:54.749 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:23:54.762 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:23:54.763 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:23:54.764 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:54.774 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:23:54.785 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T21:23:54.797 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:23:54.806 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:54.806 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:54.810 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_764439/img_v2 testimg_import 2023-09-23T21:23:54.826 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:23:54.837 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:23:54.837 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:23:54.838 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T21:23:54.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:54 smithi121 ceph-mon[219709]: failed to encode map e679 with expected crc 2023-09-23T21:23:54.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:54 smithi121 ceph-mon[219709]: osdmap e679: 8 total, 7 up, 8 in 2023-09-23T21:23:54.838 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:54 smithi121 ceph-mon[219709]: failed to encode map e679 with expected crc 2023-09-23T21:23:54.838 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:23:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:23:54] "GET /metrics HTTP/1.1" 200 34278 "" "Prometheus/2.43.0" 2023-09-23T21:23:54.839 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:54 smithi121 ceph-mon[211526]: failed to encode map e679 with expected crc 2023-09-23T21:23:54.839 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:54 smithi121 ceph-mon[211526]: osdmap e679: 8 total, 7 up, 8 in 2023-09-23T21:23:54.839 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:54 smithi121 ceph-mon[211526]: failed to encode map e679 with expected crc 2023-09-23T21:23:54.841 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:54.858 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:23:54.886 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:23:54.893 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:23:54.896 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd info testimg_import 2023-09-23T21:23:54.896 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep order 2023-09-23T21:23:54.896 INFO:tasks.workunit.client.0.smithi121.stderr:+ awk '{print $2}' 2023-09-23T21:23:54.897 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep 20 2023-09-23T21:23:54.907 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:23:54.907 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:23:54.907 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:23:54.907 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:23:54.907 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:23:54.908 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:23:54.908 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T21:23:54.908 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:23:54.908 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:23:54.932 INFO:tasks.workunit.client.0.smithi121.stdout:20 2023-09-23T21:23:54.933 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:54.933 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg_import 2023-09-23T21:23:54.947 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:54.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:55.037 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:23:55.040 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg 2023-09-23T21:23:55.052 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:55.087 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:55.129 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:23:55.132 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_764439/img testimg 2023-09-23T21:23:55.134 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:55.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:55.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:55.176 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:23:55.179 INFO:tasks.workunit.client.0.smithi121.stderr:++ rbd info testimg 2023-09-23T21:23:55.180 INFO:tasks.workunit.client.0.smithi121.stderr:++ grep features 2023-09-23T21:23:55.238 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:55.267 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:55.287 INFO:tasks.workunit.client.0.smithi121.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T21:23:55.288 INFO:tasks.workunit.client.0.smithi121.stderr: op_features: ' 2023-09-23T21:23:55.288 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:55.322 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:55.325 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_764439/img_v2 testimg_import 2023-09-23T21:23:55.470 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:23:55.474 INFO:tasks.workunit.client.0.smithi121.stderr:++ rbd info testimg_import 2023-09-23T21:23:55.474 INFO:tasks.workunit.client.0.smithi121.stderr:++ grep features 2023-09-23T21:23:55.516 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:55.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:55.613 INFO:tasks.workunit.client.0.smithi121.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T21:23:55.613 INFO:tasks.workunit.client.0.smithi121.stderr: op_features: ' 2023-09-23T21:23:55.613 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' ' features: layering 2023-09-23T21:23:55.614 INFO:tasks.workunit.client.0.smithi121.stderr: op_features: ' '!=' ' features: layering 2023-09-23T21:23:55.614 INFO:tasks.workunit.client.0.smithi121.stderr: op_features: ' ']' 2023-09-23T21:23:55.614 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:55.614 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg_import 2023-09-23T21:23:55.619 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:23:55.647 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:55.682 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:23:55.685 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg 2023-09-23T21:23:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:55 smithi123 ceph-mon[193903]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:55 smithi123 ceph-mon[193903]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:55 smithi123 ceph-mon[193903]: osdmap e680: 8 total, 7 up, 8 in 2023-09-23T21:23:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:55 smithi123 ceph-mon[193903]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1168680956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:55.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:55 smithi123 ceph-mon[193903]: pgmap v2051: 83 pgs: 5 active+undersized+degraded, 7 active+undersized, 4 peering, 3 stale+active+clean, 64 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 89 KiB/s rd, 13 KiB/s wr, 120 op/s; 8/102 objects degraded (7.843%) 2023-09-23T21:23:55.750 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:23:55.753 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_764439/img testimg 2023-09-23T21:23:55.811 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:23:55.814 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:55.856 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:55.859 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_764439/img_v2 testimg_import 2023-09-23T21:23:55.898 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:23:55.899 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[211526]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[211526]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[211526]: osdmap e680: 8 total, 7 up, 8 in 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[211526]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1168680956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[211526]: pgmap v2051: 83 pgs: 5 active+undersized+degraded, 7 active+undersized, 4 peering, 3 stale+active+clean, 64 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 89 KiB/s rd, 13 KiB/s wr, 120 op/s; 8/102 objects degraded (7.843%) 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[219709]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[219709]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.900 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[219709]: osdmap e680: 8 total, 7 up, 8 in 2023-09-23T21:23:55.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[219709]: failed to encode map e680 with expected crc 2023-09-23T21:23:55.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1168680956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:23:55.901 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:55 smithi121 ceph-mon[219709]: pgmap v2051: 83 pgs: 5 active+undersized+degraded, 7 active+undersized, 4 peering, 3 stale+active+clean, 64 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 89 KiB/s rd, 13 KiB/s wr, 120 op/s; 8/102 objects degraded (7.843%) 2023-09-23T21:23:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:55.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:23:55.919 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:23:55.922 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd info testimg_import 2023-09-23T21:23:55.922 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep 'stripe unit' 2023-09-23T21:23:55.923 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T21:23:55.956 INFO:tasks.workunit.client.0.smithi121.stdout: stripe unit: 4 KiB 2023-09-23T21:23:55.957 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd info testimg_import 2023-09-23T21:23:55.957 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep 'stripe count' 2023-09-23T21:23:55.957 INFO:tasks.workunit.client.0.smithi121.stderr:+ awk '{print $3}' 2023-09-23T21:23:55.957 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep 1000 2023-09-23T21:23:55.994 INFO:tasks.workunit.client.0.smithi121.stdout:1000 2023-09-23T21:23:55.995 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/img_v2 2023-09-23T21:23:55.995 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg_import 2023-09-23T21:23:56.003 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:56.032 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:56.034 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:23:56.035 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:23:56.093 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T21:23:56.097 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg 2023-09-23T21:23:56.127 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:56.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:56.184 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T21:23:56.187 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_764439/img testimg 2023-09-23T21:23:56.232 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:23:56.245 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:23:56.248 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap create testimg@snap1 2023-09-23T21:23:56.261 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:56.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:23:56.306 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:23:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:56.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:23:56.358 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:23:56.359 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:23:56.410 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:56.437 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:56.489 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:56.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:56.590 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:56 smithi121 ceph-mon[211526]: Health check failed: Degraded data redundancy: 8/102 objects degraded (7.843%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:23:56.596 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:56.616 INFO:tasks.workunit.client.0.smithi121.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T21:23:56.623 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap create testimg@snap2 2023-09-23T21:23:56.629 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:56.745 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:56.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:56.826 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:56.848 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:56.874 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:56 smithi123 ceph-mon[193903]: Health check failed: Degraded data redundancy: 8/102 objects degraded (7.843%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:23:56.876 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:56.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:56 smithi121 ceph-mon[219709]: Health check failed: Degraded data redundancy: 8/102 objects degraded (7.843%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T21:23:57.004 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:57.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:57.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:57.049 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T21:23:57.108 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:57.135 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:57.182 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:57.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:57.286 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:57.313 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:57.426 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:23:57.512 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:57.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:57.614 INFO:tasks.workunit.client.0.smithi121.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T21:23:57.616 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:57.620 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap protect testimg@snap2 2023-09-23T21:23:57.644 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:57.665 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_764439/snap_protect 2023-09-23T21:23:57.695 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:57.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:57.799 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:57.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: osdmap e681: 8 total, 7 up, 8 in 2023-09-23T21:23:57.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: failed to encode map e681 with expected crc 2023-09-23T21:23:57.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: failed to encode map e681 with expected crc 2023-09-23T21:23:57.822 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: pgmap v2053: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 273 KiB/s rd, 24 KiB/s wr, 348 op/s; 15/96 objects degraded (15.625%) 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: osdmap e681: 8 total, 7 up, 8 in 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: failed to encode map e681 with expected crc 2023-09-23T21:23:57.823 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: failed to encode map e681 with expected crc 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: pgmap v2053: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 273 KiB/s rd, 24 KiB/s wr, 348 op/s; 15/96 objects degraded (15.625%) 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.824 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:57.824 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:57.867 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:57.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:57.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:57.896 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T21:23:57.900 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_764439/snap_protect testimg_import 2023-09-23T21:23:57.971 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:23:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: osdmap e681: 8 total, 7 up, 8 in 2023-09-23T21:23:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: failed to encode map e681 with expected crc 2023-09-23T21:23:57.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: failed to encode map e681 with expected crc 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: pgmap v2053: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 273 KiB/s rd, 24 KiB/s wr, 348 op/s; 15/96 objects degraded (15.625%) 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:23:57.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:23:57.996 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:58.036 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T21:23:58.335 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:23:58.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:58.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:23:58.410 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:23:58.440 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:58.469 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:58.618 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:23:58.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:58.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:23:58.668 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:58 smithi123 ceph-mon[193903]: failed to encode map e682 with expected crc 2023-09-23T21:23:58.668 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:58 smithi123 ceph-mon[193903]: failed to encode map e682 with expected crc 2023-09-23T21:23:58.668 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:58 smithi123 ceph-mon[193903]: osdmap e682: 8 total, 7 up, 8 in 2023-09-23T21:23:58.722 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:58.751 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:58.800 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:58.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:58.843 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:23:58.843 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:23:58.846 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:58 smithi121 ceph-mon[211526]: failed to encode map e682 with expected crc 2023-09-23T21:23:58.846 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:58 smithi121 ceph-mon[211526]: failed to encode map e682 with expected crc 2023-09-23T21:23:58.847 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:58 smithi121 ceph-mon[211526]: osdmap e682: 8 total, 7 up, 8 in 2023-09-23T21:23:58.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:58 smithi121 ceph-mon[219709]: failed to encode map e682 with expected crc 2023-09-23T21:23:58.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:58 smithi121 ceph-mon[219709]: failed to encode map e682 with expected crc 2023-09-23T21:23:58.847 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:58 smithi121 ceph-mon[219709]: osdmap e682: 8 total, 7 up, 8 in 2023-09-23T21:23:58.905 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:23:58.932 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:59.077 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:23:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:59.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:23:59.181 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:23:59.210 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:59.258 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:23:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:59.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:23:59.361 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:23:59.388 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:59.454 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:23:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:23:59.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:23:59.498 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T21:23:59.560 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:23:59.587 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:23:59.630 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:23:59.645 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T21:23:59.649 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd info testimg_import@snap1 2023-09-23T21:23:59.649 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep 'protected: False' 2023-09-23T21:23:59.687 INFO:tasks.workunit.client.0.smithi121.stdout: protected: False 2023-09-23T21:23:59.687 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd info testimg_import@snap2 2023-09-23T21:23:59.688 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep 'protected: True' 2023-09-23T21:23:59.723 INFO:tasks.workunit.client.0.smithi121.stdout: protected: True 2023-09-23T21:23:59.723 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/snap_protect 2023-09-23T21:23:59.724 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T21:23:59.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:59 smithi123 ceph-mon[193903]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:59 smithi123 ceph-mon[193903]: osdmap e683: 8 total, 7 up, 8 in 2023-09-23T21:23:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:59 smithi123 ceph-mon[193903]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:59 smithi123 ceph-mon[193903]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2294808909' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:59 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:59.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:23:59 smithi123 ceph-mon[193903]: pgmap v2056: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 363 KiB/s rd, 39 KiB/s wr, 476 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:23:59.770 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T21:23:59.813 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap purge testimg 2023-09-23T21:23:59.839 INFO:tasks.thrashosds.thrasher:in_osds: [4, 6, 2, 7, 3, 5, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 2, 3, 4, 6, 7, 1] 2023-09-23T21:23:59.839 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T21:23:59.839 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T21:23:59.839 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:23:59.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[211526]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[211526]: osdmap e683: 8 total, 7 up, 8 in 2023-09-23T21:23:59.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[211526]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[211526]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2294808909' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:59.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:59.865 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[211526]: pgmap v2056: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 363 KiB/s rd, 39 KiB/s wr, 476 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:23:59.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[219709]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[219709]: osdmap e683: 8 total, 7 up, 8 in 2023-09-23T21:23:59.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[219709]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[219709]: failed to encode map e683 with expected crc 2023-09-23T21:23:59.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2294808909' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:59.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:23:59.866 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:23:59 smithi121 ceph-mon[219709]: pgmap v2056: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 363 KiB/s rd, 39 KiB/s wr, 476 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:23:59.900 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:00.407 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:24:00.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:00.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:24:00.512 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:24:00.541 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:00.586 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:24:00.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:00.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:24:00.614 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:00.662 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[211526]: failed to encode map e684 with expected crc 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[211526]: failed to encode map e684 with expected crc 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[211526]: osdmap e684: 8 total, 7 up, 8 in 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[219709]: failed to encode map e684 with expected crc 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[219709]: failed to encode map e684 with expected crc 2023-09-23T21:24:00.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:24:00.664 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:00 smithi121 ceph-mon[219709]: osdmap e684: 8 total, 7 up, 8 in 2023-09-23T21:24:00.689 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:24:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:00 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T21:24:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:00 smithi123 ceph-mon[193903]: failed to encode map e684 with expected crc 2023-09-23T21:24:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:00 smithi123 ceph-mon[193903]: failed to encode map e684 with expected crc 2023-09-23T21:24:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:00 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:24:00.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:00 smithi123 ceph-mon[193903]: osdmap e684: 8 total, 7 up, 8 in 2023-09-23T21:24:00.731 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:00.880 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:24:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:00.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:24:00.942 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:24:00.963 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:24:00.982 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:24:00.984 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:24:01.004 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:24:01.014 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:01.025 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:24:01.048 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:24:01.062 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:24:01.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:01.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:24:01.069 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:24:01.069 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T21:24:01.070 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T21:24:01.070 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T21:24:01.070 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T21:24:01.070 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T21:24:01.070 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T21:24:01.070 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T21:24:01.070 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T21:24:01.167 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:24:01.193 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:01.290 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29157 13.d 13:ba073a5b:::benchmark_data_smithi121_621349_object29156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.335153+0000", 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.95229465499995, 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.090469696, 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.291 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29157 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335153+0000", 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.292 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335153+0000", 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335151+0000", 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335199+0000", 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7856999999999999e-05 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.293 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335212+0000", 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2886e-05 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335218+0000", 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.003e-06 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.294 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335933+0000", 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00071500999999999997 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.335942+0000", 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9309999999999997e-06 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.410242+0000", 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0743003360000001 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.295 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.410356+0000", 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000113804 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.410462+0000", 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000105679 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415027+0000", 2023-09-23T21:24:01.296 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045649250000000001 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425478+0000", 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010451577 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425533+0000", 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5121000000000003e-05 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.297 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425583+0000", 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9027999999999999e-05 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425623+0000", 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0564e-05 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.298 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29163 13.d 13:bd0f7f4a:::benchmark_data_smithi121_621349_object29162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.94896378600004, 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.087340899, 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29163 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.299 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.300 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338484+0000", 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338517+0000", 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3837999999999997e-05 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338524+0000", 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.764e-06 2023-09-23T21:24:01.301 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.338529+0000", 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.34e-06 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.340130+0000", 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016010110000000001 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.302 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.340139+0000", 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4180000000000003e-06 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.411491+0000", 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0713520459999999 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.411569+0000", 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.7754000000000001e-05 2023-09-23T21:24:01.303 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.411661+0000", 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.1989000000000003e-05 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415037+0000", 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0033760830000000002 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.304 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425672+0000", 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.01063513 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425768+0000", 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5523e-05 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425798+0000", 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.0213e-05 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.305 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425825+0000", 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7376e-05 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29167 13.d 13:b8b3085c:::benchmark_data_smithi121_621349_object29166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.344226+0000", 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.94322231199999, 2023-09-23T21:24:01.306 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0817864850000001, 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29167 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344226+0000", 2023-09-23T21:24:01.307 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344226+0000", 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344224+0000", 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.308 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344292+0000", 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7875999999999999e-05 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344313+0000", 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1047e-05 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344323+0000", 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8160000000000005e-06 2023-09-23T21:24:01.309 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344393+0000", 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0419999999999993e-05 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.344405+0000", 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1279e-05 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.310 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.412407+0000", 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0680027459999999 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.412493+0000", 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5641000000000002e-05 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.412577+0000", 2023-09-23T21:24:01.311 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3573999999999996e-05 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416806+0000", 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042291220000000001 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425728+0000", 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0089224749999999992 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.312 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425945+0000", 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00021669500000000001 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.425990+0000", 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4672999999999997e-05 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426012+0000", 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2650000000000002e-05 2023-09-23T21:24:01.313 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29184 13.d 13:b1899bbe:::benchmark_data_smithi121_621349_object29183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.352357+0000", 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.93509122700004, 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.073765817, 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.314 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29184 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352357+0000", 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352357+0000", 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.315 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352355+0000", 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352398+0000", 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2236e-05 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.316 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352421+0000", 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.3785999999999999e-05 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352429+0000", 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1829999999999997e-06 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352523+0000", 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4435e-05 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.317 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.352563+0000", 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.0054e-05 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.413400+0000", 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0608368429999999 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.318 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.413493+0000", 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3127000000000005e-05 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.413606+0000", 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011261500000000001 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.417613+0000", 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0040076959999999998 2023-09-23T21:24:01.319 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426059+0000", 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0084452539999999993 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426080+0000", 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1024000000000001e-05 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.320 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426104+0000", 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4549000000000002e-05 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426123+0000", 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.8369999999999999e-05 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.321 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29197 13.d 13:b78b711b:::benchmark_data_smithi121_621349_object29196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.362494+0000", 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.92495443200005, 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0637530639999999, 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29197 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.322 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362494+0000", 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362494+0000", 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362491+0000", 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999971 2023-09-23T21:24:01.323 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362525+0000", 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4663000000000003e-05 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362542+0000", 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6629999999999998e-05 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.324 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362549+0000", 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3860000000000001e-06 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362612+0000", 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3017999999999999e-05 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.362624+0000", 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1355999999999999e-05 2023-09-23T21:24:01.325 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.414206+0000", 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0515827600000001 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.414326+0000", 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119527 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.326 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.414446+0000", 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000119704 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.419652+0000", 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0052065820000000004 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426167+0000", 2023-09-23T21:24:01.327 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0065143290000000001 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426189+0000", 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.2019999999999999e-05 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426213+0000", 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4711000000000001e-05 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.328 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426247+0000", 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3303000000000003e-05 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29198 13.d 13:b1f63f48:::benchmark_data_smithi121_621349_object29197:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.364267+0000", 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.92318078999995, 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.062194168, 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.329 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29198 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364267+0000", 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.330 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364267+0000", 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364266+0000", 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364319+0000", 2023-09-23T21:24:01.331 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3294000000000002e-05 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364336+0000", 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6586e-05 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364342+0000", 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3879999999999997e-06 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.332 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364395+0000", 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2553999999999999e-05 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.364403+0000", 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6710000000000005e-06 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415080+0000", 2023-09-23T21:24:01.333 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0506764390000001 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415203+0000", 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000122834 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415360+0000", 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015729000000000001 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.334 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.419658+0000", 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042984269999999996 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426326+0000", 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0066676870000000003 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426380+0000", 2023-09-23T21:24:01.335 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3671999999999997e-05 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426424+0000", 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4722999999999999e-05 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426461+0000", 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6999000000000003e-05 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.336 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29204 13.d 13:bda7deaf:::benchmark_data_smithi121_621349_object29203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.366715+0000", 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.92073298699995, 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0599175940000001, 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29204 2023-09-23T21:24:01.337 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366715+0000", 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366715+0000", 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.338 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366714+0000", 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366785+0000", 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0659000000000002e-05 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366797+0000", 2023-09-23T21:24:01.339 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2165e-05 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.366803+0000", 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3450000000000002e-06 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368437+0000", 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016336370000000001 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.340 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368443+0000", 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4799999999999998e-06 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.415992+0000", 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.047549072 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416070+0000", 2023-09-23T21:24:01.341 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8032000000000001e-05 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416163+0000", 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2848000000000003e-05 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.419663+0000", 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035002639999999999 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.342 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426506+0000", 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0068424100000000002 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426545+0000", 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9292000000000003e-05 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426596+0000", 2023-09-23T21:24:01.343 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0487e-05 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426633+0000", 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6959000000000003e-05 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29207 13.d 13:ba8daafe:::benchmark_data_smithi121_621349_object29206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-09-23T21:24:01.344 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:36.367965+0000", 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 564.91948348100004, 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.058811653, 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29207 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.367965+0000", 2023-09-23T21:24:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.367965+0000", 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.367964+0000", 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.346 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368049+0000", 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5740000000000002e-05 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368061+0000", 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1823e-05 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368067+0000", 2023-09-23T21:24:01.347 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1789999999999996e-06 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368459+0000", 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00039199200000000002 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:36.368466+0000", 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5799999999999997e-06 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.348 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416834+0000", 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.048368228 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.416945+0000", 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000110939 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,7", 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.417030+0000", 2023-09-23T21:24:01.349 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4515e-05 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.420777+0000", 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037476340000000001 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426676+0000", 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058991240000000004 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.350 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426712+0000", 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5243999999999997e-05 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426748+0000", 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6112999999999998e-05 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.351 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.426776+0000", 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8425999999999999e-05 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82776 14.9 14:9f17e60f:::benchmark_data_smithi121_684892_object82775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.046431+0000", 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 252.241016731, 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.162395522, 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.352 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82776 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046431+0000", 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.353 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046431+0000", 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046430+0000", 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999981 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046479+0000", 2023-09-23T21:24:01.354 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9322e-05 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046491+0000", 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2165e-05 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046498+0000", 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.63e-06 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.355 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046566+0000", 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7835000000000004e-05 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.046580+0000", 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4644000000000001e-05 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.182033+0000", 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1354527400000001 2023-09-23T21:24:01.356 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.182235+0000", 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00020247700000000001 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.185181+0000", 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029460900000000002 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.357 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.197562+0000", 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012380406 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.197634+0000", 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1888e-05 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.201505+0000", 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038713459999999999 2023-09-23T21:24:01.358 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.205146+0000", 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003640505 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.205201+0000", 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5662999999999999e-05 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.359 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:50.208827+0000", 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0036255250000000001 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82782 14.3 14:ca144c45:::benchmark_data_smithi121_684892_object82781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.075782+0000", 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 252.21166580400001, 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.204296958, 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.360 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82782 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075782+0000", 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.361 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075782+0000", 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075781+0000", 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075838+0000", 2023-09-23T21:24:01.362 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6950000000000002e-05 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075848+0000", 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9799999999999993e-06 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075857+0000", 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.388e-06 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.363 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075910+0000", 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.3829000000000002e-05 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.075917+0000", 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0659999999999998e-06 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939495+0000", 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86357769500000003 2023-09-23T21:24:01.364 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939502+0000", 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.567e-06 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.227405+0000", 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.287903112 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.365 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.227578+0000", 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000173096 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.227776+0000", 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00019853500000000001 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.238175+0000", 2023-09-23T21:24:01.366 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010398154999999999 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.250085+0000", 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011910732 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.259611+0000", 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095251820000000001 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.367 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.276319+0000", 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.016708074999999999 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.276371+0000", 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2423999999999998e-05 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.280079+0000", 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003708154 2023-09-23T21:24:01.368 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82750 14.d 14:bde3e4ca:::benchmark_data_smithi121_684892_object82749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929044+0000", 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 251.35840389500001, 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1129718479999999, 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.369 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82750 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929044+0000", 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929044+0000", 2023-09-23T21:24:01.370 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929043+0000", 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929154+0000", 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011115700000000001 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.371 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929167+0000", 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3256e-05 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929176+0000", 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.5490000000000006e-06 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929348+0000", 2023-09-23T21:24:01.372 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017200599999999999 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op must wait for map", 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929357+0000", 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2399999999999996e-06 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939839+0000", 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010482152999999999 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.373 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939848+0000", 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1969999999999996e-06 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.022667+0000", 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0828196860000001 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.374 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.024357+0000", 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001689664 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.024620+0000", 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00026303900000000001 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040525+0000", 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.015905097999999999 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.375 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040582+0000", 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6847e-05 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040690+0000", 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107917 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041614+0000", 2023-09-23T21:24:01.376 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00092387299999999997 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041825+0000", 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000211048 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041891+0000", 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6196000000000001e-05 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.377 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.042016+0000", 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012504399999999999 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82756 14.d 14:bbd11e56:::benchmark_data_smithi121_684892_object82755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929186+0000", 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 251.35826215899999, 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1161783830000001, 2023-09-23T21:24:01.378 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82756 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929186+0000", 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.379 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929186+0000", 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929185+0000", 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.380 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929224+0000", 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8248999999999999e-05 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929233+0000", 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3559999999999999e-06 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929238+0000", 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1250000000000001e-06 2023-09-23T21:24:01.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929414+0000", 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017563499999999999 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929420+0000", 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5549999999999999e-06 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.382 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939865+0000", 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010444759 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939871+0000", 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8429999999999998e-06 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.027891+0000", 2023-09-23T21:24:01.383 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0880204819999999 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.028035+0000", 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143812 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.028207+0000", 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017190700000000001 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.384 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040556+0000", 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.012348675 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040608+0000", 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2113000000000001e-05 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040716+0000", 2023-09-23T21:24:01.385 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010831399999999999 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041620+0000", 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00090397299999999997 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045178+0000", 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035583540000000001 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.386 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045234+0000", 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5095e-05 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045364+0000", 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013066799999999999 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.387 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82763 14.d 14:b667732c:::benchmark_data_smithi121_684892_object82762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929244+0000", 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 251.35820448699999, 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1163038169999999, 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82763 2023-09-23T21:24:01.388 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929244+0000", 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929244+0000", 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.389 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929243+0000", 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929306+0000", 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3299999999999994e-05 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929314+0000", 2023-09-23T21:24:01.390 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3209999999999999e-06 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929319+0000", 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8319999999999996e-06 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929444+0000", 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012564 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.391 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929451+0000", 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4729999999999997e-06 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939889+0000", 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010438215000000001 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939895+0000", 2023-09-23T21:24:01.392 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7130000000000002e-06 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.030421+0000", 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0905261260000001 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.030565+0000", 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000144411 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.393 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.030732+0000", 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016633699999999999 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040635+0000", 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0099034659999999997 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040743+0000", 2023-09-23T21:24:01.394 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000107513 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041626+0000", 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00088339600000000005 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.043700+0000", 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002074186 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.395 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045399+0000", 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016993799999999999 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045441+0000", 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1952000000000001e-05 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.396 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045547+0000", 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00010592899999999999 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82785 14.d 14:b7c2ba0e:::benchmark_data_smithi121_684892_object82784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 251.35811381100001, 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1163883370000001, 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.397 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82785 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.398 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929334+0000", 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929369+0000", 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5110000000000001e-05 2023-09-23T21:24:01.399 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929376+0000", 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8369999999999999e-06 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929380+0000", 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3649999999999997e-06 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.400 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929475+0000", 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5144999999999998e-05 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting_for_map not empty", 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.929482+0000", 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0609999999999998e-06 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939914+0000", 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010431874000000001 2023-09-23T21:24:01.401 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.939920+0000", 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5099999999999998e-06 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.032921+0000", 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0930011749999999 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.402 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.033068+0000", 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00014687200000000001 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5,6,7", 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.033247+0000", 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017891000000000001 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040662+0000", 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0074158979999999998 2023-09-23T21:24:01.403 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.040793+0000", 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013099699999999999 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.041632+0000", 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00083883899999999997 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.404 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.043734+0000", 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0021020280000000001 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045579+0000", 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0018450109999999999 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045619+0000", 2023-09-23T21:24:01.405 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9159000000000002e-05 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:51.045723+0000", 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000104026 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100246 14.3 14:cc38e1d4:::benchmark_data_smithi121_684892_object14929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:24:01.406 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.435396+0000", 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 210.85205218199999, 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7781351010000002, 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100246 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.407 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435396+0000", 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435396+0000", 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435394+0000", 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.408 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435402+0000", 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.498e-06 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435415+0000", 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.306e-05 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.435421+0000", 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3099999999999997e-06 2023-09-23T21:24:01.409 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.460157+0000", 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024736042 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.461010+0000", 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00085269499999999999 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.410 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.483883+0000", 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.022873072000000001 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.483970+0000", 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6707999999999997e-05 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.484100+0000", 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013049700000000001 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.411 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.497294+0000", 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.013194188000000001 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504390+0000", 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070955189999999998 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504398+0000", 2023-09-23T21:24:01.412 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4169999999999992e-06 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488059+0000", 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98366098099999999 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488067+0000", 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3329999999999999e-06 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.413 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118250+0000", 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63018284499999999 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118407+0000", 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015762799999999999 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.213461+0000", 2023-09-23T21:24:01.414 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.095054270999999996 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.213531+0000", 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.9522000000000003e-05 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100261 14.a 14:53619746:::benchmark_data_smithi121_684892_object14944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.456188+0000", 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 210.83126046699999, 2023-09-23T21:24:01.415 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2422673900000001, 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100261 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456188+0000", 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.416 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456188+0000", 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456187+0000", 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.417 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456189+0000", 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.069e-06 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456194+0000", 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7820000000000001e-06 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456197+0000", 2023-09-23T21:24:01.418 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2509999999999999e-06 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456237+0000", 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.9221999999999998e-05 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456979+0000", 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00074260199999999998 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.419 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.481936+0000", 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024957032000000001 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.482046+0000", 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000109895 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.482209+0000", 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000162895 2023-09-23T21:24:01.420 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.486894+0000", 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0046854690000000003 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.526405+0000", 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.039510085 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.421 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.526414+0000", 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6690000000000001e-06 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.698361+0000", 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.171946878 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.698455+0000", 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3826000000000006e-05 2023-09-23T21:24:01.422 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100262 14.3 14:cdb0c55f:::benchmark_data_smithi121_684892_object14945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 210.83123878800001, 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.761741352, 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.423 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100262 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:24:01.424 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456209+0000", 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456211+0000", 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1469999999999999e-06 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.425 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456216+0000", 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9259999999999999e-06 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.456219+0000", 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2210000000000002e-06 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.461047+0000", 2023-09-23T21:24:01.426 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048276259999999998 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.461643+0000", 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00059657600000000003 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.492320+0000", 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.030677013999999999 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.427 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.492430+0000", 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000109405 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.492588+0000", 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015804099999999999 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504428+0000", 2023-09-23T21:24:01.428 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011839861 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.504434+0000", 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8289999999999996e-06 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488092+0000", 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.98365893699999996 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.429 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488098+0000", 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2140000000000004e-06 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118481+0000", 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63038356299999998 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118622+0000", 2023-09-23T21:24:01.430 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140362 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.217866+0000", 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.099244350999999995 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.217951+0000", 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4611999999999996e-05 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.431 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100269 14.a 14:5ecb1e34:::benchmark_data_smithi121_684892_object14952:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.473910+0000", 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 210.813538034, 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.350025773, 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100269 2023-09-23T21:24:01.432 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473910+0000", 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473910+0000", 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.433 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473909+0000", 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473916+0000", 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9379999999999992e-06 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473933+0000", 2023-09-23T21:24:01.434 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6674e-05 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473941+0000", 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.537e-06 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.473995+0000", 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4669000000000001e-05 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.435 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.474764+0000", 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00076869500000000001 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.495964+0000", 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.021199908 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.496032+0000", 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8529999999999996e-05 2023-09-23T21:24:01.436 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,6,7", 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.496123+0000", 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0557999999999996e-05 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.499899+0000", 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0037764249999999999 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.437 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.533301+0000", 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.033401695000000002 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.533308+0000", 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3289999999999996e-06 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.787436+0000", 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.254127725 2023-09-23T21:24:01.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for missing object", 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.787602+0000", 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000165642 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.823870+0000", 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.036268056 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.439 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.823936+0000", 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.5850000000000001e-05 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100280 14.9 14:92d6aeda:::benchmark_data_smithi121_684892_object14963:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.543047+0000", 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 210.74440124899999, 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.815507285, 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.440 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100280 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543047+0000", 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.441 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543047+0000", 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543046+0000", 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543051+0000", 2023-09-23T21:24:01.442 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4709999999999998e-06 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543060+0000", 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6009999999999995e-06 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543065+0000", 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9529999999999998e-06 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.443 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543115+0000", 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.9855000000000003e-05 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543126+0000", 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1042e-05 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.485495+0000", 2023-09-23T21:24:01.444 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94236904700000002 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.485504+0000", 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0690000000000008e-06 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.077364+0000", 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.59186067399999998 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.445 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.078112+0000", 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00074786899999999996 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.170404+0000", 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.092291976999999997 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.170501+0000", 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.6801999999999999e-05 2023-09-23T21:24:01.446 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,3,6", 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.170629+0000", 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012787700000000001 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.175959+0000", 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0053303320000000001 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.447 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.175992+0000", 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2880999999999999e-05 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.184319+0000", 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0083270949999999996 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.355737+0000", 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.17141735899999999 2023-09-23T21:24:01.448 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.355801+0000", 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4055999999999998e-05 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.358554+0000", 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002753326 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.449 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100281 14.3 14:c07e95ed:::benchmark_data_smithi121_684892_object14964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.543561+0000", 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 210.743887094, 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.67612417, 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100281 2023-09-23T21:24:01.450 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543561+0000", 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543561+0000", 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:01.451 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543560+0000", 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543565+0000", 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.5770000000000004e-06 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543576+0000", 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1257000000000001e-05 2023-09-23T21:24:01.452 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543581+0000", 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2229999999999998e-06 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543629+0000", 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8146999999999998e-05 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.453 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.543639+0000", 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5459999999999997e-06 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488369+0000", 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.94473046900000002 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.488377+0000", 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.9200000000000004e-06 2023-09-23T21:24:01.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118649+0000", 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.63027127500000002 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118806+0000", 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015786700000000001 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.455 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.118980+0000", 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017401299999999999 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.160178+0000", 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.041197960999999998 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.205774+0000", 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.045596043000000003 2023-09-23T21:24:01.456 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.208482+0000", 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027076510000000002 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.218017+0000", 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0095350959999999998 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:01.457 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.218079+0000", 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2071999999999998e-05 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.219685+0000", 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0016058649999999999 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:01.458 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:24:01.592 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:24:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:01.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:24:01.643 INFO:tasks.workunit.client.0.smithi121.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T21:24:01.651 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd snap purge testimg_import 2023-09-23T21:24:01.696 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:24:01.725 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:01.811 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:24:01.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:01.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:24:01.856 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T21:24:01.881 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:01 smithi123 ceph-mon[193903]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:01 smithi123 ceph-mon[193903]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:01 smithi123 ceph-mon[193903]: osdmap e685: 8 total, 7 up, 8 in 2023-09-23T21:24:01.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:01 smithi123 ceph-mon[193903]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/873190143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:24:01.882 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:01 smithi123 ceph-mon[193903]: pgmap v2059: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 176 KiB/s rd, 25 KiB/s wr, 244 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[211526]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[211526]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[211526]: osdmap e685: 8 total, 7 up, 8 in 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[211526]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/873190143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[211526]: pgmap v2059: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 176 KiB/s rd, 25 KiB/s wr, 244 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[219709]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[219709]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[219709]: osdmap e685: 8 total, 7 up, 8 in 2023-09-23T21:24:01.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[219709]: failed to encode map e685 with expected crc 2023-09-23T21:24:01.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/873190143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:24:01.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:01 smithi121 ceph-mon[219709]: pgmap v2059: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 176 KiB/s rd, 25 KiB/s wr, 244 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:24:01.915 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:24:01.941 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:01.987 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:24:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:01.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:24:02.095 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T21:24:02.123 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:02.213 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:02.427 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T21:24:02.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:02.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T21:24:02.511 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T21:24:02.562 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:24:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:24:02.586 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:24:02.591 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:02.686 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[219709]: failed to encode map e686 with expected crc 2023-09-23T21:24:02.686 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[219709]: osdmap e686: 8 total, 7 up, 8 in 2023-09-23T21:24:02.687 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[219709]: failed to encode map e686 with expected crc 2023-09-23T21:24:02.687 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14/102 objects degraded (13.725%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:02.687 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[211526]: failed to encode map e686 with expected crc 2023-09-23T21:24:02.687 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[211526]: osdmap e686: 8 total, 7 up, 8 in 2023-09-23T21:24:02.687 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[211526]: failed to encode map e686 with expected crc 2023-09-23T21:24:02.687 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:02 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14/102 objects degraded (13.725%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:02.709 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:02 smithi123 ceph-mon[193903]: failed to encode map e686 with expected crc 2023-09-23T21:24:02.709 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:02 smithi123 ceph-mon[193903]: osdmap e686: 8 total, 7 up, 8 in 2023-09-23T21:24:02.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:02 smithi123 ceph-mon[193903]: failed to encode map e686 with expected crc 2023-09-23T21:24:02.710 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:02 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14/102 objects degraded (13.725%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:02.738 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:24:02.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:02.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:24:02.841 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:24:02.873 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:02.923 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:24:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:02.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:24:02.947 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:03.027 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:24:03.053 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:03.131 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:24:03.133 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:24:03.203 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:24:03.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:03.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:24:03.307 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:24:03.335 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:03.647 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[219709]: failed to encode map e687 with expected crc 2023-09-23T21:24:03.655 INFO:tasks.workunit.client.0.smithi121.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T21:24:03.662 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:24:03.662 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:24:03.663 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:24:03.663 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:24:03.663 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg 2023-09-23T21:24:03.753 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:24:03.756 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd remove testimg_import 2023-09-23T21:24:03.832 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:24:03.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:03.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:24:03.843 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:24:03.847 INFO:tasks.workunit.client.0.smithi121.stderr:+ tiered=0 2023-09-23T21:24:03.847 INFO:tasks.workunit.client.0.smithi121.stderr:+ ceph osd dump 2023-09-23T21:24:03.847 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep '^pool' 2023-09-23T21:24:03.848 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep ''\''rbd'\''' 2023-09-23T21:24:03.848 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep tier 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[211526]: failed to encode map e687 with expected crc 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[211526]: failed to encode map e687 with expected crc 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[211526]: osdmap e687: 8 total, 7 up, 8 in 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[211526]: from='client.58195 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[211526]: pgmap v2062: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 80 KiB/s rd, 5.2 KiB/s wr, 105 op/s; 15/111 objects degraded (13.514%) 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[219709]: failed to encode map e687 with expected crc 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[219709]: osdmap e687: 8 total, 7 up, 8 in 2023-09-23T21:24:03.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[219709]: from='client.58195 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:24:03.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:03 smithi121 ceph-mon[219709]: pgmap v2062: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 80 KiB/s rd, 5.2 KiB/s wr, 105 op/s; 15/111 objects degraded (13.514%) 2023-09-23T21:24:03.937 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:24:03.964 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":2061,"stamp":"2023-09-23T21:24:02.636913+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":821343,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":3531,"num_read_kb":4971,"num_write":559,"num_write_kb":1331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":193,"num_bytes_recovered":1152090,"num_keys_recovered":40,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2397,"ondisk_log_size":2397,"up":225,"acting":225,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":258,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8617720,"kb_used_data":1039324,"kb_used_omap":21,"kb_used_meta":7578218,"kb_avail":741343496,"statfs":{"total":767960285184,"available":759135739904,"internally_reserved":0,"allocated":1064267776,"data_stored":1043358511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22045,"internal_metadata":7760095715},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":121,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":1227,"num_read_kb":1027,"num_write":171,"num_write_kb":123,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.018855"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":257,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608448+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:23:57.608448+0000","last_peered":"2023-09-23T21:23:57.608448+0000","last_clean":"2023-09-23T21:23:57.608448+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:23:57.608448+0000","last_undegraded":"2023-09-23T21:23:57.608448+0000","last_fullsized":"2023-09-23T21:23:57.608448+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":97,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.616499+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:23:58.616499+0000","last_peered":"2023-09-23T21:23:58.616499+0000","last_clean":"2023-09-23T21:23:58.616499+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:23:58.616499+0000","last_undegraded":"2023-09-23T21:23:58.616499+0000","last_fullsized":"2023-09-23T21:23:58.616499+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.639744+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:24:00.639744+0000","last_peered":"2023-09-23T21:24:00.639744+0000","last_clean":"2023-09-23T21:24:00.639744+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:24:00.639744+0000","last_undegraded":"2023-09-23T21:24:00.639744+0000","last_fullsized":"2023-09-23T21:24:00.639744+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608533+0000","last_change":"2023-09-23T21:23:40.010951+0000","last_active":"2023-09-23T21:23:57.608533+0000","last_peered":"2023-09-23T21:23:57.608533+0000","last_clean":"2023-09-23T21:23:57.608533+0000","last_became_active":"2023-09-23T21:23:40.010331+0000","last_became_peered":"2023-09-23T21:23:40.010331+0000","last_unstale":"2023-09-23T21:23:57.608533+0000","last_undegraded":"2023-09-23T21:23:57.608533+0000","last_fullsized":"2023-09-23T21:23:57.608533+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:06.978784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":97,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.615727+0000","last_change":"2023-09-23T21:23:40.012215+0000","last_active":"2023-09-23T21:23:58.615727+0000","last_peered":"2023-09-23T21:23:58.615727+0000","last_clean":"2023-09-23T21:23:58.615727+0000","last_became_active":"2023-09-23T21:23:40.011815+0000","last_became_peered":"2023-09-23T21:23:40.011815+0000","last_unstale":"2023-09-23T21:23:58.615727+0000","last_undegraded":"2023-09-23T21:23:58.615727+0000","last_fullsized":"2023-09-23T21:23:58.615727+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:00.486707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":684,"state":"active+undersized","last_fresh":"2023-09-23T21:23:59.628481+0000","last_change":"2023-09-23T21:23:52.582753+0000","last_active":"2023-09-23T21:23:59.628481+0000","last_peered":"2023-09-23T21:23:59.628481+0000","last_clean":"2023-09-23T21:23:50.790367+0000","last_became_active":"2023-09-23T21:23:52.582753+0000","last_became_peered":"2023-09-23T21:23:52.582753+0000","last_unstale":"2023-09-23T21:23:59.628481+0000","last_undegraded":"2023-09-23T21:23:59.628481+0000","last_fullsized":"2023-09-23T21:23:52.581081+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:28.651581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":28,"reported_epoch":682,"state":"active+undersized","last_fresh":"2023-09-23T21:23:57.608549+0000","last_change":"2023-09-23T21:23:52.581133+0000","last_active":"2023-09-23T21:23:57.608549+0000","last_peered":"2023-09-23T21:23:57.608549+0000","last_clean":"2023-09-23T21:23:50.624667+0000","last_became_active":"2023-09-23T21:23:52.581133+0000","last_became_peered":"2023-09-23T21:23:52.581133+0000","last_unstale":"2023-09-23T21:23:57.608549+0000","last_undegraded":"2023-09-23T21:23:57.608549+0000","last_fullsized":"2023-09-23T21:23:52.577591+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:20.144280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":106,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.615671+0000","last_change":"2023-09-23T21:21:21.568861+0000","last_active":"2023-09-23T21:23:58.615671+0000","last_peered":"2023-09-23T21:23:58.615671+0000","last_clean":"2023-09-23T21:23:58.615671+0000","last_became_active":"2023-09-23T21:21:21.568492+0000","last_became_peered":"2023-09-23T21:21:21.568492+0000","last_unstale":"2023-09-23T21:23:58.615671+0000","last_undegraded":"2023-09-23T21:23:58.615671+0000","last_fullsized":"2023-09-23T21:23:58.615671+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:04.238433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":682,"state":"active+undersized","last_fresh":"2023-09-23T21:23:57.606492+0000","last_change":"2023-09-23T21:23:52.586371+0000","last_active":"2023-09-23T21:23:57.606492+0000","last_peered":"2023-09-23T21:23:57.606492+0000","last_clean":"2023-09-23T21:20:53.004910+0000","last_became_active":"2023-09-23T21:23:52.586371+0000","last_became_peered":"2023-09-23T21:23:52.586371+0000","last_unstale":"2023-09-23T21:23:57.606492+0000","last_undegraded":"2023-09-23T21:23:57.606492+0000","last_fullsized":"2023-09-23T21:23:52.584341+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:05.527192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.616577+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:23:58.616577+0000","last_peered":"2023-09-23T21:23:58.616577+0000","last_clean":"2023-09-23T21:23:58.616577+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:23:58.616577+0000","last_undegraded":"2023-09-23T21:23:58.616577+0000","last_fullsized":"2023-09-23T21:23:58.616577+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":317,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606362+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:23:57.606362+0000","last_peered":"2023-09-23T21:23:57.606362+0000","last_clean":"2023-09-23T21:23:57.606362+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:23:57.606362+0000","last_undegraded":"2023-09-23T21:23:57.606362+0000","last_fullsized":"2023-09-23T21:23:57.606362+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.639855+0000","last_change":"2023-09-23T21:21:21.574115+0000","last_active":"2023-09-23T21:24:00.639855+0000","last_peered":"2023-09-23T21:24:00.639855+0000","last_clean":"2023-09-23T21:24:00.639855+0000","last_became_active":"2023-09-23T21:21:21.573804+0000","last_became_peered":"2023-09-23T21:21:21.573804+0000","last_unstale":"2023-09-23T21:24:00.639855+0000","last_undegraded":"2023-09-23T21:24:00.639855+0000","last_fullsized":"2023-09-23T21:24:00.639855+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:09.799341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":413,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608486+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:23:57.608486+0000","last_peered":"2023-09-23T21:23:57.608486+0000","last_clean":"2023-09-23T21:23:57.608486+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:23:57.608486+0000","last_undegraded":"2023-09-23T21:23:57.608486+0000","last_fullsized":"2023-09-23T21:23:57.608486+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":640,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608571+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:23:57.608571+0000","last_peered":"2023-09-23T21:23:57.608571+0000","last_clean":"2023-09-23T21:23:57.608571+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:23:57.608571+0000","last_undegraded":"2023-09-23T21:23:57.608571+0000","last_fullsized":"2023-09-23T21:23:57.608571+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":265,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606266+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:23:57.606266+0000","last_peered":"2023-09-23T21:23:57.606266+0000","last_clean":"2023-09-23T21:23:57.606266+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:23:57.606266+0000","last_undegraded":"2023-09-23T21:23:57.606266+0000","last_fullsized":"2023-09-23T21:23:57.606266+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.616243+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:23:58.616243+0000","last_peered":"2023-09-23T21:23:58.616243+0000","last_clean":"2023-09-23T21:23:58.616243+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:23:58.616243+0000","last_undegraded":"2023-09-23T21:23:58.616243+0000","last_fullsized":"2023-09-23T21:23:58.616243+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":301,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606323+0000","last_change":"2023-09-23T21:23:40.011984+0000","last_active":"2023-09-23T21:23:57.606323+0000","last_peered":"2023-09-23T21:23:57.606323+0000","last_clean":"2023-09-23T21:23:57.606323+0000","last_became_active":"2023-09-23T21:23:40.011708+0000","last_became_peered":"2023-09-23T21:23:40.011708+0000","last_unstale":"2023-09-23T21:23:57.606323+0000","last_undegraded":"2023-09-23T21:23:57.606323+0000","last_fullsized":"2023-09-23T21:23:57.606323+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:03.803493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":306,"reported_epoch":682,"state":"active+undersized","last_fresh":"2023-09-23T21:23:57.606663+0000","last_change":"2023-09-23T21:23:52.586993+0000","last_active":"2023-09-23T21:23:57.606663+0000","last_peered":"2023-09-23T21:23:57.606663+0000","last_clean":"2023-09-23T21:23:50.569991+0000","last_became_active":"2023-09-23T21:23:52.586993+0000","last_became_peered":"2023-09-23T21:23:52.586993+0000","last_unstale":"2023-09-23T21:23:57.606663+0000","last_undegraded":"2023-09-23T21:23:57.606663+0000","last_fullsized":"2023-09-23T21:23:52.585317+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:20.038162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":259,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606138+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:23:57.606138+0000","last_peered":"2023-09-23T21:23:57.606138+0000","last_clean":"2023-09-23T21:23:57.606138+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:23:57.606138+0000","last_undegraded":"2023-09-23T21:23:57.606138+0000","last_fullsized":"2023-09-23T21:23:57.606138+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608643+0000","last_change":"2023-09-23T21:23:40.011399+0000","last_active":"2023-09-23T21:23:57.608643+0000","last_peered":"2023-09-23T21:23:57.608643+0000","last_clean":"2023-09-23T21:23:57.608643+0000","last_became_active":"2023-09-23T21:23:40.010992+0000","last_became_peered":"2023-09-23T21:23:40.010992+0000","last_unstale":"2023-09-23T21:23:57.608643+0000","last_undegraded":"2023-09-23T21:23:57.608643+0000","last_fullsized":"2023-09-23T21:23:57.608643+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:35.168968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":808,"reported_epoch":684,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:59.628246+0000","last_change":"2023-09-23T21:23:52.582569+0000","last_active":"2023-09-23T21:23:59.628246+0000","last_peered":"2023-09-23T21:23:59.628246+0000","last_clean":"2023-09-23T21:23:50.789122+0000","last_became_active":"2023-09-23T21:23:52.582569+0000","last_became_peered":"2023-09-23T21:23:52.582569+0000","last_unstale":"2023-09-23T21:23:59.628246+0000","last_undegraded":"2023-09-23T21:23:52.581184+0000","last_fullsized":"2023-09-23T21:23:52.581052+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:49.441631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.616144+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:23:58.616144+0000","last_peered":"2023-09-23T21:23:58.616144+0000","last_clean":"2023-09-23T21:23:58.616144+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:23:58.616144+0000","last_undegraded":"2023-09-23T21:23:58.616144+0000","last_fullsized":"2023-09-23T21:23:58.616144+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":333,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606602+0000","last_change":"2023-09-23T21:23:40.024118+0000","last_active":"2023-09-23T21:23:57.606602+0000","last_peered":"2023-09-23T21:23:57.606602+0000","last_clean":"2023-09-23T21:23:57.606602+0000","last_became_active":"2023-09-23T21:23:40.014595+0000","last_became_peered":"2023-09-23T21:23:40.014595+0000","last_unstale":"2023-09-23T21:23:57.606602+0000","last_undegraded":"2023-09-23T21:23:57.606602+0000","last_fullsized":"2023-09-23T21:23:57.606602+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.403893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":198,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":261,"reported_epoch":682,"state":"active+undersized","last_fresh":"2023-09-23T21:23:57.605921+0000","last_change":"2023-09-23T21:23:52.586379+0000","last_active":"2023-09-23T21:23:57.605921+0000","last_peered":"2023-09-23T21:23:57.605921+0000","last_clean":"2023-09-23T21:21:17.518342+0000","last_became_active":"2023-09-23T21:23:52.586379+0000","last_became_peered":"2023-09-23T21:23:52.586379+0000","last_unstale":"2023-09-23T21:23:57.605921+0000","last_undegraded":"2023-09-23T21:23:57.605921+0000","last_fullsized":"2023-09-23T21:23:52.584221+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:00.087968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":333,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606395+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:23:57.606395+0000","last_peered":"2023-09-23T21:23:57.606395+0000","last_clean":"2023-09-23T21:23:57.606395+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:23:57.606395+0000","last_undegraded":"2023-09-23T21:23:57.606395+0000","last_fullsized":"2023-09-23T21:23:57.606395+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":68,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.639732+0000","last_change":"2023-09-23T21:20:59.524077+0000","last_active":"2023-09-23T21:24:00.639732+0000","last_peered":"2023-09-23T21:24:00.639732+0000","last_clean":"2023-09-23T21:24:00.639732+0000","last_became_active":"2023-09-23T21:20:59.523768+0000","last_became_peered":"2023-09-23T21:20:59.523768+0000","last_unstale":"2023-09-23T21:24:00.639732+0000","last_undegraded":"2023-09-23T21:24:00.639732+0000","last_fullsized":"2023-09-23T21:24:00.639732+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:58.708855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":312,"reported_epoch":682,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:57.606798+0000","last_change":"2023-09-23T21:23:52.587833+0000","last_active":"2023-09-23T21:23:57.606798+0000","last_peered":"2023-09-23T21:23:57.606798+0000","last_clean":"2023-09-23T21:23:50.570079+0000","last_became_active":"2023-09-23T21:23:52.587833+0000","last_became_peered":"2023-09-23T21:23:52.587833+0000","last_unstale":"2023-09-23T21:23:57.606798+0000","last_undegraded":"2023-09-23T21:23:52.585225+0000","last_fullsized":"2023-09-23T21:23:52.585106+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:03.249732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":56,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-23T21:23:59.628126+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:23:59.628126+0000","last_peered":"2023-09-23T21:23:59.628126+0000","last_clean":"2023-09-23T21:23:59.628126+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:23:59.628126+0000","last_undegraded":"2023-09-23T21:23:59.628126+0000","last_fullsized":"2023-09-23T21:23:59.628126+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":683,"state":"active+undersized","last_fresh":"2023-09-23T21:23:58.616312+0000","last_change":"2023-09-23T21:23:52.582698+0000","last_active":"2023-09-23T21:23:58.616312+0000","last_peered":"2023-09-23T21:23:58.616312+0000","last_clean":"2023-09-23T21:23:50.565167+0000","last_became_active":"2023-09-23T21:23:52.582698+0000","last_became_peered":"2023-09-23T21:23:52.582698+0000","last_unstale":"2023-09-23T21:23:58.616312+0000","last_undegraded":"2023-09-23T21:23:58.616312+0000","last_fullsized":"2023-09-23T21:23:52.576040+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:56.285381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608502+0000","last_change":"2023-09-23T21:23:40.007119+0000","last_active":"2023-09-23T21:23:57.608502+0000","last_peered":"2023-09-23T21:23:57.608502+0000","last_clean":"2023-09-23T21:23:57.608502+0000","last_became_active":"2023-09-23T21:23:40.006773+0000","last_became_peered":"2023-09-23T21:23:40.006773+0000","last_unstale":"2023-09-23T21:23:57.608502+0000","last_undegraded":"2023-09-23T21:23:57.608502+0000","last_fullsized":"2023-09-23T21:23:57.608502+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:05.053314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1108,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606055+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:23:57.606055+0000","last_peered":"2023-09-23T21:23:57.606055+0000","last_clean":"2023-09-23T21:23:57.606055+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:23:57.606055+0000","last_undegraded":"2023-09-23T21:23:57.606055+0000","last_fullsized":"2023-09-23T21:23:57.606055+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":109,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.616018+0000","last_change":"2023-09-23T21:23:40.015941+0000","last_active":"2023-09-23T21:23:58.616018+0000","last_peered":"2023-09-23T21:23:58.616018+0000","last_clean":"2023-09-23T21:23:58.616018+0000","last_became_active":"2023-09-23T21:23:40.005288+0000","last_became_peered":"2023-09-23T21:23:40.005288+0000","last_unstale":"2023-09-23T21:23:58.616018+0000","last_undegraded":"2023-09-23T21:23:58.616018+0000","last_fullsized":"2023-09-23T21:23:58.616018+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:04.042824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":440,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608580+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:23:57.608580+0000","last_peered":"2023-09-23T21:23:57.608580+0000","last_clean":"2023-09-23T21:23:57.608580+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:23:57.608580+0000","last_undegraded":"2023-09-23T21:23:57.608580+0000","last_fullsized":"2023-09-23T21:23:57.608580+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":997,"reported_epoch":682,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:57.651882+0000","last_change":"2023-09-23T21:23:52.580400+0000","last_active":"2023-09-23T21:23:57.651882+0000","last_peered":"2023-09-23T21:23:57.651882+0000","last_clean":"2023-09-23T21:23:50.564487+0000","last_became_active":"2023-09-23T21:23:52.580400+0000","last_became_peered":"2023-09-23T21:23:52.580400+0000","last_unstale":"2023-09-23T21:23:57.651882+0000","last_undegraded":"2023-09-23T21:23:52.575120+0000","last_fullsized":"2023-09-23T21:23:52.575000+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:08.921254+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":180,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-23T21:23:59.628292+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:23:59.628292+0000","last_peered":"2023-09-23T21:23:59.628292+0000","last_clean":"2023-09-23T21:23:59.628292+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:23:59.628292+0000","last_undegraded":"2023-09-23T21:23:59.628292+0000","last_fullsized":"2023-09-23T21:23:59.628292+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":690,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608479+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:23:57.608479+0000","last_peered":"2023-09-23T21:23:57.608479+0000","last_clean":"2023-09-23T21:23:57.608479+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:23:57.608479+0000","last_undegraded":"2023-09-23T21:23:57.608479+0000","last_fullsized":"2023-09-23T21:23:57.608479+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":319,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608285+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:23:57.608285+0000","last_peered":"2023-09-23T21:23:57.608285+0000","last_clean":"2023-09-23T21:23:57.608285+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:23:57.608285+0000","last_undegraded":"2023-09-23T21:23:57.608285+0000","last_fullsized":"2023-09-23T21:23:57.608285+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":727,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608620+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:23:57.608620+0000","last_peered":"2023-09-23T21:23:57.608620+0000","last_clean":"2023-09-23T21:23:57.608620+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:23:57.608620+0000","last_undegraded":"2023-09-23T21:23:57.608620+0000","last_fullsized":"2023-09-23T21:23:57.608620+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":965,"reported_epoch":682,"state":"active+undersized","last_fresh":"2023-09-23T21:23:57.651982+0000","last_change":"2023-09-23T21:23:52.583414+0000","last_active":"2023-09-23T21:23:57.651982+0000","last_peered":"2023-09-23T21:23:57.651982+0000","last_clean":"2023-09-23T21:23:50.564522+0000","last_became_active":"2023-09-23T21:23:52.583414+0000","last_became_peered":"2023-09-23T21:23:52.583414+0000","last_unstale":"2023-09-23T21:23:57.651982+0000","last_undegraded":"2023-09-23T21:23:57.651982+0000","last_fullsized":"2023-09-23T21:23:52.575425+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":626,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:23:11.788995+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:23:11.788995+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:21.341747+0000","scrub_duration":0.002393788,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.615569+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:23:58.615569+0000","last_peered":"2023-09-23T21:23:58.615569+0000","last_clean":"2023-09-23T21:23:58.615569+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:23:58.615569+0000","last_undegraded":"2023-09-23T21:23:58.615569+0000","last_fullsized":"2023-09-23T21:23:58.615569+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":68,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.640203+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:24:00.640203+0000","last_peered":"2023-09-23T21:24:00.640203+0000","last_clean":"2023-09-23T21:24:00.640203+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:24:00.640203+0000","last_undegraded":"2023-09-23T21:24:00.640203+0000","last_fullsized":"2023-09-23T21:24:00.640203+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":767,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-23T21:23:59.628399+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:23:59.628399+0000","last_peered":"2023-09-23T21:23:59.628399+0000","last_clean":"2023-09-23T21:23:59.628399+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:23:59.628399+0000","last_undegraded":"2023-09-23T21:23:59.628399+0000","last_fullsized":"2023-09-23T21:23:59.628399+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"686'163","reported_seq":601,"reported_epoch":686,"state":"active+clean","last_fresh":"2023-09-23T21:24:01.688086+0000","last_change":"2023-09-23T21:24:01.646806+0000","last_active":"2023-09-23T21:24:01.688086+0000","last_peered":"2023-09-23T21:24:01.688086+0000","last_clean":"2023-09-23T21:24:01.688086+0000","last_became_active":"2023-09-23T21:23:40.011996+0000","last_became_peered":"2023-09-23T21:23:40.011996+0000","last_unstale":"2023-09-23T21:24:01.688086+0000","last_undegraded":"2023-09-23T21:24:01.688086+0000","last_fullsized":"2023-09-23T21:24:01.688086+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:12.817128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088069099999999996,"stat_sum":{"num_bytes":70,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":983,"num_read_kb":861,"num_write":160,"num_write_kb":83,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"2"}]},{"pgid":"3.5","version":"0'0","reported_seq":29,"reported_epoch":683,"state":"active+undersized","last_fresh":"2023-09-23T21:23:58.616379+0000","last_change":"2023-09-23T21:23:52.581252+0000","last_active":"2023-09-23T21:23:58.616379+0000","last_peered":"2023-09-23T21:23:58.616379+0000","last_clean":"2023-09-23T21:20:22.495828+0000","last_became_active":"2023-09-23T21:23:52.581252+0000","last_became_peered":"2023-09-23T21:23:52.581252+0000","last_unstale":"2023-09-23T21:23:58.616379+0000","last_undegraded":"2023-09-23T21:23:58.616379+0000","last_fullsized":"2023-09-23T21:23:52.575647+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:02.913325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":74,"reported_epoch":685,"state":"active+undersized","last_fresh":"2023-09-23T21:24:00.639928+0000","last_change":"2023-09-23T21:23:52.586094+0000","last_active":"2023-09-23T21:24:00.639928+0000","last_peered":"2023-09-23T21:24:00.639928+0000","last_clean":"2023-09-23T21:23:50.569448+0000","last_became_active":"2023-09-23T21:23:52.586094+0000","last_became_peered":"2023-09-23T21:23:52.586094+0000","last_unstale":"2023-09-23T21:24:00.639928+0000","last_undegraded":"2023-09-23T21:24:00.639928+0000","last_fullsized":"2023-09-23T21:23:52.579014+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:51.596198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":973,"reported_epoch":682,"state":"active+undersized","last_fresh":"2023-09-23T21:23:57.608289+0000","last_change":"2023-09-23T21:23:52.581222+0000","last_active":"2023-09-23T21:23:57.608289+0000","last_peered":"2023-09-23T21:23:57.608289+0000","last_clean":"2023-09-23T21:23:50.624271+0000","last_became_active":"2023-09-23T21:23:52.581222+0000","last_became_peered":"2023-09-23T21:23:52.581222+0000","last_unstale":"2023-09-23T21:23:57.608289+0000","last_undegraded":"2023-09-23T21:23:57.608289+0000","last_fullsized":"2023-09-23T21:23:52.577241+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:41.455220+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"686'51","reported_seq":358,"reported_epoch":686,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:24:01.690184+0000","last_change":"2023-09-23T21:23:52.582675+0000","last_active":"2023-09-23T21:24:01.690184+0000","last_peered":"2023-09-23T21:24:01.690184+0000","last_clean":"2023-09-23T21:23:50.807488+0000","last_became_active":"2023-09-23T21:23:52.582675+0000","last_became_peered":"2023-09-23T21:23:52.582675+0000","last_unstale":"2023-09-23T21:24:01.690184+0000","last_undegraded":"2023-09-23T21:23:52.579396+0000","last_fullsized":"2023-09-23T21:23:52.579292+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":51,"log_dups_size":0,"ondisk_log_size":51,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":4,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:12.683162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012923380000000001,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":517,"num_read_kb":617,"num_write":56,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":2}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":605,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608514+0000","last_change":"2023-09-23T21:23:40.007033+0000","last_active":"2023-09-23T21:23:57.608514+0000","last_peered":"2023-09-23T21:23:57.608514+0000","last_clean":"2023-09-23T21:23:57.608514+0000","last_became_active":"2023-09-23T21:23:40.006711+0000","last_became_peered":"2023-09-23T21:23:40.006711+0000","last_unstale":"2023-09-23T21:23:57.608514+0000","last_undegraded":"2023-09-23T21:23:57.608514+0000","last_fullsized":"2023-09-23T21:23:57.608514+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:35.819332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.640221+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:24:00.640221+0000","last_peered":"2023-09-23T21:24:00.640221+0000","last_clean":"2023-09-23T21:24:00.640221+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:24:00.640221+0000","last_undegraded":"2023-09-23T21:24:00.640221+0000","last_fullsized":"2023-09-23T21:24:00.640221+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1088,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606221+0000","last_change":"2023-09-23T21:23:42.983841+0000","last_active":"2023-09-23T21:23:57.606221+0000","last_peered":"2023-09-23T21:23:57.606221+0000","last_clean":"2023-09-23T21:23:57.606221+0000","last_became_active":"2023-09-23T21:23:42.983351+0000","last_became_peered":"2023-09-23T21:23:42.983351+0000","last_unstale":"2023-09-23T21:23:57.606221+0000","last_undegraded":"2023-09-23T21:23:57.606221+0000","last_fullsized":"2023-09-23T21:23:57.606221+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.422807+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"680'46","reported_seq":847,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.701409+0000","last_change":"2023-09-23T21:23:54.594333+0000","last_active":"2023-09-23T21:23:57.701409+0000","last_peered":"2023-09-23T21:23:57.701409+0000","last_clean":"2023-09-23T21:23:57.701409+0000","last_became_active":"2023-09-23T21:23:40.012274+0000","last_became_peered":"2023-09-23T21:23:40.012274+0000","last_unstale":"2023-09-23T21:23:57.701409+0000","last_undegraded":"2023-09-23T21:23:57.701409+0000","last_fullsized":"2023-09-23T21:23:57.701409+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":46,"log_dups_size":0,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:30.294423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031212900000000002,"stat_sum":{"num_bytes":16,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":166,"num_read_kb":93,"num_write":50,"num_write_kb":36,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.0","version":"0'0","reported_seq":68,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.640206+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:24:00.640206+0000","last_peered":"2023-09-23T21:24:00.640206+0000","last_clean":"2023-09-23T21:24:00.640206+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:24:00.640206+0000","last_undegraded":"2023-09-23T21:24:00.640206+0000","last_fullsized":"2023-09-23T21:24:00.640206+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":685,"state":"active+undersized","last_fresh":"2023-09-23T21:24:00.640099+0000","last_change":"2023-09-23T21:23:52.582462+0000","last_active":"2023-09-23T21:24:00.640099+0000","last_peered":"2023-09-23T21:24:00.640099+0000","last_clean":"2023-09-23T21:23:50.569731+0000","last_became_active":"2023-09-23T21:23:52.582462+0000","last_became_peered":"2023-09-23T21:23:52.582462+0000","last_unstale":"2023-09-23T21:24:00.640099+0000","last_undegraded":"2023-09-23T21:24:00.640099+0000","last_fullsized":"2023-09-23T21:23:52.578875+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:46.270798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"683'106","reported_seq":930,"reported_epoch":683,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:58.623528+0000","last_change":"2023-09-23T21:23:52.587328+0000","last_active":"2023-09-23T21:23:58.623528+0000","last_peered":"2023-09-23T21:23:58.623528+0000","last_clean":"2023-09-23T21:23:50.751426+0000","last_became_active":"2023-09-23T21:23:52.587328+0000","last_became_peered":"2023-09-23T21:23:52.587328+0000","last_unstale":"2023-09-23T21:23:58.623528+0000","last_undegraded":"2023-09-23T21:23:52.575685+0000","last_fullsized":"2023-09-23T21:23:52.575586+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":2,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:42.632886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080853800000000003,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":597,"num_read_kb":549,"num_write":85,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1089,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606079+0000","last_change":"2023-09-23T21:23:42.981131+0000","last_active":"2023-09-23T21:23:57.606079+0000","last_peered":"2023-09-23T21:23:57.606079+0000","last_clean":"2023-09-23T21:23:57.606079+0000","last_became_active":"2023-09-23T21:23:42.980755+0000","last_became_peered":"2023-09-23T21:23:42.980755+0000","last_unstale":"2023-09-23T21:23:57.606079+0000","last_undegraded":"2023-09-23T21:23:57.606079+0000","last_fullsized":"2023-09-23T21:23:57.606079+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:08.065765+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":393,"reported_epoch":684,"state":"active+undersized","last_fresh":"2023-09-23T21:23:59.628401+0000","last_change":"2023-09-23T21:23:52.582467+0000","last_active":"2023-09-23T21:23:59.628401+0000","last_peered":"2023-09-23T21:23:59.628401+0000","last_clean":"2023-09-23T21:20:22.521803+0000","last_became_active":"2023-09-23T21:23:52.582467+0000","last_became_peered":"2023-09-23T21:23:52.582467+0000","last_unstale":"2023-09-23T21:23:59.628401+0000","last_undegraded":"2023-09-23T21:23:59.628401+0000","last_fullsized":"2023-09-23T21:23:52.581066+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:38.577626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":56,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-23T21:23:59.628157+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:23:59.628157+0000","last_peered":"2023-09-23T21:23:59.628157+0000","last_clean":"2023-09-23T21:23:59.628157+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:23:59.628157+0000","last_undegraded":"2023-09-23T21:23:59.628157+0000","last_fullsized":"2023-09-23T21:23:59.628157+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":732,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.639950+0000","last_change":"2023-09-23T21:23:12.024326+0000","last_active":"2023-09-23T21:24:00.639950+0000","last_peered":"2023-09-23T21:24:00.639950+0000","last_clean":"2023-09-23T21:24:00.639950+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:24:00.639950+0000","last_undegraded":"2023-09-23T21:24:00.639950+0000","last_fullsized":"2023-09-23T21:24:00.639950+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:23:12.024197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:23:12.024197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:18.945737+0000","scrub_duration":0.0017519600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"682'40","reported_seq":495,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.953854+0000","last_change":"2023-09-23T21:23:54.593272+0000","last_active":"2023-09-23T21:23:57.953854+0000","last_peered":"2023-09-23T21:23:57.953854+0000","last_clean":"2023-09-23T21:23:57.953854+0000","last_became_active":"2023-09-23T21:23:40.011597+0000","last_became_peered":"2023-09-23T21:23:40.011597+0000","last_unstale":"2023-09-23T21:23:57.953854+0000","last_undegraded":"2023-09-23T21:23:57.953854+0000","last_fullsized":"2023-09-23T21:23:57.953854+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":40,"log_dups_size":0,"ondisk_log_size":40,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:17.045171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036146000000000003,"stat_sum":{"num_bytes":112675,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":561,"num_write":35,"num_write_kb":213,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.1","version":"0'0","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608250+0000","last_change":"2023-09-23T21:23:40.009291+0000","last_active":"2023-09-23T21:23:57.608250+0000","last_peered":"2023-09-23T21:23:57.608250+0000","last_clean":"2023-09-23T21:23:57.608250+0000","last_became_active":"2023-09-23T21:23:40.008907+0000","last_became_peered":"2023-09-23T21:23:40.008907+0000","last_unstale":"2023-09-23T21:23:57.608250+0000","last_undegraded":"2023-09-23T21:23:57.608250+0000","last_fullsized":"2023-09-23T21:23:57.608250+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:18.940611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608583+0000","last_change":"2023-09-23T21:23:40.012054+0000","last_active":"2023-09-23T21:23:57.608583+0000","last_peered":"2023-09-23T21:23:57.608583+0000","last_clean":"2023-09-23T21:23:57.608583+0000","last_became_active":"2023-09-23T21:23:40.011675+0000","last_became_peered":"2023-09-23T21:23:40.011675+0000","last_unstale":"2023-09-23T21:23:57.608583+0000","last_undegraded":"2023-09-23T21:23:57.608583+0000","last_fullsized":"2023-09-23T21:23:57.608583+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:02.744420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":767,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-09-23T21:23:59.628520+0000","last_change":"2023-09-23T21:23:42.981963+0000","last_active":"2023-09-23T21:23:59.628520+0000","last_peered":"2023-09-23T21:23:59.628520+0000","last_clean":"2023-09-23T21:23:59.628520+0000","last_became_active":"2023-09-23T21:23:42.981346+0000","last_became_peered":"2023-09-23T21:23:42.981346+0000","last_unstale":"2023-09-23T21:23:59.628520+0000","last_undegraded":"2023-09-23T21:23:59.628520+0000","last_fullsized":"2023-09-23T21:23:59.628520+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:20.767852+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"682'54","reported_seq":504,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.893066+0000","last_change":"2023-09-23T21:23:54.594411+0000","last_active":"2023-09-23T21:23:57.893066+0000","last_peered":"2023-09-23T21:23:57.893066+0000","last_clean":"2023-09-23T21:23:57.893066+0000","last_became_active":"2023-09-23T21:23:40.011448+0000","last_became_peered":"2023-09-23T21:23:40.011448+0000","last_unstale":"2023-09-23T21:23:57.893066+0000","last_undegraded":"2023-09-23T21:23:57.893066+0000","last_fullsized":"2023-09-23T21:23:57.893066+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:16.274116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039724600000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":378,"num_read_kb":977,"num_write":42,"num_write_kb":358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":685,"state":"active+undersized","last_fresh":"2023-09-23T21:24:00.640448+0000","last_change":"2023-09-23T21:23:52.584103+0000","last_active":"2023-09-23T21:24:00.640448+0000","last_peered":"2023-09-23T21:24:00.640448+0000","last_clean":"2023-09-23T21:23:50.569538+0000","last_became_active":"2023-09-23T21:23:52.584103+0000","last_became_peered":"2023-09-23T21:23:52.584103+0000","last_unstale":"2023-09-23T21:24:00.640448+0000","last_undegraded":"2023-09-23T21:24:00.640448+0000","last_fullsized":"2023-09-23T21:23:52.578197+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:55.966010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":239,"reported_epoch":682,"state":"active+undersized","last_fresh":"2023-09-23T21:23:57.606426+0000","last_change":"2023-09-23T21:23:52.586866+0000","last_active":"2023-09-23T21:23:57.606426+0000","last_peered":"2023-09-23T21:23:57.606426+0000","last_clean":"2023-09-23T21:23:35.380989+0000","last_became_active":"2023-09-23T21:23:52.586866+0000","last_became_peered":"2023-09-23T21:23:52.586866+0000","last_unstale":"2023-09-23T21:23:57.606426+0000","last_undegraded":"2023-09-23T21:23:57.606426+0000","last_fullsized":"2023-09-23T21:23:52.584247+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:19.371184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":983,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608443+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:23:57.608443+0000","last_peered":"2023-09-23T21:23:57.608443+0000","last_clean":"2023-09-23T21:23:57.608443+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:23:57.608443+0000","last_undegraded":"2023-09-23T21:23:57.608443+0000","last_fullsized":"2023-09-23T21:23:57.608443+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":393,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.606696+0000","last_change":"2023-09-23T21:23:40.022217+0000","last_active":"2023-09-23T21:23:57.606696+0000","last_peered":"2023-09-23T21:23:57.606696+0000","last_clean":"2023-09-23T21:23:57.606696+0000","last_became_active":"2023-09-23T21:23:40.014583+0000","last_became_peered":"2023-09-23T21:23:40.014583+0000","last_unstale":"2023-09-23T21:23:57.606696+0000","last_undegraded":"2023-09-23T21:23:57.606696+0000","last_fullsized":"2023-09-23T21:23:57.606696+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:56.174596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"682'60","reported_seq":315,"reported_epoch":686,"state":"active+clean","last_fresh":"2023-09-23T21:24:01.680302+0000","last_change":"2023-09-23T21:24:01.646038+0000","last_active":"2023-09-23T21:24:01.680302+0000","last_peered":"2023-09-23T21:24:01.680302+0000","last_clean":"2023-09-23T21:24:01.680302+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:24:01.680302+0000","last_undegraded":"2023-09-23T21:24:01.680302+0000","last_fullsized":"2023-09-23T21:24:01.680302+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":60,"log_dups_size":0,"ondisk_log_size":60,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027136199999999998,"stat_sum":{"num_bytes":112656,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":372,"num_read_kb":471,"num_write":37,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":106,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.615586+0000","last_change":"2023-09-23T21:23:40.006639+0000","last_active":"2023-09-23T21:23:58.615586+0000","last_peered":"2023-09-23T21:23:58.615586+0000","last_clean":"2023-09-23T21:23:58.615586+0000","last_became_active":"2023-09-23T21:23:40.006309+0000","last_became_peered":"2023-09-23T21:23:40.006309+0000","last_unstale":"2023-09-23T21:23:58.615586+0000","last_undegraded":"2023-09-23T21:23:58.615586+0000","last_fullsized":"2023-09-23T21:23:58.615586+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:04.424515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1085,"reported_epoch":683,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:58.616171+0000","last_change":"2023-09-23T21:23:52.587262+0000","last_active":"2023-09-23T21:23:58.616171+0000","last_peered":"2023-09-23T21:23:58.616171+0000","last_clean":"2023-09-23T21:23:50.565038+0000","last_became_active":"2023-09-23T21:23:52.587262+0000","last_became_peered":"2023-09-23T21:23:52.587262+0000","last_unstale":"2023-09-23T21:23:58.616171+0000","last_undegraded":"2023-09-23T21:23:52.575718+0000","last_fullsized":"2023-09-23T21:23:52.575607+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":631,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:48.268642+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"684'51","reported_seq":805,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-09-23T21:24:00.632847+0000","last_change":"2023-09-23T21:24:00.633049+0000","last_active":"2023-09-23T21:24:00.632847+0000","last_peered":"2023-09-23T21:24:00.632847+0000","last_clean":"2023-09-23T21:24:00.632847+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:24:00.632847+0000","last_undegraded":"2023-09-23T21:24:00.632847+0000","last_fullsized":"2023-09-23T21:24:00.632847+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":51,"log_dups_size":0,"ondisk_log_size":51,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028938199999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":306,"num_read_kb":842,"num_write":33,"num_write_kb":302,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"1"}]},{"pgid":"3.6","version":"69'1","reported_seq":26,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608478+0000","last_change":"2023-09-23T21:23:40.017705+0000","last_active":"2023-09-23T21:23:57.608478+0000","last_peered":"2023-09-23T21:23:57.608478+0000","last_clean":"2023-09-23T21:23:57.608478+0000","last_became_active":"2023-09-23T21:23:40.007892+0000","last_became_peered":"2023-09-23T21:23:40.007892+0000","last_unstale":"2023-09-23T21:23:57.608478+0000","last_undegraded":"2023-09-23T21:23:57.608478+0000","last_fullsized":"2023-09-23T21:23:57.608478+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:09.348317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":255,"reported_epoch":684,"state":"active+undersized","last_fresh":"2023-09-23T21:23:59.628095+0000","last_change":"2023-09-23T21:23:52.587724+0000","last_active":"2023-09-23T21:23:59.628095+0000","last_peered":"2023-09-23T21:23:59.628095+0000","last_clean":"2023-09-23T21:05:12.413419+0000","last_became_active":"2023-09-23T21:23:52.587724+0000","last_became_peered":"2023-09-23T21:23:52.587724+0000","last_unstale":"2023-09-23T21:23:59.628095+0000","last_undegraded":"2023-09-23T21:23:59.628095+0000","last_fullsized":"2023-09-23T21:23:52.581508+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:01.184441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608434+0000","last_change":"2023-09-23T21:23:40.014466+0000","last_active":"2023-09-23T21:23:57.608434+0000","last_peered":"2023-09-23T21:23:57.608434+0000","last_clean":"2023-09-23T21:23:57.608434+0000","last_became_active":"2023-09-23T21:23:40.014118+0000","last_became_peered":"2023-09-23T21:23:40.014118+0000","last_unstale":"2023-09-23T21:23:57.608434+0000","last_undegraded":"2023-09-23T21:23:57.608434+0000","last_fullsized":"2023-09-23T21:23:57.608434+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:21.412255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":775,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.652156+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:23:57.652156+0000","last_peered":"2023-09-23T21:23:57.652156+0000","last_clean":"2023-09-23T21:23:57.652156+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:23:57.652156+0000","last_undegraded":"2023-09-23T21:23:57.652156+0000","last_fullsized":"2023-09-23T21:23:57.652156+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":449,"reported_epoch":682,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:57.608627+0000","last_change":"2023-09-23T21:23:52.585786+0000","last_active":"2023-09-23T21:23:57.608627+0000","last_peered":"2023-09-23T21:23:57.608627+0000","last_clean":"2023-09-23T21:23:50.789613+0000","last_became_active":"2023-09-23T21:23:52.585786+0000","last_became_peered":"2023-09-23T21:23:52.585786+0000","last_unstale":"2023-09-23T21:23:57.608627+0000","last_undegraded":"2023-09-23T21:23:52.582969+0000","last_fullsized":"2023-09-23T21:23:52.582867+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:41.504354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":285,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608357+0000","last_change":"2023-09-23T21:20:59.530029+0000","last_active":"2023-09-23T21:23:57.608357+0000","last_peered":"2023-09-23T21:23:57.608357+0000","last_clean":"2023-09-23T21:23:57.608357+0000","last_became_active":"2023-09-23T21:20:59.529692+0000","last_became_peered":"2023-09-23T21:20:59.529692+0000","last_unstale":"2023-09-23T21:23:57.608357+0000","last_undegraded":"2023-09-23T21:23:57.608357+0000","last_fullsized":"2023-09-23T21:23:57.608357+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:29.977611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":203,"reported_epoch":684,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:59.628347+0000","last_change":"2023-09-23T21:23:52.586191+0000","last_active":"2023-09-23T21:23:59.628347+0000","last_peered":"2023-09-23T21:23:59.628347+0000","last_clean":"2023-09-23T21:23:50.789785+0000","last_became_active":"2023-09-23T21:23:52.586191+0000","last_became_peered":"2023-09-23T21:23:52.586191+0000","last_unstale":"2023-09-23T21:23:59.628347+0000","last_undegraded":"2023-09-23T21:23:52.582055+0000","last_fullsized":"2023-09-23T21:23:52.581945+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:57:12.851998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":88,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.615903+0000","last_change":"2023-09-23T21:23:40.006363+0000","last_active":"2023-09-23T21:23:58.615903+0000","last_peered":"2023-09-23T21:23:58.615903+0000","last_clean":"2023-09-23T21:23:58.615903+0000","last_became_active":"2023-09-23T21:23:40.005647+0000","last_became_peered":"2023-09-23T21:23:40.005647+0000","last_unstale":"2023-09-23T21:23:58.615903+0000","last_undegraded":"2023-09-23T21:23:58.615903+0000","last_fullsized":"2023-09-23T21:23:58.615903+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:48.040008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":97,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.616783+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:23:58.616783+0000","last_peered":"2023-09-23T21:23:58.616783+0000","last_clean":"2023-09-23T21:23:58.616783+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:23:58.616783+0000","last_undegraded":"2023-09-23T21:23:58.616783+0000","last_fullsized":"2023-09-23T21:23:58.616783+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":97,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T21:23:58.616537+0000","last_change":"2023-09-23T21:23:40.009045+0000","last_active":"2023-09-23T21:23:58.616537+0000","last_peered":"2023-09-23T21:23:58.616537+0000","last_clean":"2023-09-23T21:23:58.616537+0000","last_became_active":"2023-09-23T21:23:40.008753+0000","last_became_peered":"2023-09-23T21:23:40.008753+0000","last_unstale":"2023-09-23T21:23:58.616537+0000","last_undegraded":"2023-09-23T21:23:58.616537+0000","last_fullsized":"2023-09-23T21:23:58.616537+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:28.136832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":34,"reported_epoch":682,"state":"active+undersized+degraded","last_fresh":"2023-09-23T21:23:57.608578+0000","last_change":"2023-09-23T21:23:52.581370+0000","last_active":"2023-09-23T21:23:57.608578+0000","last_peered":"2023-09-23T21:23:57.608578+0000","last_clean":"2023-09-23T21:23:50.624700+0000","last_became_active":"2023-09-23T21:23:52.581370+0000","last_became_peered":"2023-09-23T21:23:52.581370+0000","last_unstale":"2023-09-23T21:23:57.608578+0000","last_undegraded":"2023-09-23T21:23:52.577410+0000","last_fullsized":"2023-09-23T21:23:52.577294+0000","mapping_epoch":677,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:52.290218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":389,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T21:23:57.608194+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:23:57.608194+0000","last_peered":"2023-09-23T21:23:57.608194+0000","last_clean":"2023-09-23T21:23:57.608194+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:23:57.608194+0000","last_undegraded":"2023-09-23T21:23:57.608194+0000","last_fullsized":"2023-09-23T21:23:57.608194+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":89,"acting":89,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":184,"num_bytes_recovered":36346,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":29,"acting":29,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":225471,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":3531,"num_read_kb":4971,"num_write":498,"num_write_kb":1302,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":159744,"data_stored":62013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":571,"ondisk_log_size":571,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201943,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928500,"kb_used_data":7404,"kb_used_omap":0,"kb_used_meta":921088,"kb_avail":92816652,"statfs":{"total":95995035648,"available":95044251648,"internally_reserved":0,"allocated":7581696,"data_stored":5723460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":943194112},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:20 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.883,"15min":2.5030000000000001},"min":{"1min":0.26400000000000001,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":6.1920000000000002,"5min":16.468,"15min":141.33199999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.016,"15min":2.6299999999999999},"min":{"1min":0.36399999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.6960000000000002,"5min":12.962,"15min":141.31700000000001},"last":0.435}]},{"osd":1,"last update":"Sat Sep 23 21:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.7839999999999998,"5min":2.5680000000000001,"15min":2.9980000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.24099999999999999},"max":{"1min":31.233000000000001,"5min":31.233000000000001,"15min":112.355},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.0270000000000001,"5min":2.6459999999999999,"15min":3.383},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.22900000000000001},"max":{"1min":31.219000000000001,"5min":31.219000000000001,"15min":117.637},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:23:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":7.0869999999999997,"15min":5.4660000000000002},"min":{"1min":0.34899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":4.827,"5min":158.726,"15min":183.44200000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.5069999999999999,"5min":6.9669999999999996,"15min":5.7130000000000001},"min":{"1min":0.29699999999999999,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":7.9969999999999999,"5min":158.70400000000001,"15min":183.46199999999999},"last":0.42599999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:23:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":2.6160000000000001,"15min":4.2969999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.26700000000000002},"max":{"1min":8.3559999999999999,"5min":30.667999999999999,"15min":135.66200000000001},"last":0.312},{"interface":"front","average":{"1min":2.2599999999999998,"5min":2.694,"15min":4.6020000000000003},"min":{"1min":0.33200000000000002,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":10.337,"5min":30.798999999999999,"15min":135.47},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":3.8900000000000001,"15min":2.2050000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":12.282999999999999,"5min":31.695,"15min":31.695},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":4.1609999999999996,"15min":2.363},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26800000000000002},"max":{"1min":3.2959999999999998,"5min":34.994,"15min":34.994},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 21:23:50 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":4.5679999999999996,"15min":2.722},"min":{"1min":0.308,"5min":0.308,"15min":0.26900000000000002},"max":{"1min":4.9269999999999996,"5min":156.30099999999999,"15min":156.30099999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":4.5620000000000003,"15min":2.6960000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.8260000000000001,"5min":156.22,"15min":156.22},"last":0.61599999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":3.6859999999999999,"15min":2.9569999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.313},"max":{"1min":12.231999999999999,"5min":23.574999999999999,"15min":143.18700000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.71499999999999997,"5min":3.2919999999999998,"15min":2.6539999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29099999999999998},"max":{"1min":3.2599999999999998,"5min":21.081,"15min":142.70400000000001},"last":0.33000000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463564,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1224692,"kb_used_data":310380,"kb_used_omap":0,"kb_used_meta":914304,"kb_avail":92520460,"statfs":{"total":95995035648,"available":94740951040,"internally_reserved":0,"allocated":317829120,"data_stored":315993675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":936247296},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.978,"5min":1.764,"15min":2.6539999999999999},"min":{"1min":0.246,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":5.7779999999999996,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.251},{"interface":"front","average":{"1min":1.488,"5min":1.909,"15min":2.8410000000000002},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.223},"max":{"1min":5.3399999999999999,"5min":11.968,"15min":149.303},"last":0.23400000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:23:42 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.931,"15min":2.794},"min":{"1min":0.255,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":3.8999999999999999,"5min":12.013,"15min":129.37899999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.96199999999999997,"5min":2.0009999999999999,"15min":2.609},"min":{"1min":0.247,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.2909999999999999,"5min":14.366,"15min":129.40100000000001},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":7.0149999999999997,"15min":7.0570000000000004},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":54.561,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":5.3129999999999997,"5min":7.351,"15min":7.2460000000000004},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":54.603999999999999,"5min":199.27000000000001,"15min":199.27000000000001},"last":0.309}]},{"osd":3,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":2.1920000000000002,"15min":3.641},"min":{"1min":0.309,"5min":0.28399999999999997,"15min":0.218},"max":{"1min":7.6269999999999998,"5min":14.411,"15min":137.875},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.927,"15min":3.476},"min":{"1min":0.315,"5min":0.315,"15min":0.29799999999999999},"max":{"1min":5.8040000000000003,"5min":12.35,"15min":137.60900000000001},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":3.5609999999999999,"15min":2.6240000000000001},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.22600000000000001},"max":{"1min":12.218999999999999,"5min":20.408999999999999,"15min":55.494},"last":0.44900000000000001},{"interface":"front","average":{"1min":3.972,"5min":3.528,"15min":2.516},"min":{"1min":0.255,"5min":0.255,"15min":0.23999999999999999},"max":{"1min":13.308999999999999,"5min":14.622,"15min":55.305},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5470000000000002,"5min":2.6739999999999999,"15min":2.8799999999999999},"min":{"1min":0.317,"5min":0.308,"15min":0.27300000000000002},"max":{"1min":6.9100000000000001,"5min":12.051,"15min":222.90899999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":5.6890000000000001,"5min":3.3500000000000001,"15min":3.0619999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":15.478,"5min":17.927,"15min":224.655},"last":0.32000000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:23:42 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":2.7349999999999999,"15min":2.4060000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28000000000000003},"max":{"1min":10.619999999999999,"5min":12.380000000000001,"15min":165.96799999999999},"last":0.438},{"interface":"front","average":{"1min":0.89100000000000001,"5min":3.4140000000000001,"15min":2.7690000000000001},"min":{"1min":0.33800000000000002,"5min":0.307,"15min":0.251},"max":{"1min":3.4790000000000001,"5min":19.815999999999999,"15min":165.82400000000001},"last":0.42599999999999999}]}]},{"osd":1,"up_from":276,"seq":1185410974011,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661972,"kb_used_data":8652,"kb_used_omap":5,"kb_used_meta":653306,"kb_avail":93083180,"statfs":{"total":95995035648,"available":95317176320,"internally_reserved":0,"allocated":8859648,"data_stored":5604283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":668986083},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.39500000000000002,"15min":1.6359999999999999},"min":{"1min":0.26000000000000001,"5min":0.214,"15min":0.19400000000000001},"max":{"1min":2.2040000000000002,"5min":2.3410000000000002,"15min":108.47},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64700000000000002,"15min":1.758},"min":{"1min":0.246,"5min":0.23499999999999999,"15min":0.19600000000000001},"max":{"1min":2.919,"5min":6.0149999999999997,"15min":109.532},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":6.5170000000000003,"15min":8.3930000000000007},"min":{"1min":0.23899999999999999,"5min":0.23200000000000001,"15min":0.218},"max":{"1min":3.0350000000000001,"5min":130.78899999999999,"15min":171.35400000000001},"last":0.20999999999999999},{"interface":"front","average":{"1min":0.501,"5min":6.4930000000000003,"15min":7.9950000000000001},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":2.1560000000000001,"5min":133.751,"15min":170.69900000000001},"last":0.27200000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.627,"15min":3.2290000000000001},"min":{"1min":0.27600000000000002,"5min":0.254,"15min":0.23799999999999999},"max":{"1min":2.2370000000000001,"5min":17.247,"15min":99.120999999999995},"last":0.313},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.79200000000000004,"15min":3.9140000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":2.8769999999999998,"5min":10.949999999999999,"15min":150.38},"last":0.26000000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.8380000000000001,"15min":2.2949999999999999},"min":{"1min":0.34100000000000003,"5min":0.27400000000000002,"15min":0.26800000000000002},"max":{"1min":3.3100000000000001,"5min":11.034000000000001,"15min":174.84399999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.77700000000000002,"5min":1.47,"15min":2.0619999999999998},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.28199999999999997},"max":{"1min":2.996,"5min":11.057,"15min":174.82400000000001},"last":0.378}]},{"osd":5,"last update":"Sat Sep 23 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7849999999999999,"5min":2.758,"15min":2.6890000000000001},"min":{"1min":0.442,"5min":0.32200000000000001,"15min":0.31900000000000001},"max":{"1min":7.6619999999999999,"5min":107.98,"15min":110.955},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.556,"5min":3.2669999999999999,"15min":2.8199999999999998},"min":{"1min":0.41499999999999998,"5min":0.33500000000000002,"15min":0.29199999999999998},"max":{"1min":9.8879999999999999,"5min":107.961,"15min":110.89},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":3.6400000000000001,"15min":2.5299999999999998},"min":{"1min":0.45600000000000002,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":7.7910000000000004,"5min":119.999,"15min":119.999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.147,"5min":3.496,"15min":2.3889999999999998},"min":{"1min":0.40200000000000002,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":3.3519999999999999,"5min":122.328,"15min":122.328},"last":0.36399999999999999}]},{"osd":7,"last update":"Sat Sep 23 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":2.165,"15min":2.9100000000000001},"min":{"1min":0.52400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":7.5469999999999997,"5min":10.648999999999999,"15min":285.47500000000002},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.399,"5min":1.71,"15min":2.569},"min":{"1min":0.44800000000000001,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":7.5590000000000002,"5min":10.664999999999999,"15min":285.48599999999999},"last":0.47399999999999998}]}]},{"osd":0,"up_from":92,"seq":395136991750,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860388,"kb_used_data":8476,"kb_used_omap":5,"kb_used_meta":851898,"kb_avail":92884764,"statfs":{"total":95995035648,"available":95113998336,"internally_reserved":0,"allocated":8679424,"data_stored":5547399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":872344186},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:23:30 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.71599999999999997,"15min":1.45},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":5.2110000000000003,"5min":11.109,"15min":53.262},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.53900000000000003,"15min":1.6870000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20200000000000001},"max":{"1min":5.8520000000000003,"5min":6.6319999999999997,"15min":53.201999999999998},"last":0.28399999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:23:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":5.8280000000000003,"15min":6.1360000000000001},"min":{"1min":0.27300000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.262,"5min":156.477,"15min":201.32300000000001},"last":0.376},{"interface":"front","average":{"1min":2.7050000000000001,"5min":5.7830000000000004,"15min":6.1859999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.245},"max":{"1min":30.327000000000002,"5min":156.66800000000001,"15min":201.43199999999999},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.624,"15min":2.1059999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":2.444,"5min":11.077,"15min":86.989999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.57099999999999995,"15min":2.0960000000000001},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.23599999999999999},"max":{"1min":4.1820000000000004,"5min":4.1820000000000004,"15min":87.043999999999997},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:23:21 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":2.169,"15min":1.9370000000000001},"min":{"1min":0.30299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.5410000000000004,"5min":39.228000000000002,"15min":39.228000000000002},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.9430000000000001,"5min":2.1429999999999998,"15min":1.7789999999999999},"min":{"1min":0.35699999999999998,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":6.2789999999999999,"5min":39.340000000000003,"15min":39.340000000000003},"last":0.503}]},{"osd":5,"last update":"Sat Sep 23 21:23:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.7030000000000001,"15min":1.8149999999999999},"min":{"1min":0.39100000000000001,"5min":0.307,"15min":0.29499999999999998},"max":{"1min":3.8660000000000001,"5min":27.331,"15min":27.331},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.7909999999999999,"15min":1.6120000000000001},"min":{"1min":0.33500000000000002,"5min":0.32400000000000001,"15min":0.30399999999999999},"max":{"1min":6.359,"5min":33.125,"15min":33.125},"last":0.70699999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:23:21 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":2.7320000000000002,"15min":2.085},"min":{"1min":0.35599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.4109999999999996,"5min":56.213000000000001,"15min":56.213000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.677,"5min":2.8759999999999999,"15min":2.069},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.9939999999999998,"5min":53.424999999999997,"15min":53.424999999999997},"last":0.439}]},{"osd":7,"last update":"Sat Sep 23 21:23:21 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":1.956,"15min":1.7509999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":6.3230000000000004,"5min":32.411999999999999,"15min":32.411999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.8169999999999999,"15min":1.788},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":6.319,"5min":32.433999999999997,"15min":32.433999999999997},"last":0.47199999999999998}]}]},{"osd":2,"up_from":129,"seq":554050781641,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1350780,"kb_used_data":8948,"kb_used_omap":7,"kb_used_meta":1341816,"kb_avail":92394372,"statfs":{"total":95995035648,"available":94611836928,"internally_reserved":0,"allocated":9162752,"data_stored":5822319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1374019665},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":3.6030000000000002,"5min":1.2569999999999999,"15min":2.0089999999999999},"min":{"1min":0.193,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":51.968000000000004,"5min":51.968000000000004,"15min":103.021},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.7919999999999998,"5min":1.3500000000000001,"15min":2.363},"min":{"1min":0.22900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":52.241999999999997,"5min":52.241999999999997,"15min":103.154},"last":0.35499999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:23:40 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.75,"15min":2.444},"min":{"1min":0.20699999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.4470000000000001,"5min":6.4470000000000001,"15min":143.482},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.65200000000000002,"15min":2.5880000000000001},"min":{"1min":0.23300000000000001,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":1.03,"5min":5.4820000000000002,"15min":143.61000000000001},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:23:28 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.85999999999999999,"15min":2.9079999999999999},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.20499999999999999},"max":{"1min":6.4039999999999999,"5min":6.4039999999999999,"15min":143.50800000000001},"last":0.309},{"interface":"front","average":{"1min":0.434,"5min":0.70299999999999996,"15min":3.0899999999999999},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.95999999999999996,"5min":5.6790000000000003,"15min":143.27799999999999},"last":0.38600000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":2.0739999999999998,"15min":3.5049999999999999},"min":{"1min":0.312,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":8.5820000000000007,"5min":8.5820000000000007,"15min":225.80799999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.0129999999999999,"15min":3.9359999999999999},"min":{"1min":0.34999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.5690000000000008,"5min":8.5690000000000008,"15min":225.79300000000001},"last":0.624}]},{"osd":5,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":4.3879999999999999,"5min":2.0259999999999998,"15min":3.5550000000000002},"min":{"1min":0.34899999999999998,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":52.338000000000001,"5min":52.338000000000001,"15min":220.63900000000001},"last":0.76400000000000001},{"interface":"front","average":{"1min":1.603,"5min":1.601,"15min":3.5939999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29399999999999998},"max":{"1min":7.9370000000000003,"5min":8.0579999999999998,"15min":220.31999999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":2.4199999999999999,"15min":3.472},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.308},"max":{"1min":5.2030000000000003,"5min":42.363,"15min":210.74700000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.665,"5min":2.3079999999999998,"15min":2.8159999999999998},"min":{"1min":0.34200000000000003,"5min":0.308,"15min":0.308},"max":{"1min":4.4050000000000002,"5min":42.383000000000003,"15min":210.72900000000001},"last":0.52200000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":5.0970000000000004,"5min":2.5299999999999998,"15min":2.2450000000000001},"min":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":52.121000000000002,"5min":52.121000000000002,"15min":52.121000000000002},"last":0.501},{"interface":"front","average":{"1min":6.0739999999999998,"5min":2.5289999999999999,"15min":2.5550000000000002},"min":{"1min":0.52600000000000002,"5min":0.38300000000000001,"15min":0.32700000000000001},"max":{"1min":52.308999999999997,"5min":52.308999999999997,"15min":52.308999999999997},"last":0.58499999999999996}]}]},{"osd":3,"up_from":148,"seq":635655160235,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906368,"kb_used_data":9784,"kb_used_omap":2,"kb_used_meta":896573,"kb_avail":92838784,"statfs":{"total":95995035648,"available":95066914816,"internally_reserved":0,"allocated":10018816,"data_stored":6208839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":918090805},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.52700000000000002,"15min":1.633},"min":{"1min":0.23400000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":2.2909999999999999,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.1679999999999999,"5min":0.65000000000000002,"15min":1.51},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.19500000000000001},"max":{"1min":5.0720000000000001,"5min":5.0720000000000001,"15min":18.277999999999999},"last":0.25600000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:23:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.83099999999999996,"15min":5.8300000000000001},"min":{"1min":0.27100000000000002,"5min":0.214,"15min":0.214},"max":{"1min":5.1180000000000003,"5min":8.3079999999999998,"15min":237.072},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.70699999999999996,"15min":5.6130000000000004},"min":{"1min":0.25600000000000001,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":4.6920000000000002,"5min":8.5220000000000002,"15min":234.21899999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":3.907,"15min":5.4729999999999999},"min":{"1min":0.34699999999999998,"5min":0.219,"15min":0.215},"max":{"1min":5.4699999999999998,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.3919999999999999,"5min":3.915,"15min":5.1589999999999998},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":5.6050000000000004,"5min":208.024,"15min":208.024},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":3.3359999999999999,"15min":2.532},"min":{"1min":0.42299999999999999,"5min":0.27900000000000003,"15min":0.27600000000000002},"max":{"1min":6.4370000000000003,"5min":117.717,"15min":117.717},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.8820000000000001,"5min":3.7730000000000001,"15min":2.597},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":6.9870000000000001,"5min":117.771,"15min":117.771},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":2.9129999999999998,"15min":2.8769999999999998},"min":{"1min":0.41099999999999998,"5min":0.35399999999999998,"15min":0.27200000000000002},"max":{"1min":7.7119999999999997,"5min":100.277,"15min":123.458},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.7000000000000002,"15min":2.7639999999999998},"min":{"1min":0.41299999999999998,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":6.4740000000000002,"5min":100.69,"15min":123.292},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.464,"5min":4.0679999999999996,"15min":4.8070000000000004},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":6.3890000000000002,"5min":130.613,"15min":304.22399999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.0390000000000001,"5min":4.3700000000000001,"15min":4.8840000000000003},"min":{"1min":0.53600000000000003,"5min":0.33500000000000002,"15min":0.33100000000000002},"max":{"1min":5.5099999999999998,"5min":130.50999999999999,"15min":304.15800000000002},"last":0.39300000000000002}]},{"osd":7,"last update":"Sat Sep 23 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":1.927,"15min":2.2240000000000002},"min":{"1min":0.497,"5min":0.38500000000000001,"15min":0.34000000000000002},"max":{"1min":8.0440000000000005,"5min":19.298999999999999,"15min":44.253},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.6080000000000001,"5min":2.4380000000000002,"15min":2.6099999999999999},"min":{"1min":0.52700000000000002,"5min":0.40600000000000003,"15min":0.36699999999999999},"max":{"1min":10.441000000000001,"5min":13.343,"15min":45.706000000000003},"last":0.53700000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986853,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":985512,"kb_used_data":7072,"kb_used_omap":0,"kb_used_meta":978432,"kb_avail":92759640,"statfs":{"total":95995035648,"available":94985871360,"internally_reserved":0,"allocated":7241728,"data_stored":5395909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1001914368},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":2.403,"5min":2.286,"15min":3.0339999999999998},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":7.6589999999999998,"5min":12.195,"15min":226.751},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.762,"15min":2.702},"min":{"1min":0.33300000000000002,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":16.478000000000002,"5min":16.478000000000002,"15min":226.43299999999999},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.8460000000000001,"15min":2.9319999999999999},"min":{"1min":0.28499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":3.5249999999999999,"5min":16.434999999999999,"15min":119.55800000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.97699999999999998,"5min":2.3330000000000002,"15min":3.04},"min":{"1min":0.41999999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5609999999999999,"5min":12.108000000000001,"15min":116.78100000000001},"last":0.34399999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":7.1280000000000001,"15min":6.4390000000000001},"min":{"1min":0.36599999999999999,"5min":0.29499999999999998,"15min":0.248},"max":{"1min":5.1630000000000003,"5min":124.70099999999999,"15min":210.50299999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.0259999999999998,"5min":6.1630000000000003,"15min":6.4420000000000002},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":5.5860000000000003,"5min":116.47,"15min":210.59},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:23:40 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":2.823,"15min":4.5410000000000004},"min":{"1min":0.32300000000000001,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":4.4710000000000001,"5min":14.323,"15min":214.64099999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.276,"5min":2.6179999999999999,"15min":4.9169999999999998},"min":{"1min":0.42499999999999999,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":4.8949999999999996,"5min":13.673999999999999,"15min":214.54900000000001},"last":0.44800000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:23:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":3.681,"15min":2.056},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.23200000000000001},"max":{"1min":13.31,"5min":17.879999999999999,"15min":34.497999999999998},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.2210000000000001,"5min":3.3039999999999998,"15min":2.2759999999999998},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.23799999999999999},"max":{"1min":6.7309999999999999,"5min":19.425000000000001,"15min":34.883000000000003},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":4.7309999999999999,"5min":3.9260000000000002,"15min":2.613},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":31.829999999999998,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.45800000000000002},{"interface":"front","average":{"1min":4.8250000000000002,"5min":5.2110000000000003,"15min":2.9580000000000002},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":26.898,"5min":67.596999999999994,"15min":67.596999999999994},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 21:23:43 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":3.7490000000000001,"15min":2.2519999999999998},"min":{"1min":0.45100000000000001,"5min":0.35599999999999998,"15min":0.28100000000000003},"max":{"1min":8.1099999999999994,"5min":17.817,"15min":18.352},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.6559999999999999,"5min":3.5699999999999998,"15min":2.121},"min":{"1min":0.38200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.2960000000000003,"5min":30.117000000000001,"15min":30.117000000000001},"last":0.48699999999999999}]}]},{"osd":5,"up_from":39,"seq":167503725215,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1699508,"kb_used_data":678608,"kb_used_omap":0,"kb_used_meta":1020800,"kb_avail":92045644,"statfs":{"total":95995035648,"available":94254739456,"internally_reserved":0,"allocated":694894592,"data_stored":693062627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1045299200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":1.8799999999999999,"15min":2.0489999999999999},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":12.131,"5min":12.558999999999999,"15min":82.280000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":3.5099999999999998,"5min":2.1880000000000002,"15min":2.3159999999999998},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":12.641,"5min":12.641,"15min":85.236000000000004},"last":0.69099999999999995}]},{"osd":1,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.1440000000000001,"15min":3.2610000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.9240000000000004,"5min":18.859999999999999,"15min":125.23399999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.754,"5min":1.8149999999999999,"15min":2.9039999999999999},"min":{"1min":0.32200000000000001,"5min":0.27000000000000002,"15min":0.26500000000000001},"max":{"1min":12.164,"5min":12.654999999999999,"15min":125.249},"last":0.73899999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":10.749000000000001,"5min":7.0819999999999999,"15min":4.3259999999999996},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":123.444,"5min":160.749,"15min":160.749},"last":0.79400000000000004},{"interface":"front","average":{"1min":11.595000000000001,"5min":7.3310000000000004,"15min":4.5289999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":124.874,"5min":161.18000000000001,"15min":161.18000000000001},"last":0.61799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.4830000000000001,"15min":4.1299999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27800000000000002},"max":{"1min":12.028,"5min":18.826000000000001,"15min":263.887},"last":0.755},{"interface":"front","average":{"1min":3.1850000000000001,"5min":2.0870000000000002,"15min":3.7799999999999998},"min":{"1min":0.38100000000000001,"5min":0.35199999999999998,"15min":0.30099999999999999},"max":{"1min":12.189,"5min":12.683999999999999,"15min":263.70100000000002},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Sep 23 21:22:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7229999999999999,"5min":3.4830000000000001,"15min":1.974},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.25},"max":{"1min":12.212,"5min":77.804000000000002,"15min":77.804000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":6.274,"5min":4.8659999999999997,"15min":2.516},"min":{"1min":0.35299999999999998,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":19.718,"5min":77.825999999999993,"15min":77.825999999999993},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":6.3650000000000002,"5min":5.173,"15min":2.972},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":19.675000000000001,"5min":68.346000000000004,"15min":83.451999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":3.2210000000000001,"5min":4.742,"15min":3.0499999999999998},"min":{"1min":0.35499999999999998,"5min":0.30599999999999999,"15min":0.29399999999999998},"max":{"1min":11.143000000000001,"5min":72.393000000000001,"15min":83.484999999999999},"last":0.55700000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:23:46 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":3.004,"15min":2.0750000000000002},"min":{"1min":0.40999999999999998,"5min":0.36299999999999999,"15min":0.30099999999999999},"max":{"1min":9.4550000000000001,"5min":12.692,"15min":12.692},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.6060000000000001,"5min":2.7050000000000001,"15min":1.6879999999999999},"min":{"1min":0.45800000000000002,"5min":0.35099999999999998,"15min":0.32000000000000001},"max":{"1min":6.3630000000000004,"5min":18.257999999999999,"15min":18.257999999999999},"last":0.60399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":10361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":20585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":86,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":16,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":10256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":10275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":10415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:24:03.971 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-09-23T21:24:03.972 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:03 smithi123 ceph-mon[193903]: failed to encode map e687 with expected crc 2023-09-23T21:24:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:03 smithi123 ceph-mon[193903]: failed to encode map e687 with expected crc 2023-09-23T21:24:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:03 smithi123 ceph-mon[193903]: osdmap e687: 8 total, 7 up, 8 in 2023-09-23T21:24:03.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:03 smithi123 ceph-mon[193903]: from='client.58195 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:24:03.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:03 smithi123 ceph-mon[193903]: pgmap v2062: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 80 KiB/s rd, 5.2 KiB/s wr, 105 op/s; 15/111 objects degraded (13.514%) 2023-09-23T21:24:04.066 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:24:04.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:04.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:24:04.170 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T21:24:04.200 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:04.313 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T21:24:04.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:04.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T21:24:04.364 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T21:24:04.417 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:24:04.446 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_764439/sparse1 2023-09-23T21:24:04.446 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:04.452 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records in 2023-09-23T21:24:04.452 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records out 2023-09-23T21:24:04.452 INFO:tasks.workunit.client.0.smithi121.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00521658 s, 201 MB/s 2023-09-23T21:24:04.452 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_764439/sparse2 2023-09-23T21:24:04.458 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records in 2023-09-23T21:24:04.458 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records out 2023-09-23T21:24:04.458 INFO:tasks.workunit.client.0.smithi121.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00533763 s, 196 MB/s 2023-09-23T21:24:04.458 INFO:tasks.workunit.client.0.smithi121.stderr:+ truncate /tmp/rbd_import_export_764439/sparse2 -s 2M 2023-09-23T21:24:04.586 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T21:24:04.597 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse1 2023-09-23T21:24:04.635 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 0% complete...failed. 2023-09-23T21:24:04.635 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T21:24:04.638 INFO:tasks.workunit.client.0.smithi121.stderr:+ true 2023-09-23T21:24:04.638 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_764439/sparse1 2023-09-23T21:24:04.703 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:04.704 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T21:24:04.705 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:24:04.705 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:24:04.709 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd ls -l 2023-09-23T21:24:04.709 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep sparse1 2023-09-23T21:24:04.709 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T21:24:04.718 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[211526]: osdmap e688: 8 total, 7 up, 8 in 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[211526]: failed to encode map e688 with expected crc 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[211526]: failed to encode map e688 with expected crc 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1487389867' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1487389867' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[219709]: osdmap e688: 8 total, 7 up, 8 in 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[219709]: failed to encode map e688 with expected crc 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[219709]: failed to encode map e688 with expected crc 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1487389867' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:24:04.719 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:04 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1487389867' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:24:04.747 INFO:tasks.workunit.client.0.smithi121.stdout:sparse1 2 MiB 2 2023-09-23T21:24:04.747 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects sparse1 2023-09-23T21:24:04.748 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse1 2023-09-23T21:24:04.748 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse1 2023-09-23T21:24:04.748 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep block_name_prefix 2023-09-23T21:24:04.748 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk '{print $NF;}' 2023-09-23T21:24:04.751 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:24:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:04.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:24:04.790 INFO:tasks.workunit.client.0.smithi121.stderr:++ prefix=rbd_data.e371422de10f 2023-09-23T21:24:04.791 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep rbd_data.e371422de10f 2023-09-23T21:24:04.791 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sed -e 's/rbd_data.e371422de10f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:24:04.791 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sort -u 2023-09-23T21:24:04.791 INFO:tasks.workunit.client.0.smithi121.stderr:++++ get_image_data_pool sparse1 2023-09-23T21:24:04.791 INFO:tasks.workunit.client.0.smithi121.stderr:++++ image=sparse1 2023-09-23T21:24:04.792 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ rbd info sparse1 2023-09-23T21:24:04.792 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:24:04.792 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:24:04.841 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool= 2023-09-23T21:24:04.841 INFO:tasks.workunit.client.0.smithi121.stderr:++++ '[' -z ']' 2023-09-23T21:24:04.841 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool=rbd 2023-09-23T21:24:04.841 INFO:tasks.workunit.client.0.smithi121.stderr:++++ echo rbd 2023-09-23T21:24:04.842 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rados ls -p rbd 2023-09-23T21:24:04.844 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:04.855 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T21:24:04.881 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects=1 2023-09-23T21:24:04.881 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo 1 2023-09-23T21:24:04.881 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T21:24:04.881 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export sparse1 /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:04.889 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:04.900 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:04 smithi123 ceph-mon[193903]: osdmap e688: 8 total, 7 up, 8 in 2023-09-23T21:24:04.900 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:04 smithi123 ceph-mon[193903]: failed to encode map e688 with expected crc 2023-09-23T21:24:04.900 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:04 smithi123 ceph-mon[193903]: failed to encode map e688 with expected crc 2023-09-23T21:24:04.900 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1487389867' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:24:04.900 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:04 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1487389867' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T21:24:04.931 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T21:24:04.936 INFO:tasks.workunit.client.0.smithi121.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_764439/sparse1 /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:04.936 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp -l /tmp/rbd_import_export_764439/sparse1 /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:04.938 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse1 --format %b 2023-09-23T21:24:04.939 INFO:tasks.workunit.client.0.smithi121.stderr:+ origsize=2048 2023-09-23T21:24:04.939 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse1.out --format %b 2023-09-23T21:24:04.940 INFO:tasks.workunit.client.0.smithi121.stderr:+ exportsize=2048 2023-09-23T21:24:04.940 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:04.940 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:04.940 INFO:tasks.workunit.client.0.smithi121.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:24:04.940 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:04.941 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse1 2023-09-23T21:24:05.034 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T21:24:05.038 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse2 2023-09-23T21:24:05.075 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:24:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:24:04] "GET /metrics HTTP/1.1" 200 34290 "" "Prometheus/2.43.0" 2023-09-23T21:24:05.078 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 0% complete...failed. 2023-09-23T21:24:05.078 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T21:24:05.081 INFO:tasks.workunit.client.0.smithi121.stderr:+ true 2023-09-23T21:24:05.081 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_764439/sparse2 2023-09-23T21:24:05.086 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T21:24:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:05.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T21:24:05.149 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T21:24:05.149 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:24:05.150 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:24:05.152 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd ls -l 2023-09-23T21:24:05.152 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep sparse2 2023-09-23T21:24:05.152 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T21:24:05.187 INFO:tasks.workunit.client.0.smithi121.stdout:sparse2 2 MiB 2 2023-09-23T21:24:05.188 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects sparse2 2023-09-23T21:24:05.188 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse2 2023-09-23T21:24:05.188 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse2 2023-09-23T21:24:05.188 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep block_name_prefix 2023-09-23T21:24:05.188 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk '{print $NF;}' 2023-09-23T21:24:05.191 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:24:05.218 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:05.225 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:05.226 INFO:tasks.workunit.client.0.smithi121.stderr:++ prefix=rbd_data.e3a1f0a20088 2023-09-23T21:24:05.226 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep rbd_data.e3a1f0a20088 2023-09-23T21:24:05.226 INFO:tasks.workunit.client.0.smithi121.stderr:++++ get_image_data_pool sparse2 2023-09-23T21:24:05.226 INFO:tasks.workunit.client.0.smithi121.stderr:++++ image=sparse2 2023-09-23T21:24:05.226 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sed -e 's/rbd_data.e3a1f0a20088\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:24:05.226 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sort -u 2023-09-23T21:24:05.227 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ rbd info sparse2 2023-09-23T21:24:05.227 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:24:05.228 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:24:05.260 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:24:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:05.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:24:05.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool= 2023-09-23T21:24:05.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ '[' -z ']' 2023-09-23T21:24:05.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool=rbd 2023-09-23T21:24:05.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ echo rbd 2023-09-23T21:24:05.269 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rados ls -p rbd 2023-09-23T21:24:05.306 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects=0 2023-09-23T21:24:05.306 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo 0 2023-09-23T21:24:05.306 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T21:24:05.306 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export sparse2 /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:05.351 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T21:24:05.354 INFO:tasks.workunit.client.0.smithi121.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_764439/sparse2 /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:05.354 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp -l /tmp/rbd_import_export_764439/sparse2 /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:05.356 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse2 --format %b 2023-09-23T21:24:05.357 INFO:tasks.workunit.client.0.smithi121.stderr:+ origsize=2048 2023-09-23T21:24:05.358 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse2.out --format %b 2023-09-23T21:24:05.359 INFO:tasks.workunit.client.0.smithi121.stderr:+ exportsize=2048 2023-09-23T21:24:05.359 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:05.359 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:05.359 INFO:tasks.workunit.client.0.smithi121.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:24:05.359 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:05.360 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse2 2023-09-23T21:24:05.364 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:24:05.391 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:05.434 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:24:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:05.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:24:05.458 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T21:24:05.461 INFO:tasks.workunit.client.0.smithi121.stderr:+ truncate /tmp/rbd_import_export_764439/sparse1 -s 10M 2023-09-23T21:24:05.462 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T21:24:05.537 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:24:05.554 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:24:05.555 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:24:05.555 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:24:05.559 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd ls -l 2023-09-23T21:24:05.559 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep sparse1 2023-09-23T21:24:05.560 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T21:24:05.571 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:05.606 INFO:tasks.workunit.client.0.smithi121.stdout:sparse1 10 MiB 2 2023-09-23T21:24:05.606 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects sparse1 2023-09-23T21:24:05.607 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse1 2023-09-23T21:24:05.607 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse1 2023-09-23T21:24:05.607 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep block_name_prefix 2023-09-23T21:24:05.607 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk '{print $NF;}' 2023-09-23T21:24:05.643 INFO:tasks.workunit.client.0.smithi121.stderr:++ prefix=rbd_data.9672e6fed658 2023-09-23T21:24:05.645 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep rbd_data.9672e6fed658 2023-09-23T21:24:05.645 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sort -u 2023-09-23T21:24:05.645 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sed -e 's/rbd_data.9672e6fed658\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:24:05.645 INFO:tasks.workunit.client.0.smithi121.stderr:++++ get_image_data_pool sparse1 2023-09-23T21:24:05.645 INFO:tasks.workunit.client.0.smithi121.stderr:++++ image=sparse1 2023-09-23T21:24:05.646 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ rbd info sparse1 2023-09-23T21:24:05.646 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:24:05.648 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:24:05.649 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:05 smithi121 ceph-mon[211526]: pgmap v2064: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 93 KiB/s rd, 8.3 KiB/s wr, 126 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:24:05.649 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2828858539' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:24:05.649 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:05 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:24:05.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:05 smithi121 ceph-mon[219709]: pgmap v2064: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 93 KiB/s rd, 8.3 KiB/s wr, 126 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:24:05.649 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2828858539' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:24:05.657 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is set 2023-09-23T21:24:05.678 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:24:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:05.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:24:05.698 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool= 2023-09-23T21:24:05.698 INFO:tasks.workunit.client.0.smithi121.stderr:++++ '[' -z ']' 2023-09-23T21:24:05.698 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool=rbd 2023-09-23T21:24:05.698 INFO:tasks.workunit.client.0.smithi121.stderr:++++ echo rbd 2023-09-23T21:24:05.698 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rados ls -p rbd 2023-09-23T21:24:05.733 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects=1 2023-09-23T21:24:05.733 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo 1 2023-09-23T21:24:05.733 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T21:24:05.733 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export sparse1 /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:05.775 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T21:24:05.779 INFO:tasks.workunit.client.0.smithi121.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_764439/sparse1 /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:05.779 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp -l /tmp/rbd_import_export_764439/sparse1 /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:05.782 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:24:05.786 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse1 --format %b 2023-09-23T21:24:05.787 INFO:tasks.workunit.client.0.smithi121.stderr:+ origsize=2048 2023-09-23T21:24:05.787 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse1.out --format %b 2023-09-23T21:24:05.789 INFO:tasks.workunit.client.0.smithi121.stderr:+ exportsize=2048 2023-09-23T21:24:05.789 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:05.789 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:05.789 INFO:tasks.workunit.client.0.smithi121.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:24:05.789 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/sparse1.out 2023-09-23T21:24:05.790 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse1 2023-09-23T21:24:05.811 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:05.866 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:24:05.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:05.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:24:05.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:05 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:24:05.951 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:05 smithi123 ceph-mon[193903]: pgmap v2064: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 802 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 93 KiB/s rd, 8.3 KiB/s wr, 126 op/s; 14/102 objects degraded (13.725%) 2023-09-23T21:24:05.951 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2828858539' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:24:05.951 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:05 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T21:24:05.966 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T21:24:05.969 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_764439/sparse2 oflag=append conv=notrunc 2023-09-23T21:24:05.970 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:24:05.998 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:06.082 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records in 2023-09-23T21:24:06.082 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records out 2023-09-23T21:24:06.082 INFO:tasks.workunit.client.0.smithi121.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.11198 s, 18.7 MB/s 2023-09-23T21:24:06.082 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T21:24:06.094 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:24:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:06.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:24:06.152 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:24:06.152 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:24:06.152 INFO:teuthology.orchestra.run.smithi123.stdout: "complaint_time": 30, 2023-09-23T21:24:06.152 INFO:teuthology.orchestra.run.smithi123.stdout: "num_blocked_ops": 0 2023-09-23T21:24:06.152 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:24:06.161 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:24:06.162 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:24:06.162 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:24:06.165 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd ls -l 2023-09-23T21:24:06.165 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep sparse2 2023-09-23T21:24:06.165 INFO:tasks.workunit.client.0.smithi121.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T21:24:06.197 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:24:06.199 INFO:tasks.workunit.client.0.smithi121.stdout:sparse2 4 MiB 2 2023-09-23T21:24:06.199 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects sparse2 2023-09-23T21:24:06.199 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse2 2023-09-23T21:24:06.200 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse2 2023-09-23T21:24:06.200 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep block_name_prefix 2023-09-23T21:24:06.200 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk '{print $NF;}' 2023-09-23T21:24:06.226 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:06.233 INFO:tasks.workunit.client.0.smithi121.stderr:++ prefix=rbd_data.969c71c6d801 2023-09-23T21:24:06.234 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep rbd_data.969c71c6d801 2023-09-23T21:24:06.234 INFO:tasks.workunit.client.0.smithi121.stderr:++++ get_image_data_pool sparse2 2023-09-23T21:24:06.234 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sed -e 's/rbd_data.969c71c6d801\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:24:06.234 INFO:tasks.workunit.client.0.smithi121.stderr:++++ image=sparse2 2023-09-23T21:24:06.234 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sort -u 2023-09-23T21:24:06.235 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ rbd info sparse2 2023-09-23T21:24:06.235 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:24:06.235 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:24:06.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool= 2023-09-23T21:24:06.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ '[' -z ']' 2023-09-23T21:24:06.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool=rbd 2023-09-23T21:24:06.269 INFO:tasks.workunit.client.0.smithi121.stderr:++++ echo rbd 2023-09-23T21:24:06.269 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rados ls -p rbd 2023-09-23T21:24:06.300 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects='0 2023-09-23T21:24:06.300 INFO:tasks.workunit.client.0.smithi121.stderr:2 2023-09-23T21:24:06.300 INFO:tasks.workunit.client.0.smithi121.stderr:3' 2023-09-23T21:24:06.300 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo 0 2 3 2023-09-23T21:24:06.300 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T21:24:06.300 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export sparse2 /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:06.337 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T21:24:06.340 INFO:tasks.workunit.client.0.smithi121.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_764439/sparse2 /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:06.340 INFO:tasks.workunit.client.0.smithi121.stderr:+ cmp -l /tmp/rbd_import_export_764439/sparse2 /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:06.343 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse2 --format %b 2023-09-23T21:24:06.344 INFO:tasks.workunit.client.0.smithi121.stderr:+ origsize=6144 2023-09-23T21:24:06.344 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse2.out --format %b 2023-09-23T21:24:06.345 INFO:tasks.workunit.client.0.smithi121.stderr:+ exportsize=6144 2023-09-23T21:24:06.345 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:06.345 INFO:tasks.workunit.client.0.smithi121.stderr:+ difference=0 2023-09-23T21:24:06.346 INFO:tasks.workunit.client.0.smithi121.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T21:24:06.346 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/sparse2.out 2023-09-23T21:24:06.346 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse2 2023-09-23T21:24:06.369 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:24:06.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:06.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:24:06.427 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T21:24:06.430 INFO:tasks.workunit.client.0.smithi121.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T21:24:06.430 INFO:tasks.workunit.client.0.smithi121.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T21:24:06.430 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_764439/sparse1 sparse 2023-09-23T21:24:06.473 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T21:24:06.490 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T21:24:06.490 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:24:06.491 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T21:24:06.493 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects sparse 2023-09-23T21:24:06.493 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse 2023-09-23T21:24:06.494 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse 2023-09-23T21:24:06.494 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep block_name_prefix 2023-09-23T21:24:06.494 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk '{print $NF;}' 2023-09-23T21:24:06.503 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:06.531 INFO:tasks.workunit.client.0.smithi121.stderr:++ prefix=rbd_data.96c9a7c8bfe 2023-09-23T21:24:06.532 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep rbd_data.96c9a7c8bfe 2023-09-23T21:24:06.532 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sed -e 's/rbd_data.96c9a7c8bfe\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:24:06.532 INFO:tasks.workunit.client.0.smithi121.stderr:++++ get_image_data_pool sparse 2023-09-23T21:24:06.532 INFO:tasks.workunit.client.0.smithi121.stderr:++++ image=sparse 2023-09-23T21:24:06.533 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sort -u 2023-09-23T21:24:06.533 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ rbd info sparse 2023-09-23T21:24:06.533 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:24:06.533 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:24:06.568 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool= 2023-09-23T21:24:06.568 INFO:tasks.workunit.client.0.smithi121.stderr:++++ '[' -z ']' 2023-09-23T21:24:06.568 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool=rbd 2023-09-23T21:24:06.569 INFO:tasks.workunit.client.0.smithi121.stderr:++++ echo rbd 2023-09-23T21:24:06.569 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rados ls -p rbd 2023-09-23T21:24:06.600 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects=1 2023-09-23T21:24:06.600 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo 1 2023-09-23T21:24:06.600 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T21:24:06.600 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse 2023-09-23T21:24:06.684 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T21:24:06.687 INFO:tasks.workunit.client.0.smithi121.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T21:24:06.688 INFO:tasks.workunit.client.0.smithi121.stdout:zeros import through stdin to sparse image 2023-09-23T21:24:06.688 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T21:24:06.688 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd import --new-format - sparse 2023-09-23T21:24:06.733 INFO:tasks.workunit.client.0.smithi121.stderr:4+0 records in 2023-09-23T21:24:06.733 INFO:tasks.workunit.client.0.smithi121.stderr:4+0 records out 2023-09-23T21:24:06.733 INFO:tasks.workunit.client.0.smithi121.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.044606 s, 94.0 MB/s 2023-09-23T21:24:06.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T21:24:06.742 INFO:tasks.workunit.client.0.smithi121.stderr: Importing image: 100% complete...done. 2023-09-23T21:24:06.742 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:24:06.745 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects sparse 2023-09-23T21:24:06.745 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse 2023-09-23T21:24:06.746 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse 2023-09-23T21:24:06.746 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep block_name_prefix 2023-09-23T21:24:06.746 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk '{print $NF;}' 2023-09-23T21:24:06.882 INFO:tasks.workunit.client.0.smithi121.stderr:++ prefix=rbd_data.bde9ccff98e5 2023-09-23T21:24:06.883 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep rbd_data.bde9ccff98e5 2023-09-23T21:24:06.883 INFO:tasks.workunit.client.0.smithi121.stderr:++++ get_image_data_pool sparse 2023-09-23T21:24:06.883 INFO:tasks.workunit.client.0.smithi121.stderr:++++ image=sparse 2023-09-23T21:24:06.883 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sed -e 's/rbd_data.bde9ccff98e5\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:24:06.884 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sort -u 2023-09-23T21:24:06.884 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ rbd info sparse 2023-09-23T21:24:06.884 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:24:06.884 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:24:06.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[211526]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:24:06.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[211526]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[211526]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[211526]: osdmap e689: 8 total, 7 up, 8 in 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[211526]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[219709]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[219709]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[219709]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:24:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[219709]: osdmap e689: 8 total, 7 up, 8 in 2023-09-23T21:24:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:06 smithi121 ceph-mon[219709]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.918 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool= 2023-09-23T21:24:06.918 INFO:tasks.workunit.client.0.smithi121.stderr:++++ '[' -z ']' 2023-09-23T21:24:06.918 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool=rbd 2023-09-23T21:24:06.918 INFO:tasks.workunit.client.0.smithi121.stderr:++++ echo rbd 2023-09-23T21:24:06.918 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rados ls -p rbd 2023-09-23T21:24:06.949 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects= 2023-09-23T21:24:06.949 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo 2023-09-23T21:24:06.950 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T21:24:06.950 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse 2023-09-23T21:24:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:06 smithi123 ceph-mon[193903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T21:24:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:06 smithi123 ceph-mon[193903]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:06 smithi123 ceph-mon[193903]: failed to encode map e689 with expected crc 2023-09-23T21:24:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:06 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T21:24:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:06 smithi123 ceph-mon[193903]: osdmap e689: 8 total, 7 up, 8 in 2023-09-23T21:24:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:06 smithi123 ceph-mon[193903]: failed to encode map e689 with expected crc 2023-09-23T21:24:07.031 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:24:07.034 INFO:tasks.workunit.client.0.smithi121.stdout:zeros export to sparse file 2023-09-23T21:24:07.034 INFO:tasks.workunit.client.0.smithi121.stderr:+ echo 'zeros export to sparse file' 2023-09-23T21:24:07.035 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T21:24:07.073 INFO:tasks.workunit.client.0.smithi121.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T21:24:07.075 INFO:tasks.workunit.client.0.smithi121.stderr:++ rbd info sparse 2023-09-23T21:24:07.075 INFO:tasks.workunit.client.0.smithi121.stderr:++ grep block_name_prefix 2023-09-23T21:24:07.075 INFO:tasks.workunit.client.0.smithi121.stderr:++ awk '{print $NF;}' 2023-09-23T21:24:07.077 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:07.115 INFO:tasks.workunit.client.0.smithi121.stderr:+ prefix=rbd_data.97086eb29702 2023-09-23T21:24:07.115 INFO:tasks.workunit.client.0.smithi121.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T21:24:07.116 INFO:tasks.workunit.client.0.smithi121.stderr:++ get_image_data_pool sparse 2023-09-23T21:24:07.116 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse 2023-09-23T21:24:07.116 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse 2023-09-23T21:24:07.116 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep 'data_pool: ' 2023-09-23T21:24:07.116 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T21:24:07.153 INFO:tasks.workunit.client.0.smithi121.stderr:++ data_pool= 2023-09-23T21:24:07.153 INFO:tasks.workunit.client.0.smithi121.stderr:++ '[' -z ']' 2023-09-23T21:24:07.153 INFO:tasks.workunit.client.0.smithi121.stderr:++ data_pool=rbd 2023-09-23T21:24:07.153 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo rbd 2023-09-23T21:24:07.153 INFO:tasks.workunit.client.0.smithi121.stderr:+ rados -p rbd put rbd_data.97086eb29702.000000000000 - 2023-09-23T21:24:07.184 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records in 2023-09-23T21:24:07.184 INFO:tasks.workunit.client.0.smithi121.stderr:1+0 records out 2023-09-23T21:24:07.184 INFO:tasks.workunit.client.0.smithi121.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0681963 s, 61.5 MB/s 2023-09-23T21:24:07.200 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects sparse 2023-09-23T21:24:07.200 INFO:tasks.workunit.client.0.smithi121.stderr:++ image=sparse 2023-09-23T21:24:07.200 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rbd info sparse 2023-09-23T21:24:07.200 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep block_name_prefix 2023-09-23T21:24:07.201 INFO:tasks.workunit.client.0.smithi121.stderr:+++ awk '{print $NF;}' 2023-09-23T21:24:07.237 INFO:tasks.workunit.client.0.smithi121.stderr:++ prefix=rbd_data.97086eb29702 2023-09-23T21:24:07.238 INFO:tasks.workunit.client.0.smithi121.stderr:+++ grep rbd_data.97086eb29702 2023-09-23T21:24:07.238 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sed -e 's/rbd_data.97086eb29702\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T21:24:07.238 INFO:tasks.workunit.client.0.smithi121.stderr:++++ get_image_data_pool sparse 2023-09-23T21:24:07.238 INFO:tasks.workunit.client.0.smithi121.stderr:++++ image=sparse 2023-09-23T21:24:07.238 INFO:tasks.workunit.client.0.smithi121.stderr:+++ sort -u 2023-09-23T21:24:07.239 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ rbd info sparse 2023-09-23T21:24:07.239 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ grep 'data_pool: ' 2023-09-23T21:24:07.240 INFO:tasks.workunit.client.0.smithi121.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T21:24:07.276 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool= 2023-09-23T21:24:07.276 INFO:tasks.workunit.client.0.smithi121.stderr:++++ '[' -z ']' 2023-09-23T21:24:07.277 INFO:tasks.workunit.client.0.smithi121.stderr:++++ data_pool=rbd 2023-09-23T21:24:07.277 INFO:tasks.workunit.client.0.smithi121.stderr:++++ echo rbd 2023-09-23T21:24:07.277 INFO:tasks.workunit.client.0.smithi121.stderr:+++ rados ls -p rbd 2023-09-23T21:24:07.309 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T21:24:07.310 INFO:tasks.workunit.client.0.smithi121.stderr:++ objects=0 2023-09-23T21:24:07.310 INFO:tasks.workunit.client.0.smithi121.stderr:++ echo 0 2023-09-23T21:24:07.310 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T21:24:07.310 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/sparse 2023-09-23T21:24:07.310 INFO:tasks.workunit.client.0.smithi121.stderr:rm: cannot remove '/tmp/rbd_import_export_764439/sparse': No such file or directory 2023-09-23T21:24:07.310 INFO:tasks.workunit.client.0.smithi121.stderr:+ true 2023-09-23T21:24:07.310 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd export sparse /tmp/rbd_import_export_764439/sparse 2023-09-23T21:24:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:07.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T21:24:07.344 INFO:tasks.workunit.client.0.smithi121.stderr: Exporting image: 100% complete...done. 2023-09-23T21:24:07.348 INFO:tasks.workunit.client.0.smithi121.stderr:++ stat /tmp/rbd_import_export_764439/sparse --format=%b 2023-09-23T21:24:07.349 INFO:tasks.workunit.client.0.smithi121.stderr:+ '[' 0 = 0 ']' 2023-09-23T21:24:07.349 INFO:tasks.workunit.client.0.smithi121.stderr:+ rbd rm sparse 2023-09-23T21:24:07.412 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:24:07.441 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:07.481 INFO:tasks.workunit.client.0.smithi121.stderr: Removing image: 100% complete...done. 2023-09-23T21:24:07.484 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm /tmp/rbd_import_export_764439/sparse /tmp/rbd_import_export_764439/sparse1 /tmp/rbd_import_export_764439/sparse2 /tmp/rbd_import_export_764439/sparse3 2023-09-23T21:24:07.486 INFO:tasks.workunit.client.0.smithi121.stderr:rm: cannot remove '/tmp/rbd_import_export_764439/sparse3': No such file or directory 2023-09-23T21:24:07.486 INFO:tasks.workunit.client.0.smithi121.stderr:+ true 2023-09-23T21:24:07.486 INFO:tasks.workunit.client.0.smithi121.stdout:OK 2023-09-23T21:24:07.487 INFO:tasks.workunit.client.0.smithi121.stderr:+ echo OK 2023-09-23T21:24:07.487 INFO:tasks.workunit.client.0.smithi121.stderr:+ rm -rf /tmp/rbd_import_export_764439 2023-09-23T21:24:07.488 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T21:24:07.488 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T21:24:07.581 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T21:24:07.582 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T21:24:07.638 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:24:07.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:07.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:24:07.743 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T21:24:07.770 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:07.811 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T21:24:07.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:07.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T21:24:07.915 DEBUG:teuthology.orchestra.run.smithi123:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T21:24:07.942 INFO:teuthology.orchestra.run.smithi123.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:07.957 DEBUG:teuthology.parallel:result is None 2023-09-23T21:24:07.957 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:24:07.989 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:24:07.989 DEBUG:teuthology.orchestra.run.smithi121:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:24:08.046 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:24:08.046 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T21:24:08.046 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T21:24:08.046 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T21:24:08.047 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T21:24:08.047 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T21:24:08.047 DEBUG:teuthology.orchestra.run.smithi121:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:24:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:24:08.104 INFO:teuthology.orchestra.run.smithi121.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T21:24:08.104 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T21:24:08.107 INFO:teuthology.orchestra.run.smithi123.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T21:24:08.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:08.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi123 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T21:24:08.162 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T21:24:08.162 DEBUG:teuthology.orchestra.run.smithi121:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T21:24:08.214 DEBUG:teuthology.orchestra.run.smithi121:> sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T21:24:08.217 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:07 smithi121 ceph-mon[219709]: pgmap v2066: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 692 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 955 KiB/s rd, 717 KiB/s wr, 332 op/s; 14/93 objects degraded (15.054%) 2023-09-23T21:24:08.217 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:07 smithi121 ceph-mon[211526]: pgmap v2066: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 692 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 955 KiB/s rd, 717 KiB/s wr, 332 op/s; 14/93 objects degraded (15.054%) 2023-09-23T21:24:08.219 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T21:24:08.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:07 smithi123 ceph-mon[193903]: pgmap v2066: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 692 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 955 KiB/s rd, 717 KiB/s wr, 332 op/s; 14/93 objects degraded (15.054%) 2023-09-23T21:24:08.243 INFO:teuthology.orchestra.run.smithi121.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T21:24:08.540 INFO:teuthology.orchestra.run.smithi121.stderr:Error: no container with name or ID ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T21:24:08.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T21:24:08.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi121 with status 125: 'sudo docker kill -s 1 ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T21:24:08.576 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:24:08.576 INFO:teuthology.orchestra.run.smithi123.stdout: "size": 20, 2023-09-23T21:24:08.576 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 600, 2023-09-23T21:24:08.576 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [ 2023-09-23T21:24:08.576 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.576 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29430 13.4 13:2f637a96:::benchmark_data_smithi121_621349_object29429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.576 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.12571895300005, 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90678336900000001, 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29430 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.577 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447656+0000", 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447655+0000", 2023-09-23T21:24:08.578 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447720+0000", 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4391000000000001e-05 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447728+0000", 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4300000000000006e-06 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.579 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447738+0000", 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0118999999999999e-05 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447838+0000", 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9207000000000001e-05 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.580 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.447846+0000", 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2439999999999993e-06 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338557+0000", 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.890711217 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338700+0000", 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000143361 2023-09-23T21:24:08.581 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.338864+0000", 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016315700000000001 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342396+0000", 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0035323749999999999 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.582 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354238+0000", 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011842363 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354340+0000", 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000101962 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.583 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354396+0000", 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6174000000000001e-05 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354439+0000", 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2994999999999997e-05 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.584 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29438 13.4 13:2ddaff29:::benchmark_data_smithi121_621349_object29437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.12177260199996, 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.90300633600000002, 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29438 2023-09-23T21:24:08.585 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451602+0000", 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.586 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451601+0000", 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451653+0000", 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2207999999999999e-05 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.587 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451664+0000", 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0692e-05 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.451669+0000", 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2660000000000001e-06 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.452098+0000", 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00042897200000000001 2023-09-23T21:24:08.588 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.452110+0000", 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.202e-05 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.339881+0000", 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88777067300000001 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.589 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.339995+0000", 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114695 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340148+0000", 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00015272199999999999 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343455+0000", 2023-09-23T21:24:08.590 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003307454 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354283+0000", 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.010827448999999999 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354522+0000", 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00023920500000000001 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.591 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354571+0000", 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.8539999999999999e-05 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354609+0000", 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8046999999999997e-05 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.592 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29448 13.4 13:22d32e47:::benchmark_data_smithi121_621349_object29447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.11702938500002, 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89842967699999998, 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29448 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.593 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456346+0000", 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.594 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456345+0000", 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456397+0000", 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2176999999999999e-05 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456411+0000", 2023-09-23T21:24:08.595 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.4219000000000001e-05 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456417+0000", 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.6210000000000001e-06 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456475+0000", 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8056000000000001e-05 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.596 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.456485+0000", 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0071000000000001e-05 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340736+0000", 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88425111199999995 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.597 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340816+0000", 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0597000000000001e-05 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.340916+0000", 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.9607999999999999e-05 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343463+0000", 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.002547278 2023-09-23T21:24:08.598 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354482+0000", 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011018857 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354722+0000", 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00024004300000000001 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.599 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354754+0000", 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.1299e-05 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354775+0000", 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1764e-05 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29452 13.4 13:23edd8ec:::benchmark_data_smithi121_621349_object29451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.600 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.11545599999999, 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.89701267699999998, 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29452 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.601 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457919+0000", 2023-09-23T21:24:08.602 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999995 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457953+0000", 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4428000000000003e-05 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457961+0000", 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.6809999999999998e-06 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.603 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.457965+0000", 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6999999999999999e-06 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.458018+0000", 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2782999999999999e-05 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.604 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.458027+0000", 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.3030000000000005e-06 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341620+0000", 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88359242400000004 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341748+0000", 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012769800000000001 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.605 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.341830+0000", 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2670999999999995e-05 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345316+0000", 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0034856290000000001 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.606 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354632+0000", 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0093166240000000008 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354861+0000", 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00022811500000000001 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354897+0000", 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6327e-05 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.607 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354932+0000", 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4768999999999999e-05 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29475 13.4 13:21b7c357:::benchmark_data_smithi121_621349_object29474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.10094363200005, 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88263625400000001, 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.608 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29475 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.609 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472431+0000", 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472430+0000", 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.610 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472465+0000", 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4814000000000002e-05 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472485+0000", 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.0435e-05 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472493+0000", 2023-09-23T21:24:08.611 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.5240000000000003e-06 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472551+0000", 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.8548999999999998e-05 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.472562+0000", 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0855e-05 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.612 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342433+0000", 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.86987049900000002 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342547+0000", 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000114554 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.613 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.342679+0000", 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000131984 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347274+0000", 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0045946049999999999 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354817+0000", 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0075430949999999997 2023-09-23T21:24:08.614 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.354987+0000", 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00017053899999999999 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355035+0000", 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7305999999999997e-05 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.615 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355068+0000", 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2978000000000002e-05 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29478 13.4 13:2842156f:::benchmark_data_smithi121_621349_object29477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.10016611699996, 2023-09-23T21:24:08.616 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.88206223699999997, 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29478 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:24:08.617 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473209+0000", 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473208+0000", 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.618 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473278+0000", 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0062999999999996e-05 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473296+0000", 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7852000000000001e-05 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.619 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473303+0000", 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.3479999999999998e-06 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473375+0000", 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.1407000000000003e-05 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.473396+0000", 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.1634999999999999e-05 2023-09-23T21:24:08.620 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343498+0000", 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.87010185600000001 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343568+0000", 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0224999999999999e-05 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.621 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.343651+0000", 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.2723000000000001e-05 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347284+0000", 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003633168 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355131+0000", 2023-09-23T21:24:08.622 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0078466939999999995 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355175+0000", 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4586999999999998e-05 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355226+0000", 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0695000000000002e-05 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.623 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355271+0000", 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4932000000000003e-05 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29505 13.4 13:2e31c810:::benchmark_data_smithi121_621349_object29504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.08322768200003, 2023-09-23T21:24:08.624 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.865266541, 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29505 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:24:08.625 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490147+0000", 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490146+0000", 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.626 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490797+0000", 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00065088600000000004 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490813+0000", 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6189999999999999e-05 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.627 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490820+0000", 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6730000000000003e-06 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490881+0000", 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0927999999999997e-05 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.490890+0000", 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.4599999999999992e-06 2023-09-23T21:24:08.628 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344328+0000", 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85343784700000003 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344457+0000", 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000128563 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.629 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.344619+0000", 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016246899999999999 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349443+0000", 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0048236540000000001 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355307+0000", 2023-09-23T21:24:08.630 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0058641479999999996 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355337+0000", 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9658999999999998e-05 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355380+0000", 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3547999999999997e-05 2023-09-23T21:24:08.631 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355414+0000", 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3769999999999997e-05 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29527 13.4 13:29cd74c5:::benchmark_data_smithi121_621349_object29526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:24:08.632 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.068361568, 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.85053144899999999, 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29527 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.633 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505013+0000", 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505012+0000", 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.634 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505051+0000", 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8639000000000001e-05 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505066+0000", 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5812999999999999e-05 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.635 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505073+0000", 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4250000000000003e-06 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505137+0000", 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4097999999999995e-05 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.505144+0000", 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.2559999999999996e-06 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.636 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345347+0000", 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84020272100000004 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345451+0000", 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103905 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.345554+0000", 2023-09-23T21:24:08.637 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103387 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349453+0000", 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003898641 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355452+0000", 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0059990449999999997 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.638 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355477+0000", 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.5531999999999999e-05 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355516+0000", 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8699000000000003e-05 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355545+0000", 2023-09-23T21:24:08.639 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8785999999999998e-05 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29536 13.4 13:2bfcf8e1:::benchmark_data_smithi121_621349_object29535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.06314532099998, 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84544850299999996, 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.640 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29536 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510230+0000", 2023-09-23T21:24:08.641 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510216+0000", 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999866 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510277+0000", 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0891999999999999e-05 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.642 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510295+0000", 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7482999999999998e-05 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510301+0000", 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4749999999999998e-06 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510359+0000", 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.7927000000000003e-05 2023-09-23T21:24:08.643 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.510368+0000", 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9919999999999996e-06 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346282+0000", 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83591370700000001 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.644 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346367+0000", 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.4648999999999995e-05 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.346467+0000", 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000100895 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351395+0000", 2023-09-23T21:24:08.645 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0049271159999999996 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355574+0000", 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041792699999999997 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355598+0000", 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4198000000000001e-05 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.646 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355645+0000", 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.6449999999999997e-05 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355678+0000", 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3657e-05 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.647 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29540 13.4 13:221f9937:::benchmark_data_smithi121_621349_object29539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.06189002300005, 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84431284100000004, 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29540 2023-09-23T21:24:08.648 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511485+0000", 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.649 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511484+0000", 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511579+0000", 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5329999999999997e-05 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511591+0000", 2023-09-23T21:24:08.650 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.145e-05 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511606+0000", 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5325e-05 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511668+0000", 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.1458999999999996e-05 2023-09-23T21:24:08.651 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.511677+0000", 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0380000000000005e-06 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347329+0000", 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.835651904 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.652 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347442+0000", 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00011388899999999999 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.347532+0000", 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9306000000000002e-05 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351403+0000", 2023-09-23T21:24:08.653 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0038715400000000001 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355708+0000", 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.004304446 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355732+0000", 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4731000000000001e-05 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.654 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355770+0000", 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.7104000000000003e-05 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355798+0000", 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8242000000000002e-05 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29542 13.4 13:25d6cd22:::benchmark_data_smithi121_621349_object29541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.05978003400003, 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.842315747, 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29542 2023-09-23T21:24:08.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513595+0000", 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.657 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513594+0000", 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513646+0000", 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.1647999999999997e-05 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.658 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513656+0000", 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0375e-05 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513663+0000", 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6549999999999998e-06 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513727+0000", 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.3992999999999995e-05 2023-09-23T21:24:08.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.513736+0000", 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.2660000000000007e-06 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348465+0000", 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.83472863200000003 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.660 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348553+0000", 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8218000000000001e-05 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.348636+0000", 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3065000000000005e-05 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.661 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352933+0000", 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0042976840000000004 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355826+0000", 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0028929429999999998 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.662 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355851+0000", 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4511e-05 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355883+0000", 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2462000000000002e-05 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355911+0000", 2023-09-23T21:24:08.663 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7293000000000001e-05 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29561 13.4 13:2d5ffecf:::benchmark_data_smithi121_621349_object29560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.04530076599997, 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82794979099999999, 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.664 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29561 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.665 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528074+0000", 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528073+0000", 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.666 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528107+0000", 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.4207e-05 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528127+0000", 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9539999999999999e-05 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528133+0000", 2023-09-23T21:24:08.667 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4559999999999997e-06 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528211+0000", 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8057000000000002e-05 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.528224+0000", 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2646000000000001e-05 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.668 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349494+0000", 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.82127059199999997 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349617+0000", 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012300200000000001 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.349779+0000", 2023-09-23T21:24:08.669 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00016171800000000001 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352944+0000", 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031647020000000001 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355939+0000", 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0029952239999999999 2023-09-23T21:24:08.670 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355964+0000", 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4788999999999999e-05 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.355997+0000", 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3025999999999999e-05 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.671 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356024+0000", 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7231999999999999e-05 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29582 13.4 13:29c0d8e7:::benchmark_data_smithi121_621349_object29581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.025194167, 2023-09-23T21:24:08.672 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80795742199999998, 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29582 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:24:08.673 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548181+0000", 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548179+0000", 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.674 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548222+0000", 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.2599999999999999e-05 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548233+0000", 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1537e-05 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.675 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548238+0000", 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.7700000000000001e-06 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548305+0000", 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6441999999999998e-05 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548314+0000", 2023-09-23T21:24:08.676 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.0410000000000003e-06 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350381+0000", 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80206683400000001 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.677 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350462+0000", 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1507000000000004e-05 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.350589+0000", 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00012726299999999999 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.678 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353762+0000", 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031722870000000002 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356053+0000", 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022911889999999999 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356078+0000", 2023-09-23T21:24:08.679 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.4754999999999999e-05 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356110+0000", 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.2641999999999997e-05 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356138+0000", 2023-09-23T21:24:08.680 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8017e-05 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29585 13.4 13:234913ee:::benchmark_data_smithi121_621349_object29584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.02439933799997, 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80729236000000004, 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.681 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29585 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:24:08.682 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548976+0000", 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.548975+0000", 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.683 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549041+0000", 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6148000000000004e-05 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549057+0000", 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5491000000000001e-05 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.684 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549064+0000", 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.8620000000000003e-06 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549105+0000", 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.104e-05 2023-09-23T21:24:08.685 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.549112+0000", 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2879999999999998e-06 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351443+0000", 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.80233120000000002 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.686 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351547+0000", 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000103696 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.351687+0000", 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000140529 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.687 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353772+0000", 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0020847539999999999 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356167+0000", 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0023954800000000002 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.688 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356196+0000", 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.8331999999999999e-05 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356229+0000", 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.3577000000000001e-05 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.689 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356268+0000", 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.8692000000000002e-05 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47003.0:29594 13.4 13:299d3c93:::benchmark_data_smithi121_621349_object29593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 571.01310164300003, 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.79612242, 2023-09-23T21:24:08.690 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47003", 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3875050078", 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 29594 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:24:08.691 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560273+0000", 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560272+0000", 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.999999 2023-09-23T21:24:08.692 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560317+0000", 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.4302999999999999e-05 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560329+0000", 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.2690999999999999e-05 2023-09-23T21:24:08.693 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560334+0000", 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0429999999999998e-06 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560389+0000", 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.4363000000000001e-05 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.694 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:37.560395+0000", 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.6510000000000004e-06 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352241+0000", 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.79184583500000005 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352323+0000", 2023-09-23T21:24:08.695 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1592000000000001e-05 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 2,5", 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.352403+0000", 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0591000000000001e-05 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.353852+0000", 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0014487339999999999 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.696 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356303+0000", 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024512969999999999 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356331+0000", 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.7708e-05 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.697 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356366+0000", 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.5049999999999998e-05 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:14:38.356396+0000", 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 2.9581999999999999e-05 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.698 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:82785 14.d 14.705d43ed (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:19:49.093456+0000", 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 259.479919435, 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84462227700000003, 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "reached pg", 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 82785 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.699 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093456+0000", 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093456+0000", 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.700 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093452+0000", 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999967 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093496+0000", 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.3893000000000001e-05 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093515+0000", 2023-09-23T21:24:08.701 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.9273e-05 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093524+0000", 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.9099999999999994e-06 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093575+0000", 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0584999999999998e-05 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.702 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.093589+0000", 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.3782e-05 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.938071+0000", 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.84448238600000003 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.703 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:19:49.938078+0000", 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.8669999999999996e-06 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:100286 14.4 14:26363854:::benchmark_data_smithi121_684892_object14969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e616)", 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:20:30.555543+0000", 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 218.01783234600001, 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.651170026, 2023-09-23T21:24:08.704 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 100286 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.705 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555543+0000", 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555543+0000", 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555541+0000", 2023-09-23T21:24:08.706 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555548+0000", 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.7730000000000002e-06 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555563+0000", 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.5026e-05 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.707 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555571+0000", 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.1629999999999996e-06 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555633+0000", 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.2039999999999996e-05 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:30.555644+0000", 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.0987e-05 2023-09-23T21:24:08.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.486375+0000", 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.930731062 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for peered", 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:31.486385+0000", 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.8640000000000008e-06 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.709 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.183175+0000", 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.69678952400000005 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.183809+0000", 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00063464999999999995 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.191729+0000", 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0079193400000000004 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.710 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.711 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.713 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.191848+0000", 2023-09-23T21:24:08.713 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118924 2023-09-23T21:24:08.713 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.713 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.713 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.192006+0000", 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000158794 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.197596+0000", 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.005589152 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.198336+0000", 2023-09-23T21:24:08.714 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00074035799999999999 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.201513+0000", 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0031775050000000002 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.203954+0000", 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0024403469999999998 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.715 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.204009+0000", 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.5402999999999998e-05 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:20:32.206713+0000", 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0027034440000000002 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.716 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:170190 14.2 14:4b09ddcc:::benchmark_data_smithi121_684892_object84873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e656)", 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:22:59.548641+0000", 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 69.024734144999996, 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.090053011000000002, 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 170190 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.717 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.548641+0000", 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.548641+0000", 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.718 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.548639+0000", 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.548646+0000", 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 7.0380000000000002e-06 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.719 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.548663+0000", 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.6708e-05 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.548671+0000", 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.0690000000000002e-06 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.550938+0000", 2023-09-23T21:24:08.720 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0022668039999999999 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.551627+0000", 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00068891700000000004 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.584608+0000", 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032980730999999999 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.721 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.584691+0000", 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.3583000000000003e-05 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3,4,5", 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.584809+0000", 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000118152 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.722 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.588969+0000", 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0041598140000000004 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.595077+0000", 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0061077340000000001 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.599780+0000", 2023-09-23T21:24:08.723 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0047034809999999998 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.603121+0000", 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003341143 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638285+0000", 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.035163962 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.724 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638371+0000", 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.6019000000000004e-05 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638694+0000", 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000322415 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.725 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:170194 14.2 14:41118a08:::benchmark_data_smithi121_684892_object84877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e656)", 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:22:59.564171+0000", 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 69.009203819000007, 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.086200542000000005, 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:08.726 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 170194 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.564171+0000", 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.564171+0000", 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.727 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.564170+0000", 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.564177+0000", 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.725e-06 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.728 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.564188+0000", 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1499e-05 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.564206+0000", 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.7957000000000001e-05 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.729 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.589178+0000", 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.024972023999999999 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.590410+0000", 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.001231609 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.619928+0000", 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.029518250999999999 2023-09-23T21:24:08.730 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.620023+0000", 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 9.5497999999999999e-05 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3,4,5", 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.620179+0000", 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000155904 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.731 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.627271+0000", 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0070914319999999999 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638771+0000", 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.011499977999999999 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638807+0000", 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 3.6291000000000002e-05 2023-09-23T21:24:08.732 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.642729+0000", 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0039221450000000001 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.650037+0000", 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.007307928 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.733 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.650078+0000", 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4.1260000000000001e-05 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.650372+0000", 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00029337699999999999 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.734 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "description": "osd_op(client.47438.0:170204 14.2 14:414e8281:::benchmark_data_smithi121_684892_object84887:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e656)", 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "initiated_at": "2023-09-23T21:22:59.588656+0000", 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "age": 68.984719364, 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.090140910000000005, 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "type_data": { 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "client_info": { 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "client": "client.47438", 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "client_addr": "172.21.15.121:0/3565311630", 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: "tid": 170204 2023-09-23T21:24:08.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: "events": [ 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "initiated", 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.588656+0000", 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "throttled", 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.588656+0000", 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0 2023-09-23T21:24:08.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "header_read", 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.588654+0000", 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 4294967295.9999986 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "all_read", 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.588660+0000", 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.0000000000000002e-06 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.737 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "dispatched", 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.588672+0000", 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 1.1934e-05 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "queued_for_pg", 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.588679+0000", 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 6.4400000000000002e-06 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.738 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.590445+0000", 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.0017658909999999999 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for degraded object", 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.591227+0000", 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.000782177 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "reached_pg", 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.623511+0000", 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.032284248000000001 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.739 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "started", 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.623600+0000", 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 8.8848e-05 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "waiting for subops from 1,3,4,5", 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.623731+0000", 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00013138000000000001 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "op_commit", 2023-09-23T21:24:08.740 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638238+0000", 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.014506763000000001 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638841+0000", 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00060282200000000004 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.638892+0000", 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.0813999999999998e-05 2023-09-23T21:24:08.741 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.642762+0000", 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.003870634 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "sub_op_commit_rec", 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.678426+0000", 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.035663540000000001 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.742 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "commit_sent", 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.678479+0000", 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 5.2711999999999999e-05 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: { 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: "event": "done", 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: "time": "2023-09-23T21:22:59.678797+0000", 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: "duration": 0.00031797299999999999 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: } 2023-09-23T21:24:08.743 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2023-09-23T21:24:08.744 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:24:08.745 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0: {} 2023-09-23T21:24:08.745 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1: {} 2023-09-23T21:24:08.745 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2: {} 2023-09-23T21:24:08.745 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3: {} 2023-09-23T21:24:08.745 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4: {} 2023-09-23T21:24:08.745 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6: {} 2023-09-23T21:24:08.745 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7: {} 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T21:24:08.746 INFO:teuthology.orchestra.run.smithi121.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T21:24:08.985 INFO:teuthology.orchestra.run.smithi121.stderr:set pool 1 pgp_num to 11 2023-09-23T21:24:09.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:08 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14/93 objects degraded (15.054%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3062913333' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:24:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1708247324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T21:24:09.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:08 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14/93 objects degraded (15.054%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:09.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3062913333' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:24:09.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1708247324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T21:24:09.172 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T21:24:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:08 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14/93 objects degraded (15.054%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:09.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3062913333' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T21:24:09.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1708247324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T21:24:09.242 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T21:24:09.519 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T21:24:09.984 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T21:24:10.251 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[211526]: failed to encode map e690 with expected crc 2023-09-23T21:24:10.251 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[211526]: failed to encode map e690 with expected crc 2023-09-23T21:24:10.251 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1708247324' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T21:24:10.251 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[211526]: osdmap e690: 8 total, 7 up, 8 in 2023-09-23T21:24:10.251 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[211526]: pgmap v2068: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 464 op/s; 15/90 objects degraded (16.667%) 2023-09-23T21:24:10.252 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[219709]: failed to encode map e690 with expected crc 2023-09-23T21:24:10.252 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[219709]: failed to encode map e690 with expected crc 2023-09-23T21:24:10.252 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1708247324' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T21:24:10.252 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[219709]: osdmap e690: 8 total, 7 up, 8 in 2023-09-23T21:24:10.252 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:09 smithi121 ceph-mon[219709]: pgmap v2068: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 464 op/s; 15/90 objects degraded (16.667%) 2023-09-23T21:24:10.360 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:09 smithi123 ceph-mon[193903]: failed to encode map e690 with expected crc 2023-09-23T21:24:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:09 smithi123 ceph-mon[193903]: failed to encode map e690 with expected crc 2023-09-23T21:24:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1708247324' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T21:24:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:09 smithi123 ceph-mon[193903]: osdmap e690: 8 total, 7 up, 8 in 2023-09-23T21:24:10.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:09 smithi123 ceph-mon[193903]: pgmap v2068: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 464 op/s; 15/90 objects degraded (16.667%) 2023-09-23T21:24:10.783 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:24:10 smithi123 conmon[172845]: logger=cleanup t=2023-09-23T21:24:10.483016616Z level=info msg="Completed cleanup jobs" duration=234.235906ms 2023-09-23T21:24:10.985 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:24:10.985 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:24:11.583 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T21:24:11.739 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:11 smithi121 ceph-mon[211526]: pgmap v2069: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 356 op/s; 15/90 objects degraded (16.667%) 2023-09-23T21:24:11.739 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:11 smithi121 ceph-mon[219709]: pgmap v2069: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 356 op/s; 15/90 objects degraded (16.667%) 2023-09-23T21:24:11.927 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:11.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:11 smithi123 ceph-mon[193903]: pgmap v2069: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 356 op/s; 15/90 objects degraded (16.667%) 2023-09-23T21:24:12.983 INFO:teuthology.orchestra.run.smithi121.stderr:noscrub is unset 2023-09-23T21:24:13.191 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4208905531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:24:13.192 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:12 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:13.384 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:24:13.385 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:24:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4208905531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:24:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:12 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4208905531' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T21:24:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:12 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:13.732 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T21:24:13.948 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T21:24:13.974 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[211526]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-09-23T21:24:13.974 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[211526]: failed to encode map e691 with expected crc 2023-09-23T21:24:13.974 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4208905531' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:24:13.974 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[211526]: osdmap e691: 8 total, 7 up, 8 in 2023-09-23T21:24:13.974 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[211526]: pgmap v2071: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 334 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:13.975 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[219709]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-09-23T21:24:13.975 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[219709]: failed to encode map e691 with expected crc 2023-09-23T21:24:13.975 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4208905531' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:24:13.975 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[219709]: osdmap e691: 8 total, 7 up, 8 in 2023-09-23T21:24:13.975 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:13 smithi121 ceph-mon[219709]: pgmap v2071: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 334 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:14.088 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:13 smithi123 ceph-mon[193903]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-09-23T21:24:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:13 smithi123 ceph-mon[193903]: failed to encode map e691 with expected crc 2023-09-23T21:24:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:13 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4208905531' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T21:24:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:13 smithi123 ceph-mon[193903]: osdmap e691: 8 total, 7 up, 8 in 2023-09-23T21:24:14.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:13 smithi123 ceph-mon[193903]: pgmap v2071: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 334 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:14.289 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:14.974 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:24:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:24:14] "GET /metrics HTTP/1.1" 200 34297 "" "Prometheus/2.43.0" 2023-09-23T21:24:15.165 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:14 smithi123 ceph-mon[193903]: failed to encode map e691 with expected crc 2023-09-23T21:24:15.166 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:14 smithi123 ceph-mon[193903]: failed to encode map e691 with expected crc 2023-09-23T21:24:15.166 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:14 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:15.268 INFO:tasks.thrashosds.thrasher:reviving osd 2023-09-23T21:24:15.269 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-09-23T21:24:15.269 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-09-23T21:24:15.269 DEBUG:teuthology.orchestra.run.smithi123:osd.5> sudo journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T21:24:15.273 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl start ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T21:24:15.333 INFO:journalctl@ceph.osd.5.smithi123.stdout:-- Logs begin at Sat 2023-09-23 20:08:24 UTC. -- 2023-09-23T21:24:15.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:14 smithi121 ceph-mon[219709]: failed to encode map e691 with expected crc 2023-09-23T21:24:15.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:14 smithi121 ceph-mon[219709]: failed to encode map e691 with expected crc 2023-09-23T21:24:15.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:14 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:14 smithi121 ceph-mon[211526]: failed to encode map e691 with expected crc 2023-09-23T21:24:15.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:14 smithi121 ceph-mon[211526]: failed to encode map e691 with expected crc 2023-09-23T21:24:15.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:14 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:15.478 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:15 smithi123 systemd[1]: Starting Ceph osd.5 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:24:15.692 INFO:teuthology.orchestra.run.smithi123.stdout:ERROR: (22) Invalid argument 2023-09-23T21:24:15.692 INFO:teuthology.orchestra.run.smithi123.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:24:15.986 INFO:teuthology.orchestra.run.smithi121.stderr:nodeep-scrub is unset 2023-09-23T21:24:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:15 smithi123 ceph-mon[193903]: pgmap v2072: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 373 KiB/s rd, 903 KiB/s wr, 160 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:15 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1974076044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:24:16.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:24:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:15 smithi121 ceph-mon[211526]: pgmap v2072: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 373 KiB/s rd, 903 KiB/s wr, 160 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:15 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1974076044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:24:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:24:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:15 smithi121 ceph-mon[219709]: pgmap v2072: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 373 KiB/s rd, 903 KiB/s wr, 160 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:15 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1974076044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:24:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T21:24:16.414 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T21:24:16.840 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:17.078 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[211526]: failed to encode map e692 with expected crc 2023-09-23T21:24:17.078 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[211526]: failed to encode map e692 with expected crc 2023-09-23T21:24:17.078 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:24:17.079 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[211526]: osdmap e692: 8 total, 7 up, 8 in 2023-09-23T21:24:17.079 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[219709]: failed to encode map e692 with expected crc 2023-09-23T21:24:17.079 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[219709]: failed to encode map e692 with expected crc 2023-09-23T21:24:17.079 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:24:17.079 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:16 smithi121 ceph-mon[219709]: osdmap e692: 8 total, 7 up, 8 in 2023-09-23T21:24:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:16 smithi123 ceph-mon[193903]: failed to encode map e692 with expected crc 2023-09-23T21:24:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:16 smithi123 ceph-mon[193903]: failed to encode map e692 with expected crc 2023-09-23T21:24:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:16 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T21:24:17.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:16 smithi123 ceph-mon[193903]: osdmap e692: 8 total, 7 up, 8 in 2023-09-23T21:24:17.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T21:24:17.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-23T21:24:17.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2023-09-23T21:24:17.230 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2023-09-23T21:24:17.230 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2023-09-23T21:24:17.230 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T21:24:17.230 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-23T21:24:17.230 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:16 smithi123 bash[455209]: --> ceph-volume lvm activate successful for osd ID: 5 2023-09-23T21:24:18.064 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:24:18.197 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:17 smithi121 ceph-mon[211526]: pgmap v2074: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.1 KiB/s rd, 0 B/s wr, 2 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:18.197 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:17 smithi121 ceph-mon[219709]: pgmap v2074: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.1 KiB/s rd, 0 B/s wr, 2 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:18.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:17 smithi123 ceph-mon[193903]: pgmap v2074: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.1 KiB/s rd, 0 B/s wr, 2 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:18.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:18 smithi123 ceph-osd[455760]: -- 172.21.15.123:0/2107143329 <== mon.1 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a6d83e8b60 con 0x55a6d83af400 2023-09-23T21:24:18.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:18 smithi123 bash[455695]: 5ef7ab6c8fa127ae1318f00977d37efaa010e4b322b3ba7e7b93e8b33b2932ec 2023-09-23T21:24:18.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:18 smithi123 systemd[1]: Started Ceph osd.5 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T21:24:18.414 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:24:18.414 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:24:18.418 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:18.826 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:18 smithi123 conmon[455736]: 2023-09-23T21:24:18.567+0000 7f82d2f29540 -1 Falling back to public interface 2023-09-23T21:24:19.101 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T21:24:19.451 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:19.692 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:19 smithi123 ceph-mon[193903]: pgmap v2075: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.1 KiB/s rd, 0 B/s wr, 2 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:19.866 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:19 smithi121 ceph-mon[211526]: pgmap v2075: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.1 KiB/s rd, 0 B/s wr, 2 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:19.867 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:19 smithi121 ceph-mon[219709]: pgmap v2075: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 1.1 KiB/s rd, 0 B/s wr, 2 op/s; 14/84 objects degraded (16.667%) 2023-09-23T21:24:19.931 INFO:teuthology.orchestra.run.smithi123.stderr:no valid command found; 10 closest matches: 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:0 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:1 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:2 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:abort 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:assert 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:config diff 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:config diff get 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:config get 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:config help [] 2023-09-23T21:24:19.932 INFO:teuthology.orchestra.run.smithi123.stderr:config set ... 2023-09-23T21:24:19.933 INFO:teuthology.orchestra.run.smithi123.stderr:admin_socket: invalid command 2023-09-23T21:24:20.539 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T21:24:20.539 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-09-23T21:24:21.121 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:24:21.122 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:24:21.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T21:24:21.912 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:21 smithi121 ceph-mon[219709]: pgmap v2076: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:21.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:21 smithi121 ceph-mon[211526]: pgmap v2076: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:21.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:21 smithi123 ceph-mon[193903]: pgmap v2076: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:22.107 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:23.714 INFO:teuthology.orchestra.run.smithi121.stdout:ERROR: (22) Invalid argument 2023-09-23T21:24:23.714 INFO:teuthology.orchestra.run.smithi121.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T21:24:23.737 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:23 smithi121 ceph-mon[211526]: pgmap v2077: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:23.738 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:23 smithi121 ceph-mon[219709]: pgmap v2077: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:23.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:23 smithi123 ceph-mon[193903]: pgmap v2077: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:25.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:24:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:24:24] "GET /metrics HTTP/1.1" 200 34297 "" "Prometheus/2.43.0" 2023-09-23T21:24:25.540 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:24:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:25 smithi121 ceph-mon[211526]: pgmap v2078: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:25.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:25 smithi121 ceph-mon[219709]: pgmap v2078: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:25.919 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:25 smithi123 ceph-mon[193903]: pgmap v2078: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:27.533 INFO:teuthology.orchestra.run.smithi123.stderr:no valid command found; 10 closest matches: 2023-09-23T21:24:27.533 INFO:teuthology.orchestra.run.smithi123.stderr:0 2023-09-23T21:24:27.533 INFO:teuthology.orchestra.run.smithi123.stderr:1 2023-09-23T21:24:27.533 INFO:teuthology.orchestra.run.smithi123.stderr:2 2023-09-23T21:24:27.533 INFO:teuthology.orchestra.run.smithi123.stderr:abort 2023-09-23T21:24:27.533 INFO:teuthology.orchestra.run.smithi123.stderr:assert 2023-09-23T21:24:27.534 INFO:teuthology.orchestra.run.smithi123.stderr:bluefs debug_inject_read_zeros 2023-09-23T21:24:27.534 INFO:teuthology.orchestra.run.smithi123.stderr:bluefs files list 2023-09-23T21:24:27.534 INFO:teuthology.orchestra.run.smithi123.stderr:bluefs stats 2023-09-23T21:24:27.534 INFO:teuthology.orchestra.run.smithi123.stderr:bluestore allocator dump block 2023-09-23T21:24:27.534 INFO:teuthology.orchestra.run.smithi123.stderr:bluestore allocator fragmentation block 2023-09-23T21:24:27.534 INFO:teuthology.orchestra.run.smithi123.stderr:admin_socket: invalid command 2023-09-23T21:24:27.979 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:27 smithi123 conmon[455736]: 2023-09-23T21:24:27.760+0000 7f82d2f29540 -1 osd.5 676 log_to_monitors true 2023-09-23T21:24:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T21:24:28.244 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-09-23T21:24:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[211526]: pgmap v2079: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:28.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[211526]: from='osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T21:24:28.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[211526]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T21:24:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[219709]: pgmap v2079: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:28.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[219709]: from='osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T21:24:28.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:28 smithi121 ceph-mon[219709]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T21:24:28.978 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:24:28 smithi123 conmon[455736]: 2023-09-23T21:24:28.506+0000 7f82c450a700 -1 osd.5 676 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T21:24:28.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:28 smithi123 ceph-mon[193903]: pgmap v2079: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:28 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:28 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:28 smithi123 ceph-mon[193903]: from='osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T21:24:28.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:28 smithi123 ceph-mon[193903]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T21:24:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[211526]: failed to encode map e693 with expected crc 2023-09-23T21:24:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[211526]: failed to encode map e693 with expected crc 2023-09-23T21:24:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[211526]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T21:24:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[211526]: osdmap e693: 8 total, 7 up, 8 in 2023-09-23T21:24:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[211526]: from='osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T21:24:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[211526]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T21:24:29.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[219709]: failed to encode map e693 with expected crc 2023-09-23T21:24:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[219709]: failed to encode map e693 with expected crc 2023-09-23T21:24:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[219709]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T21:24:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[219709]: osdmap e693: 8 total, 7 up, 8 in 2023-09-23T21:24:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[219709]: from='osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T21:24:29.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:29 smithi121 ceph-mon[219709]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T21:24:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:29 smithi123 ceph-mon[193903]: failed to encode map e693 with expected crc 2023-09-23T21:24:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:29 smithi123 ceph-mon[193903]: failed to encode map e693 with expected crc 2023-09-23T21:24:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:29 smithi123 ceph-mon[193903]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T21:24:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:29 smithi123 ceph-mon[193903]: osdmap e693: 8 total, 7 up, 8 in 2023-09-23T21:24:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:29 smithi123 ceph-mon[193903]: from='osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T21:24:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:29 smithi123 ceph-mon[193903]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2023-09-23T21:24:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: pgmap v2081: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:24:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163] boot 2023-09-23T21:24:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T21:24:30.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[211526]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: pgmap v2081: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163] boot 2023-09-23T21:24:30.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T21:24:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T21:24:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:30 smithi121 ceph-mon[219709]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: pgmap v2081: 83 pgs: 9 active+undersized+degraded, 15 active+undersized, 59 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: osd.5 [v2:172.21.15.123:6808/3144145163,v1:172.21.15.123:6809/3144145163] boot 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: failed to encode map e694 with expected crc 2023-09-23T21:24:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:30 smithi123 ceph-mon[193903]: failed to encode map e694 with expected crc 2023-09-23T21:24:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e677 with expected crc 2023-09-23T21:24:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e694 with expected crc 2023-09-23T21:24:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[211526]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e677 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e694 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.916 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:31 smithi121 ceph-mon[219709]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e677 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e694 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:31.981 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:31 smithi123 ceph-mon[193903]: failed to encode map e695 with expected crc 2023-09-23T21:24:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: pgmap v2084: 83 pgs: 8 active+undersized+degraded, 14 active+undersized, 61 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T21:24:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T21:24:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: failed to encode map e697 with expected crc 2023-09-23T21:24:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: failed to encode map e697 with expected crc 2023-09-23T21:24:32.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[211526]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T21:24:32.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: pgmap v2084: 83 pgs: 8 active+undersized+degraded, 14 active+undersized, 61 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: failed to encode map e697 with expected crc 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: failed to encode map e697 with expected crc 2023-09-23T21:24:32.915 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:32 smithi121 ceph-mon[219709]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: pgmap v2084: 83 pgs: 8 active+undersized+degraded, 14 active+undersized, 61 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: failed to encode map e696 with expected crc 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: failed to encode map e697 with expected crc 2023-09-23T21:24:32.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: failed to encode map e697 with expected crc 2023-09-23T21:24:32.980 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:32 smithi123 ceph-mon[193903]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T21:24:33.245 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T21:24:33.619 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:24:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:33 smithi123 ceph-mon[193903]: failed to encode map e696 with expected crc 2023-09-23T21:24:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:33 smithi123 ceph-mon[193903]: failed to encode map e697 with expected crc 2023-09-23T21:24:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:33 smithi123 ceph-mon[193903]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T21:24:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:33 smithi123 ceph-mon[193903]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:33 smithi123 ceph-mon[193903]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:33 smithi123 ceph-mon[193903]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[211526]: failed to encode map e696 with expected crc 2023-09-23T21:24:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[211526]: failed to encode map e697 with expected crc 2023-09-23T21:24:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[211526]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T21:24:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[211526]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[211526]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[211526]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[219709]: failed to encode map e696 with expected crc 2023-09-23T21:24:33.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[219709]: failed to encode map e697 with expected crc 2023-09-23T21:24:33.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[219709]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T21:24:33.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[219709]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[219709]: failed to encode map e698 with expected crc 2023-09-23T21:24:33.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:33 smithi121 ceph-mon[219709]: failed to encode map e698 with expected crc 2023-09-23T21:24:34.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:34 smithi121 ceph-mon[219709]: failed to encode map e698 with expected crc 2023-09-23T21:24:34.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:34 smithi121 ceph-mon[219709]: failed to encode map e698 with expected crc 2023-09-23T21:24:34.776 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:34 smithi121 ceph-mon[219709]: pgmap v2088: 83 pgs: 5 active+undersized+degraded, 10 active+undersized, 68 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 7/84 objects degraded (8.333%); 0 B/s, 1 objects/s recovering 2023-09-23T21:24:34.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:34 smithi121 ceph-mon[211526]: failed to encode map e698 with expected crc 2023-09-23T21:24:34.776 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:34 smithi121 ceph-mon[211526]: failed to encode map e698 with expected crc 2023-09-23T21:24:34.777 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:34 smithi121 ceph-mon[211526]: pgmap v2088: 83 pgs: 5 active+undersized+degraded, 10 active+undersized, 68 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 7/84 objects degraded (8.333%); 0 B/s, 1 objects/s recovering 2023-09-23T21:24:34.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:34 smithi123 ceph-mon[193903]: failed to encode map e698 with expected crc 2023-09-23T21:24:34.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:34 smithi123 ceph-mon[193903]: failed to encode map e698 with expected crc 2023-09-23T21:24:34.973 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:34 smithi123 ceph-mon[193903]: pgmap v2088: 83 pgs: 5 active+undersized+degraded, 10 active+undersized, 68 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 7/84 objects degraded (8.333%); 0 B/s, 1 objects/s recovering 2023-09-23T21:24:35.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:24:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:24:34] "GET /metrics HTTP/1.1" 200 34296 "" "Prometheus/2.43.0" 2023-09-23T21:24:35.189 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2023-09-23T21:24:35.189 INFO:teuthology.orchestra.run.smithi123.stdout: "ops": [], 2023-09-23T21:24:35.190 INFO:teuthology.orchestra.run.smithi123.stdout: "num_ops": 0 2023-09-23T21:24:35.190 INFO:teuthology.orchestra.run.smithi123.stdout:} 2023-09-23T21:24:35.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:35 smithi123 ceph-mon[193903]: pgmap v2089: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 822 KiB/s, 3 objects/s recovering 2023-09-23T21:24:35.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:35 smithi123 ceph-mon[193903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded) 2023-09-23T21:24:35.713 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:35 smithi123 ceph-mon[193903]: Cluster is now healthy 2023-09-23T21:24:35.726 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T21:24:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:35 smithi121 ceph-mon[211526]: pgmap v2089: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 822 KiB/s, 3 objects/s recovering 2023-09-23T21:24:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:35 smithi121 ceph-mon[211526]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded) 2023-09-23T21:24:35.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:35 smithi121 ceph-mon[211526]: Cluster is now healthy 2023-09-23T21:24:35.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:35 smithi121 ceph-mon[219709]: pgmap v2089: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 822 KiB/s, 3 objects/s recovering 2023-09-23T21:24:35.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:35 smithi121 ceph-mon[219709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/84 objects degraded (8.333%), 5 pgs degraded) 2023-09-23T21:24:35.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:35 smithi121 ceph-mon[219709]: Cluster is now healthy 2023-09-23T21:24:36.090 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:37.737 INFO:teuthology.orchestra.run.smithi121.stdout:mon.a: {} 2023-09-23T21:24:37.747 INFO:teuthology.orchestra.run.smithi121.stdout:mon.b: {} 2023-09-23T21:24:37.757 INFO:teuthology.orchestra.run.smithi121.stdout:mon.c: {} 2023-09-23T21:24:37.758 INFO:teuthology.orchestra.run.smithi121.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T21:24:37.758 INFO:teuthology.orchestra.run.smithi121.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T21:24:37.758 INFO:teuthology.orchestra.run.smithi121.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T21:24:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[219709]: failed to encode map e699 with expected crc 2023-09-23T21:24:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[219709]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T21:24:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[219709]: failed to encode map e699 with expected crc 2023-09-23T21:24:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[219709]: pgmap v2091: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 687 KiB/s, 2 objects/s recovering 2023-09-23T21:24:38.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/696727092' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:24:38.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[211526]: failed to encode map e699 with expected crc 2023-09-23T21:24:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[211526]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T21:24:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[211526]: failed to encode map e699 with expected crc 2023-09-23T21:24:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[211526]: pgmap v2091: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 687 KiB/s, 2 objects/s recovering 2023-09-23T21:24:38.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:37 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/696727092' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:24:38.456 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T21:24:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:37 smithi123 ceph-mon[193903]: failed to encode map e699 with expected crc 2023-09-23T21:24:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:37 smithi123 ceph-mon[193903]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T21:24:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:37 smithi123 ceph-mon[193903]: failed to encode map e699 with expected crc 2023-09-23T21:24:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:37 smithi123 ceph-mon[193903]: pgmap v2091: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 687 KiB/s, 2 objects/s recovering 2023-09-23T21:24:38.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:37 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/696727092' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:24:38.805 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:39.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:38 smithi121 ceph-mon[211526]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:38 smithi121 ceph-mon[211526]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.398 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:38 smithi121 ceph-mon[211526]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.398 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:38 smithi121 ceph-mon[219709]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.398 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:38 smithi121 ceph-mon[219709]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.399 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:38 smithi121 ceph-mon[219709]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:38 smithi123 ceph-mon[193903]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:38 smithi123 ceph-mon[193903]: failed to encode map e699 with expected crc 2023-09-23T21:24:39.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:38 smithi123 ceph-mon[193903]: failed to encode map e699 with expected crc 2023-09-23T21:24:40.175 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:39 smithi121 ceph-mon[219709]: pgmap v2092: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 546 KiB/s, 2 objects/s recovering 2023-09-23T21:24:40.175 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:39 smithi121 ceph-mon[211526]: pgmap v2092: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 546 KiB/s, 2 objects/s recovering 2023-09-23T21:24:40.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:39 smithi123 ceph-mon[193903]: pgmap v2092: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 546 KiB/s, 2 objects/s recovering 2023-09-23T21:24:40.998 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T21:24:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:40 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3119543202' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:41.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:40 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:40 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3119543202' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:41.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:40 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:41.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:40 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3119543202' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:41.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:40 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:41.705 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T21:24:42.061 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:42.288 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[211526]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[211526]: osdmap e700: 8 total, 8 up, 8 in 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[211526]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[211526]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[211526]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[211526]: pgmap v2094: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 512 KiB/s, 1 objects/s recovering 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[219709]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[219709]: osdmap e700: 8 total, 8 up, 8 in 2023-09-23T21:24:42.289 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[219709]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.290 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[219709]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.290 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[219709]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.290 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:41 smithi121 ceph-mon[219709]: pgmap v2094: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 512 KiB/s, 1 objects/s recovering 2023-09-23T21:24:42.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:42 smithi123 ceph-mon[193903]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:42 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:24:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:42 smithi123 ceph-mon[193903]: osdmap e700: 8 total, 8 up, 8 in 2023-09-23T21:24:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:42 smithi123 ceph-mon[193903]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:42 smithi123 ceph-mon[193903]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:42 smithi123 ceph-mon[193903]: failed to encode map e700 with expected crc 2023-09-23T21:24:42.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:42 smithi123 ceph-mon[193903]: pgmap v2094: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 512 KiB/s, 1 objects/s recovering 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:24:43.729 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:43 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:24:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T21:24:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-09-23T21:24:43.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:43 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T21:24:44.399 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T21:24:44.482 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: pgmap v2095: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/71903070' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: failed to encode map e701 with expected crc 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:24:44.483 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:44 smithi123 ceph-mon[193903]: osdmap e701: 8 total, 8 up, 8 in 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: pgmap v2095: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/71903070' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: failed to encode map e701 with expected crc 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:24:44.777 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[219709]: osdmap e701: 8 total, 8 up, 8 in 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: pgmap v2095: 83 pgs: 83 active+clean; 4.6 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/71903070' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: failed to encode map e701 with expected crc 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T21:24:44.778 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:44 smithi121 ceph-mon[211526]: osdmap e701: 8 total, 8 up, 8 in 2023-09-23T21:24:45.156 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:24:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:24:44] "GET /metrics HTTP/1.1" 200 34311 "" "Prometheus/2.43.0" 2023-09-23T21:24:45.172 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T21:24:45.524 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:45.747 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[211526]: failed to encode map e701 with expected crc 2023-09-23T21:24:45.747 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[211526]: failed to encode map e701 with expected crc 2023-09-23T21:24:45.747 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[211526]: failed to encode map e702 with expected crc 2023-09-23T21:24:45.747 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[211526]: osdmap e702: 8 total, 8 up, 8 in 2023-09-23T21:24:45.747 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[219709]: failed to encode map e701 with expected crc 2023-09-23T21:24:45.748 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[219709]: failed to encode map e701 with expected crc 2023-09-23T21:24:45.748 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[219709]: failed to encode map e702 with expected crc 2023-09-23T21:24:45.748 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:45 smithi121 ceph-mon[219709]: osdmap e702: 8 total, 8 up, 8 in 2023-09-23T21:24:45.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:45 smithi123 ceph-mon[193903]: failed to encode map e701 with expected crc 2023-09-23T21:24:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:45 smithi123 ceph-mon[193903]: failed to encode map e701 with expected crc 2023-09-23T21:24:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:45 smithi123 ceph-mon[193903]: failed to encode map e702 with expected crc 2023-09-23T21:24:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:45 smithi123 ceph-mon[193903]: osdmap e702: 8 total, 8 up, 8 in 2023-09-23T21:24:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[219709]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[219709]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[219709]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[219709]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[219709]: pgmap v2098: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:46.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[219709]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[211526]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[211526]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[211526]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[211526]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[211526]: pgmap v2098: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:46.915 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:46 smithi121 ceph-mon[211526]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:46 smithi123 ceph-mon[193903]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:46 smithi123 ceph-mon[193903]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:46 smithi123 ceph-mon[193903]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:46 smithi123 ceph-mon[193903]: failed to encode map e702 with expected crc 2023-09-23T21:24:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:46 smithi123 ceph-mon[193903]: pgmap v2098: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:46.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:46 smithi123 ceph-mon[193903]: failed to encode map e702 with expected crc 2023-09-23T21:24:47.999 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T21:24:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:47 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1034684973' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:48.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:47 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:48.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:47 smithi121 ceph-mon[211526]: pgmap v2099: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:47 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1034684973' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:47 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:48.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:47 smithi121 ceph-mon[219709]: pgmap v2099: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:47 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1034684973' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:47 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:48.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:47 smithi123 ceph-mon[193903]: pgmap v2099: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:48.724 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T21:24:48.990 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[211526]: failed to encode map e703 with expected crc 2023-09-23T21:24:48.990 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[211526]: failed to encode map e703 with expected crc 2023-09-23T21:24:48.990 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:24:48.991 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[211526]: osdmap e703: 8 total, 8 up, 8 in 2023-09-23T21:24:48.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[219709]: failed to encode map e703 with expected crc 2023-09-23T21:24:48.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[219709]: failed to encode map e703 with expected crc 2023-09-23T21:24:48.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:24:48.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[219709]: osdmap e703: 8 total, 8 up, 8 in 2023-09-23T21:24:48.991 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[219709]: failed to encode map e703 with expected crc 2023-09-23T21:24:49.092 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:49.303 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:48 smithi121 ceph-mon[211526]: failed to encode map e703 with expected crc 2023-09-23T21:24:49.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:48 smithi123 ceph-mon[193903]: failed to encode map e703 with expected crc 2023-09-23T21:24:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:48 smithi123 ceph-mon[193903]: failed to encode map e703 with expected crc 2023-09-23T21:24:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:48 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:24:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:48 smithi123 ceph-mon[193903]: osdmap e703: 8 total, 8 up, 8 in 2023-09-23T21:24:49.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:48 smithi123 ceph-mon[193903]: failed to encode map e703 with expected crc 2023-09-23T21:24:50.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:49 smithi121 ceph-mon[211526]: pgmap v2101: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:50.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:49 smithi121 ceph-mon[219709]: pgmap v2101: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:50.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:49 smithi123 ceph-mon[193903]: pgmap v2101: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:51.006 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T21:24:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:50 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1141610134' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:51.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:50 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:50 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1141610134' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:51.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:50 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:51.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:50 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1141610134' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:51.480 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:50 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T21:24:51.735 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T21:24:52.009 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[211526]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.010 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[211526]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.010 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:24:52.010 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[211526]: osdmap e704: 8 total, 8 up, 8 in 2023-09-23T21:24:52.010 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[211526]: pgmap v2103: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:52.010 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[211526]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.010 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[211526]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[219709]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[219709]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:24:52.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[219709]: osdmap e704: 8 total, 8 up, 8 in 2023-09-23T21:24:52.011 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[219709]: pgmap v2103: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:52.012 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[219709]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.012 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:52 smithi121 ceph-mon[219709]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.087 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:52.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:52 smithi123 ceph-mon[193903]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:52 smithi123 ceph-mon[193903]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:52 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T21:24:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:52 smithi123 ceph-mon[193903]: osdmap e704: 8 total, 8 up, 8 in 2023-09-23T21:24:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:52 smithi123 ceph-mon[193903]: pgmap v2103: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:52 smithi123 ceph-mon[193903]: failed to encode map e704 with expected crc 2023-09-23T21:24:52.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:52 smithi123 ceph-mon[193903]: failed to encode map e704 with expected crc 2023-09-23T21:24:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:53 smithi121 ceph-mon[211526]: pgmap v2104: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:53 smithi121 ceph-mon[219709]: pgmap v2104: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:53 smithi123 ceph-mon[193903]: pgmap v2104: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:54.584 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T21:24:54.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:24:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:24:54] "GET /metrics HTTP/1.1" 200 34311 "" "Prometheus/2.43.0" 2023-09-23T21:24:54.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:54 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1604172170' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:24:54.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:54 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1604172170' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:24:54.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:54 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1604172170' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:24:55.281 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T21:24:55.586 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[211526]: failed to encode map e705 with expected crc 2023-09-23T21:24:55.586 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1604172170' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:24:55.586 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[211526]: osdmap e705: 8 total, 8 up, 8 in 2023-09-23T21:24:55.586 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[211526]: failed to encode map e705 with expected crc 2023-09-23T21:24:55.586 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[211526]: pgmap v2106: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:55.632 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:55.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[219709]: failed to encode map e705 with expected crc 2023-09-23T21:24:55.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1604172170' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:24:55.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[219709]: osdmap e705: 8 total, 8 up, 8 in 2023-09-23T21:24:55.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[219709]: failed to encode map e705 with expected crc 2023-09-23T21:24:55.862 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:55 smithi121 ceph-mon[219709]: pgmap v2106: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:55.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:55 smithi123 ceph-mon[193903]: failed to encode map e705 with expected crc 2023-09-23T21:24:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1604172170' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:24:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:55 smithi123 ceph-mon[193903]: osdmap e705: 8 total, 8 up, 8 in 2023-09-23T21:24:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:55 smithi123 ceph-mon[193903]: failed to encode map e705 with expected crc 2023-09-23T21:24:55.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:55 smithi123 ceph-mon[193903]: pgmap v2106: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:57.997 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T21:24:58.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[211526]: pgmap v2107: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3246354581' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:24:58.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:24:58.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:58.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[219709]: pgmap v2107: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:58.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3246354581' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:24:58.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:57 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:24:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:24:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:57 smithi123 ceph-mon[193903]: pgmap v2107: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:24:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:57 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3246354581' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T21:24:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:57 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:24:59.058 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T21:24:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[219709]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[219709]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3246354581' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:24:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[219709]: osdmap e706: 8 total, 8 up, 8 in 2023-09-23T21:24:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[219709]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[219709]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[211526]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[211526]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3246354581' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:24:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[211526]: osdmap e706: 8 total, 8 up, 8 in 2023-09-23T21:24:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[211526]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:24:58 smithi121 ceph-mon[211526]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.442 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:24:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:58 smithi123 ceph-mon[193903]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:58 smithi123 ceph-mon[193903]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3246354581' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T21:24:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:58 smithi123 ceph-mon[193903]: osdmap e706: 8 total, 8 up, 8 in 2023-09-23T21:24:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:58 smithi123 ceph-mon[193903]: failed to encode map e706 with expected crc 2023-09-23T21:24:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:24:58 smithi123 ceph-mon[193903]: failed to encode map e706 with expected crc 2023-09-23T21:25:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[219709]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[219709]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[219709]: osdmap e707: 8 total, 8 up, 8 in 2023-09-23T21:25:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[219709]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[219709]: pgmap v2110: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:00.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[211526]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[211526]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[211526]: osdmap e707: 8 total, 8 up, 8 in 2023-09-23T21:25:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[211526]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:00 smithi121 ceph-mon[211526]: pgmap v2110: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:00 smithi123 ceph-mon[193903]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:00 smithi123 ceph-mon[193903]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:00 smithi123 ceph-mon[193903]: osdmap e707: 8 total, 8 up, 8 in 2023-09-23T21:25:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:00 smithi123 ceph-mon[193903]: failed to encode map e707 with expected crc 2023-09-23T21:25:00.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:00 smithi123 ceph-mon[193903]: pgmap v2110: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:01.897 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T21:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[211526]: pgmap v2111: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/613583305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[219709]: pgmap v2111: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:02.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/613583305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:25:02.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:01 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:01 smithi123 ceph-mon[193903]: pgmap v2111: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:01 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/613583305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:25:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:01 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:02.716 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T21:25:03.062 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/613583305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: osdmap e708: 8 total, 8 up, 8 in 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:25:03.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:03.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/613583305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:25:03.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: osdmap e708: 8 total, 8 up, 8 in 2023-09-23T21:25:03.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:25:03.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:25:03.165 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:02 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: failed to encode map e708 with expected crc 2023-09-23T21:25:03.229 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/613583305' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:25:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: osdmap e708: 8 total, 8 up, 8 in 2023-09-23T21:25:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:25:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:25:03.230 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:02 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:25:04.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:03 smithi121 ceph-mon[211526]: pgmap v2113: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:04.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:03 smithi121 ceph-mon[219709]: pgmap v2113: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:04.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:03 smithi123 ceph-mon[193903]: pgmap v2113: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:04.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:25:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:25:04] "GET /metrics HTTP/1.1" 200 34298 "" "Prometheus/2.43.0" 2023-09-23T21:25:05.012 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T21:25:05.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:05 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1716605308' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:25:05.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:05 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1716605308' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:25:05.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:05 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1716605308' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T21:25:05.742 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T21:25:06.020 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[211526]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.021 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[211526]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.021 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1716605308' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:25:06.021 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[211526]: osdmap e709: 8 total, 8 up, 8 in 2023-09-23T21:25:06.021 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[211526]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.021 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[211526]: pgmap v2115: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:06.096 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:06.308 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[219709]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.308 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[219709]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.308 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1716605308' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:25:06.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[219709]: osdmap e709: 8 total, 8 up, 8 in 2023-09-23T21:25:06.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[219709]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.309 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:06 smithi121 ceph-mon[219709]: pgmap v2115: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:06.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:06 smithi123 ceph-mon[193903]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:06 smithi123 ceph-mon[193903]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:06 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1716605308' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T21:25:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:06 smithi123 ceph-mon[193903]: osdmap e709: 8 total, 8 up, 8 in 2023-09-23T21:25:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:06 smithi123 ceph-mon[193903]: failed to encode map e709 with expected crc 2023-09-23T21:25:06.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:06 smithi123 ceph-mon[193903]: pgmap v2115: 83 pgs: 83 active+clean; 4.6 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[211526]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[211526]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[211526]: osdmap e710: 8 total, 8 up, 8 in 2023-09-23T21:25:07.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[211526]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[219709]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[219709]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[219709]: osdmap e710: 8 total, 8 up, 8 in 2023-09-23T21:25:07.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:07 smithi121 ceph-mon[219709]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:07 smithi123 ceph-mon[193903]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:07 smithi123 ceph-mon[193903]: failed to encode map e710 with expected crc 2023-09-23T21:25:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:07 smithi123 ceph-mon[193903]: osdmap e710: 8 total, 8 up, 8 in 2023-09-23T21:25:07.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:07 smithi123 ceph-mon[193903]: failed to encode map e710 with expected crc 2023-09-23T21:25:08.040 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T21:25:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:08 smithi121 ceph-mon[211526]: pgmap v2117: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:08.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:08 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4283359592' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:25:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:08 smithi121 ceph-mon[219709]: pgmap v2117: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:08.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:08 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4283359592' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:25:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:08 smithi123 ceph-mon[193903]: pgmap v2117: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:08 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4283359592' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:25:08.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T21:25:09.162 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[219709]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4283359592' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:25:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[219709]: osdmap e711: 8 total, 8 up, 8 in 2023-09-23T21:25:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[219709]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[219709]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[219709]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[219709]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[211526]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4283359592' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:25:09.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[211526]: osdmap e711: 8 total, 8 up, 8 in 2023-09-23T21:25:09.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[211526]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[211526]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[211526]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.164 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:09 smithi121 ceph-mon[211526]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.164 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:09 smithi123 ceph-mon[193903]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:09 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4283359592' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:25:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:09 smithi123 ceph-mon[193903]: osdmap e711: 8 total, 8 up, 8 in 2023-09-23T21:25:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:09 smithi123 ceph-mon[193903]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:09 smithi123 ceph-mon[193903]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:09 smithi123 ceph-mon[193903]: failed to encode map e711 with expected crc 2023-09-23T21:25:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:09 smithi123 ceph-mon[193903]: failed to encode map e711 with expected crc 2023-09-23T21:25:10.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:10 smithi121 ceph-mon[211526]: pgmap v2119: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:10.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:10 smithi121 ceph-mon[219709]: pgmap v2119: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:10.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:10 smithi123 ceph-mon[193903]: pgmap v2119: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:11.042 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T21:25:11.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:11 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4014390839' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:25:11.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:11 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4014390839' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:25:11.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:11 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4014390839' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T21:25:11.747 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T21:25:12.046 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.046 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/4014390839' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:25:12.046 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: osdmap e712: 8 total, 8 up, 8 in 2023-09-23T21:25:12.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: pgmap v2121: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:12.047 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[219709]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.109 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:12.315 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/4014390839' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: osdmap e712: 8 total, 8 up, 8 in 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: pgmap v2121: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:12.316 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:12 smithi121 ceph-mon[211526]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/4014390839' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: osdmap e712: 8 total, 8 up, 8 in 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: failed to encode map e712 with expected crc 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: pgmap v2121: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:12.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:12 smithi123 ceph-mon[193903]: failed to encode map e712 with expected crc 2023-09-23T21:25:13.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:13 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:13.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:13 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:13.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:13 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:14.061 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T21:25:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:14 smithi121 ceph-mon[219709]: pgmap v2122: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:14 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/590156400' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:14.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:14 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:14 smithi121 ceph-mon[211526]: pgmap v2122: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:14 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/590156400' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:14.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:14 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:14.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:14 smithi123 ceph-mon[193903]: pgmap v2122: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:14 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/590156400' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:14.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:14 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:14.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T21:25:15.051 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:25:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:25:14] "GET /metrics HTTP/1.1" 200 34295 "" "Prometheus/2.43.0" 2023-09-23T21:25:15.051 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[219709]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.052 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[211526]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.052 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[211526]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.052 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:25:15.110 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:15.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[211526]: osdmap e713: 8 total, 8 up, 8 in 2023-09-23T21:25:15.322 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[211526]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.323 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[211526]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[219709]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:25:15.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[219709]: osdmap e713: 8 total, 8 up, 8 in 2023-09-23T21:25:15.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[219709]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.323 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:15 smithi121 ceph-mon[219709]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:15 smithi123 ceph-mon[193903]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:15 smithi123 ceph-mon[193903]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:15 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:25:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:15 smithi123 ceph-mon[193903]: osdmap e713: 8 total, 8 up, 8 in 2023-09-23T21:25:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:15 smithi123 ceph-mon[193903]: failed to encode map e713 with expected crc 2023-09-23T21:25:15.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:15 smithi123 ceph-mon[193903]: failed to encode map e713 with expected crc 2023-09-23T21:25:16.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:16 smithi121 ceph-mon[211526]: pgmap v2124: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:16.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:16 smithi121 ceph-mon[219709]: pgmap v2124: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:16.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:16 smithi123 ceph-mon[193903]: pgmap v2124: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:17.070 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T21:25:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:17 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/23354170' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:17.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:17 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:17 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/23354170' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:17.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:17 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:17.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:17 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/23354170' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:17.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:17 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T21:25:17.771 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T21:25:18.070 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[219709]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[219709]: osdmap e714: 8 total, 8 up, 8 in 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[219709]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[219709]: pgmap v2126: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[211526]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[211526]: osdmap e714: 8 total, 8 up, 8 in 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[211526]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.071 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[211526]: pgmap v2126: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:18.072 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[211526]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.118 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:18.331 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:18 smithi121 ceph-mon[219709]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:18 smithi123 ceph-mon[193903]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:18 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T21:25:18.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:18 smithi123 ceph-mon[193903]: osdmap e714: 8 total, 8 up, 8 in 2023-09-23T21:25:18.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:18 smithi123 ceph-mon[193903]: failed to encode map e714 with expected crc 2023-09-23T21:25:18.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:18 smithi123 ceph-mon[193903]: pgmap v2126: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:18.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:18 smithi123 ceph-mon[193903]: failed to encode map e714 with expected crc 2023-09-23T21:25:19.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:19 smithi121 ceph-mon[211526]: failed to encode map e714 with expected crc 2023-09-23T21:25:19.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:19 smithi121 ceph-mon[219709]: failed to encode map e714 with expected crc 2023-09-23T21:25:19.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:19 smithi123 ceph-mon[193903]: failed to encode map e714 with expected crc 2023-09-23T21:25:20.082 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T21:25:20.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:20 smithi121 ceph-mon[219709]: pgmap v2127: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:20.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:20 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2347909463' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:20.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:20 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:20 smithi121 ceph-mon[211526]: pgmap v2127: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:20.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:20 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2347909463' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:20.414 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:20 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:20 smithi123 ceph-mon[193903]: pgmap v2127: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:20 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2347909463' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:20.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:20 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:20.787 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T21:25:21.137 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:21.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[211526]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.346 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[211526]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:25:21.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[211526]: osdmap e715: 8 total, 8 up, 8 in 2023-09-23T21:25:21.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[211526]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[211526]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[219709]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[219709]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:25:21.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[219709]: osdmap e715: 8 total, 8 up, 8 in 2023-09-23T21:25:21.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[219709]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.348 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:21 smithi121 ceph-mon[219709]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:21 smithi123 ceph-mon[193903]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:21 smithi123 ceph-mon[193903]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:21 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:25:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:21 smithi123 ceph-mon[193903]: osdmap e715: 8 total, 8 up, 8 in 2023-09-23T21:25:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:21 smithi123 ceph-mon[193903]: failed to encode map e715 with expected crc 2023-09-23T21:25:21.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:21 smithi123 ceph-mon[193903]: failed to encode map e715 with expected crc 2023-09-23T21:25:22.415 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:22 smithi121 ceph-mon[211526]: pgmap v2129: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:22.415 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:22 smithi121 ceph-mon[219709]: pgmap v2129: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:22.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:22 smithi123 ceph-mon[193903]: pgmap v2129: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:23.092 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T21:25:23.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:23 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2141783067' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:23.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:23 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:23.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:23 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2141783067' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:23.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:23 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:23.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:23 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2141783067' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:23.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:23 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T21:25:23.887 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T21:25:24.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[211526]: failed to encode map e716 with expected crc 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[211526]: failed to encode map e716 with expected crc 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[211526]: osdmap e716: 8 total, 8 up, 8 in 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[211526]: pgmap v2131: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[219709]: failed to encode map e716 with expected crc 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[219709]: failed to encode map e716 with expected crc 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:25:24.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[219709]: osdmap e716: 8 total, 8 up, 8 in 2023-09-23T21:25:24.164 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:24 smithi121 ceph-mon[219709]: pgmap v2131: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:24.239 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:24.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:24 smithi123 ceph-mon[193903]: failed to encode map e716 with expected crc 2023-09-23T21:25:24.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:24 smithi123 ceph-mon[193903]: failed to encode map e716 with expected crc 2023-09-23T21:25:24.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:24 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T21:25:24.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:24 smithi123 ceph-mon[193903]: osdmap e716: 8 total, 8 up, 8 in 2023-09-23T21:25:24.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:24 smithi123 ceph-mon[193903]: pgmap v2131: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T21:25:25.162 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:25:24 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:25:24] "GET /metrics HTTP/1.1" 200 34295 "" "Prometheus/2.43.0" 2023-09-23T21:25:25.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:25 smithi121 ceph-mon[211526]: pgmap v2132: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:25.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:25 smithi121 ceph-mon[219709]: pgmap v2132: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:25.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:25 smithi123 ceph-mon[193903]: pgmap v2132: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:26.581 INFO:teuthology.orchestra.run.smithi121.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T21:25:26.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:26 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/92506418' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:26.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:26 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/92506418' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:26.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:26 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/92506418' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:27.305 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T21:25:27.662 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:27.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[211526]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[211526]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/92506418' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:25:27.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[211526]: osdmap e717: 8 total, 8 up, 8 in 2023-09-23T21:25:27.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[211526]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.871 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[211526]: pgmap v2134: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[219709]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[219709]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/92506418' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[219709]: osdmap e717: 8 total, 8 up, 8 in 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[219709]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:27.872 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:27 smithi121 ceph-mon[219709]: pgmap v2134: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:27.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:27 smithi123 ceph-mon[193903]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:27 smithi123 ceph-mon[193903]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:27 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/92506418' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:25:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:27 smithi123 ceph-mon[193903]: osdmap e717: 8 total, 8 up, 8 in 2023-09-23T21:25:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:27 smithi123 ceph-mon[193903]: failed to encode map e717 with expected crc 2023-09-23T21:25:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:27 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:27.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:27 smithi123 ceph-mon[193903]: pgmap v2134: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:29.588 INFO:teuthology.orchestra.run.smithi121.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T21:25:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:29 smithi121 ceph-mon[211526]: pgmap v2135: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:29 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3644771300' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:29.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:29 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:29.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:29 smithi121 ceph-mon[219709]: pgmap v2135: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:29.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:29 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3644771300' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:29.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:29 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:29.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:29 smithi123 ceph-mon[193903]: pgmap v2135: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:29 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3644771300' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:29.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:29 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T21:25:30.310 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T21:25:30.311 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T21:25:30.311 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T21:25:30.311 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T21:25:30.311 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T21:25:30.311 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T21:25:30.668 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:30.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[211526]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[211526]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.894 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[211526]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.895 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[211526]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:25:30.895 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[211526]: osdmap e718: 8 total, 8 up, 8 in 2023-09-23T21:25:30.895 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[219709]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.895 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[219709]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.895 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[219709]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.895 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[219709]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:25:30.895 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:30 smithi121 ceph-mon[219709]: osdmap e718: 8 total, 8 up, 8 in 2023-09-23T21:25:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:30 smithi123 ceph-mon[193903]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:30 smithi123 ceph-mon[193903]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:30 smithi123 ceph-mon[193903]: failed to encode map e718 with expected crc 2023-09-23T21:25:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:30 smithi123 ceph-mon[193903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T21:25:30.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:30 smithi123 ceph-mon[193903]: osdmap e718: 8 total, 8 up, 8 in 2023-09-23T21:25:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[219709]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[219709]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[219709]: osdmap e719: 8 total, 8 up, 8 in 2023-09-23T21:25:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[219709]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[219709]: pgmap v2138: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:31.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[211526]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[211526]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[211526]: osdmap e719: 8 total, 8 up, 8 in 2023-09-23T21:25:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[211526]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.914 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:31 smithi121 ceph-mon[211526]: pgmap v2138: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:31.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:31 smithi123 ceph-mon[193903]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:31 smithi123 ceph-mon[193903]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:31 smithi123 ceph-mon[193903]: osdmap e719: 8 total, 8 up, 8 in 2023-09-23T21:25:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:31 smithi123 ceph-mon[193903]: failed to encode map e719 with expected crc 2023-09-23T21:25:31.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:31 smithi123 ceph-mon[193903]: pgmap v2138: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:32.499 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:25:32.499 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":719,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T21:25:30.581371+0000","last_up_change":"2023-09-23T21:24:29.482853+0000","last_in_change":"2023-09-23T21:23:38.968570+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"690","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"582","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"688","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":688,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:30:20.536872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:30:21.339996+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":92,"up_thru":701,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6803","nonce":3792083724}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6805","nonce":3792083724}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6809","nonce":3792083724}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6807","nonce":3792083724}]},"public_addr":"172.21.15.121:6803/3792083724","cluster_addr":"172.21.15.121:6805/3792083724","heartbeat_back_addr":"172.21.15.121:6809/3792083724","heartbeat_front_addr":"172.21.15.121:6807/3792083724","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":206,"last_clean_end":228,"up_from":276,"up_thru":694,"down_at":229,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6811","nonce":3255216964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6813","nonce":3255216964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6817","nonce":3255216964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6815","nonce":3255216964}]},"public_addr":"172.21.15.121:6811/3255216964","cluster_addr":"172.21.15.121:6813/3255216964","heartbeat_back_addr":"172.21.15.121:6817/3255216964","heartbeat_front_addr":"172.21.15.121:6815/3255216964","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":123,"up_from":129,"up_thru":697,"down_at":124,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6819","nonce":2050616913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6821","nonce":2050616913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6825","nonce":2050616913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6823","nonce":2050616913}]},"public_addr":"172.21.15.121:6819/2050616913","cluster_addr":"172.21.15.121:6821/2050616913","heartbeat_back_addr":"172.21.15.121:6825/2050616913","heartbeat_front_addr":"172.21.15.121:6823/2050616913","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":141,"up_from":148,"up_thru":706,"down_at":142,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6827","nonce":3257078631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6829","nonce":3257078631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6833","nonce":3257078631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6831","nonce":3257078631}]},"public_addr":"172.21.15.121:6827/3257078631","cluster_addr":"172.21.15.121:6829/3257078631","heartbeat_back_addr":"172.21.15.121:6833/3257078631","heartbeat_front_addr":"172.21.15.121:6831/3257078631","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":709,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":676,"up_from":694,"up_thru":718,"down_at":677,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6809","nonce":3144145163}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6811","nonce":3144145163}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6815","nonce":3144145163}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6813","nonce":3144145163}]},"public_addr":"172.21.15.123:6809/3144145163","cluster_addr":"172.21.15.123:6811/3144145163","heartbeat_back_addr":"172.21.15.123:6815/3144145163","heartbeat_front_addr":"172.21.15.123:6813/3144145163","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":694,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":694,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":228},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":123},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":141},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":676},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.c","mappings":[{"from":6,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/3933818352":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/2952381553":"2023-09-24T20:34:36.437609+0000","172.21.15.121:6800/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/4233508401":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/985594636":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/2000763757":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.123:0/2412578238":"2023-09-24T20:34:36.437609+0000","172.21.15.121:6800/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/3564595757":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/167205692":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/623964166":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/719574644":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/1025991692":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6834/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6835/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.123:0/1669790000":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/85909652":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/961670864":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/2713351150":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/156999919":"2023-09-24T20:33:20.814708+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T21:25:33.210 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T21:25:33.210 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T21:25:33.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:32 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/3363753941' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:25:33.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:32 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/3363753941' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:25:33.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:32 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/3363753941' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:25:33.559 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:34.111 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:33 smithi121 ceph-mon[219709]: 1.a scrub starts 2023-09-23T21:25:34.111 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:33 smithi121 ceph-mon[219709]: 1.a scrub ok 2023-09-23T21:25:34.111 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:33 smithi121 ceph-mon[219709]: pgmap v2139: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:34.111 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:33 smithi121 ceph-mon[211526]: 1.a scrub starts 2023-09-23T21:25:34.111 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:33 smithi121 ceph-mon[211526]: 1.a scrub ok 2023-09-23T21:25:34.111 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:33 smithi121 ceph-mon[211526]: pgmap v2139: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:34.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:33 smithi123 ceph-mon[193903]: 1.a scrub starts 2023-09-23T21:25:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:33 smithi123 ceph-mon[193903]: 1.a scrub ok 2023-09-23T21:25:34.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:33 smithi123 ceph-mon[193903]: pgmap v2139: 83 pgs: 83 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:34.862 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:25:34 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:25:34] "GET /metrics HTTP/1.1" 200 34299 "" "Prometheus/2.43.0" 2023-09-23T21:25:35.248 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:25:35.248 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":719,"fsid":"ffb133f0-5a4e-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:23:56.454135+0000","modified":"2023-09-23T21:25:30.581371+0000","last_up_change":"2023-09-23T21:24:29.482853+0000","last_in_change":"2023-09-23T21:23:38.968570+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:26:57.902240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"690","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"582","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:29:26.520783+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"688","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":688,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:30:20.536872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:30:21.339996+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9367e9c4-5ec4-4c89-a35d-16375c1fee90","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":92,"up_thru":701,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6803","nonce":3792083724}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6805","nonce":3792083724}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6809","nonce":3792083724}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":3792083724},{"type":"v1","addr":"172.21.15.121:6807","nonce":3792083724}]},"public_addr":"172.21.15.121:6803/3792083724","cluster_addr":"172.21.15.121:6805/3792083724","heartbeat_back_addr":"172.21.15.121:6809/3792083724","heartbeat_front_addr":"172.21.15.121:6807/3792083724","state":["exists","up"]},{"osd":1,"uuid":"798606f1-4cc7-4397-9f99-1f3d8259eba1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":206,"last_clean_end":228,"up_from":276,"up_thru":694,"down_at":229,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6811","nonce":3255216964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6813","nonce":3255216964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6817","nonce":3255216964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":3255216964},{"type":"v1","addr":"172.21.15.121:6815","nonce":3255216964}]},"public_addr":"172.21.15.121:6811/3255216964","cluster_addr":"172.21.15.121:6813/3255216964","heartbeat_back_addr":"172.21.15.121:6817/3255216964","heartbeat_front_addr":"172.21.15.121:6815/3255216964","state":["exists","up"]},{"osd":2,"uuid":"03ecb78d-0bc7-43d1-8c25-1c6051697c2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":123,"up_from":129,"up_thru":697,"down_at":124,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6819","nonce":2050616913}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6821","nonce":2050616913}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6825","nonce":2050616913}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":2050616913},{"type":"v1","addr":"172.21.15.121:6823","nonce":2050616913}]},"public_addr":"172.21.15.121:6819/2050616913","cluster_addr":"172.21.15.121:6821/2050616913","heartbeat_back_addr":"172.21.15.121:6825/2050616913","heartbeat_front_addr":"172.21.15.121:6823/2050616913","state":["exists","up"]},{"osd":3,"uuid":"e6711d54-f7b7-4876-924e-e9b4c58f9e2f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":141,"up_from":148,"up_thru":706,"down_at":142,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6827","nonce":3257078631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6829","nonce":3257078631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6833","nonce":3257078631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3257078631},{"type":"v1","addr":"172.21.15.121:6831","nonce":3257078631}]},"public_addr":"172.21.15.121:6827/3257078631","cluster_addr":"172.21.15.121:6829/3257078631","heartbeat_back_addr":"172.21.15.121:6833/3257078631","heartbeat_front_addr":"172.21.15.121:6831/3257078631","state":["exists","up"]},{"osd":4,"uuid":"84bd1437-f644-42b0-8bc8-6a3e1814ad9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":709,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6801","nonce":4231264785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6803","nonce":4231264785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6807","nonce":4231264785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":4231264785},{"type":"v1","addr":"172.21.15.123:6805","nonce":4231264785}]},"public_addr":"172.21.15.123:6801/4231264785","cluster_addr":"172.21.15.123:6803/4231264785","heartbeat_back_addr":"172.21.15.123:6807/4231264785","heartbeat_front_addr":"172.21.15.123:6805/4231264785","state":["exists","up"]},{"osd":5,"uuid":"eaf000e3-5147-4aa8-a0d3-65c6a38bd803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":39,"last_clean_end":676,"up_from":694,"up_thru":718,"down_at":677,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6809","nonce":3144145163}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6811","nonce":3144145163}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6815","nonce":3144145163}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":3144145163},{"type":"v1","addr":"172.21.15.123:6813","nonce":3144145163}]},"public_addr":"172.21.15.123:6809/3144145163","cluster_addr":"172.21.15.123:6811/3144145163","heartbeat_back_addr":"172.21.15.123:6815/3144145163","heartbeat_front_addr":"172.21.15.123:6813/3144145163","state":["exists","up"]},{"osd":6,"uuid":"29ce2c8b-96a4-4b71-97b2-c307f944ea0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":694,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6817","nonce":3633151390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6819","nonce":3633151390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6823","nonce":3633151390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":3633151390},{"type":"v1","addr":"172.21.15.123:6821","nonce":3633151390}]},"public_addr":"172.21.15.123:6817/3633151390","cluster_addr":"172.21.15.123:6819/3633151390","heartbeat_back_addr":"172.21.15.123:6823/3633151390","heartbeat_front_addr":"172.21.15.123:6821/3633151390","state":["exists","up"]},{"osd":7,"uuid":"69e6dd32-d849-40d2-bef3-a0cf635c0346","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":694,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6825","nonce":4186046382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6827","nonce":4186046382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6831","nonce":4186046382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":4186046382},{"type":"v1","addr":"172.21.15.123:6829","nonce":4186046382}]},"public_addr":"172.21.15.123:6825/4186046382","cluster_addr":"172.21.15.123:6827/4186046382","heartbeat_back_addr":"172.21.15.123:6831/4186046382","heartbeat_front_addr":"172.21.15.123:6829/4186046382","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:07.544345+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:31.462450+0000","dead_epoch":228},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:26:55.021633+0000","dead_epoch":123},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:19.545779+0000","dead_epoch":141},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:27:41.530063+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:04.162166+0000","dead_epoch":676},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:26.872740+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:28:49.121319+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.c","mappings":[{"from":6,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/3933818352":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/2952381553":"2023-09-24T20:34:36.437609+0000","172.21.15.121:6800/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/4233508401":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/1290253295":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/985594636":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/2000763757":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3020663534":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6800/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.123:0/2412578238":"2023-09-24T20:34:36.437609+0000","172.21.15.121:6800/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/3564595757":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/167205692":"2023-09-24T20:33:20.814708+0000","172.21.15.123:0/623964166":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/2517434125":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/232072490":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6801/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1448037439":"2023-09-24T20:33:20.814708+0000","172.21.15.121:0/3148587077":"2023-09-24T20:24:49.573230+0000","172.21.15.121:6800/40316177":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/3404394206":"2023-09-24T20:24:24.560243+0000","172.21.15.121:6801/3124261962":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/199329623":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6800/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/719574644":"2023-09-24T20:35:42.236940+0000","172.21.15.123:0/1025991692":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6834/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.123:6835/1638010485":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1167207050":"2023-09-24T20:29:24.862767+0000","172.21.15.121:6801/1535070110":"2023-09-24T20:24:24.560243+0000","172.21.15.123:0/1669790000":"2023-09-24T20:34:36.437609+0000","172.21.15.121:0/1502674050":"2023-09-24T20:24:24.560243+0000","172.21.15.121:0/85909652":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/3188450480":"2023-09-24T20:24:49.573230+0000","172.21.15.121:0/2776087744":"2023-09-24T20:29:24.862767+0000","172.21.15.121:0/961670864":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/2713351150":"2023-09-24T20:35:42.236940+0000","172.21.15.121:0/156999919":"2023-09-24T20:33:20.814708+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T21:25:35.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:35 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2791939432' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:25:35.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:35 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2791939432' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:25:35.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:35 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2791939432' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T21:25:35.943 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T21:25:35.944 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T21:25:35.944 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T21:25:35.945 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T21:25:35.945 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T21:25:35.946 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T21:25:35.946 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T21:25:35.946 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T21:25:36.310 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:36 smithi121 ceph-mon[211526]: pgmap v2140: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:36.311 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:36 smithi121 ceph-mon[219709]: pgmap v2140: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:36.728 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:36 smithi123 ceph-mon[193903]: pgmap v2140: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:36.967 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:37.088 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:37.118 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:37.248 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:37.328 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:38.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:37 smithi121 ceph-mon[211526]: pgmap v2141: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:38.414 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:37 smithi121 ceph-mon[219709]: pgmap v2141: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:38.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:37 smithi123 ceph-mon[193903]: pgmap v2141: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:38.484 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:38.485 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:38.619 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:39.856 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:39 smithi121 ceph-mon[219709]: pgmap v2142: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:39.856 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:39 smithi121 ceph-mon[211526]: pgmap v2142: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:39.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:39 smithi123 ceph-mon[193903]: pgmap v2142: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:41.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:41 smithi121 ceph-mon[219709]: pgmap v2143: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:41.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:41 smithi121 ceph-mon[211526]: pgmap v2143: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:41.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:41 smithi123 ceph-mon[193903]: pgmap v2143: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:43.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:42 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:43.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:42 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:43.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:42 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:43.751 INFO:teuthology.orchestra.run.smithi121.stdout:395136991771 2023-09-23T21:25:43.751 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T21:25:44.318 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:43 smithi121 ceph-mon[211526]: pgmap v2144: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:44.319 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:43 smithi121 ceph-mon[219709]: pgmap v2144: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:44.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:43 smithi123 ceph-mon[193903]: pgmap v2144: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:45.089 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:25:44 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:25:44] "GET /metrics HTTP/1.1" 200 34296 "" "Prometheus/2.43.0" 2023-09-23T21:25:45.662 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:45 smithi121 ceph-mon[219709]: pgmap v2145: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:45.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:45 smithi121 ceph-mon[211526]: pgmap v2145: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:45.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:45 smithi123 ceph-mon[193903]: pgmap v2145: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:47.025 INFO:teuthology.orchestra.run.smithi121.stdout:236223201965 2023-09-23T21:25:47.026 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T21:25:47.658 INFO:teuthology.orchestra.run.smithi121.stdout:2980707303441 2023-09-23T21:25:47.659 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T21:25:48.284 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:47 smithi121 ceph-mon[211526]: pgmap v2146: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:48.284 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:48 smithi121 ceph-mon[219709]: pgmap v2146: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:48.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:47 smithi123 ceph-mon[193903]: pgmap v2146: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:48.822 INFO:teuthology.orchestra.run.smithi121.stdout:635655160257 2023-09-23T21:25:48.822 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T21:25:49.015 INFO:teuthology.orchestra.run.smithi121.stdout:554050781663 2023-09-23T21:25:49.015 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T21:25:49.061 INFO:teuthology.orchestra.run.smithi121.stdout:201863463585 2023-09-23T21:25:49.061 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T21:25:49.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:49 smithi121 ceph-mon[219709]: pgmap v2147: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:49.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:49 smithi121 ceph-mon[211526]: pgmap v2147: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:49.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:49 smithi123 ceph-mon[193903]: pgmap v2147: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:50.122 INFO:teuthology.orchestra.run.smithi121.stdout:1185410974032 2023-09-23T21:25:50.122 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T21:25:50.152 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:50.179 INFO:teuthology.orchestra.run.smithi121.stdout:133143986875 2023-09-23T21:25:50.180 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T21:25:50.607 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:50.758 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:51.019 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:51.020 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:51.832 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:51 smithi121 ceph-mon[211526]: pgmap v2148: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:51.834 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:51 smithi121 ceph-mon[219709]: pgmap v2148: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:51.953 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:51.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:51 smithi123 ceph-mon[193903]: pgmap v2148: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:53.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:53 smithi121 ceph-mon[211526]: pgmap v2149: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:53.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:53 smithi121 ceph-mon[219709]: pgmap v2149: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:53.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:53 smithi123 ceph-mon[193903]: pgmap v2149: 83 pgs: 83 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:54.913 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:25:54 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:25:54] "GET /metrics HTTP/1.1" 200 34296 "" "Prometheus/2.43.0" 2023-09-23T21:25:55.081 INFO:teuthology.orchestra.run.smithi121.stdout:2980707303442 2023-09-23T21:25:55.126 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:55.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:55 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/257548524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T21:25:55.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:55 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/257548524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T21:25:55.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:55 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/257548524' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T21:25:56.004 INFO:teuthology.orchestra.run.smithi121.stdout:395136991773 2023-09-23T21:25:56.412 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:56 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1808646932' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T21:25:56.413 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:56 smithi121 ceph-mon[211526]: pgmap v2150: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:56.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:56 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1808646932' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T21:25:56.413 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:56 smithi121 ceph-mon[219709]: pgmap v2150: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:56.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:56 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1808646932' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T21:25:56.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:56 smithi123 ceph-mon[193903]: pgmap v2150: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:57.002 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:25:57.273 INFO:teuthology.orchestra.run.smithi121.stdout:554050781665 2023-09-23T21:25:57.353 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2980707303441 got 2980707303442 for osd.5 2023-09-23T21:25:57.354 DEBUG:teuthology.parallel:result is None 2023-09-23T21:25:58.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2690114226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T21:25:58.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:58.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[211526]: pgmap v2151: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:58.123 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2623372386' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T21:25:58.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2690114226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T21:25:58.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:58.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[219709]: pgmap v2151: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:58.124 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:58 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2623372386' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T21:25:58.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2690114226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T21:25:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:58 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T21:25:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:58 smithi123 ceph-mon[193903]: pgmap v2151: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:25:58.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:58 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2623372386' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T21:25:58.556 INFO:teuthology.orchestra.run.smithi121.stdout:236223201967 2023-09-23T21:25:59.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/649797500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T21:25:59.335 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:25:59 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/2390106072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T21:25:59.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/649797500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T21:25:59.335 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:25:59 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/2390106072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T21:25:59.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/649797500' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T21:25:59.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:25:59 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/2390106072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T21:25:59.580 INFO:teuthology.orchestra.run.smithi121.stdout:201863463588 2023-09-23T21:25:59.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201965 got 236223201967 for osd.7 2023-09-23T21:25:59.693 DEBUG:teuthology.parallel:result is None 2023-09-23T21:25:59.875 INFO:tasks.cephadm.ceph_manager.ceph:need seq 554050781663 got 554050781665 for osd.2 2023-09-23T21:25:59.875 DEBUG:teuthology.parallel:result is None 2023-09-23T21:26:00.070 INFO:teuthology.orchestra.run.smithi121.stdout:635655160258 2023-09-23T21:26:00.164 INFO:tasks.cephadm.ceph_manager.ceph:need seq 395136991771 got 395136991773 for osd.0 2023-09-23T21:26:00.164 DEBUG:teuthology.parallel:result is None 2023-09-23T21:26:00.306 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:00 smithi121 ceph-mon[211526]: pgmap v2152: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:00.306 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:00 smithi121 ceph-mon[219709]: pgmap v2152: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:00.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:00 smithi123 ceph-mon[193903]: pgmap v2152: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:01.793 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:01 smithi121 ceph-mon[211526]: pgmap v2153: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:01.794 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:01 smithi121 ceph-mon[219709]: pgmap v2153: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:01.817 INFO:tasks.cephadm.ceph_manager.ceph:need seq 635655160257 got 635655160258 for osd.3 2023-09-23T21:26:01.817 DEBUG:teuthology.parallel:result is None 2023-09-23T21:26:01.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:01 smithi123 ceph-mon[193903]: pgmap v2153: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:02.075 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463585 got 201863463588 for osd.6 2023-09-23T21:26:02.076 DEBUG:teuthology.parallel:result is None 2023-09-23T21:26:02.093 INFO:teuthology.orchestra.run.smithi121.stdout:133143986878 2023-09-23T21:26:02.451 INFO:teuthology.orchestra.run.smithi121.stdout:1185410974035 2023-09-23T21:26:03.235 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/1169554736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T21:26:03.235 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:02 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:26:03.235 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:02 smithi121 ceph-mon[211526]: from='client.? 172.21.15.121:0/831744200' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T21:26:03.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/1169554736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T21:26:03.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:02 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:26:03.235 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:02 smithi121 ceph-mon[219709]: from='client.? 172.21.15.121:0/831744200' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T21:26:03.236 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986875 got 133143986878 for osd.4 2023-09-23T21:26:03.236 DEBUG:teuthology.parallel:result is None 2023-09-23T21:26:03.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/1169554736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T21:26:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:02 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:26:03.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:02 smithi123 ceph-mon[193903]: from='client.? 172.21.15.121:0/831744200' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T21:26:03.544 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1185410974032 got 1185410974035 for osd.1 2023-09-23T21:26:03.544 DEBUG:teuthology.parallel:result is None 2023-09-23T21:26:03.544 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T21:26:03.544 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:26:04.183 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:26:04.403 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:04 smithi121 ceph-mon[219709]: pgmap v2154: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:04.403 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:04 smithi121 ceph-mon[211526]: pgmap v2154: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:04.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:04 smithi123 ceph-mon[193903]: pgmap v2154: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:05.163 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:04 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:26:04] "GET /metrics HTTP/1.1" 200 34296 "" "Prometheus/2.43.0" 2023-09-23T21:26:05.663 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:05 smithi121 ceph-mon[211526]: pgmap v2155: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:05.663 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:05 smithi121 ceph-mon[219709]: pgmap v2155: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:05.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:05 smithi123 ceph-mon[193903]: pgmap v2155: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:06.126 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:26:06.128 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:26:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:06 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:06 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:06.913 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:06 smithi121 ceph-mon[211526]: from='client.38961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:26:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:06 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:06.913 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:06 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:06.914 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:06 smithi121 ceph-mon[219709]: from='client.38961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:26:06.978 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:06 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:06 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:06.979 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:06 smithi123 ceph-mon[193903]: from='client.38961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:26:07.168 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":2155,"stamp":"2023-09-23T21:26:05.498148+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5987,"num_read_kb":13204,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":889402,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":252,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6791668,"kb_used_data":178228,"kb_used_omap":100,"kb_used_meta":6613275,"kb_avail":743169548,"statfs":{"total":767960285184,"available":761005617152,"internally_reserved":0,"allocated":182505472,"data_stored":161200636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102871,"internal_metadata":6771994153},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003099"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":293,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587026+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:25:29.587026+0000","last_peered":"2023-09-23T21:25:29.587026+0000","last_clean":"2023-09-23T21:25:29.587026+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:25:29.587026+0000","last_undegraded":"2023-09-23T21:25:29.587026+0000","last_fullsized":"2023-09-23T21:25:29.587026+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.562479+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:25:30.562479+0000","last_peered":"2023-09-23T21:25:30.562479+0000","last_clean":"2023-09-23T21:25:30.562479+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:25:30.562479+0000","last_undegraded":"2023-09-23T21:25:30.562479+0000","last_fullsized":"2023-09-23T21:25:30.562479+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":102,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.594950+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:25:30.594950+0000","last_peered":"2023-09-23T21:25:30.594950+0000","last_clean":"2023-09-23T21:25:30.594950+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:25:30.594950+0000","last_undegraded":"2023-09-23T21:25:30.594950+0000","last_fullsized":"2023-09-23T21:25:30.594950+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589039+0000","last_change":"2023-09-23T21:23:40.010951+0000","last_active":"2023-09-23T21:25:30.589039+0000","last_peered":"2023-09-23T21:25:30.589039+0000","last_clean":"2023-09-23T21:25:30.589039+0000","last_became_active":"2023-09-23T21:23:40.010331+0000","last_became_peered":"2023-09-23T21:23:40.010331+0000","last_unstale":"2023-09-23T21:25:30.589039+0000","last_undegraded":"2023-09-23T21:25:30.589039+0000","last_fullsized":"2023-09-23T21:25:30.589039+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:06.978784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583176+0000","last_change":"2023-09-23T21:23:40.012215+0000","last_active":"2023-09-23T21:25:29.583176+0000","last_peered":"2023-09-23T21:25:29.583176+0000","last_clean":"2023-09-23T21:25:29.583176+0000","last_became_active":"2023-09-23T21:23:40.011815+0000","last_became_peered":"2023-09-23T21:23:40.011815+0000","last_unstale":"2023-09-23T21:25:29.583176+0000","last_undegraded":"2023-09-23T21:25:29.583176+0000","last_fullsized":"2023-09-23T21:25:29.583176+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:00.486707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":241,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587646+0000","last_change":"2023-09-23T21:24:30.512880+0000","last_active":"2023-09-23T21:25:29.587646+0000","last_peered":"2023-09-23T21:25:29.587646+0000","last_clean":"2023-09-23T21:25:29.587646+0000","last_became_active":"2023-09-23T21:24:30.512547+0000","last_became_peered":"2023-09-23T21:24:30.512547+0000","last_unstale":"2023-09-23T21:25:29.587646+0000","last_undegraded":"2023-09-23T21:25:29.587646+0000","last_fullsized":"2023-09-23T21:25:29.587646+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:29.648160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":74,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588756+0000","last_change":"2023-09-23T21:24:30.508862+0000","last_active":"2023-09-23T21:25:30.588756+0000","last_peered":"2023-09-23T21:25:30.588756+0000","last_clean":"2023-09-23T21:25:30.588756+0000","last_became_active":"2023-09-23T21:24:30.508395+0000","last_became_peered":"2023-09-23T21:24:30.508395+0000","last_unstale":"2023-09-23T21:25:30.588756+0000","last_undegraded":"2023-09-23T21:25:30.588756+0000","last_fullsized":"2023-09-23T21:25:30.588756+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:18.972901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":151,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583256+0000","last_change":"2023-09-23T21:24:45.398853+0000","last_active":"2023-09-23T21:25:29.583256+0000","last_peered":"2023-09-23T21:25:29.583256+0000","last_clean":"2023-09-23T21:25:29.583256+0000","last_became_active":"2023-09-23T21:24:45.398549+0000","last_became_peered":"2023-09-23T21:24:45.398549+0000","last_unstale":"2023-09-23T21:25:29.583256+0000","last_undegraded":"2023-09-23T21:25:29.583256+0000","last_fullsized":"2023-09-23T21:25:29.583256+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.109131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588166+0000","last_change":"2023-09-23T21:24:30.516861+0000","last_active":"2023-09-23T21:25:29.588166+0000","last_peered":"2023-09-23T21:25:29.588166+0000","last_clean":"2023-09-23T21:25:29.588166+0000","last_became_active":"2023-09-23T21:24:30.516517+0000","last_became_peered":"2023-09-23T21:24:30.516517+0000","last_unstale":"2023-09-23T21:25:29.588166+0000","last_undegraded":"2023-09-23T21:25:29.588166+0000","last_fullsized":"2023-09-23T21:25:29.588166+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:23.519206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":115,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583262+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:25:29.583262+0000","last_peered":"2023-09-23T21:25:29.583262+0000","last_clean":"2023-09-23T21:25:29.583262+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:25:29.583262+0000","last_undegraded":"2023-09-23T21:25:29.583262+0000","last_fullsized":"2023-09-23T21:25:29.583262+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":353,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593821+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:25:29.593821+0000","last_peered":"2023-09-23T21:25:29.593821+0000","last_clean":"2023-09-23T21:25:29.593821+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:25:29.593821+0000","last_undegraded":"2023-09-23T21:25:29.593821+0000","last_fullsized":"2023-09-23T21:25:29.593821+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.589484+0000","last_change":"2023-09-23T21:24:45.399248+0000","last_active":"2023-09-23T21:25:29.589484+0000","last_peered":"2023-09-23T21:25:29.589484+0000","last_clean":"2023-09-23T21:25:29.589484+0000","last_became_active":"2023-09-23T21:24:45.398853+0000","last_became_peered":"2023-09-23T21:24:45.398853+0000","last_unstale":"2023-09-23T21:25:29.589484+0000","last_undegraded":"2023-09-23T21:25:29.589484+0000","last_fullsized":"2023-09-23T21:25:29.589484+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:31.756276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":449,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587637+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:25:29.587637+0000","last_peered":"2023-09-23T21:25:29.587637+0000","last_clean":"2023-09-23T21:25:29.587637+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:25:29.587637+0000","last_undegraded":"2023-09-23T21:25:29.587637+0000","last_fullsized":"2023-09-23T21:25:29.587637+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":676,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588573+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:25:29.588573+0000","last_peered":"2023-09-23T21:25:29.588573+0000","last_clean":"2023-09-23T21:25:29.588573+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:25:29.588573+0000","last_undegraded":"2023-09-23T21:25:29.588573+0000","last_fullsized":"2023-09-23T21:25:29.588573+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":301,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593845+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:25:29.593845+0000","last_peered":"2023-09-23T21:25:29.593845+0000","last_clean":"2023-09-23T21:25:29.593845+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:25:29.593845+0000","last_undegraded":"2023-09-23T21:25:29.593845+0000","last_fullsized":"2023-09-23T21:25:29.593845+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583829+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:25:29.583829+0000","last_peered":"2023-09-23T21:25:29.583829+0000","last_clean":"2023-09-23T21:25:29.583829+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:25:29.583829+0000","last_undegraded":"2023-09-23T21:25:29.583829+0000","last_fullsized":"2023-09-23T21:25:29.583829+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":310,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.590222+0000","last_change":"2023-09-23T21:24:45.640327+0000","last_active":"2023-09-23T21:25:29.590222+0000","last_peered":"2023-09-23T21:25:29.590222+0000","last_clean":"2023-09-23T21:25:29.590222+0000","last_became_active":"2023-09-23T21:24:45.640011+0000","last_became_peered":"2023-09-23T21:24:45.640011+0000","last_unstale":"2023-09-23T21:25:29.590222+0000","last_undegraded":"2023-09-23T21:25:29.590222+0000","last_fullsized":"2023-09-23T21:25:29.590222+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:40.743533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":332,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.600687+0000","last_change":"2023-09-23T21:25:30.600687+0000","last_active":"2023-09-23T21:25:30.600687+0000","last_peered":"2023-09-23T21:25:30.600687+0000","last_clean":"2023-09-23T21:25:30.600687+0000","last_became_active":"2023-09-23T21:25:30.600363+0000","last_became_peered":"2023-09-23T21:25:30.600363+0000","last_unstale":"2023-09-23T21:25:30.600687+0000","last_undegraded":"2023-09-23T21:25:30.600687+0000","last_fullsized":"2023-09-23T21:25:30.600687+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:35.803017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":295,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593560+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:25:29.593560+0000","last_peered":"2023-09-23T21:25:29.593560+0000","last_clean":"2023-09-23T21:25:29.593560+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:25:29.593560+0000","last_undegraded":"2023-09-23T21:25:29.593560+0000","last_fullsized":"2023-09-23T21:25:29.593560+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589115+0000","last_change":"2023-09-23T21:23:40.011399+0000","last_active":"2023-09-23T21:25:30.589115+0000","last_peered":"2023-09-23T21:25:30.589115+0000","last_clean":"2023-09-23T21:25:30.589115+0000","last_became_active":"2023-09-23T21:23:40.010992+0000","last_became_peered":"2023-09-23T21:23:40.010992+0000","last_unstale":"2023-09-23T21:25:30.589115+0000","last_undegraded":"2023-09-23T21:25:30.589115+0000","last_fullsized":"2023-09-23T21:25:30.589115+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:35.168968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":852,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586827+0000","last_change":"2023-09-23T21:24:30.514405+0000","last_active":"2023-09-23T21:25:29.586827+0000","last_peered":"2023-09-23T21:25:29.586827+0000","last_clean":"2023-09-23T21:25:29.586827+0000","last_became_active":"2023-09-23T21:24:30.514073+0000","last_became_peered":"2023-09-23T21:24:30.514073+0000","last_unstale":"2023-09-23T21:25:29.586827+0000","last_undegraded":"2023-09-23T21:25:29.586827+0000","last_fullsized":"2023-09-23T21:25:29.586827+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:30.858579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583958+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:25:29.583958+0000","last_peered":"2023-09-23T21:25:29.583958+0000","last_clean":"2023-09-23T21:25:29.583958+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:25:29.583958+0000","last_undegraded":"2023-09-23T21:25:29.583958+0000","last_fullsized":"2023-09-23T21:25:29.583958+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":369,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593904+0000","last_change":"2023-09-23T21:23:40.024118+0000","last_active":"2023-09-23T21:25:29.593904+0000","last_peered":"2023-09-23T21:25:29.593904+0000","last_clean":"2023-09-23T21:25:29.593904+0000","last_became_active":"2023-09-23T21:23:40.014595+0000","last_became_peered":"2023-09-23T21:23:40.014595+0000","last_unstale":"2023-09-23T21:25:29.593904+0000","last_undegraded":"2023-09-23T21:25:29.593904+0000","last_fullsized":"2023-09-23T21:25:29.593904+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.403893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":198,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587802+0000","last_change":"2023-09-23T21:24:30.514437+0000","last_active":"2023-09-23T21:25:29.587802+0000","last_peered":"2023-09-23T21:25:29.587802+0000","last_clean":"2023-09-23T21:25:29.587802+0000","last_became_active":"2023-09-23T21:24:30.510816+0000","last_became_peered":"2023-09-23T21:24:30.510816+0000","last_unstale":"2023-09-23T21:25:29.587802+0000","last_undegraded":"2023-09-23T21:25:29.587802+0000","last_fullsized":"2023-09-23T21:25:29.587802+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:03.265668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":369,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593855+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:25:29.593855+0000","last_peered":"2023-09-23T21:25:29.593855+0000","last_clean":"2023-09-23T21:25:29.593855+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:25:29.593855+0000","last_undegraded":"2023-09-23T21:25:29.593855+0000","last_fullsized":"2023-09-23T21:25:29.593855+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":354,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587962+0000","last_change":"2023-09-23T21:25:06.323889+0000","last_active":"2023-09-23T21:25:29.587962+0000","last_peered":"2023-09-23T21:25:29.587962+0000","last_clean":"2023-09-23T21:25:29.587962+0000","last_became_active":"2023-09-23T21:25:06.323536+0000","last_became_peered":"2023-09-23T21:25:06.323536+0000","last_unstale":"2023-09-23T21:25:29.587962+0000","last_undegraded":"2023-09-23T21:25:29.587962+0000","last_fullsized":"2023-09-23T21:25:29.587962+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:15.262384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":358,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.594037+0000","last_change":"2023-09-23T21:24:30.511297+0000","last_active":"2023-09-23T21:25:29.594037+0000","last_peered":"2023-09-23T21:25:29.594037+0000","last_clean":"2023-09-23T21:25:29.594037+0000","last_became_active":"2023-09-23T21:24:30.510906+0000","last_became_peered":"2023-09-23T21:24:30.510906+0000","last_unstale":"2023-09-23T21:25:29.594037+0000","last_undegraded":"2023-09-23T21:25:29.594037+0000","last_fullsized":"2023-09-23T21:25:29.594037+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:23.467975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":90,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587995+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:25:29.587995+0000","last_peered":"2023-09-23T21:25:29.587995+0000","last_clean":"2023-09-23T21:25:29.587995+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:25:29.587995+0000","last_undegraded":"2023-09-23T21:25:29.587995+0000","last_fullsized":"2023-09-23T21:25:29.587995+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":118,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.600992+0000","last_change":"2023-09-23T21:25:30.600992+0000","last_active":"2023-09-23T21:25:30.600992+0000","last_peered":"2023-09-23T21:25:30.600992+0000","last_clean":"2023-09-23T21:25:30.600992+0000","last_became_active":"2023-09-23T21:25:30.600543+0000","last_became_peered":"2023-09-23T21:25:30.600543+0000","last_unstale":"2023-09-23T21:25:30.600992+0000","last_undegraded":"2023-09-23T21:25:30.600992+0000","last_fullsized":"2023-09-23T21:25:30.600992+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:40.282384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588889+0000","last_change":"2023-09-23T21:23:40.007119+0000","last_active":"2023-09-23T21:25:30.588889+0000","last_peered":"2023-09-23T21:25:30.588889+0000","last_clean":"2023-09-23T21:25:30.588889+0000","last_became_active":"2023-09-23T21:23:40.006773+0000","last_became_peered":"2023-09-23T21:23:40.006773+0000","last_unstale":"2023-09-23T21:25:30.588889+0000","last_undegraded":"2023-09-23T21:25:30.588889+0000","last_fullsized":"2023-09-23T21:25:30.588889+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:05.053314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1144,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593368+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:25:29.593368+0000","last_peered":"2023-09-23T21:25:29.593368+0000","last_clean":"2023-09-23T21:25:29.593368+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:25:29.593368+0000","last_undegraded":"2023-09-23T21:25:29.593368+0000","last_fullsized":"2023-09-23T21:25:29.593368+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":145,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583716+0000","last_change":"2023-09-23T21:23:40.015941+0000","last_active":"2023-09-23T21:25:29.583716+0000","last_peered":"2023-09-23T21:25:29.583716+0000","last_clean":"2023-09-23T21:25:29.583716+0000","last_became_active":"2023-09-23T21:23:40.005288+0000","last_became_peered":"2023-09-23T21:23:40.005288+0000","last_unstale":"2023-09-23T21:25:29.583716+0000","last_undegraded":"2023-09-23T21:25:29.583716+0000","last_fullsized":"2023-09-23T21:25:29.583716+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:04.042824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":476,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587732+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:25:29.587732+0000","last_peered":"2023-09-23T21:25:29.587732+0000","last_clean":"2023-09-23T21:25:29.587732+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:25:29.587732+0000","last_undegraded":"2023-09-23T21:25:29.587732+0000","last_fullsized":"2023-09-23T21:25:29.587732+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":923,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.597742+0000","last_change":"2023-09-23T21:25:30.597742+0000","last_active":"2023-09-23T21:25:30.597742+0000","last_peered":"2023-09-23T21:25:30.597742+0000","last_clean":"2023-09-23T21:25:30.597742+0000","last_became_active":"2023-09-23T21:25:30.597323+0000","last_became_peered":"2023-09-23T21:25:30.597323+0000","last_unstale":"2023-09-23T21:25:30.597742+0000","last_undegraded":"2023-09-23T21:25:30.597742+0000","last_fullsized":"2023-09-23T21:25:30.597742+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:27.117905+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.589412+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:25:29.589412+0000","last_peered":"2023-09-23T21:25:29.589412+0000","last_clean":"2023-09-23T21:25:29.589412+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:25:29.589412+0000","last_undegraded":"2023-09-23T21:25:29.589412+0000","last_fullsized":"2023-09-23T21:25:29.589412+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":726,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588835+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:25:29.588835+0000","last_peered":"2023-09-23T21:25:29.588835+0000","last_clean":"2023-09-23T21:25:29.588835+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:25:29.588835+0000","last_undegraded":"2023-09-23T21:25:29.588835+0000","last_fullsized":"2023-09-23T21:25:29.588835+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":355,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587879+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:25:29.587879+0000","last_peered":"2023-09-23T21:25:29.587879+0000","last_clean":"2023-09-23T21:25:29.587879+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:25:29.587879+0000","last_undegraded":"2023-09-23T21:25:29.587879+0000","last_fullsized":"2023-09-23T21:25:29.587879+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":763,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587352+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:25:29.587352+0000","last_peered":"2023-09-23T21:25:29.587352+0000","last_clean":"2023-09-23T21:25:29.587352+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:25:29.587352+0000","last_undegraded":"2023-09-23T21:25:29.587352+0000","last_fullsized":"2023-09-23T21:25:29.587352+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":893,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:33.419785+0000","last_change":"2023-09-23T21:25:33.419744+0000","last_active":"2023-09-23T21:25:33.419785+0000","last_peered":"2023-09-23T21:25:33.419785+0000","last_clean":"2023-09-23T21:25:33.419785+0000","last_became_active":"2023-09-23T21:25:30.599404+0000","last_became_peered":"2023-09-23T21:25:30.599404+0000","last_unstale":"2023-09-23T21:25:33.419785+0000","last_undegraded":"2023-09-23T21:25:33.419785+0000","last_fullsized":"2023-09-23T21:25:33.419785+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:25:33.419652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:25:33.419652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:45.035941+0000","scrub_duration":0.0016204489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583017+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:25:29.583017+0000","last_peered":"2023-09-23T21:25:29.583017+0000","last_clean":"2023-09-23T21:25:29.583017+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:25:29.583017+0000","last_undegraded":"2023-09-23T21:25:29.583017+0000","last_fullsized":"2023-09-23T21:25:29.583017+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":102,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.595472+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:25:30.595472+0000","last_peered":"2023-09-23T21:25:30.595472+0000","last_clean":"2023-09-23T21:25:30.595472+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:25:30.595472+0000","last_undegraded":"2023-09-23T21:25:30.595472+0000","last_fullsized":"2023-09-23T21:25:30.595472+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":801,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586981+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:25:29.586981+0000","last_peered":"2023-09-23T21:25:29.586981+0000","last_clean":"2023-09-23T21:25:29.586981+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:25:29.586981+0000","last_undegraded":"2023-09-23T21:25:29.586981+0000","last_fullsized":"2023-09-23T21:25:29.586981+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"689'224","reported_seq":780,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591932+0000","last_change":"2023-09-23T21:24:03.654416+0000","last_active":"2023-09-23T21:25:29.591932+0000","last_peered":"2023-09-23T21:25:29.591932+0000","last_clean":"2023-09-23T21:25:29.591932+0000","last_became_active":"2023-09-23T21:23:40.011996+0000","last_became_peered":"2023-09-23T21:23:40.011996+0000","last_unstale":"2023-09-23T21:25:29.591932+0000","last_undegraded":"2023-09-23T21:25:29.591932+0000","last_fullsized":"2023-09-23T21:25:29.591932+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":224,"log_dups_size":0,"ondisk_log_size":224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:12.817128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028358000000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1177,"num_read_kb":2026,"num_write":233,"num_write_kb":1133,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":90,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591401+0000","last_change":"2023-09-23T21:24:30.508875+0000","last_active":"2023-09-23T21:25:29.591401+0000","last_peered":"2023-09-23T21:25:29.591401+0000","last_clean":"2023-09-23T21:25:29.591401+0000","last_became_active":"2023-09-23T21:24:30.508556+0000","last_became_peered":"2023-09-23T21:24:30.508556+0000","last_unstale":"2023-09-23T21:25:29.591401+0000","last_undegraded":"2023-09-23T21:25:29.591401+0000","last_fullsized":"2023-09-23T21:25:29.591401+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:11.944192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591960+0000","last_change":"2023-09-23T21:24:30.517186+0000","last_active":"2023-09-23T21:25:29.591960+0000","last_peered":"2023-09-23T21:25:29.591960+0000","last_clean":"2023-09-23T21:25:29.591960+0000","last_became_active":"2023-09-23T21:24:30.516912+0000","last_became_peered":"2023-09-23T21:24:30.516912+0000","last_unstale":"2023-09-23T21:25:29.591960+0000","last_undegraded":"2023-09-23T21:25:29.591960+0000","last_fullsized":"2023-09-23T21:25:29.591960+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:17.698045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":895,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.596398+0000","last_change":"2023-09-23T21:25:30.596398+0000","last_active":"2023-09-23T21:25:30.596398+0000","last_peered":"2023-09-23T21:25:30.596398+0000","last_clean":"2023-09-23T21:25:30.596398+0000","last_became_active":"2023-09-23T21:25:30.596051+0000","last_became_peered":"2023-09-23T21:25:30.596051+0000","last_unstale":"2023-09-23T21:25:30.596398+0000","last_undegraded":"2023-09-23T21:25:30.596398+0000","last_fullsized":"2023-09-23T21:25:30.596398+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:29.370699+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"689'121","reported_seq":803,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.599693+0000","last_change":"2023-09-23T21:25:30.599693+0000","last_active":"2023-09-23T21:25:30.599693+0000","last_peered":"2023-09-23T21:25:30.599693+0000","last_clean":"2023-09-23T21:25:30.599693+0000","last_became_active":"2023-09-23T21:25:30.599144+0000","last_became_peered":"2023-09-23T21:25:30.599144+0000","last_unstale":"2023-09-23T21:25:30.599693+0000","last_undegraded":"2023-09-23T21:25:30.599693+0000","last_fullsized":"2023-09-23T21:25:30.599693+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:16.872425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012923380000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1233,"num_read_kb":1266,"num_write":106,"num_write_kb":140,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":641,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588419+0000","last_change":"2023-09-23T21:23:40.007033+0000","last_active":"2023-09-23T21:25:29.588419+0000","last_peered":"2023-09-23T21:25:29.588419+0000","last_clean":"2023-09-23T21:25:29.588419+0000","last_became_active":"2023-09-23T21:23:40.006711+0000","last_became_peered":"2023-09-23T21:23:40.006711+0000","last_unstale":"2023-09-23T21:25:29.588419+0000","last_undegraded":"2023-09-23T21:25:29.588419+0000","last_fullsized":"2023-09-23T21:25:29.588419+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:35.819332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":102,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591843+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:25:29.591843+0000","last_peered":"2023-09-23T21:25:29.591843+0000","last_clean":"2023-09-23T21:25:29.591843+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:25:29.591843+0000","last_undegraded":"2023-09-23T21:25:29.591843+0000","last_fullsized":"2023-09-23T21:25:29.591843+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593412+0000","last_change":"2023-09-23T21:23:42.983841+0000","last_active":"2023-09-23T21:25:29.593412+0000","last_peered":"2023-09-23T21:25:29.593412+0000","last_clean":"2023-09-23T21:25:29.593412+0000","last_became_active":"2023-09-23T21:23:42.983351+0000","last_became_peered":"2023-09-23T21:23:42.983351+0000","last_unstale":"2023-09-23T21:25:29.593412+0000","last_undegraded":"2023-09-23T21:25:29.593412+0000","last_fullsized":"2023-09-23T21:25:29.593412+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.422807+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"689'92","reported_seq":1137,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589206+0000","last_change":"2023-09-23T21:24:03.649812+0000","last_active":"2023-09-23T21:25:30.589206+0000","last_peered":"2023-09-23T21:25:30.589206+0000","last_clean":"2023-09-23T21:25:30.589206+0000","last_became_active":"2023-09-23T21:23:40.012274+0000","last_became_peered":"2023-09-23T21:23:40.012274+0000","last_unstale":"2023-09-23T21:25:30.589206+0000","last_undegraded":"2023-09-23T21:25:30.589206+0000","last_fullsized":"2023-09-23T21:25:30.589206+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:30.294423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032399600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":580,"num_read_kb":441,"num_write":77,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591517+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:25:29.591517+0000","last_peered":"2023-09-23T21:25:29.591517+0000","last_clean":"2023-09-23T21:25:29.591517+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:25:29.591517+0000","last_undegraded":"2023-09-23T21:25:29.591517+0000","last_fullsized":"2023-09-23T21:25:29.591517+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":126,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.592322+0000","last_change":"2023-09-23T21:24:30.507847+0000","last_active":"2023-09-23T21:25:29.592322+0000","last_peered":"2023-09-23T21:25:29.592322+0000","last_clean":"2023-09-23T21:25:29.592322+0000","last_became_active":"2023-09-23T21:24:30.507507+0000","last_became_peered":"2023-09-23T21:24:30.507507+0000","last_unstale":"2023-09-23T21:25:29.592322+0000","last_undegraded":"2023-09-23T21:25:29.592322+0000","last_fullsized":"2023-09-23T21:25:29.592322+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:51.885487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"689'175","reported_seq":1150,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.599628+0000","last_change":"2023-09-23T21:25:30.599628+0000","last_active":"2023-09-23T21:25:30.599628+0000","last_peered":"2023-09-23T21:25:30.599628+0000","last_clean":"2023-09-23T21:25:30.599628+0000","last_became_active":"2023-09-23T21:25:30.599132+0000","last_became_peered":"2023-09-23T21:25:30.599132+0000","last_unstale":"2023-09-23T21:25:30.599628+0000","last_undegraded":"2023-09-23T21:25:30.599628+0000","last_fullsized":"2023-09-23T21:25:30.599628+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:30.509053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080853800000000003,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":841,"num_read_kb":768,"num_write":143,"num_write_kb":4181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1125,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593136+0000","last_change":"2023-09-23T21:23:42.981131+0000","last_active":"2023-09-23T21:25:29.593136+0000","last_peered":"2023-09-23T21:25:29.593136+0000","last_clean":"2023-09-23T21:25:29.593136+0000","last_became_active":"2023-09-23T21:23:42.980755+0000","last_became_peered":"2023-09-23T21:23:42.980755+0000","last_unstale":"2023-09-23T21:25:29.593136+0000","last_undegraded":"2023-09-23T21:25:29.593136+0000","last_fullsized":"2023-09-23T21:25:29.593136+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:08.065765+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":404,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583057+0000","last_change":"2023-09-23T21:24:59.008039+0000","last_active":"2023-09-23T21:25:29.583057+0000","last_peered":"2023-09-23T21:25:29.583057+0000","last_clean":"2023-09-23T21:25:29.583057+0000","last_became_active":"2023-09-23T21:24:59.007765+0000","last_became_peered":"2023-09-23T21:24:59.007765+0000","last_unstale":"2023-09-23T21:25:29.583057+0000","last_undegraded":"2023-09-23T21:25:29.583057+0000","last_fullsized":"2023-09-23T21:25:29.583057+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:37.732988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":90,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588051+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:25:29.588051+0000","last_peered":"2023-09-23T21:25:29.588051+0000","last_clean":"2023-09-23T21:25:29.588051+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:25:29.588051+0000","last_undegraded":"2023-09-23T21:25:29.588051+0000","last_fullsized":"2023-09-23T21:25:29.588051+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":766,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591526+0000","last_change":"2023-09-23T21:23:12.024326+0000","last_active":"2023-09-23T21:25:29.591526+0000","last_peered":"2023-09-23T21:25:29.591526+0000","last_clean":"2023-09-23T21:25:29.591526+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:25:29.591526+0000","last_undegraded":"2023-09-23T21:25:29.591526+0000","last_fullsized":"2023-09-23T21:25:29.591526+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:23:12.024197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:23:12.024197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:18.945737+0000","scrub_duration":0.0017519600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"689'95","reported_seq":701,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593235+0000","last_change":"2023-09-23T21:24:03.659565+0000","last_active":"2023-09-23T21:25:29.593235+0000","last_peered":"2023-09-23T21:25:29.593235+0000","last_clean":"2023-09-23T21:25:29.593235+0000","last_became_active":"2023-09-23T21:23:40.011597+0000","last_became_peered":"2023-09-23T21:23:40.011597+0000","last_unstale":"2023-09-23T21:25:29.593235+0000","last_undegraded":"2023-09-23T21:25:29.593235+0000","last_fullsized":"2023-09-23T21:25:29.593235+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":95,"log_dups_size":0,"ondisk_log_size":95,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:17.045171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000319396,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":2857,"num_write":87,"num_write_kb":2282,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589728+0000","last_change":"2023-09-23T21:23:40.009291+0000","last_active":"2023-09-23T21:25:30.589728+0000","last_peered":"2023-09-23T21:25:30.589728+0000","last_clean":"2023-09-23T21:25:30.589728+0000","last_became_active":"2023-09-23T21:23:40.008907+0000","last_became_peered":"2023-09-23T21:23:40.008907+0000","last_unstale":"2023-09-23T21:25:30.589728+0000","last_undegraded":"2023-09-23T21:25:30.589728+0000","last_fullsized":"2023-09-23T21:25:30.589728+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:18.940611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589077+0000","last_change":"2023-09-23T21:23:40.012054+0000","last_active":"2023-09-23T21:25:30.589077+0000","last_peered":"2023-09-23T21:25:30.589077+0000","last_clean":"2023-09-23T21:25:30.589077+0000","last_became_active":"2023-09-23T21:23:40.011675+0000","last_became_peered":"2023-09-23T21:23:40.011675+0000","last_unstale":"2023-09-23T21:25:30.589077+0000","last_undegraded":"2023-09-23T21:25:30.589077+0000","last_fullsized":"2023-09-23T21:25:30.589077+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:02.744420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":801,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586796+0000","last_change":"2023-09-23T21:23:42.981963+0000","last_active":"2023-09-23T21:25:29.586796+0000","last_peered":"2023-09-23T21:25:29.586796+0000","last_clean":"2023-09-23T21:25:29.586796+0000","last_became_active":"2023-09-23T21:23:42.981346+0000","last_became_peered":"2023-09-23T21:23:42.981346+0000","last_unstale":"2023-09-23T21:25:29.586796+0000","last_undegraded":"2023-09-23T21:25:29.586796+0000","last_fullsized":"2023-09-23T21:25:29.586796+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:20.767852+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"689'105","reported_seq":713,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593965+0000","last_change":"2023-09-23T21:24:03.659887+0000","last_active":"2023-09-23T21:25:29.593965+0000","last_peered":"2023-09-23T21:25:29.593965+0000","last_clean":"2023-09-23T21:25:29.593965+0000","last_became_active":"2023-09-23T21:23:40.011448+0000","last_became_peered":"2023-09-23T21:23:40.011448+0000","last_unstale":"2023-09-23T21:25:29.593965+0000","last_undegraded":"2023-09-23T21:25:29.593965+0000","last_fullsized":"2023-09-23T21:25:29.593965+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:16.274116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037762899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":682,"num_read_kb":1215,"num_write":94,"num_write_kb":385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":117,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591753+0000","last_change":"2023-09-23T21:24:30.511008+0000","last_active":"2023-09-23T21:25:29.591753+0000","last_peered":"2023-09-23T21:25:29.591753+0000","last_clean":"2023-09-23T21:25:29.591753+0000","last_became_active":"2023-09-23T21:24:30.510625+0000","last_became_peered":"2023-09-23T21:24:30.510625+0000","last_unstale":"2023-09-23T21:25:29.591753+0000","last_undegraded":"2023-09-23T21:25:29.591753+0000","last_fullsized":"2023-09-23T21:25:29.591753+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:50.890363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":257,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.590427+0000","last_change":"2023-09-23T21:24:45.640521+0000","last_active":"2023-09-23T21:25:29.590427+0000","last_peered":"2023-09-23T21:25:29.590427+0000","last_clean":"2023-09-23T21:25:29.590427+0000","last_became_active":"2023-09-23T21:24:45.640198+0000","last_became_peered":"2023-09-23T21:24:45.640198+0000","last_unstale":"2023-09-23T21:25:29.590427+0000","last_undegraded":"2023-09-23T21:25:29.590427+0000","last_fullsized":"2023-09-23T21:25:29.590427+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:02.374401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":1019,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588753+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:25:29.588753+0000","last_peered":"2023-09-23T21:25:29.588753+0000","last_clean":"2023-09-23T21:25:29.588753+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:25:29.588753+0000","last_undegraded":"2023-09-23T21:25:29.588753+0000","last_fullsized":"2023-09-23T21:25:29.588753+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":311,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583871+0000","last_change":"2023-09-23T21:24:59.003848+0000","last_active":"2023-09-23T21:25:29.583871+0000","last_peered":"2023-09-23T21:25:29.583871+0000","last_clean":"2023-09-23T21:25:29.583871+0000","last_became_active":"2023-09-23T21:24:59.003528+0000","last_became_peered":"2023-09-23T21:24:59.003528+0000","last_unstale":"2023-09-23T21:25:29.583871+0000","last_undegraded":"2023-09-23T21:25:29.583871+0000","last_fullsized":"2023-09-23T21:25:29.583871+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:06.635575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"689'98","reported_seq":502,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591983+0000","last_change":"2023-09-23T21:24:03.654574+0000","last_active":"2023-09-23T21:25:29.591983+0000","last_peered":"2023-09-23T21:25:29.591983+0000","last_clean":"2023-09-23T21:25:29.591983+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:25:29.591983+0000","last_undegraded":"2023-09-23T21:25:29.591983+0000","last_fullsized":"2023-09-23T21:25:29.591983+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027368,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":619,"num_read_kb":2739,"num_write":55,"num_write_kb":2178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583103+0000","last_change":"2023-09-23T21:23:40.006639+0000","last_active":"2023-09-23T21:25:29.583103+0000","last_peered":"2023-09-23T21:25:29.583103+0000","last_clean":"2023-09-23T21:25:29.583103+0000","last_became_active":"2023-09-23T21:23:40.006309+0000","last_became_peered":"2023-09-23T21:23:40.006309+0000","last_unstale":"2023-09-23T21:25:29.583103+0000","last_undegraded":"2023-09-23T21:25:29.583103+0000","last_fullsized":"2023-09-23T21:25:29.583103+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:04.424515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1130,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583589+0000","last_change":"2023-09-23T21:24:30.511083+0000","last_active":"2023-09-23T21:25:29.583589+0000","last_peered":"2023-09-23T21:25:29.583589+0000","last_clean":"2023-09-23T21:25:29.583589+0000","last_became_active":"2023-09-23T21:24:30.510633+0000","last_became_peered":"2023-09-23T21:24:30.510633+0000","last_unstale":"2023-09-23T21:25:29.583589+0000","last_undegraded":"2023-09-23T21:25:29.583589+0000","last_fullsized":"2023-09-23T21:25:29.583589+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:57.269339+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"689'66","reported_seq":872,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587750+0000","last_change":"2023-09-23T21:24:03.657789+0000","last_active":"2023-09-23T21:25:29.587750+0000","last_peered":"2023-09-23T21:25:29.587750+0000","last_clean":"2023-09-23T21:25:29.587750+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:25:29.587750+0000","last_undegraded":"2023-09-23T21:25:29.587750+0000","last_fullsized":"2023-09-23T21:25:29.587750+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031051700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":342,"num_read_kb":1892,"num_write":51,"num_write_kb":1333,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":63,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588998+0000","last_change":"2023-09-23T21:23:40.017705+0000","last_active":"2023-09-23T21:25:30.588998+0000","last_peered":"2023-09-23T21:25:30.588998+0000","last_clean":"2023-09-23T21:25:30.588998+0000","last_became_active":"2023-09-23T21:23:40.007892+0000","last_became_peered":"2023-09-23T21:23:40.007892+0000","last_unstale":"2023-09-23T21:25:30.588998+0000","last_undegraded":"2023-09-23T21:25:30.588998+0000","last_fullsized":"2023-09-23T21:25:30.588998+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:09.348317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":264,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587789+0000","last_change":"2023-09-23T21:25:06.031614+0000","last_active":"2023-09-23T21:25:29.587789+0000","last_peered":"2023-09-23T21:25:29.587789+0000","last_clean":"2023-09-23T21:25:29.587789+0000","last_became_active":"2023-09-23T21:25:06.031295+0000","last_became_peered":"2023-09-23T21:25:06.031295+0000","last_unstale":"2023-09-23T21:25:29.587789+0000","last_undegraded":"2023-09-23T21:25:29.587789+0000","last_fullsized":"2023-09-23T21:25:29.587789+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:41.309598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":58,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.590511+0000","last_change":"2023-09-23T21:23:40.014466+0000","last_active":"2023-09-23T21:25:29.590511+0000","last_peered":"2023-09-23T21:25:29.590511+0000","last_clean":"2023-09-23T21:25:29.590511+0000","last_became_active":"2023-09-23T21:23:40.014118+0000","last_became_peered":"2023-09-23T21:23:40.014118+0000","last_unstale":"2023-09-23T21:25:29.590511+0000","last_undegraded":"2023-09-23T21:25:29.590511+0000","last_fullsized":"2023-09-23T21:25:29.590511+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:21.412255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":812,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.590487+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:25:29.590487+0000","last_peered":"2023-09-23T21:25:29.590487+0000","last_clean":"2023-09-23T21:25:29.590487+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:25:29.590487+0000","last_undegraded":"2023-09-23T21:25:29.590487+0000","last_fullsized":"2023-09-23T21:25:29.590487+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":369,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.597922+0000","last_change":"2023-09-23T21:25:30.597922+0000","last_active":"2023-09-23T21:25:30.597922+0000","last_peered":"2023-09-23T21:25:30.597922+0000","last_clean":"2023-09-23T21:25:30.597922+0000","last_became_active":"2023-09-23T21:25:30.597592+0000","last_became_peered":"2023-09-23T21:25:30.597592+0000","last_unstale":"2023-09-23T21:25:30.597922+0000","last_undegraded":"2023-09-23T21:25:30.597922+0000","last_fullsized":"2023-09-23T21:25:30.597922+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:32.759345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":331,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588195+0000","last_change":"2023-09-23T21:24:45.399000+0000","last_active":"2023-09-23T21:25:29.588195+0000","last_peered":"2023-09-23T21:25:29.588195+0000","last_clean":"2023-09-23T21:25:29.588195+0000","last_became_active":"2023-09-23T21:24:45.398564+0000","last_became_peered":"2023-09-23T21:24:45.398564+0000","last_unstale":"2023-09-23T21:25:29.588195+0000","last_undegraded":"2023-09-23T21:25:29.588195+0000","last_fullsized":"2023-09-23T21:25:29.588195+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:25.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":134,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.600360+0000","last_change":"2023-09-23T21:25:30.600360+0000","last_active":"2023-09-23T21:25:30.600360+0000","last_peered":"2023-09-23T21:25:30.600360+0000","last_clean":"2023-09-23T21:25:30.600360+0000","last_became_active":"2023-09-23T21:25:30.599915+0000","last_became_peered":"2023-09-23T21:25:30.599915+0000","last_unstale":"2023-09-23T21:25:30.600360+0000","last_undegraded":"2023-09-23T21:25:30.600360+0000","last_fullsized":"2023-09-23T21:25:30.600360+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:46.090965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":171,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586927+0000","last_change":"2023-09-23T21:25:06.323886+0000","last_active":"2023-09-23T21:25:29.586927+0000","last_peered":"2023-09-23T21:25:29.586927+0000","last_clean":"2023-09-23T21:25:29.586927+0000","last_became_active":"2023-09-23T21:25:06.323433+0000","last_became_peered":"2023-09-23T21:25:06.323433+0000","last_unstale":"2023-09-23T21:25:29.586927+0000","last_undegraded":"2023-09-23T21:25:29.586927+0000","last_fullsized":"2023-09-23T21:25:29.586927+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:40.608942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583698+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:25:29.583698+0000","last_peered":"2023-09-23T21:25:29.583698+0000","last_clean":"2023-09-23T21:25:29.583698+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:25:29.583698+0000","last_undegraded":"2023-09-23T21:25:29.583698+0000","last_fullsized":"2023-09-23T21:25:29.583698+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583231+0000","last_change":"2023-09-23T21:23:40.009045+0000","last_active":"2023-09-23T21:25:29.583231+0000","last_peered":"2023-09-23T21:25:29.583231+0000","last_clean":"2023-09-23T21:25:29.583231+0000","last_became_active":"2023-09-23T21:23:40.008753+0000","last_became_peered":"2023-09-23T21:23:40.008753+0000","last_unstale":"2023-09-23T21:25:29.583231+0000","last_undegraded":"2023-09-23T21:25:29.583231+0000","last_fullsized":"2023-09-23T21:25:29.583231+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:28.136832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":80,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588832+0000","last_change":"2023-09-23T21:24:30.508673+0000","last_active":"2023-09-23T21:25:30.588832+0000","last_peered":"2023-09-23T21:25:30.588832+0000","last_clean":"2023-09-23T21:25:30.588832+0000","last_became_active":"2023-09-23T21:24:30.508002+0000","last_became_peered":"2023-09-23T21:24:30.508002+0000","last_unstale":"2023-09-23T21:25:30.588832+0000","last_undegraded":"2023-09-23T21:25:30.588832+0000","last_fullsized":"2023-09-23T21:25:30.588832+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:22.538829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":425,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586679+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:25:29.586679+0000","last_peered":"2023-09-23T21:25:29.586679+0000","last_clean":"2023-09-23T21:25:29.586679+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:25:29.586679+0000","last_undegraded":"2023-09-23T21:25:29.586679+0000","last_fullsized":"2023-09-23T21:25:29.586679+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":36346,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":853056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13204,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201969,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":933760,"kb_used_data":11768,"kb_used_omap":0,"kb_used_meta":921984,"kb_avail":92811392,"statfs":{"total":95995035648,"available":95038865408,"internally_reserved":0,"allocated":12050432,"data_stored":10195082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":944111616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.381,"15min":2.347},"min":{"1min":0.35999999999999999,"5min":0.252,"15min":0.214},"max":{"1min":0.72499999999999998,"5min":10.522,"15min":141.33199999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.4359999999999999,"15min":2.48},"min":{"1min":0.51600000000000001,"5min":0.29799999999999999,"15min":0.23799999999999999},"max":{"1min":0.79800000000000004,"5min":12.962,"15min":141.31700000000001},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 23 21:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.8500000000000001,"15min":2.504},"min":{"1min":0.432,"5min":0.28000000000000003,"15min":0.24099999999999999},"max":{"1min":0.73999999999999999,"5min":31.233000000000001,"15min":76.007999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.0550000000000002,"15min":2.883},"min":{"1min":0.443,"5min":0.23799999999999999,"15min":0.22900000000000001},"max":{"1min":0.73899999999999999,"5min":31.219000000000001,"15min":117.637},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 21:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":3.9609999999999999,"15min":4.593},"min":{"1min":0.372,"5min":0.252,"15min":0.224},"max":{"1min":0.80300000000000005,"5min":157.65700000000001,"15min":158.726},"last":0.436},{"interface":"front","average":{"1min":0.55800000000000005,"5min":4.0039999999999996,"15min":4.8600000000000003},"min":{"1min":0.41099999999999998,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":0.70599999999999996,"5min":157.642,"15min":158.70400000000001},"last":0.65700000000000003}]},{"osd":3,"last update":"Sat Sep 23 21:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":2.056,"15min":4.0819999999999999},"min":{"1min":0.41899999999999998,"5min":0.312,"15min":0.28799999999999998},"max":{"1min":0.74099999999999999,"5min":30.667999999999999,"15min":135.66200000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":2.202,"15min":4.3369999999999997},"min":{"1min":0.35799999999999998,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":0.71899999999999997,"5min":30.798999999999999,"15min":135.47},"last":0.60299999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:25:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":2.2749999999999999,"15min":2.2029999999999998},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":0.91700000000000004,"5min":13.821999999999999,"15min":31.695},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.847,"15min":2.3180000000000001},"min":{"1min":0.434,"5min":0.26900000000000002,"15min":0.26800000000000002},"max":{"1min":0.89900000000000002,"5min":27.821999999999999,"15min":34.994},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Sep 23 21:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:25:55 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":2.234,"15min":2.7370000000000001},"min":{"1min":0.42999999999999999,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":0.997,"5min":14.066000000000001,"15min":143.18700000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":2.4279999999999999,"15min":2.4209999999999998},"min":{"1min":0.40100000000000002,"5min":0.33000000000000002,"15min":0.29099999999999998},"max":{"1min":0.80700000000000005,"5min":21.081,"15min":142.70400000000001},"last":0.63400000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463589,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927028,"kb_used_data":7340,"kb_used_omap":0,"kb_used_meta":919680,"kb_avail":92818124,"statfs":{"total":95995035648,"available":95045758976,"internally_reserved":0,"allocated":7516160,"data_stored":5672764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":941752320},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.4319999999999999,"15min":2.5390000000000001},"min":{"1min":0.29299999999999998,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":1.079,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.4730000000000001,"15min":2.6989999999999998},"min":{"1min":0.33200000000000002,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":1.5189999999999999,"5min":11.968,"15min":149.303},"last":0.623}]},{"osd":1,"last update":"Sat Sep 23 21:25:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.2829999999999999,"15min":2.5990000000000002},"min":{"1min":0.33600000000000002,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":0.86099999999999999,"5min":12.013,"15min":129.37899999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.431,"15min":2.5030000000000001},"min":{"1min":0.372,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.92100000000000004,"5min":14.366,"15min":129.40100000000001},"last":0.64200000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":6.5830000000000002,"15min":6.4989999999999997},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25800000000000001},"max":{"1min":0.95199999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.76400000000000001,"5min":6.758,"15min":6.7450000000000001},"min":{"1min":0.41899999999999998,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":4.1589999999999998,"5min":199.27000000000001,"15min":199.27000000000001},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.6639999999999999,"15min":3.4300000000000002},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.218},"max":{"1min":1.008,"5min":14.411,"15min":137.875},"last":0.435},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.5760000000000001,"15min":3.274},"min":{"1min":0.41099999999999998,"5min":0.315,"15min":0.29799999999999999},"max":{"1min":0.99199999999999999,"5min":12.35,"15min":137.60900000000001},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":2.4660000000000002,"15min":2.5259999999999998},"min":{"1min":0.29999999999999999,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":0.77200000000000002,"5min":20.408999999999999,"15min":55.494},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.6669999999999998,"15min":2.4529999999999998},"min":{"1min":0.33600000000000002,"5min":0.255,"15min":0.23999999999999999},"max":{"1min":0.96999999999999997,"5min":14.622,"15min":55.305},"last":0.70699999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 21:25:45 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.8939999999999999,"15min":2.391},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.28000000000000003},"max":{"1min":0.86399999999999999,"5min":12.779,"15min":165.96799999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.67500000000000004,"5min":2.073,"15min":2.7480000000000002},"min":{"1min":0.46400000000000002,"5min":0.307,"15min":0.251},"max":{"1min":0.93200000000000005,"5min":14.457000000000001,"15min":165.82400000000001},"last":0.73499999999999999}]}]},{"osd":1,"up_from":276,"seq":1185410974036,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":662280,"kb_used_data":8960,"kb_used_omap":5,"kb_used_meta":653306,"kb_avail":93082872,"statfs":{"total":95995035648,"available":95316860928,"internally_reserved":0,"allocated":9175040,"data_stored":5848596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":668986083},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39900000000000002,"15min":1.343},"min":{"1min":0.251,"5min":0.214,"15min":0.19400000000000001},"max":{"1min":0.54000000000000004,"5min":2.3410000000000002,"15min":108.47},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.32700000000000001,"5min":0.63600000000000001,"15min":1.446},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.19600000000000001},"max":{"1min":0.44800000000000001,"5min":6.0149999999999997,"15min":109.532},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":4.4349999999999996,"15min":7.5430000000000001},"min":{"1min":0.28699999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.54800000000000004,"5min":130.78899999999999,"15min":171.35400000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":4.3849999999999998,"15min":7.2619999999999996},"min":{"1min":0.27200000000000002,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":0.50800000000000001,"5min":133.751,"15min":170.69900000000001},"last":0.376}]},{"osd":3,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.624,"15min":2.4420000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":0.52900000000000003,"5min":17.247,"15min":99.120999999999995},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.70799999999999996,"15min":3.177},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":0.54300000000000004,"5min":10.949999999999999,"15min":150.38},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.3029999999999999,"15min":2.1760000000000002},"min":{"1min":0.374,"5min":0.27400000000000002,"15min":0.26800000000000002},"max":{"1min":0.91400000000000003,"5min":9.5800000000000001,"15min":174.84399999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.63700000000000001,"5min":1.05,"15min":1.877},"min":{"1min":0.49099999999999999,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":0.85099999999999998,"5min":8.5109999999999992,"15min":174.82400000000001},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 23 21:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.64300000000000002}]},{"osd":6,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.784,"15min":2.2770000000000001},"min":{"1min":0.442,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":0.92400000000000004,"5min":20.265999999999998,"15min":119.999},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.5269999999999999,"15min":2.2250000000000001},"min":{"1min":0.46600000000000003,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":0.79400000000000004,"5min":20.361999999999998,"15min":122.328},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.6080000000000001,"15min":2.7719999999999998},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":0.85799999999999998,"5min":10.648999999999999,"15min":285.47500000000002},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2709999999999999,"15min":2.379},"min":{"1min":0.504,"5min":0.38600000000000001,"15min":0.34899999999999998},"max":{"1min":0.878,"5min":10.664999999999999,"15min":285.48599999999999},"last":0.68899999999999995}]}]},{"osd":0,"up_from":92,"seq":395136991776,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":864832,"kb_used_data":8824,"kb_used_omap":5,"kb_used_meta":855994,"kb_avail":92880320,"statfs":{"total":95995035648,"available":95109447680,"internally_reserved":0,"allocated":9035776,"data_stored":5826789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":876538490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.67900000000000005,"15min":1.038},"min":{"1min":0.221,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.97599999999999998,"5min":11.109,"15min":11.782999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.54000000000000004,"15min":1.175},"min":{"1min":0.23599999999999999,"5min":0.20799999999999999,"15min":0.20200000000000001},"max":{"1min":0.70599999999999996,"5min":6.6319999999999997,"15min":15.266},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":2.375,"15min":5.3129999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.19800000000000001},"max":{"1min":0.91500000000000004,"5min":96.641000000000005,"15min":201.32300000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.3109999999999999,"15min":5.3899999999999997},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.245},"max":{"1min":0.78800000000000003,"5min":98.347999999999999,"15min":201.43199999999999},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.60499999999999998,"15min":1.698},"min":{"1min":0.28000000000000003,"5min":0.217,"15min":0.217},"max":{"1min":0.95399999999999996,"5min":11.077,"15min":86.989999999999995},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.57699999999999996,"15min":1.6839999999999999},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.23599999999999999},"max":{"1min":0.78800000000000003,"5min":4.1820000000000004,"15min":87.043999999999997},"last":0.61799999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.802,"15min":1.8720000000000001},"min":{"1min":0.43099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.04,"5min":39.228000000000002,"15min":39.228000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.8260000000000001,"15min":1.6579999999999999},"min":{"1min":0.32800000000000001,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":0.879,"5min":39.340000000000003,"15min":39.340000000000003},"last":0.71599999999999997}]},{"osd":5,"last update":"Sat Sep 23 21:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.495},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.68400000000000005}]},{"osd":6,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.855,"15min":1.9059999999999999},"min":{"1min":0.41199999999999998,"5min":0.34999999999999998,"15min":0.29999999999999999},"max":{"1min":0.93600000000000005,"5min":56.213000000000001,"15min":56.213000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.194,"15min":1.849},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.85399999999999998,"5min":53.424999999999997,"15min":53.424999999999997},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.6850000000000001,"15min":1.599},"min":{"1min":0.35199999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.88100000000000001,"5min":32.411999999999999,"15min":32.411999999999999},"last":0.755},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.544,"15min":1.591},"min":{"1min":0.441,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89600000000000002,"5min":32.433999999999997,"15min":32.433999999999997},"last":0.746}]}]},{"osd":2,"up_from":129,"seq":554050781667,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1359348,"kb_used_data":13420,"kb_used_omap":7,"kb_used_meta":1345912,"kb_avail":92385804,"statfs":{"total":95995035648,"available":94603063296,"internally_reserved":0,"allocated":13742080,"data_stored":10306997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1378213969},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.1799999999999999,"15min":1.7470000000000001},"min":{"1min":0.22500000000000001,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":0.46700000000000003,"5min":51.968000000000004,"15min":103.021},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.34000000000000002,"5min":1.288,"15min":1.974},"min":{"1min":0.26800000000000002,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":0.443,"5min":52.241999999999997,"15min":103.154},"last":0.308}]},{"osd":1,"last update":"Sat Sep 23 21:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.61599999999999999,"15min":2.1760000000000002},"min":{"1min":0.247,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.50800000000000001,"5min":6.4470000000000001,"15min":143.482},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.54600000000000004,"15min":2.3700000000000001},"min":{"1min":0.23899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.54900000000000004,"5min":3.27,"15min":143.61000000000001},"last":0.29599999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.69499999999999995,"15min":2.1909999999999998},"min":{"1min":0.28799999999999998,"5min":0.22600000000000001,"15min":0.20499999999999999},"max":{"1min":0.54400000000000004,"5min":6.4039999999999999,"15min":143.50800000000001},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.58399999999999996,"15min":2.4199999999999999},"min":{"1min":0.309,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52000000000000002,"5min":4.2670000000000003,"15min":143.27799999999999},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.5620000000000001,"15min":3.371},"min":{"1min":0.441,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":0.72699999999999998,"5min":8.5820000000000007,"15min":225.80799999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.609,"15min":3.6360000000000001},"min":{"1min":0.34999999999999998,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":0.76500000000000001,"5min":8.5690000000000008,"15min":225.79300000000001},"last":0.65700000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.66800000000000004}]},{"osd":6,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.5640000000000001,"15min":3.3180000000000001},"min":{"1min":0.38800000000000001,"5min":0.38,"15min":0.31900000000000001},"max":{"1min":0.81899999999999995,"5min":10.148,"15min":210.74700000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.4770000000000001,"15min":2.6629999999999998},"min":{"1min":0.41999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.81899999999999995,"5min":10.634,"15min":210.72900000000001},"last":0.70699999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":2.1920000000000002,"15min":2.0680000000000001},"min":{"1min":0.47699999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":0.75,"5min":52.121000000000002,"15min":52.121000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":2.2610000000000001,"15min":2.2010000000000001},"min":{"1min":0.45600000000000002,"5min":0.39800000000000002,"15min":0.32700000000000001},"max":{"1min":0.84399999999999997,"5min":52.308999999999997,"15min":52.308999999999997},"last":0.69799999999999995}]}]},{"osd":3,"up_from":148,"seq":635655160260,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906764,"kb_used_data":10180,"kb_used_omap":2,"kb_used_meta":896573,"kb_avail":92838388,"statfs":{"total":95995035648,"available":95066509312,"internally_reserved":0,"allocated":10424320,"data_stored":6494439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":918090805},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.53300000000000003,"15min":1.3049999999999999},"min":{"1min":0.25900000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":0.46899999999999997,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.64300000000000002,"15min":1.234},"min":{"1min":0.26300000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.495,"5min":5.0720000000000001,"15min":18.277999999999999},"last":0.26100000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.79900000000000004,"15min":2.8650000000000002},"min":{"1min":0.253,"5min":0.214,"15min":0.214},"max":{"1min":0.56799999999999995,"5min":8.3079999999999998,"15min":147.12100000000001},"last":0.311},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.68799999999999994,"15min":2.839},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":0.53900000000000003,"5min":8.5220000000000002,"15min":151.14699999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":3.8490000000000002,"15min":4.6529999999999996},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.215},"max":{"1min":0.61799999999999999,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.42399999999999999,"5min":3.8650000000000002,"15min":4.4089999999999998},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.217},"max":{"1min":0.52600000000000002,"5min":208.024,"15min":208.024},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.8540000000000001,"15min":2.3069999999999999},"min":{"1min":0.39600000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.83299999999999996,"5min":13.640000000000001,"15min":117.717},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":2.0539999999999998,"15min":2.3820000000000001},"min":{"1min":0.46100000000000002,"5min":0.29999999999999999,"15min":0.28799999999999998},"max":{"1min":0.75600000000000001,"5min":11.968999999999999,"15min":117.771},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.4299999999999999,"15min":3.3410000000000002},"min":{"1min":0.41399999999999998,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":0.73999999999999999,"5min":7.7889999999999997,"15min":130.613},"last":0.626},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.5209999999999999,"15min":3.488},"min":{"1min":0.47199999999999998,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":0.79100000000000004,"5min":8.8940000000000001,"15min":130.50999999999999},"last":0.54500000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.508,"15min":1.9630000000000001},"min":{"1min":0.45500000000000002,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":0.81200000000000006,"5min":19.298999999999999,"15min":44.253},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.867,"15min":2.3260000000000001},"min":{"1min":0.47899999999999998,"5min":0.41699999999999998,"15min":0.36699999999999999},"max":{"1min":0.91100000000000003,"5min":10.441000000000001,"15min":45.706000000000003},"last":0.65800000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986879,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986304,"kb_used_data":7352,"kb_used_omap":0,"kb_used_meta":978944,"kb_avail":92758848,"statfs":{"total":95995035648,"available":94985060352,"internally_reserved":0,"allocated":7528448,"data_stored":5673351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1002438656},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.8069999999999999,"15min":2.8610000000000002},"min":{"1min":0.32000000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":0.84699999999999998,"5min":12.195,"15min":226.751},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.54100000000000004,"5min":1.387,"15min":2.504},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.83899999999999997,"5min":16.478000000000002,"15min":226.43299999999999},"last":0.61799999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.345,"15min":2.5049999999999999},"min":{"1min":0.40600000000000003,"5min":0.246,"15min":0.246},"max":{"1min":6.4859999999999998,"5min":16.434999999999999,"15min":119.55800000000001},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.6379999999999999,"15min":2.5750000000000002},"min":{"1min":0.35999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.04,"5min":12.108000000000001,"15min":116.78100000000001},"last":0.60199999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":4.6500000000000004,"15min":5.7460000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.248},"max":{"1min":0.80500000000000005,"5min":124.70099999999999,"15min":210.50299999999999},"last":0.629},{"interface":"front","average":{"1min":0.59999999999999998,"5min":3.952,"15min":5.7990000000000004},"min":{"1min":0.437,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.78300000000000003,"5min":116.47,"15min":210.59},"last":0.64300000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.853,"15min":3.8359999999999999},"min":{"1min":0.441,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":5.7839999999999998,"5min":12.973000000000001,"15min":214.64099999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.9279999999999999,"15min":4.2080000000000002},"min":{"1min":0.45700000000000002,"5min":0.33200000000000002,"15min":0.32500000000000001},"max":{"1min":6.1520000000000001,"5min":13.673999999999999,"15min":214.54900000000001},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.496}]},{"osd":6,"last update":"Sat Sep 23 21:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":3.3180000000000001,"15min":2.3900000000000001},"min":{"1min":0.35099999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.76000000000000001,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.63200000000000001,"5min":3.931,"15min":2.7890000000000001},"min":{"1min":0.39100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.89000000000000001,"5min":67.596999999999994,"15min":67.596999999999994},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":2.3889999999999998,"15min":2.2160000000000002},"min":{"1min":0.46000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.84799999999999998,"5min":16.998999999999999,"15min":18.352},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.66700000000000004,"5min":2.0870000000000002,"15min":2.0670000000000002},"min":{"1min":0.40999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.90300000000000002,"5min":16.664999999999999,"15min":30.117000000000001},"last":0.65400000000000003}]}]},{"osd":5,"up_from":694,"seq":2980707303445,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":151352,"kb_used_data":110384,"kb_used_omap":78,"kb_used_meta":40881,"kb_avail":93593800,"statfs":{"total":95995035648,"available":95840051200,"internally_reserved":0,"allocated":113033216,"data_stored":111182618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":80826,"internal_metadata":41862214},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.751},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.55200000000000005}]},{"osd":1,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.77700000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.72699999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.65300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:26:07.175 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T21:26:07.740 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:26:08.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:08 smithi121 ceph-mon[219709]: pgmap v2156: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:08.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:08.347 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:08 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:08.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:08 smithi121 ceph-mon[211526]: pgmap v2156: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:08.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:08.347 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:08 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:08 smithi123 ceph-mon[193903]: pgmap v2156: 83 pgs: 83 active+clean; 4.6 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:08.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:08.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:08 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:09.121 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:26:09.121 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:09 smithi121 ceph-mon[211526]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:26:09.121 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:09 smithi121 ceph-mon[211526]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:09.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:26:09.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:09 smithi121 ceph-mon[219709]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:26:09.121 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:09 smithi121 ceph-mon[219709]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:09.478 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:26:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:09 smithi123 ceph-mon[193903]: from='mgr.25066 172.21.15.121:0/969734570' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:26:09.479 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:09 smithi123 ceph-mon[193903]: from='mgr.25066 ' entity='mgr.y' 2023-09-23T21:26:09.537 INFO:teuthology.orchestra.run.smithi121.stdout: 2023-09-23T21:26:09.538 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2023-09-23T21:26:10.267 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:10 smithi121 ceph-mon[211526]: pgmap v2157: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:10.267 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:10 smithi121 ceph-mon[219709]: pgmap v2157: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:10.268 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":2157,"stamp":"2023-09-23T21:26:09.499126+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5987,"num_read_kb":13204,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":889402,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":252,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6762932,"kb_used_data":149428,"kb_used_omap":100,"kb_used_meta":6613339,"kb_avail":743198284,"statfs":{"total":767960285184,"available":761035042816,"internally_reserved":0,"allocated":153014272,"data_stored":131709436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102871,"internal_metadata":6772059689},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003071"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":293,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587026+0000","last_change":"2023-09-23T21:20:59.531531+0000","last_active":"2023-09-23T21:25:29.587026+0000","last_peered":"2023-09-23T21:25:29.587026+0000","last_clean":"2023-09-23T21:25:29.587026+0000","last_became_active":"2023-09-23T21:20:59.531111+0000","last_became_peered":"2023-09-23T21:20:59.531111+0000","last_unstale":"2023-09-23T21:25:29.587026+0000","last_undegraded":"2023-09-23T21:25:29.587026+0000","last_fullsized":"2023-09-23T21:25:29.587026+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:39.027808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.562479+0000","last_change":"2023-09-23T21:20:59.528200+0000","last_active":"2023-09-23T21:25:30.562479+0000","last_peered":"2023-09-23T21:25:30.562479+0000","last_clean":"2023-09-23T21:25:30.562479+0000","last_became_active":"2023-09-23T21:20:59.527928+0000","last_became_peered":"2023-09-23T21:20:59.527928+0000","last_unstale":"2023-09-23T21:25:30.562479+0000","last_undegraded":"2023-09-23T21:25:30.562479+0000","last_fullsized":"2023-09-23T21:25:30.562479+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:19.413801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":102,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.594950+0000","last_change":"2023-09-23T21:20:59.524092+0000","last_active":"2023-09-23T21:25:30.594950+0000","last_peered":"2023-09-23T21:25:30.594950+0000","last_clean":"2023-09-23T21:25:30.594950+0000","last_became_active":"2023-09-23T21:20:59.523780+0000","last_became_peered":"2023-09-23T21:20:59.523780+0000","last_unstale":"2023-09-23T21:25:30.594950+0000","last_undegraded":"2023-09-23T21:25:30.594950+0000","last_fullsized":"2023-09-23T21:25:30.594950+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:20.921496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589039+0000","last_change":"2023-09-23T21:23:40.010951+0000","last_active":"2023-09-23T21:25:30.589039+0000","last_peered":"2023-09-23T21:25:30.589039+0000","last_clean":"2023-09-23T21:25:30.589039+0000","last_became_active":"2023-09-23T21:23:40.010331+0000","last_became_peered":"2023-09-23T21:23:40.010331+0000","last_unstale":"2023-09-23T21:25:30.589039+0000","last_undegraded":"2023-09-23T21:25:30.589039+0000","last_fullsized":"2023-09-23T21:25:30.589039+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:06.978784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583176+0000","last_change":"2023-09-23T21:23:40.012215+0000","last_active":"2023-09-23T21:25:29.583176+0000","last_peered":"2023-09-23T21:25:29.583176+0000","last_clean":"2023-09-23T21:25:29.583176+0000","last_became_active":"2023-09-23T21:23:40.011815+0000","last_became_peered":"2023-09-23T21:23:40.011815+0000","last_unstale":"2023-09-23T21:25:29.583176+0000","last_undegraded":"2023-09-23T21:25:29.583176+0000","last_fullsized":"2023-09-23T21:25:29.583176+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:00.486707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":241,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587646+0000","last_change":"2023-09-23T21:24:30.512880+0000","last_active":"2023-09-23T21:25:29.587646+0000","last_peered":"2023-09-23T21:25:29.587646+0000","last_clean":"2023-09-23T21:25:29.587646+0000","last_became_active":"2023-09-23T21:24:30.512547+0000","last_became_peered":"2023-09-23T21:24:30.512547+0000","last_unstale":"2023-09-23T21:25:29.587646+0000","last_undegraded":"2023-09-23T21:25:29.587646+0000","last_fullsized":"2023-09-23T21:25:29.587646+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:29.648160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":74,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588756+0000","last_change":"2023-09-23T21:24:30.508862+0000","last_active":"2023-09-23T21:25:30.588756+0000","last_peered":"2023-09-23T21:25:30.588756+0000","last_clean":"2023-09-23T21:25:30.588756+0000","last_became_active":"2023-09-23T21:24:30.508395+0000","last_became_peered":"2023-09-23T21:24:30.508395+0000","last_unstale":"2023-09-23T21:25:30.588756+0000","last_undegraded":"2023-09-23T21:25:30.588756+0000","last_fullsized":"2023-09-23T21:25:30.588756+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:18.972901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":151,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583256+0000","last_change":"2023-09-23T21:24:45.398853+0000","last_active":"2023-09-23T21:25:29.583256+0000","last_peered":"2023-09-23T21:25:29.583256+0000","last_clean":"2023-09-23T21:25:29.583256+0000","last_became_active":"2023-09-23T21:24:45.398549+0000","last_became_peered":"2023-09-23T21:24:45.398549+0000","last_unstale":"2023-09-23T21:25:29.583256+0000","last_undegraded":"2023-09-23T21:25:29.583256+0000","last_fullsized":"2023-09-23T21:25:29.583256+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.109131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588166+0000","last_change":"2023-09-23T21:24:30.516861+0000","last_active":"2023-09-23T21:25:29.588166+0000","last_peered":"2023-09-23T21:25:29.588166+0000","last_clean":"2023-09-23T21:25:29.588166+0000","last_became_active":"2023-09-23T21:24:30.516517+0000","last_became_peered":"2023-09-23T21:24:30.516517+0000","last_unstale":"2023-09-23T21:25:29.588166+0000","last_undegraded":"2023-09-23T21:25:29.588166+0000","last_fullsized":"2023-09-23T21:25:29.588166+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:23.519206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":115,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583262+0000","last_change":"2023-09-23T21:20:31.490619+0000","last_active":"2023-09-23T21:25:29.583262+0000","last_peered":"2023-09-23T21:25:29.583262+0000","last_clean":"2023-09-23T21:25:29.583262+0000","last_became_active":"2023-09-23T21:20:31.490316+0000","last_became_peered":"2023-09-23T21:20:31.490316+0000","last_unstale":"2023-09-23T21:25:29.583262+0000","last_undegraded":"2023-09-23T21:25:29.583262+0000","last_fullsized":"2023-09-23T21:25:29.583262+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:09.916320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":353,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593821+0000","last_change":"2023-09-23T21:20:31.616723+0000","last_active":"2023-09-23T21:25:29.593821+0000","last_peered":"2023-09-23T21:25:29.593821+0000","last_clean":"2023-09-23T21:25:29.593821+0000","last_became_active":"2023-09-23T21:20:31.516026+0000","last_became_peered":"2023-09-23T21:20:31.516026+0000","last_unstale":"2023-09-23T21:25:29.593821+0000","last_undegraded":"2023-09-23T21:25:29.593821+0000","last_fullsized":"2023-09-23T21:25:29.593821+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:03.558778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.589484+0000","last_change":"2023-09-23T21:24:45.399248+0000","last_active":"2023-09-23T21:25:29.589484+0000","last_peered":"2023-09-23T21:25:29.589484+0000","last_clean":"2023-09-23T21:25:29.589484+0000","last_became_active":"2023-09-23T21:24:45.398853+0000","last_became_peered":"2023-09-23T21:24:45.398853+0000","last_unstale":"2023-09-23T21:25:29.589484+0000","last_undegraded":"2023-09-23T21:25:29.589484+0000","last_fullsized":"2023-09-23T21:25:29.589484+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:31.756276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"83'22","reported_seq":449,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587637+0000","last_change":"2023-09-23T21:19:18.470554+0000","last_active":"2023-09-23T21:25:29.587637+0000","last_peered":"2023-09-23T21:25:29.587637+0000","last_clean":"2023-09-23T21:25:29.587637+0000","last_became_active":"2023-09-23T21:19:18.470224+0000","last_became_peered":"2023-09-23T21:19:18.470224+0000","last_unstale":"2023-09-23T21:25:29.587637+0000","last_undegraded":"2023-09-23T21:25:29.587637+0000","last_fullsized":"2023-09-23T21:25:29.587637+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.636868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":676,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588573+0000","last_change":"2023-09-23T21:20:59.527388+0000","last_active":"2023-09-23T21:25:29.588573+0000","last_peered":"2023-09-23T21:25:29.588573+0000","last_clean":"2023-09-23T21:25:29.588573+0000","last_became_active":"2023-09-23T21:20:59.527045+0000","last_became_peered":"2023-09-23T21:20:59.527045+0000","last_unstale":"2023-09-23T21:25:29.588573+0000","last_undegraded":"2023-09-23T21:25:29.588573+0000","last_fullsized":"2023-09-23T21:25:29.588573+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:58.914768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":301,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593845+0000","last_change":"2023-09-23T21:10:15.599296+0000","last_active":"2023-09-23T21:25:29.593845+0000","last_peered":"2023-09-23T21:25:29.593845+0000","last_clean":"2023-09-23T21:25:29.593845+0000","last_became_active":"2023-09-23T21:10:15.586465+0000","last_became_peered":"2023-09-23T21:10:15.586465+0000","last_unstale":"2023-09-23T21:25:29.593845+0000","last_undegraded":"2023-09-23T21:25:29.593845+0000","last_fullsized":"2023-09-23T21:25:29.593845+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.534349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583829+0000","last_change":"2023-09-23T21:20:59.513912+0000","last_active":"2023-09-23T21:25:29.583829+0000","last_peered":"2023-09-23T21:25:29.583829+0000","last_clean":"2023-09-23T21:25:29.583829+0000","last_became_active":"2023-09-23T21:20:59.513582+0000","last_became_peered":"2023-09-23T21:20:59.513582+0000","last_unstale":"2023-09-23T21:25:29.583829+0000","last_undegraded":"2023-09-23T21:25:29.583829+0000","last_fullsized":"2023-09-23T21:25:29.583829+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:24.538601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":310,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.590222+0000","last_change":"2023-09-23T21:24:45.640327+0000","last_active":"2023-09-23T21:25:29.590222+0000","last_peered":"2023-09-23T21:25:29.590222+0000","last_clean":"2023-09-23T21:25:29.590222+0000","last_became_active":"2023-09-23T21:24:45.640011+0000","last_became_peered":"2023-09-23T21:24:45.640011+0000","last_unstale":"2023-09-23T21:25:29.590222+0000","last_undegraded":"2023-09-23T21:25:29.590222+0000","last_fullsized":"2023-09-23T21:25:29.590222+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:40.743533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":332,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.600687+0000","last_change":"2023-09-23T21:25:30.600687+0000","last_active":"2023-09-23T21:25:30.600687+0000","last_peered":"2023-09-23T21:25:30.600687+0000","last_clean":"2023-09-23T21:25:30.600687+0000","last_became_active":"2023-09-23T21:25:30.600363+0000","last_became_peered":"2023-09-23T21:25:30.600363+0000","last_unstale":"2023-09-23T21:25:30.600687+0000","last_undegraded":"2023-09-23T21:25:30.600687+0000","last_fullsized":"2023-09-23T21:25:30.600687+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:35.803017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":295,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593560+0000","last_change":"2023-09-23T21:10:15.578962+0000","last_active":"2023-09-23T21:25:29.593560+0000","last_peered":"2023-09-23T21:25:29.593560+0000","last_clean":"2023-09-23T21:25:29.593560+0000","last_became_active":"2023-09-23T21:10:15.578569+0000","last_became_peered":"2023-09-23T21:10:15.578569+0000","last_unstale":"2023-09-23T21:25:29.593560+0000","last_undegraded":"2023-09-23T21:25:29.593560+0000","last_fullsized":"2023-09-23T21:25:29.593560+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:30.167226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589115+0000","last_change":"2023-09-23T21:23:40.011399+0000","last_active":"2023-09-23T21:25:30.589115+0000","last_peered":"2023-09-23T21:25:30.589115+0000","last_clean":"2023-09-23T21:25:30.589115+0000","last_became_active":"2023-09-23T21:23:40.010992+0000","last_became_peered":"2023-09-23T21:23:40.010992+0000","last_unstale":"2023-09-23T21:25:30.589115+0000","last_undegraded":"2023-09-23T21:25:30.589115+0000","last_fullsized":"2023-09-23T21:25:30.589115+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:35.168968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":852,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586827+0000","last_change":"2023-09-23T21:24:30.514405+0000","last_active":"2023-09-23T21:25:29.586827+0000","last_peered":"2023-09-23T21:25:29.586827+0000","last_clean":"2023-09-23T21:25:29.586827+0000","last_became_active":"2023-09-23T21:24:30.514073+0000","last_became_peered":"2023-09-23T21:24:30.514073+0000","last_unstale":"2023-09-23T21:25:29.586827+0000","last_undegraded":"2023-09-23T21:25:29.586827+0000","last_fullsized":"2023-09-23T21:25:29.586827+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:30.858579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583958+0000","last_change":"2023-09-23T21:20:59.516654+0000","last_active":"2023-09-23T21:25:29.583958+0000","last_peered":"2023-09-23T21:25:29.583958+0000","last_clean":"2023-09-23T21:25:29.583958+0000","last_became_active":"2023-09-23T21:20:59.516342+0000","last_became_peered":"2023-09-23T21:20:59.516342+0000","last_unstale":"2023-09-23T21:25:29.583958+0000","last_undegraded":"2023-09-23T21:25:29.583958+0000","last_fullsized":"2023-09-23T21:25:29.583958+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:23.755185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":369,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593904+0000","last_change":"2023-09-23T21:23:40.024118+0000","last_active":"2023-09-23T21:25:29.593904+0000","last_peered":"2023-09-23T21:25:29.593904+0000","last_clean":"2023-09-23T21:25:29.593904+0000","last_became_active":"2023-09-23T21:23:40.014595+0000","last_became_peered":"2023-09-23T21:23:40.014595+0000","last_unstale":"2023-09-23T21:25:29.593904+0000","last_undegraded":"2023-09-23T21:25:29.593904+0000","last_fullsized":"2023-09-23T21:25:29.593904+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.403893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":198,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587802+0000","last_change":"2023-09-23T21:24:30.514437+0000","last_active":"2023-09-23T21:25:29.587802+0000","last_peered":"2023-09-23T21:25:29.587802+0000","last_clean":"2023-09-23T21:25:29.587802+0000","last_became_active":"2023-09-23T21:24:30.510816+0000","last_became_peered":"2023-09-23T21:24:30.510816+0000","last_unstale":"2023-09-23T21:25:29.587802+0000","last_undegraded":"2023-09-23T21:25:29.587802+0000","last_fullsized":"2023-09-23T21:25:29.587802+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:03.265668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":369,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593855+0000","last_change":"2023-09-23T21:20:59.618055+0000","last_active":"2023-09-23T21:25:29.593855+0000","last_peered":"2023-09-23T21:25:29.593855+0000","last_clean":"2023-09-23T21:25:29.593855+0000","last_became_active":"2023-09-23T21:20:59.537993+0000","last_became_peered":"2023-09-23T21:20:59.537993+0000","last_unstale":"2023-09-23T21:25:29.593855+0000","last_undegraded":"2023-09-23T21:25:29.593855+0000","last_fullsized":"2023-09-23T21:25:29.593855+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:03.575305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":354,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587962+0000","last_change":"2023-09-23T21:25:06.323889+0000","last_active":"2023-09-23T21:25:29.587962+0000","last_peered":"2023-09-23T21:25:29.587962+0000","last_clean":"2023-09-23T21:25:29.587962+0000","last_became_active":"2023-09-23T21:25:06.323536+0000","last_became_peered":"2023-09-23T21:25:06.323536+0000","last_unstale":"2023-09-23T21:25:29.587962+0000","last_undegraded":"2023-09-23T21:25:29.587962+0000","last_fullsized":"2023-09-23T21:25:29.587962+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:15.262384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":358,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.594037+0000","last_change":"2023-09-23T21:24:30.511297+0000","last_active":"2023-09-23T21:25:29.594037+0000","last_peered":"2023-09-23T21:25:29.594037+0000","last_clean":"2023-09-23T21:25:29.594037+0000","last_became_active":"2023-09-23T21:24:30.510906+0000","last_became_peered":"2023-09-23T21:24:30.510906+0000","last_unstale":"2023-09-23T21:25:29.594037+0000","last_undegraded":"2023-09-23T21:25:29.594037+0000","last_fullsized":"2023-09-23T21:25:29.594037+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:23.467975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":90,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587995+0000","last_change":"2023-09-23T21:21:21.574086+0000","last_active":"2023-09-23T21:25:29.587995+0000","last_peered":"2023-09-23T21:25:29.587995+0000","last_clean":"2023-09-23T21:25:29.587995+0000","last_became_active":"2023-09-23T21:21:21.573791+0000","last_became_peered":"2023-09-23T21:21:21.573791+0000","last_unstale":"2023-09-23T21:25:29.587995+0000","last_undegraded":"2023-09-23T21:25:29.587995+0000","last_fullsized":"2023-09-23T21:25:29.587995+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.829694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":118,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.600992+0000","last_change":"2023-09-23T21:25:30.600992+0000","last_active":"2023-09-23T21:25:30.600992+0000","last_peered":"2023-09-23T21:25:30.600992+0000","last_clean":"2023-09-23T21:25:30.600992+0000","last_became_active":"2023-09-23T21:25:30.600543+0000","last_became_peered":"2023-09-23T21:25:30.600543+0000","last_unstale":"2023-09-23T21:25:30.600992+0000","last_undegraded":"2023-09-23T21:25:30.600992+0000","last_fullsized":"2023-09-23T21:25:30.600992+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:40.282384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588889+0000","last_change":"2023-09-23T21:23:40.007119+0000","last_active":"2023-09-23T21:25:30.588889+0000","last_peered":"2023-09-23T21:25:30.588889+0000","last_clean":"2023-09-23T21:25:30.588889+0000","last_became_active":"2023-09-23T21:23:40.006773+0000","last_became_peered":"2023-09-23T21:23:40.006773+0000","last_unstale":"2023-09-23T21:25:30.588889+0000","last_undegraded":"2023-09-23T21:25:30.588889+0000","last_fullsized":"2023-09-23T21:25:30.588889+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:05.053314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"78'161","reported_seq":1144,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593368+0000","last_change":"2023-09-23T21:20:33.970325+0000","last_active":"2023-09-23T21:25:29.593368+0000","last_peered":"2023-09-23T21:25:29.593368+0000","last_clean":"2023-09-23T21:25:29.593368+0000","last_became_active":"2023-09-23T21:20:33.969950+0000","last_became_peered":"2023-09-23T21:20:33.969950+0000","last_unstale":"2023-09-23T21:25:29.593368+0000","last_undegraded":"2023-09-23T21:25:29.593368+0000","last_fullsized":"2023-09-23T21:25:29.593368+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:01.214172+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:01.214172+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:13.743408+0000","scrub_duration":0.0015501359999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":145,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583716+0000","last_change":"2023-09-23T21:23:40.015941+0000","last_active":"2023-09-23T21:25:29.583716+0000","last_peered":"2023-09-23T21:25:29.583716+0000","last_clean":"2023-09-23T21:25:29.583716+0000","last_became_active":"2023-09-23T21:23:40.005288+0000","last_became_peered":"2023-09-23T21:23:40.005288+0000","last_unstale":"2023-09-23T21:25:29.583716+0000","last_undegraded":"2023-09-23T21:25:29.583716+0000","last_fullsized":"2023-09-23T21:25:29.583716+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:04.042824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":476,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587732+0000","last_change":"2023-09-23T21:20:31.526494+0000","last_active":"2023-09-23T21:25:29.587732+0000","last_peered":"2023-09-23T21:25:29.587732+0000","last_clean":"2023-09-23T21:25:29.587732+0000","last_became_active":"2023-09-23T21:20:31.526034+0000","last_became_peered":"2023-09-23T21:20:31.526034+0000","last_unstale":"2023-09-23T21:25:29.587732+0000","last_undegraded":"2023-09-23T21:25:29.587732+0000","last_fullsized":"2023-09-23T21:25:29.587732+0000","mapping_epoch":616,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":617,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:10.577663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"78'161","reported_seq":923,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.597742+0000","last_change":"2023-09-23T21:25:30.597742+0000","last_active":"2023-09-23T21:25:30.597742+0000","last_peered":"2023-09-23T21:25:30.597742+0000","last_clean":"2023-09-23T21:25:30.597742+0000","last_became_active":"2023-09-23T21:25:30.597323+0000","last_became_peered":"2023-09-23T21:25:30.597323+0000","last_unstale":"2023-09-23T21:25:30.597742+0000","last_undegraded":"2023-09-23T21:25:30.597742+0000","last_fullsized":"2023-09-23T21:25:30.597742+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:26.234946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:26.234946+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:27.117905+0000","scrub_duration":0.029846851000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.589412+0000","last_change":"2023-09-23T21:20:59.531170+0000","last_active":"2023-09-23T21:25:29.589412+0000","last_peered":"2023-09-23T21:25:29.589412+0000","last_clean":"2023-09-23T21:25:29.589412+0000","last_became_active":"2023-09-23T21:20:59.530858+0000","last_became_peered":"2023-09-23T21:20:59.530858+0000","last_unstale":"2023-09-23T21:25:29.589412+0000","last_undegraded":"2023-09-23T21:25:29.589412+0000","last_fullsized":"2023-09-23T21:25:29.589412+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:28.024569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":726,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588835+0000","last_change":"2023-09-23T21:20:59.531703+0000","last_active":"2023-09-23T21:25:29.588835+0000","last_peered":"2023-09-23T21:25:29.588835+0000","last_clean":"2023-09-23T21:25:29.588835+0000","last_became_active":"2023-09-23T21:20:59.531401+0000","last_became_peered":"2023-09-23T21:20:59.531401+0000","last_unstale":"2023-09-23T21:25:29.588835+0000","last_undegraded":"2023-09-23T21:25:29.588835+0000","last_fullsized":"2023-09-23T21:25:29.588835+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:21.372810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":355,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587879+0000","last_change":"2023-09-23T21:10:15.586505+0000","last_active":"2023-09-23T21:25:29.587879+0000","last_peered":"2023-09-23T21:25:29.587879+0000","last_clean":"2023-09-23T21:25:29.587879+0000","last_became_active":"2023-09-23T21:10:15.586145+0000","last_became_peered":"2023-09-23T21:10:15.586145+0000","last_unstale":"2023-09-23T21:25:29.587879+0000","last_undegraded":"2023-09-23T21:25:29.587879+0000","last_fullsized":"2023-09-23T21:25:29.587879+0000","mapping_epoch":432,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:53.231514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":763,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587352+0000","last_change":"2023-09-23T21:21:21.571046+0000","last_active":"2023-09-23T21:25:29.587352+0000","last_peered":"2023-09-23T21:25:29.587352+0000","last_clean":"2023-09-23T21:25:29.587352+0000","last_became_active":"2023-09-23T21:21:21.570734+0000","last_became_peered":"2023-09-23T21:21:21.570734+0000","last_unstale":"2023-09-23T21:25:29.587352+0000","last_undegraded":"2023-09-23T21:25:29.587352+0000","last_fullsized":"2023-09-23T21:25:29.587352+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:47.631885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"78'161","reported_seq":893,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:33.419785+0000","last_change":"2023-09-23T21:25:33.419744+0000","last_active":"2023-09-23T21:25:33.419785+0000","last_peered":"2023-09-23T21:25:33.419785+0000","last_clean":"2023-09-23T21:25:33.419785+0000","last_became_active":"2023-09-23T21:25:30.599404+0000","last_became_peered":"2023-09-23T21:25:30.599404+0000","last_unstale":"2023-09-23T21:25:33.419785+0000","last_undegraded":"2023-09-23T21:25:33.419785+0000","last_fullsized":"2023-09-23T21:25:33.419785+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:25:33.419652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:25:33.419652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:45.035941+0000","scrub_duration":0.0016204489999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583017+0000","last_change":"2023-09-23T21:20:59.514816+0000","last_active":"2023-09-23T21:25:29.583017+0000","last_peered":"2023-09-23T21:25:29.583017+0000","last_clean":"2023-09-23T21:25:29.583017+0000","last_became_active":"2023-09-23T21:20:59.513913+0000","last_became_peered":"2023-09-23T21:20:59.513913+0000","last_unstale":"2023-09-23T21:25:29.583017+0000","last_undegraded":"2023-09-23T21:25:29.583017+0000","last_fullsized":"2023-09-23T21:25:29.583017+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.841666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":102,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.595472+0000","last_change":"2023-09-23T21:20:59.526672+0000","last_active":"2023-09-23T21:25:30.595472+0000","last_peered":"2023-09-23T21:25:30.595472+0000","last_clean":"2023-09-23T21:25:30.595472+0000","last_became_active":"2023-09-23T21:20:59.526338+0000","last_became_peered":"2023-09-23T21:20:59.526338+0000","last_unstale":"2023-09-23T21:25:30.595472+0000","last_undegraded":"2023-09-23T21:25:30.595472+0000","last_fullsized":"2023-09-23T21:25:30.595472+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:49.136670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"78'161","reported_seq":801,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586981+0000","last_change":"2023-09-23T21:20:33.995769+0000","last_active":"2023-09-23T21:25:29.586981+0000","last_peered":"2023-09-23T21:25:29.586981+0000","last_clean":"2023-09-23T21:25:29.586981+0000","last_became_active":"2023-09-23T21:20:33.995379+0000","last_became_peered":"2023-09-23T21:20:33.995379+0000","last_unstale":"2023-09-23T21:25:29.586981+0000","last_undegraded":"2023-09-23T21:25:29.586981+0000","last_fullsized":"2023-09-23T21:25:29.586981+0000","mapping_epoch":619,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":620,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:59.557037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:59.557037+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:49.754309+0000","scrub_duration":0.021944829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"689'224","reported_seq":780,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591932+0000","last_change":"2023-09-23T21:24:03.654416+0000","last_active":"2023-09-23T21:25:29.591932+0000","last_peered":"2023-09-23T21:25:29.591932+0000","last_clean":"2023-09-23T21:25:29.591932+0000","last_became_active":"2023-09-23T21:23:40.011996+0000","last_became_peered":"2023-09-23T21:23:40.011996+0000","last_unstale":"2023-09-23T21:25:29.591932+0000","last_undegraded":"2023-09-23T21:25:29.591932+0000","last_fullsized":"2023-09-23T21:25:29.591932+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":224,"log_dups_size":0,"ondisk_log_size":224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:12.817128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028358000000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1177,"num_read_kb":2026,"num_write":233,"num_write_kb":1133,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":90,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591401+0000","last_change":"2023-09-23T21:24:30.508875+0000","last_active":"2023-09-23T21:25:29.591401+0000","last_peered":"2023-09-23T21:25:29.591401+0000","last_clean":"2023-09-23T21:25:29.591401+0000","last_became_active":"2023-09-23T21:24:30.508556+0000","last_became_peered":"2023-09-23T21:24:30.508556+0000","last_unstale":"2023-09-23T21:25:29.591401+0000","last_undegraded":"2023-09-23T21:25:29.591401+0000","last_fullsized":"2023-09-23T21:25:29.591401+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:11.944192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591960+0000","last_change":"2023-09-23T21:24:30.517186+0000","last_active":"2023-09-23T21:25:29.591960+0000","last_peered":"2023-09-23T21:25:29.591960+0000","last_clean":"2023-09-23T21:25:29.591960+0000","last_became_active":"2023-09-23T21:24:30.516912+0000","last_became_peered":"2023-09-23T21:24:30.516912+0000","last_unstale":"2023-09-23T21:25:29.591960+0000","last_undegraded":"2023-09-23T21:25:29.591960+0000","last_fullsized":"2023-09-23T21:25:29.591960+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:17.698045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"78'161","reported_seq":895,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.596398+0000","last_change":"2023-09-23T21:25:30.596398+0000","last_active":"2023-09-23T21:25:30.596398+0000","last_peered":"2023-09-23T21:25:30.596398+0000","last_clean":"2023-09-23T21:25:30.596398+0000","last_became_active":"2023-09-23T21:25:30.596051+0000","last_became_peered":"2023-09-23T21:25:30.596051+0000","last_unstale":"2023-09-23T21:25:30.596398+0000","last_undegraded":"2023-09-23T21:25:30.596398+0000","last_fullsized":"2023-09-23T21:25:30.596398+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:00.586176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:00.586176+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:29.370699+0000","scrub_duration":0.0038920790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"689'121","reported_seq":803,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.599693+0000","last_change":"2023-09-23T21:25:30.599693+0000","last_active":"2023-09-23T21:25:30.599693+0000","last_peered":"2023-09-23T21:25:30.599693+0000","last_clean":"2023-09-23T21:25:30.599693+0000","last_became_active":"2023-09-23T21:25:30.599144+0000","last_became_peered":"2023-09-23T21:25:30.599144+0000","last_unstale":"2023-09-23T21:25:30.599693+0000","last_undegraded":"2023-09-23T21:25:30.599693+0000","last_fullsized":"2023-09-23T21:25:30.599693+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:16.872425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012923380000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1233,"num_read_kb":1266,"num_write":106,"num_write_kb":140,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":641,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588419+0000","last_change":"2023-09-23T21:23:40.007033+0000","last_active":"2023-09-23T21:25:29.588419+0000","last_peered":"2023-09-23T21:25:29.588419+0000","last_clean":"2023-09-23T21:25:29.588419+0000","last_became_active":"2023-09-23T21:23:40.006711+0000","last_became_peered":"2023-09-23T21:23:40.006711+0000","last_unstale":"2023-09-23T21:25:29.588419+0000","last_undegraded":"2023-09-23T21:25:29.588419+0000","last_fullsized":"2023-09-23T21:25:29.588419+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:35.819332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":102,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591843+0000","last_change":"2023-09-23T21:20:59.523105+0000","last_active":"2023-09-23T21:25:29.591843+0000","last_peered":"2023-09-23T21:25:29.591843+0000","last_clean":"2023-09-23T21:25:29.591843+0000","last_became_active":"2023-09-23T21:20:59.522387+0000","last_became_peered":"2023-09-23T21:20:59.522387+0000","last_unstale":"2023-09-23T21:25:29.591843+0000","last_undegraded":"2023-09-23T21:25:29.591843+0000","last_fullsized":"2023-09-23T21:25:29.591843+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:40.916368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"78'161","reported_seq":1124,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593412+0000","last_change":"2023-09-23T21:23:42.983841+0000","last_active":"2023-09-23T21:25:29.593412+0000","last_peered":"2023-09-23T21:25:29.593412+0000","last_clean":"2023-09-23T21:25:29.593412+0000","last_became_active":"2023-09-23T21:23:42.983351+0000","last_became_peered":"2023-09-23T21:23:42.983351+0000","last_unstale":"2023-09-23T21:25:29.593412+0000","last_undegraded":"2023-09-23T21:25:29.593412+0000","last_fullsized":"2023-09-23T21:25:29.593412+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":4,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:24.382759+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:24.382759+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:05.422807+0000","scrub_duration":0.0048642520000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"689'92","reported_seq":1137,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589206+0000","last_change":"2023-09-23T21:24:03.649812+0000","last_active":"2023-09-23T21:25:30.589206+0000","last_peered":"2023-09-23T21:25:30.589206+0000","last_clean":"2023-09-23T21:25:30.589206+0000","last_became_active":"2023-09-23T21:23:40.012274+0000","last_became_peered":"2023-09-23T21:23:40.012274+0000","last_unstale":"2023-09-23T21:25:30.589206+0000","last_undegraded":"2023-09-23T21:25:30.589206+0000","last_fullsized":"2023-09-23T21:25:30.589206+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:30.294423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032399600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":580,"num_read_kb":441,"num_write":77,"num_write_kb":1064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591517+0000","last_change":"2023-09-23T21:20:59.527283+0000","last_active":"2023-09-23T21:25:29.591517+0000","last_peered":"2023-09-23T21:25:29.591517+0000","last_clean":"2023-09-23T21:25:29.591517+0000","last_became_active":"2023-09-23T21:20:59.527003+0000","last_became_peered":"2023-09-23T21:20:59.527003+0000","last_unstale":"2023-09-23T21:25:29.591517+0000","last_undegraded":"2023-09-23T21:25:29.591517+0000","last_fullsized":"2023-09-23T21:25:29.591517+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:02.896644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":126,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.592322+0000","last_change":"2023-09-23T21:24:30.507847+0000","last_active":"2023-09-23T21:25:29.592322+0000","last_peered":"2023-09-23T21:25:29.592322+0000","last_clean":"2023-09-23T21:25:29.592322+0000","last_became_active":"2023-09-23T21:24:30.507507+0000","last_became_peered":"2023-09-23T21:24:30.507507+0000","last_unstale":"2023-09-23T21:25:29.592322+0000","last_undegraded":"2023-09-23T21:25:29.592322+0000","last_fullsized":"2023-09-23T21:25:29.592322+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:51.885487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"689'175","reported_seq":1150,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.599628+0000","last_change":"2023-09-23T21:25:30.599628+0000","last_active":"2023-09-23T21:25:30.599628+0000","last_peered":"2023-09-23T21:25:30.599628+0000","last_clean":"2023-09-23T21:25:30.599628+0000","last_became_active":"2023-09-23T21:25:30.599132+0000","last_became_peered":"2023-09-23T21:25:30.599132+0000","last_unstale":"2023-09-23T21:25:30.599628+0000","last_undegraded":"2023-09-23T21:25:30.599628+0000","last_fullsized":"2023-09-23T21:25:30.599628+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:30.509053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080853800000000003,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":841,"num_read_kb":768,"num_write":143,"num_write_kb":4181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"78'161","reported_seq":1125,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593136+0000","last_change":"2023-09-23T21:23:42.981131+0000","last_active":"2023-09-23T21:25:29.593136+0000","last_peered":"2023-09-23T21:25:29.593136+0000","last_clean":"2023-09-23T21:25:29.593136+0000","last_became_active":"2023-09-23T21:23:42.980755+0000","last_became_peered":"2023-09-23T21:23:42.980755+0000","last_unstale":"2023-09-23T21:25:29.593136+0000","last_undegraded":"2023-09-23T21:25:29.593136+0000","last_fullsized":"2023-09-23T21:25:29.593136+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":0,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:18:55.033178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:18:55.033178+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:08.065765+0000","scrub_duration":0.0043722550000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":404,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583057+0000","last_change":"2023-09-23T21:24:59.008039+0000","last_active":"2023-09-23T21:25:29.583057+0000","last_peered":"2023-09-23T21:25:29.583057+0000","last_clean":"2023-09-23T21:25:29.583057+0000","last_became_active":"2023-09-23T21:24:59.007765+0000","last_became_peered":"2023-09-23T21:24:59.007765+0000","last_unstale":"2023-09-23T21:25:29.583057+0000","last_undegraded":"2023-09-23T21:25:29.583057+0000","last_fullsized":"2023-09-23T21:25:29.583057+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:37.732988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":90,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588051+0000","last_change":"2023-09-23T21:21:21.575359+0000","last_active":"2023-09-23T21:25:29.588051+0000","last_peered":"2023-09-23T21:25:29.588051+0000","last_clean":"2023-09-23T21:25:29.588051+0000","last_became_active":"2023-09-23T21:21:21.575075+0000","last_became_peered":"2023-09-23T21:21:21.575075+0000","last_unstale":"2023-09-23T21:25:29.588051+0000","last_undegraded":"2023-09-23T21:25:29.588051+0000","last_fullsized":"2023-09-23T21:25:29.588051+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:28.125868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"78'161","reported_seq":766,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591526+0000","last_change":"2023-09-23T21:23:12.024326+0000","last_active":"2023-09-23T21:25:29.591526+0000","last_peered":"2023-09-23T21:25:29.591526+0000","last_clean":"2023-09-23T21:25:29.591526+0000","last_became_active":"2023-09-23T21:21:21.570976+0000","last_became_peered":"2023-09-23T21:21:21.570976+0000","last_unstale":"2023-09-23T21:25:29.591526+0000","last_undegraded":"2023-09-23T21:25:29.591526+0000","last_fullsized":"2023-09-23T21:25:29.591526+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:23:12.024197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:23:12.024197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:18.945737+0000","scrub_duration":0.0017519600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"689'95","reported_seq":701,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593235+0000","last_change":"2023-09-23T21:24:03.659565+0000","last_active":"2023-09-23T21:25:29.593235+0000","last_peered":"2023-09-23T21:25:29.593235+0000","last_clean":"2023-09-23T21:25:29.593235+0000","last_became_active":"2023-09-23T21:23:40.011597+0000","last_became_peered":"2023-09-23T21:23:40.011597+0000","last_unstale":"2023-09-23T21:25:29.593235+0000","last_undegraded":"2023-09-23T21:25:29.593235+0000","last_fullsized":"2023-09-23T21:25:29.593235+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":95,"log_dups_size":0,"ondisk_log_size":95,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:29:17.045171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000319396,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":513,"num_read_kb":2857,"num_write":87,"num_write_kb":2282,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589728+0000","last_change":"2023-09-23T21:23:40.009291+0000","last_active":"2023-09-23T21:25:30.589728+0000","last_peered":"2023-09-23T21:25:30.589728+0000","last_clean":"2023-09-23T21:25:30.589728+0000","last_became_active":"2023-09-23T21:23:40.008907+0000","last_became_peered":"2023-09-23T21:23:40.008907+0000","last_unstale":"2023-09-23T21:25:30.589728+0000","last_undegraded":"2023-09-23T21:25:30.589728+0000","last_fullsized":"2023-09-23T21:25:30.589728+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:18.940611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.589077+0000","last_change":"2023-09-23T21:23:40.012054+0000","last_active":"2023-09-23T21:25:30.589077+0000","last_peered":"2023-09-23T21:25:30.589077+0000","last_clean":"2023-09-23T21:25:30.589077+0000","last_became_active":"2023-09-23T21:23:40.011675+0000","last_became_peered":"2023-09-23T21:23:40.011675+0000","last_unstale":"2023-09-23T21:25:30.589077+0000","last_undegraded":"2023-09-23T21:25:30.589077+0000","last_fullsized":"2023-09-23T21:25:30.589077+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:02.744420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"78'161","reported_seq":801,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586796+0000","last_change":"2023-09-23T21:23:42.981963+0000","last_active":"2023-09-23T21:25:29.586796+0000","last_peered":"2023-09-23T21:25:29.586796+0000","last_clean":"2023-09-23T21:25:29.586796+0000","last_became_active":"2023-09-23T21:23:42.981346+0000","last_became_peered":"2023-09-23T21:23:42.981346+0000","last_unstale":"2023-09-23T21:25:29.586796+0000","last_undegraded":"2023-09-23T21:25:29.586796+0000","last_fullsized":"2023-09-23T21:25:29.586796+0000","mapping_epoch":674,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:20.535510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:20.535510+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:20.767852+0000","scrub_duration":0.015384745999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"689'105","reported_seq":713,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.593965+0000","last_change":"2023-09-23T21:24:03.659887+0000","last_active":"2023-09-23T21:25:29.593965+0000","last_peered":"2023-09-23T21:25:29.593965+0000","last_clean":"2023-09-23T21:25:29.593965+0000","last_became_active":"2023-09-23T21:23:40.011448+0000","last_became_peered":"2023-09-23T21:23:40.011448+0000","last_unstale":"2023-09-23T21:25:29.593965+0000","last_undegraded":"2023-09-23T21:25:29.593965+0000","last_fullsized":"2023-09-23T21:25:29.593965+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:16.274116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037762899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":682,"num_read_kb":1215,"num_write":94,"num_write_kb":385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":117,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591753+0000","last_change":"2023-09-23T21:24:30.511008+0000","last_active":"2023-09-23T21:25:29.591753+0000","last_peered":"2023-09-23T21:25:29.591753+0000","last_clean":"2023-09-23T21:25:29.591753+0000","last_became_active":"2023-09-23T21:24:30.510625+0000","last_became_peered":"2023-09-23T21:24:30.510625+0000","last_unstale":"2023-09-23T21:25:29.591753+0000","last_undegraded":"2023-09-23T21:25:29.591753+0000","last_fullsized":"2023-09-23T21:25:29.591753+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:50.890363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":257,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.590427+0000","last_change":"2023-09-23T21:24:45.640521+0000","last_active":"2023-09-23T21:25:29.590427+0000","last_peered":"2023-09-23T21:25:29.590427+0000","last_clean":"2023-09-23T21:25:29.590427+0000","last_became_active":"2023-09-23T21:24:45.640198+0000","last_became_peered":"2023-09-23T21:24:45.640198+0000","last_unstale":"2023-09-23T21:25:29.590427+0000","last_undegraded":"2023-09-23T21:25:29.590427+0000","last_fullsized":"2023-09-23T21:25:29.590427+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:02.374401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"78'161","reported_seq":1019,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588753+0000","last_change":"2023-09-23T21:21:21.573089+0000","last_active":"2023-09-23T21:25:29.588753+0000","last_peered":"2023-09-23T21:25:29.588753+0000","last_clean":"2023-09-23T21:25:29.588753+0000","last_became_active":"2023-09-23T21:21:21.572783+0000","last_became_peered":"2023-09-23T21:21:21.572783+0000","last_unstale":"2023-09-23T21:25:29.588753+0000","last_undegraded":"2023-09-23T21:25:29.588753+0000","last_fullsized":"2023-09-23T21:25:29.588753+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:02.336318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:02.336318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:00.853418+0000","scrub_duration":0.018304702999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":311,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583871+0000","last_change":"2023-09-23T21:24:59.003848+0000","last_active":"2023-09-23T21:25:29.583871+0000","last_peered":"2023-09-23T21:25:29.583871+0000","last_clean":"2023-09-23T21:25:29.583871+0000","last_became_active":"2023-09-23T21:24:59.003528+0000","last_became_peered":"2023-09-23T21:24:59.003528+0000","last_unstale":"2023-09-23T21:25:29.583871+0000","last_undegraded":"2023-09-23T21:25:29.583871+0000","last_fullsized":"2023-09-23T21:25:29.583871+0000","mapping_epoch":706,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":707,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:06.635575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"689'98","reported_seq":502,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.591983+0000","last_change":"2023-09-23T21:24:03.654574+0000","last_active":"2023-09-23T21:25:29.591983+0000","last_peered":"2023-09-23T21:25:29.591983+0000","last_clean":"2023-09-23T21:25:29.591983+0000","last_became_active":"2023-09-23T21:20:59.528856+0000","last_became_peered":"2023-09-23T21:20:59.528856+0000","last_unstale":"2023-09-23T21:25:29.591983+0000","last_undegraded":"2023-09-23T21:25:29.591983+0000","last_fullsized":"2023-09-23T21:25:29.591983+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":98,"log_dups_size":0,"ondisk_log_size":98,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:50.478298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027368,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":619,"num_read_kb":2739,"num_write":55,"num_write_kb":2178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583103+0000","last_change":"2023-09-23T21:23:40.006639+0000","last_active":"2023-09-23T21:25:29.583103+0000","last_peered":"2023-09-23T21:25:29.583103+0000","last_clean":"2023-09-23T21:25:29.583103+0000","last_became_active":"2023-09-23T21:23:40.006309+0000","last_became_peered":"2023-09-23T21:23:40.006309+0000","last_unstale":"2023-09-23T21:25:29.583103+0000","last_undegraded":"2023-09-23T21:25:29.583103+0000","last_fullsized":"2023-09-23T21:25:29.583103+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:04.424515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"78'161","reported_seq":1130,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583589+0000","last_change":"2023-09-23T21:24:30.511083+0000","last_active":"2023-09-23T21:25:29.583589+0000","last_peered":"2023-09-23T21:25:29.583589+0000","last_clean":"2023-09-23T21:25:29.583589+0000","last_became_active":"2023-09-23T21:24:30.510633+0000","last_became_peered":"2023-09-23T21:24:30.510633+0000","last_unstale":"2023-09-23T21:25:29.583589+0000","last_undegraded":"2023-09-23T21:25:29.583589+0000","last_fullsized":"2023-09-23T21:25:29.583589+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":582,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":3,"last_scrub":"78'161","last_scrub_stamp":"2023-09-23T21:19:25.411715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:26:58.272483+0000","last_clean_scrub_stamp":"2023-09-23T21:19:25.411715+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:57.269339+0000","scrub_duration":0.030778138999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"689'66","reported_seq":872,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587750+0000","last_change":"2023-09-23T21:24:03.657789+0000","last_active":"2023-09-23T21:25:29.587750+0000","last_peered":"2023-09-23T21:25:29.587750+0000","last_clean":"2023-09-23T21:25:29.587750+0000","last_became_active":"2023-09-23T21:12:51.653285+0000","last_became_peered":"2023-09-23T21:12:51.653285+0000","last_unstale":"2023-09-23T21:25:29.587750+0000","last_undegraded":"2023-09-23T21:25:29.587750+0000","last_fullsized":"2023-09-23T21:25:29.587750+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:29:27.041758+0000","last_clean_scrub_stamp":"2023-09-23T20:29:27.041758+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:56.542836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031051700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":342,"num_read_kb":1892,"num_write":51,"num_write_kb":1333,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":63,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588998+0000","last_change":"2023-09-23T21:23:40.017705+0000","last_active":"2023-09-23T21:25:30.588998+0000","last_peered":"2023-09-23T21:25:30.588998+0000","last_clean":"2023-09-23T21:25:30.588998+0000","last_became_active":"2023-09-23T21:23:40.007892+0000","last_became_peered":"2023-09-23T21:23:40.007892+0000","last_unstale":"2023-09-23T21:25:30.588998+0000","last_undegraded":"2023-09-23T21:25:30.588998+0000","last_fullsized":"2023-09-23T21:25:30.588998+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:09.348317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":264,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.587789+0000","last_change":"2023-09-23T21:25:06.031614+0000","last_active":"2023-09-23T21:25:29.587789+0000","last_peered":"2023-09-23T21:25:29.587789+0000","last_clean":"2023-09-23T21:25:29.587789+0000","last_became_active":"2023-09-23T21:25:06.031295+0000","last_became_peered":"2023-09-23T21:25:06.031295+0000","last_unstale":"2023-09-23T21:25:29.587789+0000","last_undegraded":"2023-09-23T21:25:29.587789+0000","last_fullsized":"2023-09-23T21:25:29.587789+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:41.309598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":59,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588786+0000","last_change":"2023-09-23T21:23:40.014466+0000","last_active":"2023-09-23T21:25:30.588786+0000","last_peered":"2023-09-23T21:25:30.588786+0000","last_clean":"2023-09-23T21:25:30.588786+0000","last_became_active":"2023-09-23T21:23:40.014118+0000","last_became_peered":"2023-09-23T21:23:40.014118+0000","last_unstale":"2023-09-23T21:25:30.588786+0000","last_undegraded":"2023-09-23T21:25:30.588786+0000","last_fullsized":"2023-09-23T21:25:30.588786+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:21.412255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":812,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.590487+0000","last_change":"2023-09-23T21:20:59.519203+0000","last_active":"2023-09-23T21:25:29.590487+0000","last_peered":"2023-09-23T21:25:29.590487+0000","last_clean":"2023-09-23T21:25:29.590487+0000","last_became_active":"2023-09-23T21:20:59.518832+0000","last_became_peered":"2023-09-23T21:20:59.518832+0000","last_unstale":"2023-09-23T21:25:29.590487+0000","last_undegraded":"2023-09-23T21:25:29.590487+0000","last_fullsized":"2023-09-23T21:25:29.590487+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:47.088421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":369,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.597922+0000","last_change":"2023-09-23T21:25:30.597922+0000","last_active":"2023-09-23T21:25:30.597922+0000","last_peered":"2023-09-23T21:25:30.597922+0000","last_clean":"2023-09-23T21:25:30.597922+0000","last_became_active":"2023-09-23T21:25:30.597592+0000","last_became_peered":"2023-09-23T21:25:30.597592+0000","last_unstale":"2023-09-23T21:25:30.597922+0000","last_undegraded":"2023-09-23T21:25:30.597922+0000","last_fullsized":"2023-09-23T21:25:30.597922+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:32.759345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":331,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.588195+0000","last_change":"2023-09-23T21:24:45.399000+0000","last_active":"2023-09-23T21:25:29.588195+0000","last_peered":"2023-09-23T21:25:29.588195+0000","last_clean":"2023-09-23T21:25:29.588195+0000","last_became_active":"2023-09-23T21:24:45.398564+0000","last_became_peered":"2023-09-23T21:24:45.398564+0000","last_unstale":"2023-09-23T21:25:29.588195+0000","last_undegraded":"2023-09-23T21:25:29.588195+0000","last_fullsized":"2023-09-23T21:25:29.588195+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:25.762025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":134,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.600360+0000","last_change":"2023-09-23T21:25:30.600360+0000","last_active":"2023-09-23T21:25:30.600360+0000","last_peered":"2023-09-23T21:25:30.600360+0000","last_clean":"2023-09-23T21:25:30.600360+0000","last_became_active":"2023-09-23T21:25:30.599915+0000","last_became_peered":"2023-09-23T21:25:30.599915+0000","last_unstale":"2023-09-23T21:25:30.600360+0000","last_undegraded":"2023-09-23T21:25:30.600360+0000","last_fullsized":"2023-09-23T21:25:30.600360+0000","mapping_epoch":718,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":719,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:46.090965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":171,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586927+0000","last_change":"2023-09-23T21:25:06.323886+0000","last_active":"2023-09-23T21:25:29.586927+0000","last_peered":"2023-09-23T21:25:29.586927+0000","last_clean":"2023-09-23T21:25:29.586927+0000","last_became_active":"2023-09-23T21:25:06.323433+0000","last_became_peered":"2023-09-23T21:25:06.323433+0000","last_unstale":"2023-09-23T21:25:29.586927+0000","last_undegraded":"2023-09-23T21:25:29.586927+0000","last_fullsized":"2023-09-23T21:25:29.586927+0000","mapping_epoch":709,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":710,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:40.608942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583698+0000","last_change":"2023-09-23T21:21:21.569587+0000","last_active":"2023-09-23T21:25:29.583698+0000","last_peered":"2023-09-23T21:25:29.583698+0000","last_clean":"2023-09-23T21:25:29.583698+0000","last_became_active":"2023-09-23T21:21:21.569222+0000","last_became_peered":"2023-09-23T21:21:21.569222+0000","last_unstale":"2023-09-23T21:25:29.583698+0000","last_undegraded":"2023-09-23T21:25:29.583698+0000","last_fullsized":"2023-09-23T21:25:29.583698+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:36.018446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.583231+0000","last_change":"2023-09-23T21:23:40.009045+0000","last_active":"2023-09-23T21:25:29.583231+0000","last_peered":"2023-09-23T21:25:29.583231+0000","last_clean":"2023-09-23T21:25:29.583231+0000","last_became_active":"2023-09-23T21:23:40.008753+0000","last_became_peered":"2023-09-23T21:23:40.008753+0000","last_unstale":"2023-09-23T21:25:29.583231+0000","last_undegraded":"2023-09-23T21:25:29.583231+0000","last_fullsized":"2023-09-23T21:25:29.583231+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:17:28.136832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":80,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-09-23T21:25:30.588832+0000","last_change":"2023-09-23T21:24:30.508673+0000","last_active":"2023-09-23T21:25:30.588832+0000","last_peered":"2023-09-23T21:25:30.588832+0000","last_clean":"2023-09-23T21:25:30.588832+0000","last_became_active":"2023-09-23T21:24:30.508002+0000","last_became_peered":"2023-09-23T21:24:30.508002+0000","last_unstale":"2023-09-23T21:25:30.588832+0000","last_undegraded":"2023-09-23T21:25:30.588832+0000","last_fullsized":"2023-09-23T21:25:30.588832+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:21.089733+0000","last_clean_scrub_stamp":"2023-09-23T20:30:21.089733+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:22.538829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":425,"reported_epoch":718,"state":"active+clean","last_fresh":"2023-09-23T21:25:29.586679+0000","last_change":"2023-09-23T21:21:21.572914+0000","last_active":"2023-09-23T21:25:29.586679+0000","last_peered":"2023-09-23T21:25:29.586679+0000","last_clean":"2023-09-23T21:25:29.586679+0000","last_became_active":"2023-09-23T21:21:21.572619+0000","last_became_peered":"2023-09-23T21:21:21.572619+0000","last_unstale":"2023-09-23T21:25:29.586679+0000","last_undegraded":"2023-09-23T21:25:29.586679+0000","last_fullsized":"2023-09-23T21:25:29.586679+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:30:22.093910+0000","last_clean_scrub_stamp":"2023-09-23T20:30:22.093910+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:23.854342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":36346,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20862,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":853056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13204,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201970,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":933760,"kb_used_data":11768,"kb_used_omap":0,"kb_used_meta":921984,"kb_avail":92811392,"statfs":{"total":95995035648,"available":95038865408,"internally_reserved":0,"allocated":12050432,"data_stored":10195082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":944111616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.381,"15min":2.347},"min":{"1min":0.35999999999999999,"5min":0.252,"15min":0.214},"max":{"1min":0.72499999999999998,"5min":10.522,"15min":141.33199999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.4359999999999999,"15min":2.48},"min":{"1min":0.51600000000000001,"5min":0.29799999999999999,"15min":0.23799999999999999},"max":{"1min":0.79800000000000004,"5min":12.962,"15min":141.31700000000001},"last":0.72199999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.8500000000000001,"15min":2.504},"min":{"1min":0.432,"5min":0.28000000000000003,"15min":0.24099999999999999},"max":{"1min":0.73999999999999999,"5min":31.233000000000001,"15min":76.007999999999996},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.0550000000000002,"15min":2.883},"min":{"1min":0.443,"5min":0.23799999999999999,"15min":0.22900000000000001},"max":{"1min":0.73899999999999999,"5min":31.219000000000001,"15min":117.637},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Sep 23 21:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":3.9609999999999999,"15min":4.593},"min":{"1min":0.372,"5min":0.252,"15min":0.224},"max":{"1min":0.80300000000000005,"5min":157.65700000000001,"15min":158.726},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":4.0039999999999996,"15min":4.8600000000000003},"min":{"1min":0.41099999999999998,"5min":0.27200000000000002,"15min":0.23200000000000001},"max":{"1min":0.70599999999999996,"5min":157.642,"15min":158.70400000000001},"last":0.47599999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":2.056,"15min":4.0819999999999999},"min":{"1min":0.41899999999999998,"5min":0.312,"15min":0.28799999999999998},"max":{"1min":0.74099999999999999,"5min":30.667999999999999,"15min":135.66200000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58899999999999997,"5min":2.202,"15min":4.3369999999999997},"min":{"1min":0.35799999999999998,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":0.71899999999999997,"5min":30.798999999999999,"15min":135.47},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 23 21:25:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":2.2749999999999999,"15min":2.2029999999999998},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":0.91700000000000004,"5min":13.821999999999999,"15min":31.695},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.847,"15min":2.3180000000000001},"min":{"1min":0.434,"5min":0.26900000000000002,"15min":0.26800000000000002},"max":{"1min":0.89900000000000002,"5min":27.821999999999999,"15min":34.994},"last":0.65200000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:25:29 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.85899999999999999}]},{"osd":6,"last update":"Sat Sep 23 21:25:55 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":2.234,"15min":2.7370000000000001},"min":{"1min":0.42999999999999999,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":0.997,"5min":14.066000000000001,"15min":143.18700000000001},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":2.4279999999999999,"15min":2.4209999999999998},"min":{"1min":0.40100000000000002,"5min":0.33000000000000002,"15min":0.29099999999999998},"max":{"1min":0.80700000000000005,"5min":21.081,"15min":142.70400000000001},"last":0.68899999999999995}]}]},{"osd":6,"up_from":47,"seq":201863463590,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927028,"kb_used_data":7340,"kb_used_omap":0,"kb_used_meta":919680,"kb_avail":92818124,"statfs":{"total":95995035648,"available":95045758976,"internally_reserved":0,"allocated":7516160,"data_stored":5672764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":941752320},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.4319999999999999,"15min":2.5390000000000001},"min":{"1min":0.29299999999999998,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":1.079,"5min":12.313000000000001,"15min":149.61600000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.4730000000000001,"15min":2.6989999999999998},"min":{"1min":0.33200000000000002,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":1.5189999999999999,"5min":11.968,"15min":149.303},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:25:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.2829999999999999,"15min":2.5990000000000002},"min":{"1min":0.33600000000000002,"5min":0.247,"15min":0.22700000000000001},"max":{"1min":0.86099999999999999,"5min":12.013,"15min":129.37899999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.431,"15min":2.5030000000000001},"min":{"1min":0.372,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.92100000000000004,"5min":14.366,"15min":129.40100000000001},"last":0.53600000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":6.5830000000000002,"15min":6.4989999999999997},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25800000000000001},"max":{"1min":0.95199999999999996,"5min":199.08000000000001,"15min":199.08000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.76400000000000001,"5min":6.758,"15min":6.7450000000000001},"min":{"1min":0.41899999999999998,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":4.1589999999999998,"5min":199.27000000000001,"15min":199.27000000000001},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.6639999999999999,"15min":3.4300000000000002},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.218},"max":{"1min":1.008,"5min":14.411,"15min":137.875},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.5760000000000001,"15min":3.274},"min":{"1min":0.41099999999999998,"5min":0.315,"15min":0.29799999999999999},"max":{"1min":0.99199999999999999,"5min":12.35,"15min":137.60900000000001},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":2.4660000000000002,"15min":2.5259999999999998},"min":{"1min":0.29999999999999999,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":0.77200000000000002,"5min":20.408999999999999,"15min":55.494},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":2.6669999999999998,"15min":2.4529999999999998},"min":{"1min":0.33600000000000002,"5min":0.255,"15min":0.23999999999999999},"max":{"1min":0.96999999999999997,"5min":14.622,"15min":55.305},"last":0.50600000000000001}]},{"osd":5,"last update":"Sat Sep 23 21:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.008,"5min":1.008,"15min":1.008},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.37}]},{"osd":7,"last update":"Sat Sep 23 21:25:45 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.8939999999999999,"15min":2.391},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.28000000000000003},"max":{"1min":0.86399999999999999,"5min":12.779,"15min":165.96799999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":2.073,"15min":2.7480000000000002},"min":{"1min":0.46400000000000002,"5min":0.307,"15min":0.251},"max":{"1min":0.93200000000000005,"5min":14.457000000000001,"15min":165.82400000000001},"last":0.44800000000000001}]}]},{"osd":1,"up_from":276,"seq":1185410974037,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":662280,"kb_used_data":8960,"kb_used_omap":5,"kb_used_meta":653306,"kb_avail":93082872,"statfs":{"total":95995035648,"available":95316860928,"internally_reserved":0,"allocated":9175040,"data_stored":5848596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":668986083},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39900000000000002,"15min":1.343},"min":{"1min":0.251,"5min":0.214,"15min":0.19400000000000001},"max":{"1min":0.54000000000000004,"5min":2.3410000000000002,"15min":108.47},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.32700000000000001,"5min":0.63600000000000001,"15min":1.446},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.19600000000000001},"max":{"1min":0.44800000000000001,"5min":6.0149999999999997,"15min":109.532},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":4.4349999999999996,"15min":7.5430000000000001},"min":{"1min":0.28699999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.54800000000000004,"5min":130.78899999999999,"15min":171.35400000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.38200000000000001,"5min":4.3849999999999998,"15min":7.2619999999999996},"min":{"1min":0.27200000000000002,"5min":0.20899999999999999,"15min":0.182},"max":{"1min":0.50800000000000001,"5min":133.751,"15min":170.69900000000001},"last":0.33900000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.624,"15min":2.4420000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":0.52900000000000003,"5min":17.247,"15min":99.120999999999995},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.70799999999999996,"15min":3.177},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":0.54300000000000004,"5min":10.949999999999999,"15min":150.38},"last":0.39100000000000001}]},{"osd":4,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.3029999999999999,"15min":2.1760000000000002},"min":{"1min":0.374,"5min":0.27400000000000002,"15min":0.26800000000000002},"max":{"1min":0.91400000000000003,"5min":9.5800000000000001,"15min":174.84399999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.63700000000000001,"5min":1.05,"15min":1.877},"min":{"1min":0.49099999999999999,"5min":0.29899999999999999,"15min":0.29199999999999998},"max":{"1min":0.85099999999999998,"5min":8.5109999999999992,"15min":174.82400000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 21:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.784,"15min":2.2770000000000001},"min":{"1min":0.442,"5min":0.33900000000000002,"15min":0.28999999999999998},"max":{"1min":0.92400000000000004,"5min":20.265999999999998,"15min":119.999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.5269999999999999,"15min":2.2250000000000001},"min":{"1min":0.46600000000000003,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":0.79400000000000004,"5min":20.361999999999998,"15min":122.328},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.6080000000000001,"15min":2.7719999999999998},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":0.85799999999999998,"5min":10.648999999999999,"15min":285.47500000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.2709999999999999,"15min":2.379},"min":{"1min":0.504,"5min":0.38600000000000001,"15min":0.34899999999999998},"max":{"1min":0.878,"5min":10.664999999999999,"15min":285.48599999999999},"last":0.53500000000000003}]}]},{"osd":0,"up_from":92,"seq":395136991777,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":864832,"kb_used_data":8824,"kb_used_omap":5,"kb_used_meta":855994,"kb_avail":92880320,"statfs":{"total":95995035648,"available":95109447680,"internally_reserved":0,"allocated":9035776,"data_stored":5826789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":876538490},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.67900000000000005,"15min":1.038},"min":{"1min":0.221,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.97599999999999998,"5min":11.109,"15min":11.782999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.54000000000000004,"15min":1.175},"min":{"1min":0.23599999999999999,"5min":0.20799999999999999,"15min":0.20200000000000001},"max":{"1min":0.70599999999999996,"5min":6.6319999999999997,"15min":15.266},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":2.375,"15min":5.3129999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.19800000000000001},"max":{"1min":0.91500000000000004,"5min":96.641000000000005,"15min":201.32300000000001},"last":0.439},{"interface":"front","average":{"1min":0.42199999999999999,"5min":2.3109999999999999,"15min":5.3899999999999997},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.245},"max":{"1min":0.78800000000000003,"5min":98.347999999999999,"15min":201.43199999999999},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.60499999999999998,"15min":1.698},"min":{"1min":0.28000000000000003,"5min":0.217,"15min":0.217},"max":{"1min":0.95399999999999996,"5min":11.077,"15min":86.989999999999995},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.57699999999999996,"15min":1.6839999999999999},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.23599999999999999},"max":{"1min":0.78800000000000003,"5min":4.1820000000000004,"15min":87.043999999999997},"last":0.45800000000000002}]},{"osd":4,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.802,"15min":1.8720000000000001},"min":{"1min":0.43099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.04,"5min":39.228000000000002,"15min":39.228000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.8260000000000001,"15min":1.6579999999999999},"min":{"1min":0.32800000000000001,"5min":0.28699999999999998,"15min":0.28000000000000003},"max":{"1min":0.879,"5min":39.340000000000003,"15min":39.340000000000003},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 23 21:25:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":1.855,"15min":1.9059999999999999},"min":{"1min":0.41199999999999998,"5min":0.34999999999999998,"15min":0.29999999999999999},"max":{"1min":0.93600000000000005,"5min":56.213000000000001,"15min":56.213000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.194,"15min":1.849},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.85399999999999998,"5min":53.424999999999997,"15min":53.424999999999997},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:25:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.6850000000000001,"15min":1.599},"min":{"1min":0.35199999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.88100000000000001,"5min":32.411999999999999,"15min":32.411999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":1.544,"15min":1.591},"min":{"1min":0.441,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.89600000000000002,"5min":32.433999999999997,"15min":32.433999999999997},"last":0.52800000000000002}]}]},{"osd":2,"up_from":129,"seq":554050781668,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1359348,"kb_used_data":13420,"kb_used_omap":7,"kb_used_meta":1345912,"kb_avail":92385804,"statfs":{"total":95995035648,"available":94603063296,"internally_reserved":0,"allocated":13742080,"data_stored":10306997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1378213969},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.1799999999999999,"15min":1.7470000000000001},"min":{"1min":0.22500000000000001,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":0.46700000000000003,"5min":51.968000000000004,"15min":103.021},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.34000000000000002,"5min":1.288,"15min":1.974},"min":{"1min":0.26800000000000002,"5min":0.22900000000000001,"15min":0.218},"max":{"1min":0.443,"5min":52.241999999999997,"15min":103.154},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Sep 23 21:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.61599999999999999,"15min":2.1760000000000002},"min":{"1min":0.247,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.50800000000000001,"5min":6.4470000000000001,"15min":143.482},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.54600000000000004,"15min":2.3700000000000001},"min":{"1min":0.23899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.54900000000000004,"5min":3.27,"15min":143.61000000000001},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.69499999999999995,"15min":2.1909999999999998},"min":{"1min":0.28799999999999998,"5min":0.22600000000000001,"15min":0.20499999999999999},"max":{"1min":0.54400000000000004,"5min":6.4039999999999999,"15min":143.50800000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.58399999999999996,"15min":2.4199999999999999},"min":{"1min":0.309,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52000000000000002,"5min":4.2670000000000003,"15min":143.27799999999999},"last":0.29999999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.5620000000000001,"15min":3.371},"min":{"1min":0.441,"5min":0.307,"15min":0.27800000000000002},"max":{"1min":0.72699999999999998,"5min":8.5820000000000007,"15min":225.80799999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.609,"15min":3.6360000000000001},"min":{"1min":0.34999999999999998,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":0.76500000000000001,"5min":8.5690000000000008,"15min":225.79300000000001},"last":0.41299999999999998}]},{"osd":5,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.629}]},{"osd":6,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.5640000000000001,"15min":3.3180000000000001},"min":{"1min":0.38800000000000001,"5min":0.38,"15min":0.31900000000000001},"max":{"1min":0.81899999999999995,"5min":10.148,"15min":210.74700000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.4770000000000001,"15min":2.6629999999999998},"min":{"1min":0.41999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.81899999999999995,"5min":10.634,"15min":210.72900000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 21:25:15 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":2.1920000000000002,"15min":2.0680000000000001},"min":{"1min":0.47699999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":0.75,"5min":52.121000000000002,"15min":52.121000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":2.2610000000000001,"15min":2.2010000000000001},"min":{"1min":0.45600000000000002,"5min":0.39800000000000002,"15min":0.32700000000000001},"max":{"1min":0.84399999999999997,"5min":52.308999999999997,"15min":52.308999999999997},"last":0.52700000000000002}]}]},{"osd":3,"up_from":148,"seq":635655160261,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906764,"kb_used_data":10180,"kb_used_omap":2,"kb_used_meta":896573,"kb_avail":92838388,"statfs":{"total":95995035648,"available":95066509312,"internally_reserved":0,"allocated":10424320,"data_stored":6494439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":918090805},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.53300000000000003,"15min":1.3049999999999999},"min":{"1min":0.25900000000000001,"5min":0.19800000000000001,"15min":0.17999999999999999},"max":{"1min":0.46899999999999997,"5min":4.7430000000000003,"15min":25.722999999999999},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.64300000000000002,"15min":1.234},"min":{"1min":0.26300000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.495,"5min":5.0720000000000001,"15min":18.277999999999999},"last":0.30099999999999999}]},{"osd":1,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.79900000000000004,"15min":2.8650000000000002},"min":{"1min":0.253,"5min":0.214,"15min":0.214},"max":{"1min":0.56799999999999995,"5min":8.3079999999999998,"15min":147.12100000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.68799999999999994,"15min":2.839},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.20999999999999999},"max":{"1min":0.53900000000000003,"5min":8.5220000000000002,"15min":151.14699999999999},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":3.8490000000000002,"15min":4.6529999999999996},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.215},"max":{"1min":0.61799999999999999,"5min":207.59999999999999,"15min":207.59999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":3.8650000000000002,"15min":4.4089999999999998},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.217},"max":{"1min":0.52600000000000002,"5min":208.024,"15min":208.024},"last":0.504}]},{"osd":4,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.8540000000000001,"15min":2.3069999999999999},"min":{"1min":0.39600000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.83299999999999996,"5min":13.640000000000001,"15min":117.717},"last":0.622},{"interface":"front","average":{"1min":0.58199999999999996,"5min":2.0539999999999998,"15min":2.3820000000000001},"min":{"1min":0.46100000000000002,"5min":0.29999999999999999,"15min":0.28799999999999998},"max":{"1min":0.75600000000000001,"5min":11.968999999999999,"15min":117.771},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Sep 23 21:25:33 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.60299999999999998}]},{"osd":6,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.4299999999999999,"15min":3.3410000000000002},"min":{"1min":0.41399999999999998,"5min":0.33000000000000002,"15min":0.316},"max":{"1min":0.73999999999999999,"5min":7.7889999999999997,"15min":130.613},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.5209999999999999,"15min":3.488},"min":{"1min":0.47199999999999998,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":0.79100000000000004,"5min":8.8940000000000001,"15min":130.50999999999999},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 21:25:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.508,"15min":1.9630000000000001},"min":{"1min":0.45500000000000002,"5min":0.36799999999999999,"15min":0.34000000000000002},"max":{"1min":0.81200000000000006,"5min":19.298999999999999,"15min":44.253},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.867,"15min":2.3260000000000001},"min":{"1min":0.47899999999999998,"5min":0.41699999999999998,"15min":0.36699999999999999},"max":{"1min":0.91100000000000003,"5min":10.441000000000001,"15min":45.706000000000003},"last":0.58299999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986880,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986304,"kb_used_data":7352,"kb_used_omap":0,"kb_used_meta":978944,"kb_avail":92758848,"statfs":{"total":95995035648,"available":94985060352,"internally_reserved":0,"allocated":7528448,"data_stored":5673351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1002438656},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.8069999999999999,"15min":2.8610000000000002},"min":{"1min":0.32000000000000001,"5min":0.224,"15min":0.20699999999999999},"max":{"1min":0.84699999999999998,"5min":12.195,"15min":226.751},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":1.387,"15min":2.504},"min":{"1min":0.32500000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.83899999999999997,"5min":16.478000000000002,"15min":226.43299999999999},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.345,"15min":2.5049999999999999},"min":{"1min":0.40600000000000003,"5min":0.246,"15min":0.246},"max":{"1min":6.4859999999999998,"5min":16.434999999999999,"15min":119.55800000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.6379999999999999,"15min":2.5750000000000002},"min":{"1min":0.35999999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.04,"5min":12.108000000000001,"15min":116.78100000000001},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 23 21:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":4.6500000000000004,"15min":5.7460000000000004},"min":{"1min":0.40699999999999997,"5min":0.30599999999999999,"15min":0.248},"max":{"1min":0.80500000000000005,"5min":124.70099999999999,"15min":210.50299999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":3.952,"15min":5.7990000000000004},"min":{"1min":0.437,"5min":0.28799999999999998,"15min":0.26800000000000002},"max":{"1min":0.78300000000000003,"5min":116.47,"15min":210.59},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.853,"15min":3.8359999999999999},"min":{"1min":0.441,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":5.7839999999999998,"5min":12.973000000000001,"15min":214.64099999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.9279999999999999,"15min":4.2080000000000002},"min":{"1min":0.45700000000000002,"5min":0.33200000000000002,"15min":0.32500000000000001},"max":{"1min":6.1520000000000001,"5min":13.673999999999999,"15min":214.54900000000001},"last":0.64700000000000002}]},{"osd":5,"last update":"Sat Sep 23 21:25:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.82599999999999996}]},{"osd":6,"last update":"Sat Sep 23 21:25:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":3.3180000000000001,"15min":2.3900000000000001},"min":{"1min":0.35099999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.76000000000000001,"5min":67.697999999999993,"15min":67.697999999999993},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":3.931,"15min":2.7890000000000001},"min":{"1min":0.39100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.89000000000000001,"5min":67.596999999999994,"15min":67.596999999999994},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Sep 23 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":2.3889999999999998,"15min":2.2160000000000002},"min":{"1min":0.46000000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.84799999999999998,"5min":16.998999999999999,"15min":18.352},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.66700000000000004,"5min":2.0870000000000002,"15min":2.0670000000000002},"min":{"1min":0.40999999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.90300000000000002,"5min":16.664999999999999,"15min":30.117000000000001},"last":0.85799999999999998}]}]},{"osd":5,"up_from":694,"seq":2980707303446,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":122616,"kb_used_data":81584,"kb_used_omap":78,"kb_used_meta":40945,"kb_avail":93622536,"statfs":{"total":95995035648,"available":95869476864,"internally_reserved":0,"allocated":83542016,"data_stored":81691418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":80826,"internal_metadata":41927750},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.502}]},{"osd":1,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.59299999999999997}]},{"osd":2,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53900000000000003}]},{"osd":6,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.57899999999999996}]},{"osd":7,"last update":"Sat Sep 23 21:25:30 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.442},{"interface":"front","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.64100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5405,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T21:26:10.274 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T21:26:10.274 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T21:26:10.340 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=1164e90def6b4284b737a144248cdc75 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T21:26:10.342 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T21:26:10.354 INFO:tasks.cephadm:Teardown begin 2023-09-23T21:26:10.354 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:26:10.420 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:26:10.466 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:10 smithi123 ceph-mon[193903]: pgmap v2157: 83 pgs: 83 active+clean; 4.6 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-09-23T21:26:10.563 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T21:26:10.563 DEBUG:teuthology.orchestra.run.smithi121:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T21:26:10.580 DEBUG:teuthology.orchestra.run.smithi123:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T21:26:10.600 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T21:26:10.600 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T21:26:10.600 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T21:26:10.912 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:10 smithi121 systemd[1]: Stopping Ceph mon.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:11.162 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:10 smithi121 conmon[211495]: 2023-09-23T21:26:10.911+0000 7fdc56471700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:10 smithi121 conmon[211495]: 2023-09-23T21:26:10.911+0000 7fdc56471700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T21:26:11.163 INFO:journalctl@ceph.mon.a.smithi121.stdout:Sep 23 21:26:11 smithi121 bash[793687]: 41cbfbe73b6ea1e161be620558526c576fa2e3554d617a5ddcf12c2091ee6c8a 2023-09-23T21:26:11.815 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T21:26:11.864 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:11.864 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T21:26:11.864 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T21:26:11.864 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T21:26:12.163 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:11 smithi121 systemd[1]: Stopping Ceph mon.c for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:12.454 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[198376]: [23/Sep/2023:21:26:12] ENGINE Bus STOPPING 2023-09-23T21:26:12.454 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[198376]: [23/Sep/2023:21:26:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T21:26:12.454 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[198376]: [23/Sep/2023:21:26:12] ENGINE Bus STOPPED 2023-09-23T21:26:12.454 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[198376]: [23/Sep/2023:21:26:12] ENGINE Bus STARTING 2023-09-23T21:26:12.454 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[198376]: [23/Sep/2023:21:26:12] ENGINE Serving on http://:::9283 2023-09-23T21:26:12.455 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[198376]: [23/Sep/2023:21:26:12] ENGINE Bus STARTED 2023-09-23T21:26:12.455 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[219682]: 2023-09-23T21:26:12.177+0000 7f54ae147700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:12.455 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:12 smithi121 conmon[219682]: 2023-09-23T21:26:12.177+0000 7f54ae147700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T21:26:12.455 INFO:journalctl@ceph.mon.c.smithi121.stdout:Sep 23 21:26:12 smithi121 bash[793991]: 828ec0e17fe967965a3e8392decf1d53622aa9149206cf5e071a252eec920aa7 2023-09-23T21:26:13.045 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T21:26:13.084 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:13.084 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T21:26:13.084 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T21:26:13.084 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T21:26:13.407 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:13 smithi123 systemd[1]: Stopping Ceph mon.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:13.408 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:13 smithi123 conmon[193880]: 2023-09-23T21:26:13.405+0000 7f1e0eed8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:13.408 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:13 smithi123 conmon[193880]: 2023-09-23T21:26:13.405+0000 7f1e0eed8700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T21:26:14.228 INFO:journalctl@ceph.mon.b.smithi123.stdout:Sep 23 21:26:13 smithi123 bash[460180]: 5eccbfe5a4b758c5f36cf1ddc0550ba21a2429fd99cf8820d2cb8acdbb36ea82 2023-09-23T21:26:14.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T21:26:14.552 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:14.552 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T21:26:14.552 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T21:26:14.552 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T21:26:14.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:14 smithi121 systemd[1]: Stopping Ceph mgr.y for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:14.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:14 smithi121 conmon[198376]: ::ffff:172.21.15.123 - - [23/Sep/2023:21:26:14] "GET /metrics HTTP/1.1" 200 34297 "" "Prometheus/2.43.0" 2023-09-23T21:26:14.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:14 smithi121 conmon[198376]: 2023-09-23T21:26:14.836+0000 7f47d101f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:14.837 INFO:journalctl@ceph.mgr.y.smithi121.stdout:Sep 23 21:26:14 smithi121 conmon[198376]: 2023-09-23T21:26:14.836+0000 7f47d101f700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-09-23T21:26:15.702 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T21:26:15.740 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:15.740 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T21:26:15.740 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T21:26:15.741 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T21:26:16.166 INFO:journalctl@ceph.mgr.x.smithi123.stdout:Sep 23 21:26:15 smithi123 systemd[1]: Stopping Ceph mgr.x for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:16.904 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T21:26:16.941 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:16.942 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T21:26:16.942 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T21:26:16.942 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T21:26:17.226 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 21:26:16 smithi121 systemd[1]: Stopping Ceph osd.0 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:17.663 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 21:26:17 smithi121 conmon[236522]: 2023-09-23T21:26:17.225+0000 7f667e621700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:17.663 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 21:26:17 smithi121 conmon[236522]: 2023-09-23T21:26:17.225+0000 7f667e621700 -1 osd.0 719 *** Got signal Terminated *** 2023-09-23T21:26:17.663 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 21:26:17 smithi121 conmon[236522]: 2023-09-23T21:26:17.226+0000 7f667e621700 -1 osd.0 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:26:23.163 INFO:journalctl@ceph.osd.0.smithi121.stdout:Sep 23 21:26:22 smithi121 bash[794599]: 2acc40f5254ffa5ee96fbb41ddcdbe21488c529fbec9810a798572d4ac9edd6e 2023-09-23T21:26:25.258 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T21:26:25.299 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:25.299 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T21:26:25.299 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T21:26:25.299 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T21:26:25.609 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 21:26:25 smithi121 systemd[1]: Stopping Ceph osd.1 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:25.912 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 21:26:25 smithi121 conmon[393696]: 2023-09-23T21:26:25.608+0000 7f378471a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:25.913 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 21:26:25 smithi121 conmon[393696]: 2023-09-23T21:26:25.608+0000 7f378471a700 -1 osd.1 719 *** Got signal Terminated *** 2023-09-23T21:26:25.913 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 21:26:25 smithi121 conmon[393696]: 2023-09-23T21:26:25.608+0000 7f378471a700 -1 osd.1 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:26:31.663 INFO:journalctl@ceph.osd.1.smithi121.stdout:Sep 23 21:26:31 smithi121 bash[795183]: 1baf61c043a7b1fbcf0037176efa6395864fc54098a8b1e0bcac4b5ce7a0b0f0 2023-09-23T21:26:33.316 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T21:26:33.358 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:33.359 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T21:26:33.359 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T21:26:33.359 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T21:26:33.663 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 21:26:33 smithi121 systemd[1]: Stopping Ceph osd.2 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:34.163 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 21:26:33 smithi121 conmon[260103]: 2023-09-23T21:26:33.663+0000 7ffbcd9cd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:34.163 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 21:26:33 smithi121 conmon[260103]: 2023-09-23T21:26:33.663+0000 7ffbcd9cd700 -1 osd.2 719 *** Got signal Terminated *** 2023-09-23T21:26:34.163 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 21:26:33 smithi121 conmon[260103]: 2023-09-23T21:26:33.663+0000 7ffbcd9cd700 -1 osd.2 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:26:39.663 INFO:journalctl@ceph.osd.2.smithi121.stdout:Sep 23 21:26:39 smithi121 bash[795767]: 97892133c587b4d3f535ea944318951689195933f0feafb99af69bf2d90370db 2023-09-23T21:26:41.363 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T21:26:41.403 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:41.403 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T21:26:41.404 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T21:26:41.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T21:26:41.662 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:41 smithi121 systemd[1]: Stopping Ceph osd.3 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:42.162 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:41 smithi121 conmon[269821]: 2023-09-23T21:26:41.720+0000 7fb7f8434700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:42.163 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:41 smithi121 conmon[269821]: 2023-09-23T21:26:41.720+0000 7fb7f8434700 -1 osd.3 719 *** Got signal Terminated *** 2023-09-23T21:26:42.163 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:41 smithi121 conmon[269821]: 2023-09-23T21:26:41.720+0000 7fb7f8434700 -1 osd.3 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:26:43.729 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:43 smithi123 conmon[141153]: 2023-09-23T21:26:43.310+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:44.728 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:44 smithi123 conmon[141153]: 2023-09-23T21:26:44.326+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:44.912 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:44 smithi121 conmon[269821]: 2023-09-23T21:26:44.475+0000 7fb7f0a22700 -1 osd.3 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:18.415259+0000 front 2023-09-23T21:26:18.415160+0000 (oldest deadline 2023-09-23T21:26:44.315362+0000) 2023-09-23T21:26:45.228 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:44 smithi123 conmon[455736]: 2023-09-23T21:26:44.795+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:45.728 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:45 smithi123 conmon[141153]: 2023-09-23T21:26:45.351+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:45.912 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:45 smithi121 conmon[269821]: 2023-09-23T21:26:45.473+0000 7fb7f0a22700 -1 osd.3 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:18.415259+0000 front 2023-09-23T21:26:18.415160+0000 (oldest deadline 2023-09-23T21:26:44.315362+0000) 2023-09-23T21:26:46.051 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:45 smithi123 conmon[455736]: 2023-09-23T21:26:45.783+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:46.310 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:46 smithi123 conmon[131622]: 2023-09-23T21:26:46.050+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:46.729 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:46 smithi123 conmon[141153]: 2023-09-23T21:26:46.352+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:46.729 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:46 smithi123 conmon[112995]: 2023-09-23T21:26:46.309+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:46.913 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:46 smithi121 conmon[269821]: 2023-09-23T21:26:46.436+0000 7fb7f0a22700 -1 osd.3 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:18.415259+0000 front 2023-09-23T21:26:18.415160+0000 (oldest deadline 2023-09-23T21:26:44.315362+0000) 2023-09-23T21:26:47.228 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:47 smithi123 conmon[131622]: 2023-09-23T21:26:47.035+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:47.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:46 smithi123 conmon[455736]: 2023-09-23T21:26:46.794+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:47.729 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:47 smithi123 conmon[112995]: 2023-09-23T21:26:47.340+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:47.729 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:47 smithi123 conmon[141153]: 2023-09-23T21:26:47.348+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:48.074 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:47 smithi123 conmon[455736]: 2023-09-23T21:26:47.809+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:48.162 INFO:journalctl@ceph.osd.3.smithi121.stdout:Sep 23 21:26:47 smithi121 bash[796362]: 5bca5bb9cc3f5ea5e93c93fe5cf62ed4a7e1fa08a11e1130e83da0f0a2e4732d 2023-09-23T21:26:48.328 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:48 smithi123 conmon[112995]: 2023-09-23T21:26:48.301+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:48.328 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:48 smithi123 conmon[131622]: 2023-09-23T21:26:48.074+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:48.728 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:48 smithi123 conmon[141153]: 2023-09-23T21:26:48.328+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:49.228 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:49 smithi123 conmon[131622]: 2023-09-23T21:26:49.025+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:49.229 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:48 smithi123 conmon[455736]: 2023-09-23T21:26:48.801+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:49.269 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T21:26:49.308 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:49.308 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T21:26:49.309 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T21:26:49.309 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T21:26:49.729 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:49 smithi123 conmon[141153]: 2023-09-23T21:26:49.308+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:49.729 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:49 smithi123 conmon[112995]: 2023-09-23T21:26:49.269+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:49.729 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:49 smithi123 systemd[1]: Stopping Ceph osd.4 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:50.055 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:49 smithi123 conmon[455736]: 2023-09-23T21:26:49.796+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:50.056 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:49 smithi123 conmon[112995]: 2023-09-23T21:26:49.769+0000 7fe444235700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:50.056 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:49 smithi123 conmon[112995]: 2023-09-23T21:26:49.769+0000 7fe444235700 -1 osd.4 719 *** Got signal Terminated *** 2023-09-23T21:26:50.056 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:49 smithi123 conmon[112995]: 2023-09-23T21:26:49.769+0000 7fe444235700 -1 osd.4 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:26:50.478 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:50 smithi123 conmon[112995]: 2023-09-23T21:26:50.279+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:50.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:50 smithi123 conmon[131622]: 2023-09-23T21:26:50.054+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:50.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:50 smithi123 conmon[141153]: 2023-09-23T21:26:50.289+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:51.071 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:50 smithi123 conmon[455736]: 2023-09-23T21:26:50.776+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:51.479 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:51 smithi123 conmon[112995]: 2023-09-23T21:26:51.278+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:51.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:51 smithi123 conmon[131622]: 2023-09-23T21:26:51.070+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:51.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:51 smithi123 conmon[141153]: 2023-09-23T21:26:51.248+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:52.070 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:51 smithi123 conmon[455736]: 2023-09-23T21:26:51.795+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:52.070 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:51 smithi123 conmon[455736]: 2023-09-23T21:26:51.795+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:52.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:52 smithi123 conmon[131622]: 2023-09-23T21:26:52.070+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:52.479 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:52 smithi123 conmon[112995]: 2023-09-23T21:26:52.293+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:52.479 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:52 smithi123 conmon[112995]: 2023-09-23T21:26:52.293+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:25.635443+0000 front 2023-09-23T21:26:25.635366+0000 (oldest deadline 2023-09-23T21:26:51.435939+0000) 2023-09-23T21:26:52.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:52 smithi123 conmon[141153]: 2023-09-23T21:26:52.296+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:52.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:52 smithi123 conmon[141153]: 2023-09-23T21:26:52.296+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:53.037 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:52 smithi123 conmon[455736]: 2023-09-23T21:26:52.773+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:53.037 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:52 smithi123 conmon[455736]: 2023-09-23T21:26:52.773+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:53.288 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:53 smithi123 conmon[131622]: 2023-09-23T21:26:53.036+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:53.728 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:53 smithi123 conmon[141153]: 2023-09-23T21:26:53.288+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:53.729 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:53 smithi123 conmon[141153]: 2023-09-23T21:26:53.288+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:53.729 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:53 smithi123 conmon[112995]: 2023-09-23T21:26:53.307+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:53.729 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:53 smithi123 conmon[112995]: 2023-09-23T21:26:53.307+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:25.635443+0000 front 2023-09-23T21:26:25.635366+0000 (oldest deadline 2023-09-23T21:26:51.435939+0000) 2023-09-23T21:26:54.036 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:53 smithi123 conmon[455736]: 2023-09-23T21:26:53.770+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:54.036 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:53 smithi123 conmon[455736]: 2023-09-23T21:26:53.770+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:54.339 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[131622]: 2023-09-23T21:26:54.034+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:54.340 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[131622]: 2023-09-23T21:26:54.034+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:26:54.340 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[141153]: 2023-09-23T21:26:54.253+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:54.340 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[141153]: 2023-09-23T21:26:54.253+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:54.728 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[112995]: 2023-09-23T21:26:54.339+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:21.534919+0000 front 2023-09-23T21:26:21.534945+0000 (oldest deadline 2023-09-23T21:26:45.634820+0000) 2023-09-23T21:26:54.729 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[112995]: 2023-09-23T21:26:54.339+0000 7fe441835700 -1 osd.4 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:25.635443+0000 front 2023-09-23T21:26:25.635366+0000 (oldest deadline 2023-09-23T21:26:51.435939+0000) 2023-09-23T21:26:55.052 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[455736]: 2023-09-23T21:26:54.753+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:55.053 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:54 smithi123 conmon[455736]: 2023-09-23T21:26:54.753+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:55.390 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:55 smithi123 conmon[131622]: 2023-09-23T21:26:55.052+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:55.391 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:55 smithi123 conmon[131622]: 2023-09-23T21:26:55.052+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:26:55.391 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:55 smithi123 conmon[141153]: 2023-09-23T21:26:55.240+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:55.391 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:55 smithi123 conmon[141153]: 2023-09-23T21:26:55.240+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:55.979 INFO:journalctl@ceph.osd.4.smithi123.stdout:Sep 23 21:26:55 smithi123 bash[460797]: 470c2a4d010b73419c9f9f90d115dc79d7ed18be7165510f6bd11cd16a4bc1e9 2023-09-23T21:26:55.979 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:55 smithi123 conmon[455736]: 2023-09-23T21:26:55.720+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:55.979 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:55 smithi123 conmon[455736]: 2023-09-23T21:26:55.720+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:56.405 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:56 smithi123 conmon[141153]: 2023-09-23T21:26:56.193+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:56.406 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:56 smithi123 conmon[141153]: 2023-09-23T21:26:56.193+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:56.406 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:56 smithi123 conmon[131622]: 2023-09-23T21:26:56.064+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:56.406 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:56 smithi123 conmon[131622]: 2023-09-23T21:26:56.064+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:26:57.070 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:56 smithi123 conmon[455736]: 2023-09-23T21:26:56.761+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:57.070 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:56 smithi123 conmon[455736]: 2023-09-23T21:26:56.761+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:57.478 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:57 smithi123 conmon[131622]: 2023-09-23T21:26:57.069+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:57.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:57 smithi123 conmon[131622]: 2023-09-23T21:26:57.069+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:26:57.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:57 smithi123 conmon[141153]: 2023-09-23T21:26:57.230+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:57.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:57 smithi123 conmon[141153]: 2023-09-23T21:26:57.230+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:57.658 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T21:26:57.700 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:26:57.700 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T21:26:57.701 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T21:26:57.701 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T21:26:57.979 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:57 smithi123 systemd[1]: Stopping Ceph osd.5 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:26:57.979 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:57 smithi123 conmon[455736]: 2023-09-23T21:26:57.789+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:57.979 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:57 smithi123 conmon[455736]: 2023-09-23T21:26:57.789+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:58.448 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[141153]: 2023-09-23T21:26:58.182+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:58.448 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[141153]: 2023-09-23T21:26:58.182+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:58.449 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[131622]: 2023-09-23T21:26:58.078+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:58.449 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[131622]: 2023-09-23T21:26:58.078+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:26:58.449 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[455736]: 2023-09-23T21:26:58.015+0000 7f82ca711700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:26:58.449 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[455736]: 2023-09-23T21:26:58.015+0000 7f82ca711700 -1 osd.5 719 *** Got signal Terminated *** 2023-09-23T21:26:58.449 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[455736]: 2023-09-23T21:26:58.015+0000 7f82ca711700 -1 osd.5 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:26:58.728 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[177538]: ts=2023-09-23T21:26:58.448Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.121:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.121:8765: connect: connection refused" 2023-09-23T21:26:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[177538]: ts=2023-09-23T21:26:58.448Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.121:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.121:8765: connect: connection refused" 2023-09-23T21:26:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[177538]: ts=2023-09-23T21:26:58.450Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.121:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.121:8765: connect: connection refused" 2023-09-23T21:26:58.729 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[177538]: ts=2023-09-23T21:26:58.452Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.121:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.121:8765: connect: connection refused" 2023-09-23T21:26:59.084 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[455736]: 2023-09-23T21:26:58.750+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:26:59.084 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:58 smithi123 conmon[455736]: 2023-09-23T21:26:58.750+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:26:59.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:59 smithi123 conmon[141153]: 2023-09-23T21:26:59.148+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:26:59.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:26:59 smithi123 conmon[141153]: 2023-09-23T21:26:59.148+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:26:59.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:59 smithi123 conmon[131622]: 2023-09-23T21:26:59.084+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:26:59.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:26:59 smithi123 conmon[131622]: 2023-09-23T21:26:59.084+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:00.107 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:59 smithi123 conmon[455736]: 2023-09-23T21:26:59.797+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:27:00.108 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:26:59 smithi123 conmon[455736]: 2023-09-23T21:26:59.797+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:27:00.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:00 smithi123 conmon[131622]: 2023-09-23T21:27:00.107+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:00.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:00 smithi123 conmon[131622]: 2023-09-23T21:27:00.107+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:00.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:00 smithi123 conmon[141153]: 2023-09-23T21:27:00.109+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:00.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:00 smithi123 conmon[141153]: 2023-09-23T21:27:00.109+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:01.066 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:27:00 smithi123 conmon[455736]: 2023-09-23T21:27:00.768+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:27:01.067 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:27:00 smithi123 conmon[455736]: 2023-09-23T21:27:00.768+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:27:01.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:01 smithi123 conmon[131622]: 2023-09-23T21:27:01.066+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:01.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:01 smithi123 conmon[131622]: 2023-09-23T21:27:01.066+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:01.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:01 smithi123 conmon[141153]: 2023-09-23T21:27:01.114+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:01.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:01 smithi123 conmon[141153]: 2023-09-23T21:27:01.114+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:02.076 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:27:01 smithi123 conmon[455736]: 2023-09-23T21:27:01.798+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:27:02.077 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:27:01 smithi123 conmon[455736]: 2023-09-23T21:27:01.798+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:27:02.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:02 smithi123 conmon[141153]: 2023-09-23T21:27:02.112+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:02.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:02 smithi123 conmon[141153]: 2023-09-23T21:27:02.112+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:02.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:02 smithi123 conmon[141153]: 2023-09-23T21:27:02.112+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:02.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:02 smithi123 conmon[131622]: 2023-09-23T21:27:02.076+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:02.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:02 smithi123 conmon[131622]: 2023-09-23T21:27:02.076+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:03.131 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:27:02 smithi123 conmon[455736]: 2023-09-23T21:27:02.804+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.979040+0000 front 2023-09-23T21:26:19.979140+0000 (oldest deadline 2023-09-23T21:26:44.078990+0000) 2023-09-23T21:27:03.131 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:27:02 smithi123 conmon[455736]: 2023-09-23T21:27:02.804+0000 7f82c5d0d700 -1 osd.5 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:29.379852+0000 front 2023-09-23T21:26:29.380039+0000 (oldest deadline 2023-09-23T21:26:51.679815+0000) 2023-09-23T21:27:03.131 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:03 smithi123 conmon[131622]: 2023-09-23T21:27:03.027+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:03.131 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:03 smithi123 conmon[131622]: 2023-09-23T21:27:03.027+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:03.131 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:03 smithi123 conmon[131622]: 2023-09-23T21:27:03.027+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:03.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:03 smithi123 conmon[141153]: 2023-09-23T21:27:03.130+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:03.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:03 smithi123 conmon[141153]: 2023-09-23T21:27:03.130+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:03.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:03 smithi123 conmon[141153]: 2023-09-23T21:27:03.130+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:03.978 INFO:journalctl@ceph.osd.5.smithi123.stdout:Sep 23 21:27:03 smithi123 bash[461383]: 5ef7ab6c8fa127ae1318f00977d37efaa010e4b322b3ba7e7b93e8b33b2932ec 2023-09-23T21:27:04.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[141153]: 2023-09-23T21:27:04.117+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:04.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[141153]: 2023-09-23T21:27:04.117+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:04.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[141153]: 2023-09-23T21:27:04.117+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:04.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[131622]: 2023-09-23T21:27:04.027+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:04.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[131622]: 2023-09-23T21:27:04.027+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:04.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[131622]: 2023-09-23T21:27:04.027+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:05.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[141153]: 2023-09-23T21:27:05.078+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:05.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[141153]: 2023-09-23T21:27:05.078+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:05.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[141153]: 2023-09-23T21:27:05.078+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:05.479 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[131622]: 2023-09-23T21:27:04.990+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:05.480 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[131622]: 2023-09-23T21:27:04.990+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:05.480 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:04 smithi123 conmon[131622]: 2023-09-23T21:27:04.990+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:05.620 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T21:27:05.660 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:27:05.660 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T21:27:05.660 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T21:27:05.660 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T21:27:05.950 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:05 smithi123 systemd[1]: Stopping Ceph osd.6 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:27:06.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[131622]: 2023-09-23T21:27:05.948+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:06.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[131622]: 2023-09-23T21:27:05.948+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:06.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[131622]: 2023-09-23T21:27:05.948+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:06.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[131622]: 2023-09-23T21:27:05.950+0000 7fe794b5e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:27:06.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[131622]: 2023-09-23T21:27:05.950+0000 7fe794b5e700 -1 osd.6 719 *** Got signal Terminated *** 2023-09-23T21:27:06.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:05 smithi123 conmon[131622]: 2023-09-23T21:27:05.950+0000 7fe794b5e700 -1 osd.6 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:27:06.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[141153]: 2023-09-23T21:27:06.046+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:06.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[141153]: 2023-09-23T21:27:06.046+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:06.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[141153]: 2023-09-23T21:27:06.046+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:07.228 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[131622]: 2023-09-23T21:27:06.948+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:07.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[131622]: 2023-09-23T21:27:06.948+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:07.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[131622]: 2023-09-23T21:27:06.948+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:07.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[141153]: 2023-09-23T21:27:06.998+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:07.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:06 smithi123 conmon[141153]: 2023-09-23T21:27:06.998+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:07.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:07 smithi123 conmon[141153]: 2023-09-23T21:27:06.998+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:08.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:07 smithi123 conmon[131622]: 2023-09-23T21:27:07.908+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:08.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:07 smithi123 conmon[131622]: 2023-09-23T21:27:07.908+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:08.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:07 smithi123 conmon[131622]: 2023-09-23T21:27:07.908+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:08.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:07 smithi123 conmon[131622]: 2023-09-23T21:27:07.908+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.221387+0000 front 2023-09-23T21:26:42.221345+0000 (oldest deadline 2023-09-23T21:27:07.521307+0000) 2023-09-23T21:27:08.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[141153]: 2023-09-23T21:27:08.036+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:08.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[141153]: 2023-09-23T21:27:08.036+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:08.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[141153]: 2023-09-23T21:27:08.036+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:08.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[141153]: 2023-09-23T21:27:08.036+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:09.228 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[131622]: 2023-09-23T21:27:08.895+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:09.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[131622]: 2023-09-23T21:27:08.895+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:09.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[131622]: 2023-09-23T21:27:08.895+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:09.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:08 smithi123 conmon[131622]: 2023-09-23T21:27:08.895+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.221387+0000 front 2023-09-23T21:26:42.221345+0000 (oldest deadline 2023-09-23T21:27:07.521307+0000) 2023-09-23T21:27:09.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[141153]: 2023-09-23T21:27:09.023+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:09.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[141153]: 2023-09-23T21:27:09.024+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:09.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[141153]: 2023-09-23T21:27:09.024+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:09.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[141153]: 2023-09-23T21:27:09.024+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:10.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[141153]: 2023-09-23T21:27:10.006+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:10.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[141153]: 2023-09-23T21:27:10.006+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:10.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[141153]: 2023-09-23T21:27:10.006+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:10.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[141153]: 2023-09-23T21:27:10.006+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:10.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[131622]: 2023-09-23T21:27:09.942+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:10.230 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[131622]: 2023-09-23T21:27:09.942+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:10.230 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[131622]: 2023-09-23T21:27:09.942+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:10.230 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:09 smithi123 conmon[131622]: 2023-09-23T21:27:09.942+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.221387+0000 front 2023-09-23T21:26:42.221345+0000 (oldest deadline 2023-09-23T21:27:07.521307+0000) 2023-09-23T21:27:11.228 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[131622]: 2023-09-23T21:27:10.899+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:20.018160+0000 front 2023-09-23T21:26:20.018238+0000 (oldest deadline 2023-09-23T21:26:45.918292+0000) 2023-09-23T21:27:11.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[131622]: 2023-09-23T21:27:10.899+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:27.519722+0000 front 2023-09-23T21:26:27.519795+0000 (oldest deadline 2023-09-23T21:26:53.419649+0000) 2023-09-23T21:27:11.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[131622]: 2023-09-23T21:27:10.899+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:37.520616+0000 front 2023-09-23T21:26:37.520649+0000 (oldest deadline 2023-09-23T21:27:02.220781+0000) 2023-09-23T21:27:11.229 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:10 smithi123 conmon[131622]: 2023-09-23T21:27:10.899+0000 7fe79295f700 -1 osd.6 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.221387+0000 front 2023-09-23T21:26:42.221345+0000 (oldest deadline 2023-09-23T21:27:07.521307+0000) 2023-09-23T21:27:11.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:11 smithi123 conmon[141153]: 2023-09-23T21:27:11.032+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:11.229 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:11 smithi123 conmon[141153]: 2023-09-23T21:27:11.032+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:11.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:11 smithi123 conmon[141153]: 2023-09-23T21:27:11.032+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:11.230 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:11 smithi123 conmon[141153]: 2023-09-23T21:27:11.032+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:12.478 INFO:journalctl@ceph.osd.6.smithi123.stdout:Sep 23 21:27:12 smithi123 bash[461967]: 3f6aba9e3a6b322c032235e5f2286b38b4e465b075b1008408c214b6189d4106 2023-09-23T21:27:12.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:12 smithi123 conmon[141153]: 2023-09-23T21:27:12.075+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:12.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:12 smithi123 conmon[141153]: 2023-09-23T21:27:12.075+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:12.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:12 smithi123 conmon[141153]: 2023-09-23T21:27:12.075+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:12.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:12 smithi123 conmon[141153]: 2023-09-23T21:27:12.075+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:13.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:13 smithi123 conmon[141153]: 2023-09-23T21:27:13.122+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:13.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:13 smithi123 conmon[141153]: 2023-09-23T21:27:13.122+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:13.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:13 smithi123 conmon[141153]: 2023-09-23T21:27:13.122+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:13.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:13 smithi123 conmon[141153]: 2023-09-23T21:27:13.122+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:13.711 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T21:27:13.750 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:27:13.751 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T21:27:13.751 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T21:27:13.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T21:27:13.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:13 smithi123 systemd[1]: Stopping Ceph osd.7 for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:27:14.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:14 smithi123 conmon[141153]: 2023-09-23T21:27:14.061+0000 7f0716373700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:27:14.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:14 smithi123 conmon[141153]: 2023-09-23T21:27:14.061+0000 7f0716373700 -1 osd.7 719 *** Got signal Terminated *** 2023-09-23T21:27:14.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:14 smithi123 conmon[141153]: 2023-09-23T21:27:14.061+0000 7f0716373700 -1 osd.7 719 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T21:27:14.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:14 smithi123 conmon[141153]: 2023-09-23T21:27:14.094+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:14.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:14 smithi123 conmon[141153]: 2023-09-23T21:27:14.094+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:14.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:14 smithi123 conmon[141153]: 2023-09-23T21:27:14.094+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:14.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:14 smithi123 conmon[141153]: 2023-09-23T21:27:14.094+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:15.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:15 smithi123 conmon[141153]: 2023-09-23T21:27:15.140+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:15.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:15 smithi123 conmon[141153]: 2023-09-23T21:27:15.140+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:15.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:15 smithi123 conmon[141153]: 2023-09-23T21:27:15.140+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:15.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:15 smithi123 conmon[141153]: 2023-09-23T21:27:15.140+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:16.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:16 smithi123 conmon[141153]: 2023-09-23T21:27:16.180+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:16.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:16 smithi123 conmon[141153]: 2023-09-23T21:27:16.180+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:16.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:16 smithi123 conmon[141153]: 2023-09-23T21:27:16.180+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:16.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:16 smithi123 conmon[141153]: 2023-09-23T21:27:16.180+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:16.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:16 smithi123 conmon[141153]: 2023-09-23T21:27:16.180+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.123:6804 osd.4 since back 2023-09-23T21:26:53.174057+0000 front 2023-09-23T21:26:53.174126+0000 (oldest deadline 2023-09-23T21:27:15.474024+0000) 2023-09-23T21:27:17.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:17 smithi123 conmon[141153]: 2023-09-23T21:27:17.186+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:17.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:17 smithi123 conmon[141153]: 2023-09-23T21:27:17.186+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:17.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:17 smithi123 conmon[141153]: 2023-09-23T21:27:17.186+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:17.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:17 smithi123 conmon[141153]: 2023-09-23T21:27:17.186+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:17.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:17 smithi123 conmon[141153]: 2023-09-23T21:27:17.186+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.123:6804 osd.4 since back 2023-09-23T21:26:53.174057+0000 front 2023-09-23T21:26:53.174126+0000 (oldest deadline 2023-09-23T21:27:15.474024+0000) 2023-09-23T21:27:18.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:18 smithi123 conmon[141153]: 2023-09-23T21:27:18.220+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2023-09-23T21:26:19.369106+0000 front 2023-09-23T21:26:19.369118+0000 (oldest deadline 2023-09-23T21:26:42.868879+0000) 2023-09-23T21:27:18.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:18 smithi123 conmon[141153]: 2023-09-23T21:27:18.220+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2023-09-23T21:26:26.270754+0000 front 2023-09-23T21:26:26.270585+0000 (oldest deadline 2023-09-23T21:26:51.570565+0000) 2023-09-23T21:27:18.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:18 smithi123 conmon[141153]: 2023-09-23T21:27:18.220+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2023-09-23T21:26:38.572297+0000 front 2023-09-23T21:26:38.572323+0000 (oldest deadline 2023-09-23T21:27:02.072310+0000) 2023-09-23T21:27:18.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:18 smithi123 conmon[141153]: 2023-09-23T21:27:18.220+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2023-09-23T21:26:42.072888+0000 front 2023-09-23T21:26:42.072832+0000 (oldest deadline 2023-09-23T21:27:07.972755+0000) 2023-09-23T21:27:18.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:18 smithi123 conmon[141153]: 2023-09-23T21:27:18.220+0000 7f071196f700 -1 osd.7 719 heartbeat_check: no reply from 172.21.15.123:6804 osd.4 since back 2023-09-23T21:26:53.174057+0000 front 2023-09-23T21:26:53.174126+0000 (oldest deadline 2023-09-23T21:27:15.474024+0000) 2023-09-23T21:27:20.479 INFO:journalctl@ceph.osd.7.smithi123.stdout:Sep 23 21:27:20 smithi123 bash[462538]: 514f59904aaaa73372d19d92177d2f3acc28f9be8c905ac3cd70ccd23f46cb4a 2023-09-23T21:27:21.634 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T21:27:21.674 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:27:21.675 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T21:27:21.675 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T21:27:21.675 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T21:27:21.979 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 systemd[1]: Stopping Ceph prometheus.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:27:22.454 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T21:27:22.454 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T21:27:22.454 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T21:27:22.454 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.985Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.987Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.987Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T21:27:22.455 INFO:journalctl@ceph.prometheus.a.smithi123.stdout:Sep 23 21:27:21 smithi123 conmon[177538]: ts=2023-09-23T21:27:21.987Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T21:27:22.627 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T21:27:22.663 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T21:27:22.663 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T21:27:22.663 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T21:27:24.147 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 21:27:23 smithi121 systemd[1]: Stopping Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:27:24.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 21:27:24 smithi121 conmon[196532]: ts=2023-09-23T21:27:24.146Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T21:27:24.912 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 21:27:24 smithi121 bash[797234]: a7c6818f0f7f6b1dd9d44f9b3c144a7d7af5bcf0baf84225bcd716477a27ae66 2023-09-23T21:27:25.413 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 21:27:25 smithi121 systemd[1]: Stopping Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:27:25.413 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 21:27:24 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T21:27:25.414 INFO:journalctl@ceph.alertmanager.a.smithi121.stdout:Sep 23 21:27:24 smithi121 systemd[1]: Stopped Ceph alertmanager.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T21:27:26.106 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 21:27:25 smithi121 bash[797525]: ab86679c1d7db3f48d201b2b719844462ea4c029f50b48e2f33e1f236b2e832f 2023-09-23T21:27:26.107 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 21:27:25 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T21:27:26.107 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 21:27:26 smithi121 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T21:27:26.107 INFO:journalctl@ceph.node-exporter.a.smithi121.stdout:Sep 23 21:27:26 smithi121 systemd[1]: Stopped Ceph node-exporter.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T21:27:36.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T21:27:37.729 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:27:37 smithi123 systemd[1]: Stopping Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:27:38.438 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:27:38 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T21:27:38.438 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:27:38 smithi123 bash[463727]: c2e1f7abecffcaa31d9bcfca24798c3ef57f315e904eb8eb508d6fe508ec66cf 2023-09-23T21:27:38.438 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:27:38 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T21:27:38.438 INFO:journalctl@ceph.grafana.a.smithi123.stdout:Sep 23 21:27:38 smithi123 systemd[1]: Stopped Ceph grafana.a for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T21:27:38.729 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 21:27:38 smithi123 systemd[1]: Stopping Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7... 2023-09-23T21:27:39.414 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 21:27:39 smithi123 bash[464018]: 5f079d4704ad7de30fdb44216bb0dc2adad076f55bde93a50b85a6e9af0caf00 2023-09-23T21:27:39.414 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 21:27:39 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T21:27:39.414 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 21:27:39 smithi123 systemd[1]: ceph-ffb133f0-5a4e-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T21:27:39.415 INFO:journalctl@ceph.node-exporter.b.smithi123.stdout:Sep 23 21:27:39 smithi123 systemd[1]: Stopped Ceph node-exporter.b for ffb133f0-5a4e-11ee-8db4-212e2dc638e7. 2023-09-23T21:27:44.767 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:27:44.805 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:27:44.834 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T21:27:44.837 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832/remote/smithi121/crash 2023-09-23T21:27:44.838 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T21:27:44.892 INFO:teuthology.orchestra.run.smithi121.stderr:tar: /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T21:27:44.893 INFO:teuthology.orchestra.run.smithi121.stderr:tar: Error is not recoverable: exiting now 2023-09-23T21:27:44.895 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832/remote/smithi123/crash 2023-09-23T21:27:44.895 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T21:27:44.946 INFO:teuthology.orchestra.run.smithi123.stderr:tar: /var/lib/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T21:27:44.947 INFO:teuthology.orchestra.run.smithi123.stderr:tar: Error is not recoverable: exiting now 2023-09-23T21:27:44.948 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T21:27:44.948 DEBUG:teuthology.orchestra.run.smithi121:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T21:27:44.978 INFO:teuthology.orchestra.run.smithi121.stderr:grep: /var/log/ceph/ffb133f0-5a4e-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T21:27:44.980 INFO:tasks.cephadm:Compressing logs... 2023-09-23T21:27:44.980 DEBUG:teuthology.orchestra.run.smithi121:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:27:45.023 DEBUG:teuthology.orchestra.run.smithi123:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:27:45.051 INFO:teuthology.orchestra.run.smithi121.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T21:27:45.067 INFO:teuthology.orchestra.run.smithi123.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T21:41:37.591 INFO:tasks.cephadm:Archiving logs... 2023-09-23T21:41:37.604 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832/remote/smithi121/log 2023-09-23T21:41:37.605 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T21:43:24.715 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832/remote/smithi123/log 2023-09-23T21:43:24.716 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T21:44:59.361 INFO:tasks.cephadm:Removing cluster... 2023-09-23T21:44:59.361 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 --force 2023-09-23T21:45:00.965 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ffb133f0-5a4e-11ee-8db4-212e2dc638e7 --force 2023-09-23T21:45:02.744 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T21:45:02.744 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T21:45:02.765 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T21:45:02.793 INFO:tasks.cephadm:Teardown complete 2023-09-23T21:45:02.793 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T21:45:02.806 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T21:45:02.806 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T21:45:02.809 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T21:45:02.938 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T21:45:02.947 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T21:45:02.948 DEBUG:teuthology.orchestra.run.smithi121:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:45:02.950 DEBUG:teuthology.orchestra.run.smithi123:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:45:02.966 INFO:teuthology.orchestra.run.smithi121.stderr:bash: ntpq: command not found 2023-09-23T21:45:02.969 INFO:teuthology.orchestra.run.smithi123.stderr:bash: ntpq: command not found 2023-09-23T21:45:02.998 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:45:02.998 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2023-09-23T21:45:02.999 INFO:teuthology.orchestra.run.smithi123.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 37 -1454us[-1415us] +/- 89ms 2023-09-23T21:45:02.999 INFO:teuthology.orchestra.run.smithi123.stdout:^* hv02.front.sepia.ceph.com 3 7 377 31 +74us[ +113us] +/- 51ms 2023-09-23T21:45:02.999 INFO:teuthology.orchestra.run.smithi123.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 36 +1101us[+1140us] +/- 80ms 2023-09-23T21:45:02.999 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:45:02.999 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:45:02.999 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2023-09-23T21:45:03.000 INFO:teuthology.orchestra.run.smithi121.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 42 -1368us[-1368us] +/- 89ms 2023-09-23T21:45:03.000 INFO:teuthology.orchestra.run.smithi121.stdout:^* hv02.front.sepia.ceph.com 3 8 377 92 +113us[ +25us] +/- 50ms 2023-09-23T21:45:03.000 INFO:teuthology.orchestra.run.smithi121.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 33 +1190us[+1190us] +/- 80ms 2023-09-23T21:45:03.000 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:45:03.000 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T21:45:03.012 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T21:45:03.013 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T21:45:03.024 DEBUG:teuthology.orchestra.run.smithi121:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T21:45:03.189 DEBUG:teuthology.orchestra.run.smithi123:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T21:45:03.305 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:45:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:45:03.340 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:45:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:45:03.372 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T21:45:03.381 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T21:45:03.399 INFO:teuthology.task.internal:Duration was 5731.468211 seconds 2023-09-23T21:45:03.399 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T21:45:03.414 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T21:45:03.414 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T21:45:03.417 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T21:45:03.486 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:45:03.507 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:45:03.950 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T21:45:03.950 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi121.front.sepia.ceph.com 2023-09-23T21:45:03.951 DEBUG:teuthology.orchestra.run.smithi121:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T21:45:03.977 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi123.front.sepia.ceph.com 2023-09-23T21:45:03.977 DEBUG:teuthology.orchestra.run.smithi123:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T21:45:04.015 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T21:45:04.016 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:45:04.019 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:45:04.181 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T21:45:04.181 DEBUG:teuthology.orchestra.run.smithi121:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T21:45:04.183 DEBUG:teuthology.orchestra.run.smithi123:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T21:45:04.574 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T21:45:04.585 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T21:45:04.585 DEBUG:teuthology.orchestra.run.smithi121:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T21:45:04.634 DEBUG:teuthology.orchestra.run.smithi123:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T21:45:04.672 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T21:45:04.683 DEBUG:teuthology.orchestra.run.smithi121:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:45:04.685 DEBUG:teuthology.orchestra.run.smithi123:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:45:04.747 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = core 2023-09-23T21:45:04.774 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = core 2023-09-23T21:45:04.789 DEBUG:teuthology.orchestra.run.smithi121:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:45:04.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:45:04.805 DEBUG:teuthology.orchestra.run.smithi123:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:45:04.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:45:04.823 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T21:45:04.837 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T21:45:04.838 DEBUG:teuthology.misc:Transferring archived files from smithi121:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832/remote/smithi121 2023-09-23T21:45:04.838 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T21:45:04.911 DEBUG:teuthology.misc:Transferring archived files from smithi123:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832/remote/smithi123 2023-09-23T21:45:04.912 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T21:45:04.962 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T21:45:04.963 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T21:45:04.965 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T21:45:05.007 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T21:45:05.019 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T21:45:05.019 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T21:45:05.031 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T21:45:05.031 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T21:45:05.033 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T21:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: 262324 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 21:45 /home/ubuntu/cephtest 2023-09-23T21:45:05.047 INFO:teuthology.orchestra.run.smithi121.stdout: 5898245 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 21:24 /home/ubuntu/cephtest/mnt.0 2023-09-23T21:45:05.048 INFO:teuthology.orchestra.run.smithi121.stdout: 5898253 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 21:24 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:45:05.048 INFO:teuthology.orchestra.run.smithi121.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T21:45:05.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:45:05.050 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi121 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T21:45:05.057 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T21:45:05.069 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T21:45:05.083 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi121 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T21:45:05.083 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T21:45:05.090 INFO:teuthology.orchestra.run.smithi123.stdout: 262324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 21:45 /home/ubuntu/cephtest 2023-09-23T21:45:05.121 DEBUG:teuthology.nuke:shortname: smithi121 2023-09-23T21:45:05.121 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T21:45:05.147 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi121.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:04:22.227064', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:45:05.151 DEBUG:teuthology.nuke:shortname: smithi123 2023-09-23T21:45:05.151 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T21:45:05.175 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi123.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401832', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 20:04:22.228906', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:45:05.204 INFO:teuthology.orchestra.console.smithi121:Power off 2023-09-23T21:45:05.204 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T21:45:05.227 DEBUG:teuthology.orchestra.console.smithi121:power off output: Chassis Power Control: Down/Off 2023-09-23T21:45:05.228 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:45:05.242 INFO:teuthology.orchestra.console.smithi123:Power off 2023-09-23T21:45:05.243 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T21:45:05.252 DEBUG:teuthology.orchestra.console.smithi121:check power output: Chassis Power is on 2023-09-23T21:45:05.264 DEBUG:teuthology.orchestra.console.smithi123:power off output: Chassis Power Control: Down/Off 2023-09-23T21:45:05.264 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:45:05.285 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is on 2023-09-23T21:45:09.254 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:45:09.286 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:45:09.364 DEBUG:teuthology.orchestra.console.smithi121:check power output: Chassis Power is on 2023-09-23T21:45:09.395 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is on 2023-09-23T21:45:13.370 DEBUG:teuthology.orchestra.console.smithi121:pexpect command: ipmitool -H smithi121.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:45:13.397 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:45:13.480 DEBUG:teuthology.orchestra.console.smithi121:check power output: Chassis Power is off 2023-09-23T21:45:13.506 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is off 2023-09-23T21:45:13.582 INFO:teuthology.orchestra.console.smithi121:Power off completed 2023-09-23T21:45:13.607 INFO:teuthology.orchestra.console.smithi123:Power off completed 2023-09-23T21:45:13.712 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 5731.468210935593 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=1164e90def6b4284b737a144248cdc75 status: fail success: false 2023-09-23T21:45:13.712 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T21:45:13.820 INFO:teuthology.run:FAIL